diff --git a/src/main/k8s/testing/secretfiles/metric_spec_config.textproto b/src/main/k8s/testing/secretfiles/metric_spec_config.textproto index a76b4f6044a..659c968d82f 100644 --- a/src/main/k8s/testing/secretfiles/metric_spec_config.textproto +++ b/src/main/k8s/testing/secretfiles/metric_spec_config.textproto @@ -1,50 +1,95 @@ # proto-file: wfa/measurement/config/reporting/metric_spec_config.proto # proto-message: MetricSpecConfig reach_params { - privacy_params { - epsilon: 0.0041 - delta: 1.0E-12 + multiple_data_provider_params { + privacy_params { + epsilon: 0.0041 + delta: 1.0E-12 + } + vid_sampling_interval { + fixed_start { + start: 0.00 + width: 0.01 + } + } + } + single_data_provider_params { + privacy_params { + epsilon: 0.0041 + delta: 1.0E-12 + } + vid_sampling_interval { + fixed_start { + start: 0.00 + width: 0.01 + } + } } } reach_and_frequency_params { - reach_privacy_params { - epsilon: 0.0033 - delta: 1.0E-12 + multiple_data_provider_params { + privacy_params { + epsilon: 0.0033 + delta: 1.0E-12 + } + frequency_privacy_params { + epsilon: 0.115 + delta: 1.0E-12 + } + vid_sampling_interval { + fixed_start { + start: 0.16 + width: 0.016666668 + } + } } - frequency_privacy_params { - epsilon: 0.115 - delta: 1.0E-12 + single_data_provider_params { + privacy_params { + epsilon: 0.0033 + delta: 1.0E-12 + } + frequency_privacy_params { + epsilon: 0.115 + delta: 1.0E-12 + } + vid_sampling_interval { + fixed_start { + start: 0.16 + width: 0.016666668 + } + } } maximum_frequency: 10 } impression_count_params { - privacy_params { - epsilon: 0.0011 - delta: 1.0E-12 + params { + privacy_params { + epsilon: 0.0011 + delta: 1.0E-12 + } + vid_sampling_interval { + fixed_start { + start: 0.47666666 + width: 0.20666666 + } + } } maximum_frequency_per_user: 60 } watch_duration_params { - privacy_params { - epsilon: 0.001 - delta: 1.0E-12 + params { + privacy_params { + epsilon: 0.001 + delta: 1.0E-12 + } + vid_sampling_interval { + fixed_start { + start: 0.68333334 + width: 0.31666666 + } + } } maximum_watch_duration_per_user { seconds: 4000 } } -reach_vid_sampling_interval { - width: 0.01 -} -reach_and_frequency_vid_sampling_interval { - start: 0.16 - width: 0.016666668 -} -impression_count_vid_sampling_interval { - start: 0.47666666 - width: 0.20666666 -} -watch_duration_vid_sampling_interval { - start: 0.68333334 - width: 0.31666666 -} diff --git a/src/main/kotlin/org/wfanet/measurement/integration/common/reporting/v2/InProcessLifeOfAReportIntegrationTest.kt b/src/main/kotlin/org/wfanet/measurement/integration/common/reporting/v2/InProcessLifeOfAReportIntegrationTest.kt index 1d38b1e5a88..289c3d6a299 100644 --- a/src/main/kotlin/org/wfanet/measurement/integration/common/reporting/v2/InProcessLifeOfAReportIntegrationTest.kt +++ b/src/main/kotlin/org/wfanet/measurement/integration/common/reporting/v2/InProcessLifeOfAReportIntegrationTest.kt @@ -49,6 +49,8 @@ import org.wfanet.measurement.api.v2alpha.MeasurementKt import org.wfanet.measurement.api.v2alpha.RequisitionSpecKt import org.wfanet.measurement.api.v2alpha.batchGetEventGroupMetadataDescriptorsRequest import org.wfanet.measurement.api.v2alpha.eventGroup as cmmsEventGroup +import java.security.SecureRandom +import kotlin.random.asKotlinRandom import org.wfanet.measurement.api.v2alpha.event_templates.testing.Person import org.wfanet.measurement.api.v2alpha.getDataProviderRequest import org.wfanet.measurement.api.v2alpha.getMeasurementConsumerRequest @@ -142,6 +144,8 @@ abstract class InProcessLifeOfAReportIntegrationTest( abstract val internalReportingServerServices: InternalReportingServer.Services + private val secureRandom = SecureRandom().asKotlinRandom() + private val reportingServerRule = object : TestRule { lateinit var reportingServer: InProcessReportingServer @@ -329,7 +333,7 @@ abstract class InProcessLifeOfAReportIntegrationTest( reach = MetricSpecKt.reachParams { privacyParams = DP_PARAMS } vidSamplingInterval = VID_SAMPLING_INTERVAL } - .withDefaults(reportingServer.metricSpecConfig) + .withDefaults(reportingServer.metricSpecConfig, secureRandom) } metricCalculationSpecId = "fed" } @@ -453,7 +457,7 @@ abstract class InProcessLifeOfAReportIntegrationTest( reach = MetricSpecKt.reachParams { privacyParams = DP_PARAMS } vidSamplingInterval = VID_SAMPLING_INTERVAL } - .withDefaults(reportingServer.metricSpecConfig) + .withDefaults(reportingServer.metricSpecConfig, secureRandom) } metricCalculationSpecId = "fed" } @@ -566,7 +570,7 @@ abstract class InProcessLifeOfAReportIntegrationTest( reach = MetricSpecKt.reachParams { privacyParams = DP_PARAMS } vidSamplingInterval = VID_SAMPLING_INTERVAL } - .withDefaults(reportingServer.metricSpecConfig) + .withDefaults(reportingServer.metricSpecConfig, secureRandom) } metricCalculationSpecId = "fed" } @@ -645,7 +649,7 @@ abstract class InProcessLifeOfAReportIntegrationTest( reach = MetricSpecKt.reachParams { privacyParams = DP_PARAMS } vidSamplingInterval = VID_SAMPLING_INTERVAL } - .withDefaults(reportingServer.metricSpecConfig) + .withDefaults(reportingServer.metricSpecConfig, secureRandom) } metricCalculationSpecId = "fed" } @@ -732,7 +736,7 @@ abstract class InProcessLifeOfAReportIntegrationTest( reach = MetricSpecKt.reachParams { privacyParams = DP_PARAMS } vidSamplingInterval = VID_SAMPLING_INTERVAL } - .withDefaults(reportingServer.metricSpecConfig) + .withDefaults(reportingServer.metricSpecConfig, secureRandom) } metricCalculationSpecId = "fed" } @@ -811,7 +815,7 @@ abstract class InProcessLifeOfAReportIntegrationTest( reach = MetricSpecKt.reachParams { privacyParams = DP_PARAMS } vidSamplingInterval = VID_SAMPLING_INTERVAL } - .withDefaults(reportingServer.metricSpecConfig) + .withDefaults(reportingServer.metricSpecConfig, secureRandom) metricFrequencySpec = MetricCalculationSpecKt.metricFrequencySpec { daily = MetricCalculationSpec.MetricFrequencySpec.Daily.getDefaultInstance() @@ -913,7 +917,7 @@ abstract class InProcessLifeOfAReportIntegrationTest( reach = MetricSpecKt.reachParams { privacyParams = DP_PARAMS } vidSamplingInterval = VID_SAMPLING_INTERVAL } - .withDefaults(reportingServer.metricSpecConfig) + .withDefaults(reportingServer.metricSpecConfig, secureRandom) groupings += MetricCalculationSpecKt.grouping { predicates += grouping1Predicate1 @@ -1005,7 +1009,7 @@ abstract class InProcessLifeOfAReportIntegrationTest( privacyParams = MetricSpecKt.differentialPrivacyParams {} } } - .withDefaults(reportingServer.metricSpecConfig) + .withDefaults(reportingServer.metricSpecConfig, secureRandom) } metricCalculationSpecId = "fed" } @@ -1088,7 +1092,7 @@ abstract class InProcessLifeOfAReportIntegrationTest( reach = MetricSpecKt.reachParams { privacyParams = DP_PARAMS } vidSamplingInterval = VID_SAMPLING_INTERVAL } - .withDefaults(reportingServer.metricSpecConfig) + .withDefaults(reportingServer.metricSpecConfig, secureRandom) } val createdMetric = @@ -1119,6 +1123,63 @@ abstract class InProcessLifeOfAReportIntegrationTest( assertThat(actualResult).reachValue().isWithin(tolerance).of(expectedResult.reach.value) } + @Test + fun `reach metric with single edp params result has the expected result`() = runBlocking { + val measurementConsumerData = inProcessCmmsComponents.getMeasurementConsumerData() + val eventGroups = listEventGroups() + val eventGroup = eventGroups.first() + val eventGroupEntries: List> = + listOf(eventGroup to "person.age_group == ${Person.AgeGroup.YEARS_18_TO_34_VALUE}") + val createdPrimitiveReportingSet: ReportingSet = + createPrimitiveReportingSets(eventGroupEntries, measurementConsumerData.name).single() + + val metric = metric { + reportingSet = createdPrimitiveReportingSet.name + timeInterval = EVENT_RANGE.toInterval() + metricSpec = + metricSpec { + reach = MetricSpecKt.reachParams { + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = DP_PARAMS + vidSamplingInterval = VID_SAMPLING_INTERVAL + } + singleDataProviderParams = MetricSpecKt.params { + privacyParams = SINGLE_DATA_PROVIDER_DP_PARAMS + vidSamplingInterval = SINGLE_DATA_PROVIDER_VID_SAMPLING_INTERVAL + } + } + } + .withDefaults(reportingServer.metricSpecConfig, secureRandom) + } + + val createdMetric = + publicMetricsClient + .withPrincipalName(measurementConsumerData.name) + .createMetric( + createMetricRequest { + parent = measurementConsumerData.name + this.metric = metric + metricId = "abc" + } + ) + + val retrievedMetric = pollForCompletedMetric(measurementConsumerData.name, createdMetric.name) + assertThat(retrievedMetric.state).isEqualTo(Metric.State.SUCCEEDED) + + val eventGroupSpecs: Iterable = + eventGroupEntries.map { (eventGroup, filter) -> + buildEventGroupSpec(eventGroup, filter, EVENT_RANGE.toInterval()) + } + val sampledVids = sampleVids(eventGroupSpecs, metric.metricSpec.reach.singleDataProviderParams.vidSamplingInterval) + val expectedResult = calculateExpectedReachMeasurementResult(sampledVids) + + val reachResult = retrievedMetric.result.reach + val actualResult = + MeasurementKt.result { reach = MeasurementKt.ResultKt.reach { value = reachResult.value } } + val tolerance = computeErrorMargin(reachResult.univariateStatistics.standardDeviation) + assertThat(actualResult).reachValue().isWithin(tolerance).of(expectedResult.reach.value) + } + @Test fun `reach-and-frequency metric has the expected result`() = runBlocking { val measurementConsumerData = inProcessCmmsComponents.getMeasurementConsumerData() @@ -1142,7 +1203,7 @@ abstract class InProcessLifeOfAReportIntegrationTest( } vidSamplingInterval = VID_SAMPLING_INTERVAL } - .withDefaults(reportingServer.metricSpecConfig) + .withDefaults(reportingServer.metricSpecConfig, secureRandom) } val createdMetric = @@ -1215,7 +1276,7 @@ abstract class InProcessLifeOfAReportIntegrationTest( impressionCount = MetricSpecKt.impressionCountParams { privacyParams = DP_PARAMS } vidSamplingInterval = VID_SAMPLING_INTERVAL } - .withDefaults(reportingServer.metricSpecConfig) + .withDefaults(reportingServer.metricSpecConfig, secureRandom) } val createdMetric = @@ -1274,7 +1335,7 @@ abstract class InProcessLifeOfAReportIntegrationTest( watchDuration = MetricSpecKt.watchDurationParams { privacyParams = DP_PARAMS } vidSamplingInterval = VID_SAMPLING_INTERVAL } - .withDefaults(reportingServer.metricSpecConfig) + .withDefaults(reportingServer.metricSpecConfig, secureRandom) } val createdMetric = @@ -1312,7 +1373,7 @@ abstract class InProcessLifeOfAReportIntegrationTest( reach = MetricSpecKt.reachParams { privacyParams = DP_PARAMS } vidSamplingInterval = VID_SAMPLING_INTERVAL } - .withDefaults(reportingServer.metricSpecConfig) + .withDefaults(reportingServer.metricSpecConfig, secureRandom) filters += "person.gender == ${Person.Gender.MALE_VALUE}" } @@ -1367,7 +1428,7 @@ abstract class InProcessLifeOfAReportIntegrationTest( reach = MetricSpecKt.reachParams { privacyParams = MetricSpecKt.differentialPrivacyParams {} } } - .withDefaults(reportingServer.metricSpecConfig) + .withDefaults(reportingServer.metricSpecConfig, secureRandom) } val deferred: MutableList> = mutableListOf() @@ -1736,12 +1797,24 @@ abstract class InProcessLifeOfAReportIntegrationTest( delta = 1e-15 } + private val SINGLE_DATA_PROVIDER_DP_PARAMS = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 1.01e-15 + } + private val VID_SAMPLING_INTERVAL = MetricSpecKt.vidSamplingInterval { start = 0.0f width = 1.0f } + private val SINGLE_DATA_PROVIDER_VID_SAMPLING_INTERVAL = + MetricSpecKt.vidSamplingInterval { + start = 0.0f + width = 0.9f + } + // For a 99.9% Confidence Interval. private const val CONFIDENCE_INTERVAL_MULTIPLIER = 3.291 diff --git a/src/main/kotlin/org/wfanet/measurement/integration/common/reporting/v2/InProcessReportingServer.kt b/src/main/kotlin/org/wfanet/measurement/integration/common/reporting/v2/InProcessReportingServer.kt index d7700a9753c..af4c7ab80a0 100644 --- a/src/main/kotlin/org/wfanet/measurement/integration/common/reporting/v2/InProcessReportingServer.kt +++ b/src/main/kotlin/org/wfanet/measurement/integration/common/reporting/v2/InProcessReportingServer.kt @@ -197,7 +197,8 @@ class InProcessReportingServer( celEnvCacheProvider, ) .withMetadataPrincipalIdentities(measurementConsumerConfigs), - MetricCalculationSpecsService(internalMetricCalculationSpecsClient, METRIC_SPEC_CONFIG) + MetricCalculationSpecsService(internalMetricCalculationSpecsClient, METRIC_SPEC_CONFIG, SecureRandom().asKotlinRandom(), + ) .withMetadataPrincipalIdentities(measurementConsumerConfigs), MetricsService( METRIC_SPEC_CONFIG, @@ -226,7 +227,8 @@ class InProcessReportingServer( internalMetricCalculationSpecsClient, PublicMetricsCoroutineStub(this@GrpcTestServerRule.channel), METRIC_SPEC_CONFIG, - ) + SecureRandom().asKotlinRandom(), + ) .withMetadataPrincipalIdentities(measurementConsumerConfigs), ) .forEach { addService(it.withVerboseLogging(verboseGrpcLogging)) } @@ -250,67 +252,79 @@ class InProcessReportingServer( private val METRIC_SPEC_CONFIG = metricSpecConfig { reachParams = MetricSpecConfigKt.reachParams { - privacyParams = - MetricSpecConfigKt.differentialPrivacyParams { - epsilon = 0.0041 - delta = 1e-12 + multipleDataProviderParams = MetricSpecConfigKt.params { + privacyParams = + MetricSpecConfigKt.differentialPrivacyParams { + epsilon = 0.0041 + delta = 1e-12 + } + vidSamplingInterval = MetricSpecConfigKt.vidSamplingInterval { + fixedStart = MetricSpecConfigKt.VidSamplingIntervalKt.fixedStart { + start = 0.0f + width = 3.0f / NUMBER_VID_BUCKETS + } } - } - reachVidSamplingInterval = - MetricSpecConfigKt.vidSamplingInterval { - start = 0.0f - width = 3.0f / NUMBER_VID_BUCKETS + } } reachAndFrequencyParams = MetricSpecConfigKt.reachAndFrequencyParams { - reachPrivacyParams = - MetricSpecConfigKt.differentialPrivacyParams { - epsilon = 0.0033 - delta = 1e-12 - } - frequencyPrivacyParams = - MetricSpecConfigKt.differentialPrivacyParams { - epsilon = 0.0033 - delta = 1e-12 + multipleDataProviderParams = MetricSpecConfigKt.params { + privacyParams = + MetricSpecConfigKt.differentialPrivacyParams { + epsilon = 0.0033 + delta = 1e-12 + } + frequencyPrivacyParams = + MetricSpecConfigKt.differentialPrivacyParams { + epsilon = 0.0033 + delta = 1e-12 + } + vidSamplingInterval = MetricSpecConfigKt.vidSamplingInterval { + fixedStart = MetricSpecConfigKt.VidSamplingIntervalKt.fixedStart { + start = 48.0f / NUMBER_VID_BUCKETS + width = 5.0f / NUMBER_VID_BUCKETS + } } + } maximumFrequency = 10 } - reachAndFrequencyVidSamplingInterval = - MetricSpecConfigKt.vidSamplingInterval { - start = 48.0f / NUMBER_VID_BUCKETS - width = 5.0f / NUMBER_VID_BUCKETS - } impressionCountParams = MetricSpecConfigKt.impressionCountParams { - privacyParams = - MetricSpecConfigKt.differentialPrivacyParams { - epsilon = 0.0011 - delta = 1e-12 + params = MetricSpecConfigKt.params { + privacyParams = + MetricSpecConfigKt.differentialPrivacyParams { + epsilon = 0.0011 + delta = 1e-12 + } + vidSamplingInterval = MetricSpecConfigKt.vidSamplingInterval { + fixedStart = MetricSpecConfigKt.VidSamplingIntervalKt.fixedStart { + start = 143.0f / NUMBER_VID_BUCKETS + width = 62.0f / NUMBER_VID_BUCKETS + } } + } maximumFrequencyPerUser = 60 } - impressionCountVidSamplingInterval = - MetricSpecConfigKt.vidSamplingInterval { - start = 143.0f / NUMBER_VID_BUCKETS - width = 62.0f / NUMBER_VID_BUCKETS - } watchDurationParams = MetricSpecConfigKt.watchDurationParams { - privacyParams = - MetricSpecConfigKt.differentialPrivacyParams { - epsilon = 0.001 - delta = 1e-12 + params = MetricSpecConfigKt.params { + privacyParams = + MetricSpecConfigKt.differentialPrivacyParams { + epsilon = 0.001 + delta = 1e-12 + } + vidSamplingInterval = MetricSpecConfigKt.vidSamplingInterval { + fixedStart = MetricSpecConfigKt.VidSamplingIntervalKt.fixedStart { + start = 205.0f / NUMBER_VID_BUCKETS + width = 95.0f / NUMBER_VID_BUCKETS + } } + } maximumWatchDurationPerUser = Durations.fromSeconds(4000) } - watchDurationVidSamplingInterval = - MetricSpecConfigKt.vidSamplingInterval { - start = 205.0f / NUMBER_VID_BUCKETS - width = 95.0f / NUMBER_VID_BUCKETS - } } } } diff --git a/src/main/kotlin/org/wfanet/measurement/reporting/deploy/v2/common/job/ReportSchedulingJobExecutor.kt b/src/main/kotlin/org/wfanet/measurement/reporting/deploy/v2/common/job/ReportSchedulingJobExecutor.kt index c0fe4e650e4..6140029c3a3 100644 --- a/src/main/kotlin/org/wfanet/measurement/reporting/deploy/v2/common/job/ReportSchedulingJobExecutor.kt +++ b/src/main/kotlin/org/wfanet/measurement/reporting/deploy/v2/common/job/ReportSchedulingJobExecutor.kt @@ -148,7 +148,8 @@ private fun run( InternalMetricCalculationSpecsCoroutineStub(channel), MetricsCoroutineStub(inProcessMetricsChannel), metricSpecConfig, - ) + SecureRandom().asKotlinRandom(), + ) val inProcessReportsServerName = InProcessServerBuilder.generateName() val inProcessReportsServer: Server = diff --git a/src/main/kotlin/org/wfanet/measurement/reporting/deploy/v2/common/server/V2AlphaPublicApiServer.kt b/src/main/kotlin/org/wfanet/measurement/reporting/deploy/v2/common/server/V2AlphaPublicApiServer.kt index 423a8e2f0a8..86235785015 100644 --- a/src/main/kotlin/org/wfanet/measurement/reporting/deploy/v2/common/server/V2AlphaPublicApiServer.kt +++ b/src/main/kotlin/org/wfanet/measurement/reporting/deploy/v2/common/server/V2AlphaPublicApiServer.kt @@ -244,7 +244,8 @@ private fun run( InternalMetricCalculationSpecsCoroutineStub(channel), MetricsCoroutineStub(inProcessChannel), metricSpecConfig, - ) + SecureRandom().asKotlinRandom(), + ) .withPrincipalsFromX509AuthorityKeyIdentifiers(principalLookup), ReportSchedulesService( InternalReportSchedulesCoroutineStub(channel), @@ -258,7 +259,8 @@ private fun run( MetricCalculationSpecsService( InternalMetricCalculationSpecsCoroutineStub(channel), metricSpecConfig, - ) + SecureRandom().asKotlinRandom(), + ) .withPrincipalsFromX509AuthorityKeyIdentifiers(principalLookup), ) CommonServer.fromFlags(commonServerFlags, SERVER_NAME, services).start().blockUntilShutdown() diff --git a/src/main/kotlin/org/wfanet/measurement/reporting/deploy/v2/postgres/PostgresMetricsService.kt b/src/main/kotlin/org/wfanet/measurement/reporting/deploy/v2/postgres/PostgresMetricsService.kt index 02eb5acfe00..deeb23e517b 100644 --- a/src/main/kotlin/org/wfanet/measurement/reporting/deploy/v2/postgres/PostgresMetricsService.kt +++ b/src/main/kotlin/org/wfanet/measurement/reporting/deploy/v2/postgres/PostgresMetricsService.kt @@ -60,12 +60,6 @@ class PostgresMetricsService( "Metric Spec missing type." } - if (!request.metric.metricSpec.hasPopulationCount()) { - grpcRequire(request.metric.metricSpec.hasVidSamplingInterval()) { - "Metric Spec missing vid sampling interval." - } - } - grpcRequire(request.metric.weightedMeasurementsCount > 0) { "Metric missing weighted measurements." } @@ -97,10 +91,6 @@ class PostgresMetricsService( "Metric Spec missing type." } - grpcRequire(it.metric.metricSpec.hasVidSamplingInterval()) { - "Metric Spec missing vid sampling interval." - } - grpcRequire(it.metric.weightedMeasurementsCount > 0) { "Metric missing weighted measurements." } diff --git a/src/main/kotlin/org/wfanet/measurement/reporting/deploy/v2/postgres/readers/MetricReader.kt b/src/main/kotlin/org/wfanet/measurement/reporting/deploy/v2/postgres/readers/MetricReader.kt index 78b44bbe304..403d762bb04 100644 --- a/src/main/kotlin/org/wfanet/measurement/reporting/deploy/v2/postgres/readers/MetricReader.kt +++ b/src/main/kotlin/org/wfanet/measurement/reporting/deploy/v2/postgres/readers/MetricReader.kt @@ -41,6 +41,7 @@ import org.wfanet.measurement.internal.reporting.v2.Measurement import org.wfanet.measurement.internal.reporting.v2.Metric import org.wfanet.measurement.internal.reporting.v2.MetricKt import org.wfanet.measurement.internal.reporting.v2.MetricSpec +import org.wfanet.measurement.internal.reporting.v2.MetricSpec.VidSamplingInterval import org.wfanet.measurement.internal.reporting.v2.MetricSpecKt import org.wfanet.measurement.internal.reporting.v2.ReportingSetKt import org.wfanet.measurement.internal.reporting.v2.StreamMetricsRequest @@ -107,6 +108,7 @@ class MetricReader(private val readContext: ReadContext) { val primitiveReportingSetBasisInfoMap: MutableMap, val state: Measurement.State, val details: Measurement.Details, + val isSingleDataProvider: Boolean, ) private data class PrimitiveReportingSetBasisInfo( @@ -135,6 +137,12 @@ class MetricReader(private val readContext: ReadContext) { Metrics.MaximumWatchDurationPerUser, Metrics.VidSamplingIntervalStart, Metrics.VidSamplingIntervalWidth, + Metrics.SingleDataProviderDifferentialPrivacyEpsilon, + Metrics.SingleDataProviderDifferentialPrivacyDelta, + Metrics.SingleDataProviderFrequencyDifferentialPrivacyEpsilon, + Metrics.SingleDataProviderFrequencyDifferentialPrivacyDelta, + Metrics.SingleDataProviderVidSamplingIntervalStart, + Metrics.SingleDataProviderVidSamplingIntervalWidth, Metrics.CreateTime, Metrics.MetricDetails, Metrics.State as MetricsState, @@ -147,6 +155,7 @@ class MetricReader(private val readContext: ReadContext) { Measurements.TimeIntervalEndExclusive AS MeasurementsTimeIntervalEndExclusive, Measurements.State as MeasurementsState, Measurements.MeasurementDetails, + Measurements.IsSingleDataProvider, PrimitiveReportingSetBases.PrimitiveReportingSetBasisId, PrimitiveReportingSets.ExternalReportingSetId AS PrimitiveExternalReportingSetId, PrimitiveReportingSetBasisFilters.Filter AS PrimitiveReportingSetBasisFilter @@ -297,6 +306,12 @@ class MetricReader(private val readContext: ReadContext) { Metrics.MaximumWatchDurationPerUser, Metrics.VidSamplingIntervalStart, Metrics.VidSamplingIntervalWidth, + Metrics.SingleDataProviderDifferentialPrivacyEpsilon, + Metrics.SingleDataProviderDifferentialPrivacyDelta, + Metrics.SingleDataProviderFrequencyDifferentialPrivacyEpsilon, + Metrics.SingleDataProviderFrequencyDifferentialPrivacyDelta, + Metrics.SingleDataProviderVidSamplingIntervalStart, + Metrics.SingleDataProviderVidSamplingIntervalWidth, Metrics.MetricDetails, Metrics.State, Metrics.CreateTime @@ -382,6 +397,12 @@ class MetricReader(private val readContext: ReadContext) { val maximumWatchDurationPerUser: PostgresInterval? = row["MaximumWatchDurationPerUser"] val vidSamplingStart: Float = row["VidSamplingIntervalStart"] val vidSamplingWidth: Float = row["VidSamplingIntervalWidth"] + val singleDataProviderDifferentialPrivacyEpsilon: Double? = row["SingleDataProviderDifferentialPrivacyEpsilon"] + val singleDataProviderDifferentialPrivacyDelta: Double? = row["SingleDataProviderDifferentialPrivacyDelta"] + val singleDataProviderFrequencyDifferentialPrivacyEpsilon: Double? = row["SingleDataProviderFrequencyDifferentialPrivacyEpsilon"] + val singleDataProviderFrequencyDifferentialPrivacyDelta: Double? = row["SingleDataProviderFrequencyDifferentialPrivacyDelta"] + val singleDataProviderVidSamplingStart: Float? = row["SingleDataProviderVidSamplingIntervalStart"] + val singleDataProviderVidSamplingWidth: Float? = row["SingleDataProviderVidSamplingIntervalWidth"] val metricDetails: Metric.Details = row.getProtoMessage("MetricDetails", Metric.Details.parser()) val createTime: Instant = row["CreateTime"] @@ -392,81 +413,24 @@ class MetricReader(private val readContext: ReadContext) { endTime = metricTimeIntervalEnd.toProtoTime() } - val vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = vidSamplingStart - width = vidSamplingWidth - } - - val metricSpec = metricSpec { - when (metricType) { - MetricSpec.TypeCase.REACH -> - reach = - MetricSpecKt.reachParams { - privacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = differentialPrivacyEpsilon - delta = differentialPrivacyDelta - } - } - MetricSpec.TypeCase.REACH_AND_FREQUENCY -> { - if ( - frequencyDifferentialPrivacyDelta == null || - frequencyDifferentialPrivacyEpsilon == null || - maximumFrequency == null - ) { - throw IllegalStateException() - } - - reachAndFrequency = - MetricSpecKt.reachAndFrequencyParams { - reachPrivacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = differentialPrivacyEpsilon - delta = differentialPrivacyDelta - } - frequencyPrivacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = frequencyDifferentialPrivacyEpsilon - delta = frequencyDifferentialPrivacyDelta - } - this.maximumFrequency = maximumFrequency - } - } - MetricSpec.TypeCase.IMPRESSION_COUNT -> { - if (maximumFrequencyPerUser == null) { - throw IllegalStateException() - } - - impressionCount = - MetricSpecKt.impressionCountParams { - privacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = differentialPrivacyEpsilon - delta = differentialPrivacyDelta - } - this.maximumFrequencyPerUser = maximumFrequencyPerUser - } - } - MetricSpec.TypeCase.WATCH_DURATION -> { - watchDuration = - MetricSpecKt.watchDurationParams { - privacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = differentialPrivacyEpsilon - delta = differentialPrivacyDelta - } - this.maximumWatchDurationPerUser = - checkNotNull(maximumWatchDurationPerUser).duration.toProtoDuration() - } - } - MetricSpec.TypeCase.POPULATION_COUNT -> { - populationCount = MetricSpec.PopulationCountParams.getDefaultInstance() - } - MetricSpec.TypeCase.TYPE_NOT_SET -> throw IllegalStateException() - } - this.vidSamplingInterval = vidSamplingInterval - } + val metricSpec = buildMetricSpec( + metricType = metricType, + differentialPrivacyEpsilon = differentialPrivacyEpsilon, + differentialPrivacyDelta = differentialPrivacyDelta, + frequencyDifferentialPrivacyEpsilon = frequencyDifferentialPrivacyEpsilon, + frequencyDifferentialPrivacyDelta = frequencyDifferentialPrivacyDelta, + maximumFrequency = maximumFrequency, + maximumFrequencyPerUser = maximumFrequencyPerUser, + maximumWatchDurationPerUser = maximumWatchDurationPerUser, + vidSamplingStart = vidSamplingStart, + vidSamplingWidth = vidSamplingWidth, + singleDataProviderDifferentialPrivacyEpsilon = singleDataProviderDifferentialPrivacyEpsilon, + singleDataProviderDifferentialPrivacyDelta = singleDataProviderDifferentialPrivacyDelta, + singleDataProviderFrequencyDifferentialPrivacyEpsilon = singleDataProviderFrequencyDifferentialPrivacyEpsilon, + singleDataProviderFrequencyDifferentialPrivacyDelta = singleDataProviderFrequencyDifferentialPrivacyDelta, + singleDataProviderVidSamplingStart = singleDataProviderVidSamplingStart, + singleDataProviderVidSamplingWidth = singleDataProviderVidSamplingWidth + ) ReportingMetric( reportingMetricKey = @@ -624,6 +588,7 @@ class MetricReader(private val readContext: ReadContext) { if (it.measurementInfo.details != Measurement.Details.getDefaultInstance()) { details = it.measurementInfo.details } + isSingleDataProvider = it.measurementInfo.isSingleDataProvider } } } @@ -658,6 +623,12 @@ class MetricReader(private val readContext: ReadContext) { val maximumWatchDurationPerUser: PostgresInterval? = row["MaximumWatchDurationPerUser"] val vidSamplingStart: Float = row["VidSamplingIntervalStart"] val vidSamplingWidth: Float = row["VidSamplingIntervalWidth"] + val singleDataProviderDifferentialPrivacyEpsilon: Double? = row["SingleDataProviderDifferentialPrivacyEpsilon"] + val singleDataProviderDifferentialPrivacyDelta: Double? = row["SingleDataProviderDifferentialPrivacyDelta"] + val singleDataProviderFrequencyDifferentialPrivacyEpsilon: Double? = row["SingleDataProviderFrequencyDifferentialPrivacyEpsilon"] + val singleDataProviderFrequencyDifferentialPrivacyDelta: Double? = row["SingleDataProviderFrequencyDifferentialPrivacyDelta"] + val singleDataProviderVidSamplingStart: Float? = row["SingleDataProviderVidSamplingIntervalStart"] + val singleDataProviderVidSamplingWidth: Float? = row["SingleDataProviderVidSamplingIntervalWidth"] val createTime: Instant = row["CreateTime"] val metricDetails: Metric.Details = row.getProtoMessage("MetricDetails", Metric.Details.parser()) @@ -672,6 +643,7 @@ class MetricReader(private val readContext: ReadContext) { row.getProtoEnum("MeasurementsState", Measurement.State::forNumber) val measurementDetails: Measurement.Details = row.getProtoMessage("MeasurementDetails", Measurement.Details.parser()) + val isSingleDataProvider: Boolean = row["IsSingleDataProvider"] val primitiveReportingSetBasisId: InternalId = row["PrimitiveReportingSetBasisId"] val primitiveExternalReportingSetId: String = row["PrimitiveExternalReportingSetId"] val primitiveReportingSetBasisFilter: String? = row["PrimitiveReportingSetBasisFilter"] @@ -684,84 +656,24 @@ class MetricReader(private val readContext: ReadContext) { endTime = metricTimeIntervalEnd.toProtoTime() } - val vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = vidSamplingStart - width = vidSamplingWidth - } - - val metricSpec = metricSpec { - when (metricType) { - MetricSpec.TypeCase.REACH -> - reach = - MetricSpecKt.reachParams { - privacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = differentialPrivacyEpsilon - delta = differentialPrivacyDelta - } - } - MetricSpec.TypeCase.REACH_AND_FREQUENCY -> { - if ( - frequencyDifferentialPrivacyDelta == null || - frequencyDifferentialPrivacyEpsilon == null || - maximumFrequency == null - ) { - throw IllegalStateException() - } - - reachAndFrequency = - MetricSpecKt.reachAndFrequencyParams { - reachPrivacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = differentialPrivacyEpsilon - delta = differentialPrivacyDelta - } - frequencyPrivacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = frequencyDifferentialPrivacyEpsilon - delta = frequencyDifferentialPrivacyDelta - } - this.maximumFrequency = maximumFrequency - } - } - MetricSpec.TypeCase.IMPRESSION_COUNT -> { - if (maximumFrequencyPerUser == null) { - throw IllegalStateException() - } - - impressionCount = - MetricSpecKt.impressionCountParams { - privacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = differentialPrivacyEpsilon - delta = differentialPrivacyDelta - } - this.maximumFrequencyPerUser = maximumFrequencyPerUser - } - } - MetricSpec.TypeCase.WATCH_DURATION -> { - watchDuration = - MetricSpecKt.watchDurationParams { - privacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = differentialPrivacyEpsilon - delta = differentialPrivacyDelta - } - this.maximumWatchDurationPerUser = - checkNotNull(maximumWatchDurationPerUser).duration.toProtoDuration() - } - } - MetricSpec.TypeCase.POPULATION_COUNT -> { - populationCount = MetricSpec.PopulationCountParams.getDefaultInstance() - } - MetricSpec.TypeCase.TYPE_NOT_SET -> throw IllegalStateException() - } - // Population metric does not have a vidSamplingInterval - if (metricType != MetricSpec.TypeCase.POPULATION_COUNT) { - this.vidSamplingInterval = vidSamplingInterval - } - } + val metricSpec = buildMetricSpec( + metricType = metricType, + differentialPrivacyEpsilon = differentialPrivacyEpsilon, + differentialPrivacyDelta = differentialPrivacyDelta, + frequencyDifferentialPrivacyEpsilon = frequencyDifferentialPrivacyEpsilon, + frequencyDifferentialPrivacyDelta = frequencyDifferentialPrivacyDelta, + maximumFrequency = maximumFrequency, + maximumFrequencyPerUser = maximumFrequencyPerUser, + maximumWatchDurationPerUser = maximumWatchDurationPerUser, + vidSamplingStart = vidSamplingStart, + vidSamplingWidth = vidSamplingWidth, + singleDataProviderDifferentialPrivacyEpsilon = singleDataProviderDifferentialPrivacyEpsilon, + singleDataProviderDifferentialPrivacyDelta = singleDataProviderDifferentialPrivacyDelta, + singleDataProviderFrequencyDifferentialPrivacyEpsilon = singleDataProviderFrequencyDifferentialPrivacyEpsilon, + singleDataProviderFrequencyDifferentialPrivacyDelta = singleDataProviderFrequencyDifferentialPrivacyDelta, + singleDataProviderVidSamplingStart = singleDataProviderVidSamplingStart, + singleDataProviderVidSamplingWidth = singleDataProviderVidSamplingWidth + ) MetricInfo( measurementConsumerId = measurementConsumerId, @@ -800,6 +712,7 @@ class MetricReader(private val readContext: ReadContext) { state = measurementState, details = measurementDetails, primitiveReportingSetBasisInfoMap = mutableMapOf(), + isSingleDataProvider = isSingleDataProvider, ) WeightedMeasurementInfo( @@ -828,4 +741,147 @@ class MetricReader(private val readContext: ReadContext) { return metricInfoMap } + + /** Builds a [MetricSpec] givens all the necessary parameters. */ + private fun buildMetricSpec( + metricType: MetricSpec.TypeCase, + differentialPrivacyEpsilon: Double, + differentialPrivacyDelta: Double, + frequencyDifferentialPrivacyEpsilon: Double?, + frequencyDifferentialPrivacyDelta: Double?, + maximumFrequency: Int?, + maximumFrequencyPerUser: Int?, + maximumWatchDurationPerUser: PostgresInterval?, + vidSamplingStart: Float, + vidSamplingWidth: Float, + singleDataProviderDifferentialPrivacyEpsilon: Double?, + singleDataProviderDifferentialPrivacyDelta: Double?, + singleDataProviderFrequencyDifferentialPrivacyEpsilon: Double?, + singleDataProviderFrequencyDifferentialPrivacyDelta: Double?, + singleDataProviderVidSamplingStart: Float?, + singleDataProviderVidSamplingWidth: Float?, + ): MetricSpec { + val vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = vidSamplingStart + width = vidSamplingWidth + } + + val singleDataProviderVidSamplingInterval: VidSamplingInterval? = + if (singleDataProviderVidSamplingStart != null && singleDataProviderVidSamplingWidth != null) { + MetricSpecKt.vidSamplingInterval { + start = singleDataProviderVidSamplingStart + width = singleDataProviderVidSamplingWidth + } + } else { + null + } + + return metricSpec { + when (metricType) { + MetricSpec.TypeCase.REACH -> + reach = + MetricSpecKt.reachParams { + multipleDataProviderParams = + MetricSpecKt.params { + privacyParams = MetricSpecKt.differentialPrivacyParams { + epsilon = differentialPrivacyEpsilon + delta = differentialPrivacyDelta + } + this.vidSamplingInterval = vidSamplingInterval + } + if (singleDataProviderVidSamplingInterval != null && singleDataProviderDifferentialPrivacyEpsilon != null && singleDataProviderDifferentialPrivacyDelta != null) { + singleDataProviderParams = + MetricSpecKt.params { + privacyParams = MetricSpecKt.differentialPrivacyParams { + epsilon = singleDataProviderDifferentialPrivacyEpsilon + delta = singleDataProviderDifferentialPrivacyDelta + } + this.vidSamplingInterval = singleDataProviderVidSamplingInterval + } + } + } + MetricSpec.TypeCase.REACH_AND_FREQUENCY -> { + if ( + frequencyDifferentialPrivacyDelta == null || + frequencyDifferentialPrivacyEpsilon == null || + maximumFrequency == null + ) { + throw IllegalStateException() + } + + reachAndFrequency = + MetricSpecKt.reachAndFrequencyParams { + multipleDataProviderParams = + MetricSpecKt.params { + privacyParams = MetricSpecKt.differentialPrivacyParams { + epsilon = differentialPrivacyEpsilon + delta = differentialPrivacyDelta + } + frequencyPrivacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = frequencyDifferentialPrivacyEpsilon + delta = frequencyDifferentialPrivacyDelta + } + this.vidSamplingInterval = vidSamplingInterval + } + if (singleDataProviderVidSamplingInterval != null + && singleDataProviderDifferentialPrivacyEpsilon != null + && singleDataProviderDifferentialPrivacyDelta != null + && singleDataProviderFrequencyDifferentialPrivacyEpsilon != null + && singleDataProviderFrequencyDifferentialPrivacyDelta != null) { + singleDataProviderParams = + MetricSpecKt.params { + privacyParams = MetricSpecKt.differentialPrivacyParams { + epsilon = singleDataProviderDifferentialPrivacyEpsilon + delta = singleDataProviderDifferentialPrivacyDelta + } + frequencyPrivacyParams = MetricSpecKt.differentialPrivacyParams { + epsilon = singleDataProviderFrequencyDifferentialPrivacyEpsilon + delta = singleDataProviderFrequencyDifferentialPrivacyDelta + } + this.vidSamplingInterval = singleDataProviderVidSamplingInterval + } + } + this.maximumFrequency = maximumFrequency + } + } + MetricSpec.TypeCase.IMPRESSION_COUNT -> { + if (maximumFrequencyPerUser == null) { + throw IllegalStateException() + } + + impressionCount = + MetricSpecKt.impressionCountParams { + params = MetricSpecKt.params { + privacyParams = MetricSpecKt.differentialPrivacyParams { + epsilon = differentialPrivacyEpsilon + delta = differentialPrivacyDelta + } + this.vidSamplingInterval = vidSamplingInterval + } + this.maximumFrequencyPerUser = maximumFrequencyPerUser + } + } + MetricSpec.TypeCase.WATCH_DURATION -> { + watchDuration = + MetricSpecKt.watchDurationParams { + params = MetricSpecKt.params { + privacyParams = MetricSpecKt.differentialPrivacyParams { + epsilon = differentialPrivacyEpsilon + delta = differentialPrivacyDelta + } + this.vidSamplingInterval = vidSamplingInterval + } + this.maximumWatchDurationPerUser = + checkNotNull(maximumWatchDurationPerUser).duration.toProtoDuration() + } + } + MetricSpec.TypeCase.POPULATION_COUNT -> { + populationCount = MetricSpec.PopulationCountParams.getDefaultInstance() + } + MetricSpec.TypeCase.TYPE_NOT_SET -> throw IllegalStateException() + } + } + } } diff --git a/src/main/kotlin/org/wfanet/measurement/reporting/deploy/v2/postgres/writers/CreateMetrics.kt b/src/main/kotlin/org/wfanet/measurement/reporting/deploy/v2/postgres/writers/CreateMetrics.kt index bede5115af2..ea6620994f2 100644 --- a/src/main/kotlin/org/wfanet/measurement/reporting/deploy/v2/postgres/writers/CreateMetrics.kt +++ b/src/main/kotlin/org/wfanet/measurement/reporting/deploy/v2/postgres/writers/CreateMetrics.kt @@ -78,6 +78,7 @@ class CreateMetrics(private val requests: List) : val cmmsCreateMeasurementRequestId: UUID, val timeIntervalStart: OffsetDateTime, val timeIntervalEndExclusive: OffsetDateTime, + val isSingleDataProvider: Boolean, ) private data class MetricMeasurementsValues( @@ -193,7 +194,7 @@ class CreateMetrics(private val requests: List) : val statement = valuesListBoundStatement( valuesStartIndex = 0, - paramCount = 21, + paramCount = 27, """ INSERT INTO Metrics ( @@ -217,7 +218,13 @@ class CreateMetrics(private val requests: List) : CreateTime, MetricDetails, MetricDetailsJson, - State + State, + SingleDataProviderDifferentialPrivacyEpsilon, + SingleDataProviderDifferentialPrivacyDelta, + SingleDataProviderFrequencyDifferentialPrivacyEpsilon, + SingleDataProviderFrequencyDifferentialPrivacyDelta, + SingleDataProviderVidSamplingIntervalStart, + SingleDataProviderVidSamplingIntervalWidth ) VALUES ${ValuesListBoundStatement.VALUES_LIST_PLACEHOLDER} """, @@ -231,12 +238,6 @@ class CreateMetrics(private val requests: List) : val metricId = idGenerator.generateInternalId() val externalMetricId: String = it.externalMetricId val reportingSetId: InternalId? = reportingSetMap[it.metric.externalReportingSetId] - val vidSamplingIntervalStart = - if (it.metric.metricSpec.typeCase == MetricSpec.TypeCase.POPULATION_COUNT) 0 - else it.metric.metricSpec.vidSamplingInterval.start - val vidSamplingIntervalWidth = - if (it.metric.metricSpec.typeCase == MetricSpec.TypeCase.POPULATION_COUNT) 0 - else it.metric.metricSpec.vidSamplingInterval.width addValuesBinding { bindValuesParam(0, measurementConsumerId) @@ -261,38 +262,78 @@ class CreateMetrics(private val requests: List) : when (it.metric.metricSpec.typeCase) { MetricSpec.TypeCase.REACH_AND_FREQUENCY -> { val reachAndFrequency = it.metric.metricSpec.reachAndFrequency - bindValuesParam(8, reachAndFrequency.reachPrivacyParams.epsilon) - bindValuesParam(9, reachAndFrequency.reachPrivacyParams.delta) - bindValuesParam(10, reachAndFrequency.frequencyPrivacyParams.epsilon) - bindValuesParam(11, reachAndFrequency.reachPrivacyParams.delta) + bindValuesParam(8, reachAndFrequency.multipleDataProviderParams.privacyParams.epsilon) + bindValuesParam(9, reachAndFrequency.multipleDataProviderParams.privacyParams.delta) + bindValuesParam(10, reachAndFrequency.multipleDataProviderParams.frequencyPrivacyParams.epsilon) + bindValuesParam(11, reachAndFrequency.multipleDataProviderParams.frequencyPrivacyParams.delta) bindValuesParam(12, null) bindValuesParam(13, null) bindValuesParam(14, reachAndFrequency.maximumFrequency) + bindValuesParam(15, reachAndFrequency.multipleDataProviderParams.vidSamplingInterval.start) + bindValuesParam(16, reachAndFrequency.multipleDataProviderParams.vidSamplingInterval.width) + if (reachAndFrequency.hasSingleDataProviderParams()) { + bindValuesParam(21, reachAndFrequency.singleDataProviderParams.privacyParams.epsilon) + bindValuesParam(22, reachAndFrequency.singleDataProviderParams.privacyParams.delta) + bindValuesParam(23, reachAndFrequency.singleDataProviderParams.frequencyPrivacyParams.epsilon) + bindValuesParam(24, reachAndFrequency.singleDataProviderParams.frequencyPrivacyParams.delta) + bindValuesParam(25, reachAndFrequency.singleDataProviderParams.vidSamplingInterval.start) + bindValuesParam(26, reachAndFrequency.singleDataProviderParams.vidSamplingInterval.width) + } else { + bindValuesParam(21, null) + bindValuesParam(22, null) + bindValuesParam(23, null) + bindValuesParam(24, null) + bindValuesParam(25, null) + bindValuesParam(26, null) + } } MetricSpec.TypeCase.REACH -> { val reach = it.metric.metricSpec.reach - bindValuesParam(8, reach.privacyParams.epsilon) - bindValuesParam(9, reach.privacyParams.delta) + bindValuesParam(8, reach.multipleDataProviderParams.privacyParams.epsilon) + bindValuesParam(9, reach.multipleDataProviderParams.privacyParams.delta) bindValuesParam(10, null) bindValuesParam(11, null) bindValuesParam(12, null) bindValuesParam(13, null) bindValuesParam(14, null) + bindValuesParam(15, reach.multipleDataProviderParams.vidSamplingInterval.start) + bindValuesParam(16, reach.multipleDataProviderParams.vidSamplingInterval.width) + bindValuesParam(23, null) + bindValuesParam(24, null) + if (reach.hasSingleDataProviderParams()) { + bindValuesParam(21, reach.singleDataProviderParams.privacyParams.epsilon) + bindValuesParam(22, reach.singleDataProviderParams.privacyParams.delta) + bindValuesParam(25, reach.singleDataProviderParams.vidSamplingInterval.start) + bindValuesParam(26, reach.singleDataProviderParams.vidSamplingInterval.width) + } else { + bindValuesParam(21, null) + bindValuesParam(22, null) + bindValuesParam(25, null) + bindValuesParam(26, null) + } } MetricSpec.TypeCase.IMPRESSION_COUNT -> { val impressionCount = it.metric.metricSpec.impressionCount - bindValuesParam(8, impressionCount.privacyParams.epsilon) - bindValuesParam(9, impressionCount.privacyParams.delta) + bindValuesParam(8, impressionCount.params.privacyParams.epsilon) + bindValuesParam(9, impressionCount.params.privacyParams.delta) bindValuesParam(10, null) bindValuesParam(11, null) bindValuesParam(12, impressionCount.maximumFrequencyPerUser) bindValuesParam(13, null) bindValuesParam(14, null) + bindValuesParam(15, impressionCount.params.vidSamplingInterval.start) + bindValuesParam(16, impressionCount.params.vidSamplingInterval.width) + bindValuesParam(21, null) + bindValuesParam(22, null) + bindValuesParam(23, null) + bindValuesParam(24, null) + bindValuesParam(25, null) + bindValuesParam(26, null) } MetricSpec.TypeCase.WATCH_DURATION -> { val watchDuration = it.metric.metricSpec.watchDuration - bindValuesParam(8, watchDuration.privacyParams.epsilon) - bindValuesParam(9, watchDuration.privacyParams.delta) + bindValuesParam(8, watchDuration.params.privacyParams.epsilon) + bindValuesParam(9, watchDuration.params.privacyParams.delta) bindValuesParam(10, null) bindValuesParam(11, null) bindValuesParam(12, null) @@ -301,6 +342,14 @@ class CreateMetrics(private val requests: List) : PostgresInterval.of(watchDuration.maximumWatchDurationPerUser.toDuration()), ) bindValuesParam(14, null) + bindValuesParam(15, watchDuration.params.vidSamplingInterval.start) + bindValuesParam(16, watchDuration.params.vidSamplingInterval.width) + bindValuesParam(21, null) + bindValuesParam(22, null) + bindValuesParam(23, null) + bindValuesParam(24, null) + bindValuesParam(25, null) + bindValuesParam(26, null) } MetricSpec.TypeCase.POPULATION_COUNT -> { bindValuesParam(8, 0) @@ -310,11 +359,17 @@ class CreateMetrics(private val requests: List) : bindValuesParam(12, null) bindValuesParam(13, null) bindValuesParam(14, null) + bindValuesParam(15, 0) + bindValuesParam(16, 0) + bindValuesParam(21, null) + bindValuesParam(22, null) + bindValuesParam(23, null) + bindValuesParam(24, null) + bindValuesParam(25, null) + bindValuesParam(26, null) } MetricSpec.TypeCase.TYPE_NOT_SET -> {} } - bindValuesParam(15, vidSamplingIntervalStart) - bindValuesParam(16, vidSamplingIntervalWidth) bindValuesParam(17, createTime) bindValuesParam(18, it.metric.details) bindValuesParam(19, it.metric.details.toJson()) @@ -397,7 +452,7 @@ class CreateMetrics(private val requests: List) : val measurementsStatement = valuesListBoundStatement( valuesStartIndex = 0, - paramCount = 9, + paramCount = 10, """ INSERT INTO Measurements ( @@ -409,7 +464,8 @@ class CreateMetrics(private val requests: List) : TimeIntervalEndExclusive, State, MeasurementDetails, - MeasurementDetailsJson + MeasurementDetailsJson, + IsSingleDataProvider ) VALUES ${ValuesListBoundStatement.VALUES_LIST_PLACEHOLDER} @@ -426,6 +482,7 @@ class CreateMetrics(private val requests: List) : bindValuesParam(6, Measurement.State.STATE_UNSPECIFIED) bindValuesParam(7, Measurement.Details.getDefaultInstance()) bindValuesParam(8, Measurement.Details.getDefaultInstance().toJson()) + bindValuesParam(9, it.isSingleDataProvider) } } } @@ -581,6 +638,7 @@ class CreateMetrics(private val requests: List) : it.measurement.timeInterval.startTime.toInstant().atOffset(ZoneOffset.UTC), timeIntervalEndExclusive = it.measurement.timeInterval.endTime.toInstant().atOffset(ZoneOffset.UTC), + isSingleDataProvider = it.measurement.isSingleDataProvider, ) ) diff --git a/src/main/kotlin/org/wfanet/measurement/reporting/service/api/v2alpha/MetricCalculationSpecsService.kt b/src/main/kotlin/org/wfanet/measurement/reporting/service/api/v2alpha/MetricCalculationSpecsService.kt index d4fff4e33b7..3644b3de1b4 100644 --- a/src/main/kotlin/org/wfanet/measurement/reporting/service/api/v2alpha/MetricCalculationSpecsService.kt +++ b/src/main/kotlin/org/wfanet/measurement/reporting/service/api/v2alpha/MetricCalculationSpecsService.kt @@ -37,6 +37,7 @@ import org.wfanet.measurement.internal.reporting.v2.createMetricCalculationSpecR import org.wfanet.measurement.internal.reporting.v2.getMetricCalculationSpecRequest import org.wfanet.measurement.internal.reporting.v2.listMetricCalculationSpecsRequest import org.wfanet.measurement.internal.reporting.v2.metricCalculationSpec as internalMetricCalculationSpec +import kotlin.random.Random import org.wfanet.measurement.reporting.v2alpha.CreateMetricCalculationSpecRequest import org.wfanet.measurement.reporting.v2alpha.GetMetricCalculationSpecRequest import org.wfanet.measurement.reporting.v2alpha.ListMetricCalculationSpecsPageToken @@ -54,6 +55,7 @@ import org.wfanet.measurement.reporting.v2alpha.metricCalculationSpec class MetricCalculationSpecsService( private val internalMetricCalculationSpecsStub: MetricCalculationSpecsCoroutineStub, private val metricSpecConfig: MetricSpecConfig, + private val secureRandom: Random, ) : MetricCalculationSpecsCoroutineImplBase() { override suspend fun createMetricCalculationSpec( request: CreateMetricCalculationSpecRequest @@ -240,7 +242,7 @@ class MetricCalculationSpecsService( val internalMetricSpecs = source.metricSpecsList.map { metricSpec -> try { - metricSpec.withDefaults(metricSpecConfig).toInternal() + metricSpec.withDefaults(metricSpecConfig, secureRandom).toInternal() } catch (e: MetricSpecDefaultsException) { failGrpc(Status.INVALID_ARGUMENT) { listOfNotNull("Invalid metric_spec.", e.message, e.cause?.message) diff --git a/src/main/kotlin/org/wfanet/measurement/reporting/service/api/v2alpha/MetricSpecDefaults.kt b/src/main/kotlin/org/wfanet/measurement/reporting/service/api/v2alpha/MetricSpecDefaults.kt index 8e73d56d9e9..06bf9470a69 100644 --- a/src/main/kotlin/org/wfanet/measurement/reporting/service/api/v2alpha/MetricSpecDefaults.kt +++ b/src/main/kotlin/org/wfanet/measurement/reporting/service/api/v2alpha/MetricSpecDefaults.kt @@ -16,11 +16,14 @@ package org.wfanet.measurement.reporting.service.api.v2alpha +import kotlin.random.Random import org.wfanet.measurement.config.reporting.MetricSpecConfig import org.wfanet.measurement.reporting.v2alpha.MetricSpec import org.wfanet.measurement.reporting.v2alpha.MetricSpecKt import org.wfanet.measurement.reporting.v2alpha.copy +private const val NUM_BUCKETS = 10000 + class MetricSpecDefaultsException(message: String? = null, cause: Throwable? = null) : Exception(message, cause) @@ -28,69 +31,31 @@ class MetricSpecDefaultsException(message: String? = null, cause: Throwable? = n * Specifies default values using [MetricSpecConfig] when optional fields in the [MetricSpec] are * not set. */ -fun MetricSpec.withDefaults(metricSpecConfig: MetricSpecConfig): MetricSpec { +fun MetricSpec.withDefaults(metricSpecConfig: MetricSpecConfig, secureRandom: Random): MetricSpec { return copy { - val defaultVidSamplingInterval: MetricSpecConfig.VidSamplingInterval = - when (typeCase) { - MetricSpec.TypeCase.REACH -> { - reach = reach.withDefaults(metricSpecConfig) - metricSpecConfig.reachVidSamplingInterval - } - MetricSpec.TypeCase.REACH_AND_FREQUENCY -> { - reachAndFrequency = reachAndFrequency.withDefaults(metricSpecConfig) - metricSpecConfig.reachAndFrequencyVidSamplingInterval - } - MetricSpec.TypeCase.IMPRESSION_COUNT -> { - impressionCount = impressionCount.withDefaults(metricSpecConfig) - metricSpecConfig.impressionCountVidSamplingInterval - } - MetricSpec.TypeCase.WATCH_DURATION -> { - watchDuration = watchDuration.withDefaults(metricSpecConfig) - metricSpecConfig.watchDurationVidSamplingInterval - } - MetricSpec.TypeCase.POPULATION_COUNT -> { - populationCount = MetricSpec.PopulationCountParams.getDefaultInstance() - MetricSpecConfig.VidSamplingInterval.getDefaultInstance() - } - MetricSpec.TypeCase.TYPE_NOT_SET -> - throw MetricSpecDefaultsException( - "Invalid metric spec type", - IllegalArgumentException("The metric type in Metric is not specified."), - ) + when (typeCase) { + MetricSpec.TypeCase.REACH -> { + reach = reach.withDefaults(this@withDefaults, metricSpecConfig, secureRandom) } - - // VID sampling interval is not needed in metric spec used to create population measurement. - if (typeCase != MetricSpec.TypeCase.POPULATION_COUNT) { - vidSamplingInterval = - if (hasVidSamplingInterval()) { - vidSamplingInterval - } else defaultVidSamplingInterval.toVidSamplingInterval() - - if (vidSamplingInterval.start < 0) { - throw MetricSpecDefaultsException( - "Invalid vidSamplingInterval", - IllegalArgumentException("vidSamplingInterval.start cannot be negative."), - ) + MetricSpec.TypeCase.REACH_AND_FREQUENCY -> { + reachAndFrequency = reachAndFrequency.withDefaults(this@withDefaults, metricSpecConfig, secureRandom) } - if (vidSamplingInterval.start >= 1) { - throw MetricSpecDefaultsException( - "Invalid vidSamplingInterval", - IllegalArgumentException("vidSamplingInterval.start must be smaller than 1."), - ) + MetricSpec.TypeCase.IMPRESSION_COUNT -> { + impressionCount = impressionCount.withDefaults(this@withDefaults, metricSpecConfig, secureRandom) } - if (vidSamplingInterval.width <= 0) { - throw MetricSpecDefaultsException( - "Invalid vidSamplingInterval", - IllegalArgumentException("vidSamplingInterval.width must be greater than 0."), - ) + MetricSpec.TypeCase.WATCH_DURATION -> { + watchDuration = watchDuration.withDefaults(this@withDefaults, metricSpecConfig, secureRandom) + } + MetricSpec.TypeCase.POPULATION_COUNT -> { + populationCount = MetricSpec.PopulationCountParams.getDefaultInstance() } - if (vidSamplingInterval.start + vidSamplingInterval.width > 1) { + MetricSpec.TypeCase.TYPE_NOT_SET -> throw MetricSpecDefaultsException( - "Invalid vidSamplingInterval", - IllegalArgumentException("vidSamplingInterval start + width cannot be greater than 1."), + "Invalid metric spec type", + IllegalArgumentException("The metric type in Metric is not specified."), ) - } } + clearVidSamplingInterval() } } @@ -99,21 +64,62 @@ fun MetricSpec.withDefaults(metricSpecConfig: MetricSpecConfig): MetricSpec { * [MetricSpec.ReachParams] are not set. */ private fun MetricSpec.ReachParams.withDefaults( - metricSpecConfig: MetricSpecConfig + metricSpec: MetricSpec, + metricSpecConfig: MetricSpecConfig, + secureRandom: Random, ): MetricSpec.ReachParams { - if (!hasPrivacyParams()) { - throw MetricSpecDefaultsException( - "Invalid privacy params", - IllegalArgumentException("privacyParams in reach is not set."), - ) - } - return copy { - privacyParams = - privacyParams.withDefaults( - metricSpecConfig.reachParams.privacyParams.epsilon, - metricSpecConfig.reachParams.privacyParams.delta, + if (hasMultipleDataProviderParams() && hasSingleDataProviderParams()) { + clearPrivacyParams() + multipleDataProviderParams = multipleDataProviderParams.copy { + privacyParams = privacyParams.withDefaults( + defaultEpsilon = metricSpecConfig.reachParams.multipleDataProviderParams.privacyParams.epsilon, + defaultDelta = metricSpecConfig.reachParams.multipleDataProviderParams.privacyParams.delta, + ) + vidSamplingInterval = + if (hasVidSamplingInterval()) { + vidSamplingInterval + } else { + metricSpecConfig.reachParams.multipleDataProviderParams.vidSamplingInterval.toVidSamplingInterval(secureRandom) + } + vidSamplingInterval.validate() + } + + singleDataProviderParams = singleDataProviderParams.copy { + privacyParams = privacyParams.withDefaults( + defaultEpsilon = metricSpecConfig.reachParams.singleDataProviderParams.privacyParams.epsilon, + defaultDelta = metricSpecConfig.reachParams.singleDataProviderParams.privacyParams.delta, + ) + vidSamplingInterval = + if (hasVidSamplingInterval()) { + vidSamplingInterval + } else { + metricSpecConfig.reachParams.singleDataProviderParams.vidSamplingInterval.toVidSamplingInterval(secureRandom) + } + vidSamplingInterval.validate() + } + } else if (hasPrivacyParams() && !hasSingleDataProviderParams() && !hasMultipleDataProviderParams()) { + val source = this + multipleDataProviderParams = multipleDataProviderParams.copy { + privacyParams = source.privacyParams.withDefaults( + defaultEpsilon = metricSpecConfig.reachParams.multipleDataProviderParams.privacyParams.epsilon, + defaultDelta = metricSpecConfig.reachParams.multipleDataProviderParams.privacyParams.delta, + ) + vidSamplingInterval = + if (metricSpec.hasVidSamplingInterval()) { + metricSpec.vidSamplingInterval + } else { + metricSpecConfig.reachParams.multipleDataProviderParams.vidSamplingInterval.toVidSamplingInterval(secureRandom) + } + vidSamplingInterval.validate() + } + clearPrivacyParams() + } else { + throw MetricSpecDefaultsException( + "Invalid privacy_params", + IllegalArgumentException("privacy_params in reach is not set, or only one of single_data_provider_params and multiple_data_provider_params set."), ) + } } } @@ -122,35 +128,81 @@ private fun MetricSpec.ReachParams.withDefaults( * [MetricSpec.ReachAndFrequencyParams] are not set. */ private fun MetricSpec.ReachAndFrequencyParams.withDefaults( - metricSpecConfig: MetricSpecConfig + metricSpec: MetricSpec, + metricSpecConfig: MetricSpecConfig, + secureRandom: Random ): MetricSpec.ReachAndFrequencyParams { - if (!hasReachPrivacyParams()) { - throw MetricSpecDefaultsException( - "Invalid privacy params", - IllegalArgumentException("reachPrivacyParams in reach-and-frequency is not set."), - ) - } - if (!hasFrequencyPrivacyParams()) { - throw MetricSpecDefaultsException( - "Invalid privacy params", - IllegalArgumentException("frequencyPrivacyParams in reach-and-frequency is not set."), - ) - } - return copy { - reachPrivacyParams = - reachPrivacyParams.withDefaults( - metricSpecConfig.reachAndFrequencyParams.reachPrivacyParams.epsilon, - metricSpecConfig.reachAndFrequencyParams.reachPrivacyParams.delta, - ) - frequencyPrivacyParams = - frequencyPrivacyParams.withDefaults( - metricSpecConfig.reachAndFrequencyParams.frequencyPrivacyParams.epsilon, - metricSpecConfig.reachAndFrequencyParams.frequencyPrivacyParams.delta, - ) if (maximumFrequency == 0) { maximumFrequency = metricSpecConfig.reachAndFrequencyParams.maximumFrequency } + + if (hasMultipleDataProviderParams() && hasSingleDataProviderParams()) { + clearReachPrivacyParams() + clearFrequencyPrivacyParams() + multipleDataProviderParams = multipleDataProviderParams.copy { + privacyParams = privacyParams.withDefaults( + defaultEpsilon = metricSpecConfig.reachAndFrequencyParams.multipleDataProviderParams.privacyParams.epsilon, + defaultDelta = metricSpecConfig.reachAndFrequencyParams.multipleDataProviderParams.privacyParams.delta, + ) + frequencyPrivacyParams = frequencyPrivacyParams.withDefaults( + defaultEpsilon = metricSpecConfig.reachAndFrequencyParams.multipleDataProviderParams.frequencyPrivacyParams.epsilon, + defaultDelta = metricSpecConfig.reachAndFrequencyParams.multipleDataProviderParams.frequencyPrivacyParams.delta, + ) + vidSamplingInterval = + if (hasVidSamplingInterval()) { + vidSamplingInterval + } else { + metricSpecConfig.reachAndFrequencyParams.multipleDataProviderParams.vidSamplingInterval.toVidSamplingInterval(secureRandom) + } + vidSamplingInterval.validate() + } + + singleDataProviderParams = singleDataProviderParams.copy { + privacyParams = privacyParams.withDefaults( + defaultEpsilon = metricSpecConfig.reachAndFrequencyParams.singleDataProviderParams.privacyParams.epsilon, + defaultDelta = metricSpecConfig.reachAndFrequencyParams.singleDataProviderParams.privacyParams.delta, + ) + frequencyPrivacyParams = frequencyPrivacyParams.withDefaults( + defaultEpsilon = metricSpecConfig.reachAndFrequencyParams.singleDataProviderParams.frequencyPrivacyParams.epsilon, + defaultDelta = metricSpecConfig.reachAndFrequencyParams.singleDataProviderParams.frequencyPrivacyParams.delta, + ) + vidSamplingInterval = + if (hasVidSamplingInterval()) { + vidSamplingInterval + } else { + metricSpecConfig.reachAndFrequencyParams.singleDataProviderParams.vidSamplingInterval.toVidSamplingInterval(secureRandom) + } + vidSamplingInterval.validate() + } + } else if (hasReachPrivacyParams() && hasFrequencyPrivacyParams() && !hasSingleDataProviderParams() && !hasMultipleDataProviderParams()) { + val source = this + multipleDataProviderParams = multipleDataProviderParams.copy { + privacyParams = source.reachPrivacyParams.withDefaults( + defaultEpsilon = metricSpecConfig.reachAndFrequencyParams.multipleDataProviderParams.privacyParams.epsilon, + defaultDelta = metricSpecConfig.reachAndFrequencyParams.multipleDataProviderParams.privacyParams.delta, + ) + frequencyPrivacyParams = + source.frequencyPrivacyParams.withDefaults( + defaultEpsilon = metricSpecConfig.reachAndFrequencyParams.multipleDataProviderParams.frequencyPrivacyParams.epsilon, + defaultDelta = metricSpecConfig.reachAndFrequencyParams.multipleDataProviderParams.frequencyPrivacyParams.delta, + ) + vidSamplingInterval = + if (metricSpec.hasVidSamplingInterval()) { + metricSpec.vidSamplingInterval + } else { + metricSpecConfig.reachAndFrequencyParams.multipleDataProviderParams.vidSamplingInterval.toVidSamplingInterval(secureRandom) + } + vidSamplingInterval.validate() + } + clearReachPrivacyParams() + clearFrequencyPrivacyParams() + } else { + throw MetricSpecDefaultsException( + "Invalid privacy_params", + IllegalArgumentException("reach_privacy_params or frequency_privacy_params in reach and frequency is not set, or only one of single_data_provider_params and multiple_data_provider_params set."), + ) + } } } @@ -159,27 +211,55 @@ private fun MetricSpec.ReachAndFrequencyParams.withDefaults( * [MetricSpec.WatchDurationParams] are not set. */ private fun MetricSpec.WatchDurationParams.withDefaults( - metricSpecConfig: MetricSpecConfig + metricSpec: MetricSpec, + metricSpecConfig: MetricSpecConfig, + secureRandom: Random, ): MetricSpec.WatchDurationParams { - if (!hasPrivacyParams()) { - throw MetricSpecDefaultsException( - "Invalid privacy params", - IllegalArgumentException("privacyParams in watch duration is not set."), - ) - } - return copy { - privacyParams = - privacyParams.withDefaults( - metricSpecConfig.watchDurationParams.privacyParams.epsilon, - metricSpecConfig.watchDurationParams.privacyParams.delta, - ) maximumWatchDurationPerUser = if (hasMaximumWatchDurationPerUser()) { maximumWatchDurationPerUser } else { metricSpecConfig.watchDurationParams.maximumWatchDurationPerUser } + + if (hasParams()) { + clearPrivacyParams() + params = params.copy { + privacyParams = privacyParams.withDefaults( + defaultEpsilon = metricSpecConfig.watchDurationParams.params.privacyParams.epsilon, + defaultDelta = metricSpecConfig.watchDurationParams.params.privacyParams.delta, + ) + vidSamplingInterval = + if (hasVidSamplingInterval()) { + vidSamplingInterval + } else { + metricSpecConfig.watchDurationParams.params.vidSamplingInterval.toVidSamplingInterval(secureRandom) + } + vidSamplingInterval.validate() + } + } else if (hasPrivacyParams() && !hasParams()) { + val source = this + params = params.copy { + privacyParams = source.privacyParams.withDefaults( + defaultEpsilon = metricSpecConfig.watchDurationParams.params.privacyParams.epsilon, + defaultDelta = metricSpecConfig.watchDurationParams.params.privacyParams.delta, + ) + vidSamplingInterval = + if (metricSpec.hasVidSamplingInterval()) { + metricSpec.vidSamplingInterval + } else { + metricSpecConfig.watchDurationParams.params.vidSamplingInterval.toVidSamplingInterval(secureRandom) + } + vidSamplingInterval.validate() + } + clearPrivacyParams() + } else { + throw MetricSpecDefaultsException( + "Invalid privacy_params", + IllegalArgumentException("privacy_params in watch duration is not set."), + ) + } } } @@ -188,27 +268,55 @@ private fun MetricSpec.WatchDurationParams.withDefaults( * [MetricSpec.ImpressionCountParams] are not set. */ private fun MetricSpec.ImpressionCountParams.withDefaults( - metricSpecConfig: MetricSpecConfig + metricSpec: MetricSpec, + metricSpecConfig: MetricSpecConfig, + secureRandom: Random, ): MetricSpec.ImpressionCountParams { - if (!hasPrivacyParams()) { - throw MetricSpecDefaultsException( - "Invalid privacy params", - IllegalArgumentException("privacyParams in impression count is not set."), - ) - } - return copy { - privacyParams = - privacyParams.withDefaults( - metricSpecConfig.impressionCountParams.privacyParams.epsilon, - metricSpecConfig.impressionCountParams.privacyParams.delta, - ) maximumFrequencyPerUser = if (hasMaximumFrequencyPerUser()) { maximumFrequencyPerUser } else { metricSpecConfig.impressionCountParams.maximumFrequencyPerUser } + + if (hasParams()) { + clearPrivacyParams() + params = params.copy { + privacyParams = privacyParams.withDefaults( + defaultEpsilon = metricSpecConfig.impressionCountParams.params.privacyParams.epsilon, + defaultDelta = metricSpecConfig.impressionCountParams.params.privacyParams.delta, + ) + vidSamplingInterval = + if (hasVidSamplingInterval()) { + vidSamplingInterval + } else { + metricSpecConfig.impressionCountParams.params.vidSamplingInterval.toVidSamplingInterval(secureRandom) + } + vidSamplingInterval.validate() + } + } else if (hasPrivacyParams() && !hasParams()) { + val source = this + params = params.copy { + privacyParams = source.privacyParams.withDefaults( + defaultEpsilon = metricSpecConfig.impressionCountParams.params.privacyParams.epsilon, + defaultDelta = metricSpecConfig.impressionCountParams.params.privacyParams.delta, + ) + vidSamplingInterval = + if (metricSpec.hasVidSamplingInterval()) { + metricSpec.vidSamplingInterval + } else { + metricSpecConfig.impressionCountParams.params.vidSamplingInterval.toVidSamplingInterval(secureRandom) + } + vidSamplingInterval.validate() + } + clearPrivacyParams() + } else { + throw MetricSpecDefaultsException( + "Invalid privacy_params", + IllegalArgumentException("privacy_params in impression count is not set."), + ) + } } } @@ -227,11 +335,54 @@ private fun MetricSpec.DifferentialPrivacyParams.withDefaults( } /** Converts an [MetricSpecConfig.VidSamplingInterval] to an [MetricSpec.VidSamplingInterval]. */ -private fun MetricSpecConfig.VidSamplingInterval.toVidSamplingInterval(): +private fun MetricSpecConfig.VidSamplingInterval.toVidSamplingInterval(secureRandom: Random): MetricSpec.VidSamplingInterval { val source = this - return MetricSpecKt.vidSamplingInterval { - start = source.start - width = source.width + if (source.hasFixedStart()) { + return MetricSpecKt.vidSamplingInterval { + start = source.fixedStart.start + width = source.fixedStart.width + } + } else { + val maxStart = NUM_BUCKETS - (source.randomStart.width * NUM_BUCKETS).toInt() + // The `- 1` is in case the rounding from source.randomStart.width * NUM_BUCKETS rounds down. This + // prevents the random start from being too big if rounding down does occur. + val randomStart = secureRandom.nextInt(maxStart - 1) + return MetricSpecKt.vidSamplingInterval { + start = randomStart.toFloat() / NUM_BUCKETS + width = source.randomStart.width + } + } +} + +/** + * Validates a [MetricSpec.VidSamplingInterval]. + * + * @throws [IllegalArgumentException] if validation fails. + */ +private fun MetricSpec.VidSamplingInterval.validate() { + if (this.start < 0) { + throw MetricSpecDefaultsException( + "Invalid vidSamplingInterval", + IllegalArgumentException("vidSamplingInterval.start cannot be negative."), + ) + } + if (this.start >= 1) { + throw MetricSpecDefaultsException( + "Invalid vidSamplingInterval", + IllegalArgumentException("vidSamplingInterval.start must be smaller than 1."), + ) + } + if (this.width <= 0) { + throw MetricSpecDefaultsException( + "Invalid vidSamplingInterval", + IllegalArgumentException("vidSamplingInterval.width must be greater than 0."), + ) + } + if (this.start + this.width > 1) { + throw MetricSpecDefaultsException( + "Invalid vidSamplingInterval", + IllegalArgumentException("vidSamplingInterval start + width cannot be greater than 1."), + ) } } diff --git a/src/main/kotlin/org/wfanet/measurement/reporting/service/api/v2alpha/MetricsService.kt b/src/main/kotlin/org/wfanet/measurement/reporting/service/api/v2alpha/MetricsService.kt index e78f9434a30..30d59da9552 100644 --- a/src/main/kotlin/org/wfanet/measurement/reporting/service/api/v2alpha/MetricsService.kt +++ b/src/main/kotlin/org/wfanet/measurement/reporting/service/api/v2alpha/MetricsService.kt @@ -128,7 +128,6 @@ import org.wfanet.measurement.internal.reporting.v2.Metric.WeightedMeasurement import org.wfanet.measurement.internal.reporting.v2.MetricKt as InternalMetricKt import org.wfanet.measurement.internal.reporting.v2.MetricKt.weightedMeasurement import org.wfanet.measurement.internal.reporting.v2.MetricSpec as InternalMetricSpec -import org.wfanet.measurement.internal.reporting.v2.MetricSpec import org.wfanet.measurement.internal.reporting.v2.MetricsGrpcKt.MetricsCoroutineStub as InternalMetricsCoroutineStub import org.wfanet.measurement.internal.reporting.v2.ReportingSet as InternalReportingSet import org.wfanet.measurement.internal.reporting.v2.ReportingSetsGrpcKt.ReportingSetsCoroutineStub as InternalReportingSetsCoroutineStub @@ -212,144 +211,98 @@ private const val BATCH_SET_MEASUREMENT_RESULTS_LIMIT = 1000 private const val BATCH_SET_MEASUREMENT_FAILURES_LIMIT = 1000 class MetricsService( - private val metricSpecConfig: MetricSpecConfig, - private val internalReportingSetsStub: InternalReportingSetsCoroutineStub, - private val internalMetricsStub: InternalMetricsCoroutineStub, - private val variances: Variances, - internalMeasurementsStub: InternalMeasurementsCoroutineStub, - dataProvidersStub: DataProvidersCoroutineStub, - measurementsStub: MeasurementsCoroutineStub, - certificatesStub: CertificatesCoroutineStub, - measurementConsumersStub: MeasurementConsumersCoroutineStub, - encryptionKeyPairStore: EncryptionKeyPairStore, - secureRandom: Random, - signingPrivateKeyDir: File, - trustedCertificates: Map, - certificateCacheExpirationDuration: Duration = Duration.ofMinutes(60), - dataProviderCacheExpirationDuration: Duration = Duration.ofMinutes(60), - keyReaderContext: @BlockingExecutor CoroutineContext = Dispatchers.IO, - cacheLoaderContext: @NonBlockingExecutor CoroutineContext = Dispatchers.Default, + private val metricSpecConfig: MetricSpecConfig, + private val internalReportingSetsStub: InternalReportingSetsCoroutineStub, + private val internalMetricsStub: InternalMetricsCoroutineStub, + private val variances: Variances, + internalMeasurementsStub: InternalMeasurementsCoroutineStub, + dataProvidersStub: DataProvidersCoroutineStub, + measurementsStub: MeasurementsCoroutineStub, + certificatesStub: CertificatesCoroutineStub, + measurementConsumersStub: MeasurementConsumersCoroutineStub, + encryptionKeyPairStore: EncryptionKeyPairStore, + private val secureRandom: Random, + signingPrivateKeyDir: File, + trustedCertificates: Map, + certificateCacheExpirationDuration: Duration = Duration.ofMinutes(60), + dataProviderCacheExpirationDuration: Duration = Duration.ofMinutes(60), + keyReaderContext: @BlockingExecutor CoroutineContext = Dispatchers.IO, + cacheLoaderContext: @NonBlockingExecutor CoroutineContext = Dispatchers.Default, ) : MetricsCoroutineImplBase() { private data class DataProviderInfo( - val dataProviderName: String, - val publicKey: SignedMessage, - val certificateName: String, + val dataProviderName: String, + val publicKey: SignedMessage, + val certificateName: String, ) private val measurementSupplier = - MeasurementSupplier( - internalReportingSetsStub, - internalMeasurementsStub, - measurementsStub, - dataProvidersStub, - certificatesStub, - measurementConsumersStub, - encryptionKeyPairStore, - secureRandom, - signingPrivateKeyDir, - trustedCertificates, - certificateCacheExpirationDuration = certificateCacheExpirationDuration, - dataProviderCacheExpirationDuration = dataProviderCacheExpirationDuration, - keyReaderContext, - cacheLoaderContext, - ) + MeasurementSupplier( + internalMeasurementsStub, + measurementsStub, + dataProvidersStub, + certificatesStub, + measurementConsumersStub, + encryptionKeyPairStore, + secureRandom, + signingPrivateKeyDir, + trustedCertificates, + certificateCacheExpirationDuration = certificateCacheExpirationDuration, + dataProviderCacheExpirationDuration = dataProviderCacheExpirationDuration, + keyReaderContext, + cacheLoaderContext, + ) private class MeasurementSupplier( - private val internalReportingSetsStub: InternalReportingSetsCoroutineStub, - private val internalMeasurementsStub: InternalMeasurementsCoroutineStub, - private val measurementsStub: MeasurementsCoroutineStub, - private val dataProvidersStub: DataProvidersCoroutineStub, - private val certificatesStub: CertificatesCoroutineStub, - private val measurementConsumersStub: MeasurementConsumersCoroutineStub, - private val encryptionKeyPairStore: EncryptionKeyPairStore, - private val secureRandom: Random, - private val signingPrivateKeyDir: File, - private val trustedCertificates: Map, - certificateCacheExpirationDuration: Duration, - dataProviderCacheExpirationDuration: Duration, - private val keyReaderContext: @BlockingExecutor CoroutineContext = Dispatchers.IO, - cacheLoaderContext: @NonBlockingExecutor CoroutineContext = Dispatchers.Default, + private val internalMeasurementsStub: InternalMeasurementsCoroutineStub, + private val measurementsStub: MeasurementsCoroutineStub, + private val dataProvidersStub: DataProvidersCoroutineStub, + private val certificatesStub: CertificatesCoroutineStub, + private val measurementConsumersStub: MeasurementConsumersCoroutineStub, + private val encryptionKeyPairStore: EncryptionKeyPairStore, + private val secureRandom: Random, + private val signingPrivateKeyDir: File, + private val trustedCertificates: Map, + certificateCacheExpirationDuration: Duration, + dataProviderCacheExpirationDuration: Duration, + private val keyReaderContext: @BlockingExecutor CoroutineContext = Dispatchers.IO, + cacheLoaderContext: @NonBlockingExecutor CoroutineContext = Dispatchers.Default, ) { private data class ResourceNameApiAuthenticationKey( - val name: String, - val apiAuthenticationKey: String, + val name: String, + val apiAuthenticationKey: String, ) private val certificateCache: LoadingCache = - LoadingCache( - Caffeine.newBuilder() - .expireAfterWrite(certificateCacheExpirationDuration) - .executor( - (cacheLoaderContext[ContinuationInterceptor] as CoroutineDispatcher).asExecutor() - ) - .buildAsync() - ) { key -> - getCertificate(name = key.name, apiAuthenticationKey = key.apiAuthenticationKey) - } + LoadingCache( + Caffeine.newBuilder() + .expireAfterWrite(certificateCacheExpirationDuration) + .executor( + (cacheLoaderContext[ContinuationInterceptor] as CoroutineDispatcher) + .asExecutor()) + .buildAsync()) { key -> + getCertificate(name = key.name, apiAuthenticationKey = key.apiAuthenticationKey) + } private val dataProviderCache: LoadingCache = - LoadingCache( - Caffeine.newBuilder() - .expireAfterWrite(dataProviderCacheExpirationDuration) - .executor( - (cacheLoaderContext[ContinuationInterceptor] as CoroutineDispatcher).asExecutor() - ) - .buildAsync() - ) { key -> - getDataProvider(name = key.name, apiAuthenticationKey = key.apiAuthenticationKey) - } + LoadingCache( + Caffeine.newBuilder() + .expireAfterWrite(dataProviderCacheExpirationDuration) + .executor( + (cacheLoaderContext[ContinuationInterceptor] as CoroutineDispatcher) + .asExecutor()) + .buildAsync()) { key -> + getDataProvider(name = key.name, apiAuthenticationKey = key.apiAuthenticationKey) + } - /** - * Creates CMM public [Measurement]s and [InternalMeasurement]s from a list of [InternalMetric]. - */ + /** Creates CMM public [Measurement]s from a list of [InternalMetric]. */ suspend fun createCmmsMeasurements( - internalMetricsList: List, - principal: MeasurementConsumerPrincipal, + internalMetricsList: List, + internalPrimitiveReportingSetMap: Map, + principal: MeasurementConsumerPrincipal, ) { val measurementConsumer: MeasurementConsumer = getMeasurementConsumer(principal) - // Gets all external IDs of primitive reporting sets from the metric list. - val externalPrimitiveReportingSetIds: Flow = flow { - buildSet { - for (internalMetric in internalMetricsList) { - for (weightedMeasurement in internalMetric.weightedMeasurementsList) { - for (primitiveReportingSetBasis in - weightedMeasurement.measurement.primitiveReportingSetBasesList) { - // Checks if the set already contains the ID - if (!contains(primitiveReportingSetBasis.externalReportingSetId)) { - // If the set doesn't contain the ID, emit it and add it to the set so it won't - // get emitted again. - emit(primitiveReportingSetBasis.externalReportingSetId) - add(primitiveReportingSetBasis.externalReportingSetId) - } - } - } - } - } - } - - val callBatchGetInternalReportingSetsRpc: - suspend (List) -> BatchGetReportingSetsResponse = - { items -> - batchGetInternalReportingSets(principal.resourceKey.measurementConsumerId, items) - } - - val internalPrimitiveReportingSetMap: Map = buildMap { - submitBatchRequests( - externalPrimitiveReportingSetIds, - BATCH_GET_REPORTING_SETS_LIMIT, - callBatchGetInternalReportingSetsRpc, - ) { response: BatchGetReportingSetsResponse -> - response.reportingSetsList - } - .collect { reportingSets: List -> - for (reportingSet in reportingSets) { - computeIfAbsent(reportingSet.externalReportingSetId) { reportingSet } - } - } - } - val dataProviderNames = mutableSetOf() for (internalPrimitiveReportingSet in internalPrimitiveReportingSetMap.values) { for (eventGroupKey in internalPrimitiveReportingSet.primitive.eventGroupKeysList) { @@ -357,7 +310,7 @@ class MetricsService( } } val dataProviderInfoMap: Map = - buildDataProviderInfoMap(principal.config.apiKey, dataProviderNames) + buildDataProviderInfoMap(principal.config.apiKey, dataProviderNames) val measurementConsumerSigningKey = getMeasurementConsumerSigningKey(principal) @@ -366,16 +319,15 @@ class MetricsService( for (weightedMeasurement in internalMetric.weightedMeasurementsList) { if (weightedMeasurement.measurement.cmmsMeasurementId.isBlank()) { emit( - buildCreateMeasurementRequest( - weightedMeasurement.measurement, - internalMetric.metricSpec, - internalPrimitiveReportingSetMap, - measurementConsumer, - principal, - dataProviderInfoMap, - measurementConsumerSigningKey, - ) - ) + buildCreateMeasurementRequest( + weightedMeasurement.measurement, + internalMetric.metricSpec, + internalPrimitiveReportingSetMap, + measurementConsumer, + principal, + dataProviderInfoMap, + measurementConsumerSigningKey, + )) } } } @@ -383,22 +335,22 @@ class MetricsService( // Create CMMS measurements. val callBatchCreateMeasurementsRpc: - suspend (List) -> BatchCreateMeasurementsResponse = - { items -> - batchCreateCmmsMeasurements(principal, items) - } + suspend (List) -> BatchCreateMeasurementsResponse = + { items -> + batchCreateCmmsMeasurements(principal, items) + } @OptIn(ExperimentalCoroutinesApi::class) val cmmsMeasurements: Flow = - submitBatchRequests( - cmmsCreateMeasurementRequests, - BATCH_KINGDOM_MEASUREMENTS_LIMIT, - callBatchCreateMeasurementsRpc, - ) { response: BatchCreateMeasurementsResponse -> - response.measurementsList - } - .map { it.asFlow() } - .flattenMerge() + submitBatchRequests( + cmmsCreateMeasurementRequests, + BATCH_KINGDOM_MEASUREMENTS_LIMIT, + callBatchCreateMeasurementsRpc, + ) { response: BatchCreateMeasurementsResponse -> + response.measurementsList + } + .map { it.asFlow() } + .flattenMerge() // Set CMMS measurement IDs. val callBatchSetCmmsMeasurementIdsRpc: suspend (List) -> Unit = { items -> @@ -406,32 +358,31 @@ class MetricsService( } submitBatchRequests( - cmmsMeasurements.map { - measurementIds { - cmmsCreateMeasurementRequestId = it.measurementReferenceId - cmmsMeasurementId = MeasurementKey.fromName(it.name)!!.measurementId - } - }, - BATCH_SET_CMMS_MEASUREMENT_IDS_LIMIT, - callBatchSetCmmsMeasurementIdsRpc, - ) { _: Unit -> - emptyList() - } - .collect {} + cmmsMeasurements.map { + measurementIds { + cmmsCreateMeasurementRequestId = it.measurementReferenceId + cmmsMeasurementId = MeasurementKey.fromName(it.name)!!.measurementId + } + }, + BATCH_SET_CMMS_MEASUREMENT_IDS_LIMIT, + callBatchSetCmmsMeasurementIdsRpc, + ) { _: Unit -> + emptyList() + } + .collect {} } /** Sets a batch of CMMS [MeasurementIds] to the [InternalMeasurement] table. */ private suspend fun batchSetCmmsMeasurementIds( - cmmsMeasurementConsumerId: String, - measurementIds: List, + cmmsMeasurementConsumerId: String, + measurementIds: List, ) { try { internalMeasurementsStub.batchSetCmmsMeasurementIds( - batchSetCmmsMeasurementIdsRequest { - this.cmmsMeasurementConsumerId = cmmsMeasurementConsumerId - this.measurementIds += measurementIds - } - ) + batchSetCmmsMeasurementIdsRequest { + this.cmmsMeasurementConsumerId = cmmsMeasurementConsumerId + this.measurementIds += measurementIds + }) } catch (e: StatusException) { throw Exception("Unable to set the CMMS measurement IDs for the measurements.", e) } @@ -439,49 +390,49 @@ class MetricsService( /** Batch create CMMS measurements. */ private suspend fun batchCreateCmmsMeasurements( - principal: MeasurementConsumerPrincipal, - createMeasurementRequests: List, + principal: MeasurementConsumerPrincipal, + createMeasurementRequests: List, ): BatchCreateMeasurementsResponse { try { return measurementsStub - .withAuthenticationKey(principal.config.apiKey) - .batchCreateMeasurements( - batchCreateMeasurementsRequest { - parent = principal.resourceKey.toName() - requests += createMeasurementRequests - } - ) + .withAuthenticationKey(principal.config.apiKey) + .batchCreateMeasurements( + batchCreateMeasurementsRequest { + parent = principal.resourceKey.toName() + requests += createMeasurementRequests + }) } catch (e: StatusException) { throw when (e.status.code) { - Status.Code.INVALID_ARGUMENT -> - Status.INVALID_ARGUMENT.withDescription("Required field unspecified or invalid.") - Status.Code.PERMISSION_DENIED -> - Status.PERMISSION_DENIED.withDescription( - "Cannot create CMMS Measurements for another MeasurementConsumer." - ) - Status.Code.FAILED_PRECONDITION -> - Status.FAILED_PRECONDITION.withDescription("Failed precondition.") - Status.Code.NOT_FOUND -> - Status.NOT_FOUND.withDescription("${principal.resourceKey.toName()} is not found.") - else -> Status.UNKNOWN.withDescription("Unable to create CMMS Measurements.") - } - .withCause(e) - .asRuntimeException() + Status.Code.INVALID_ARGUMENT -> + Status.INVALID_ARGUMENT.withDescription("Required field unspecified or invalid.") + Status.Code.PERMISSION_DENIED -> + Status.PERMISSION_DENIED.withDescription( + "Cannot create CMMS Measurements for another MeasurementConsumer.") + Status.Code.FAILED_PRECONDITION -> + Status.FAILED_PRECONDITION.withDescription("Failed precondition.") + Status.Code.NOT_FOUND -> + Status.NOT_FOUND.withDescription( + "${principal.resourceKey.toName()} is not found.") + else -> Status.UNKNOWN.withDescription("Unable to create CMMS Measurements.") + } + .withCause(e) + .asRuntimeException() } } /** Builds a CMMS [CreateMeasurementRequest]. */ private fun buildCreateMeasurementRequest( - internalMeasurement: InternalMeasurement, - metricSpec: InternalMetricSpec, - internalPrimitiveReportingSetMap: Map, - measurementConsumer: MeasurementConsumer, - principal: MeasurementConsumerPrincipal, - dataProviderInfoMap: Map, - measurementConsumerSigningKey: SigningKeyHandle, + internalMeasurement: InternalMeasurement, + metricSpec: InternalMetricSpec, + internalPrimitiveReportingSetMap: Map, + measurementConsumer: MeasurementConsumer, + principal: MeasurementConsumerPrincipal, + dataProviderInfoMap: Map, + measurementConsumerSigningKey: SigningKeyHandle, ): CreateMeasurementRequest { val eventGroupEntriesByDataProvider = - groupEventGroupEntriesByDataProvider(internalMeasurement, internalPrimitiveReportingSetMap) + groupEventGroupEntriesByDataProvider( + internalMeasurement, internalPrimitiveReportingSetMap) val packedMeasurementEncryptionPublicKey = measurementConsumer.publicKey.message return createMeasurementRequest { @@ -490,22 +441,22 @@ class MetricsService( measurementConsumerCertificate = principal.config.signingCertificateName dataProviders += - buildDataProviderEntries( - eventGroupEntriesByDataProvider, - packedMeasurementEncryptionPublicKey, - measurementConsumerSigningKey, - dataProviderInfoMap, - ) + buildDataProviderEntries( + eventGroupEntriesByDataProvider, + packedMeasurementEncryptionPublicKey, + measurementConsumerSigningKey, + dataProviderInfoMap, + ) val unsignedMeasurementSpec: MeasurementSpec = - buildUnsignedMeasurementSpec( - packedMeasurementEncryptionPublicKey, - dataProviders.map { it.value.nonceHash }, - metricSpec, - ) + buildUnsignedMeasurementSpec( + packedMeasurementEncryptionPublicKey, + dataProviders.map { it.value.nonceHash }, + metricSpec, + ) measurementSpec = - signMeasurementSpec(unsignedMeasurementSpec, measurementConsumerSigningKey) + signMeasurementSpec(unsignedMeasurementSpec, measurementConsumerSigningKey) // To help map reporting measurements to cmms measurements. measurementReferenceId = internalMeasurement.cmmsCreateMeasurementRequestId } @@ -515,27 +466,29 @@ class MetricsService( /** Gets a [SigningKeyHandle] for a [MeasurementConsumerPrincipal]. */ private suspend fun getMeasurementConsumerSigningKey( - principal: MeasurementConsumerPrincipal + principal: MeasurementConsumerPrincipal ): SigningKeyHandle { // TODO: Factor this out to a separate class similar to EncryptionKeyPairStore. val signingPrivateKeyDer: ByteString = - withContext(keyReaderContext) { - signingPrivateKeyDir.resolve(principal.config.signingPrivateKeyPath).readByteString() - } + withContext(keyReaderContext) { + signingPrivateKeyDir.resolve(principal.config.signingPrivateKeyPath).readByteString() + } val measurementConsumerCertificate: X509Certificate = - readCertificate(getSigningCertificateDer(principal)) + readCertificate(getSigningCertificateDer(principal)) val signingPrivateKey: PrivateKey = - readPrivateKey(signingPrivateKeyDer, measurementConsumerCertificate.publicKey.algorithm) + readPrivateKey(signingPrivateKeyDer, measurementConsumerCertificate.publicKey.algorithm) return SigningKeyHandle(measurementConsumerCertificate, signingPrivateKey) } /** Builds an unsigned [MeasurementSpec]. */ private fun buildUnsignedMeasurementSpec( - packedMeasurementEncryptionPublicKey: ProtoAny, - nonceHashes: List, - metricSpec: InternalMetricSpec, + packedMeasurementEncryptionPublicKey: ProtoAny, + nonceHashes: List, + metricSpec: InternalMetricSpec, ): MeasurementSpec { + val isSingleDataProvider: Boolean = nonceHashes.size == 1 + return measurementSpec { measurementPublicKey = packedMeasurementEncryptionPublicKey // TODO(world-federation-of-advertisers/cross-media-measurement#1301): Stop setting this @@ -546,34 +499,42 @@ class MetricsService( @Suppress("WHEN_ENUM_CAN_BE_NULL_IN_JAVA") // Proto enum fields are never null. when (metricSpec.typeCase) { InternalMetricSpec.TypeCase.REACH -> { - reach = metricSpec.reach.toReach() + val reachPair = metricSpec.reach.toReach(isSingleDataProvider) + reach = reachPair.first + vidSamplingInterval = reachPair.second } InternalMetricSpec.TypeCase.REACH_AND_FREQUENCY -> { - reachAndFrequency = metricSpec.reachAndFrequency.toReachAndFrequency() + val reachAndFrequencyPair = + metricSpec.reachAndFrequency.toReachAndFrequency(isSingleDataProvider) + reachAndFrequency = reachAndFrequencyPair.first + vidSamplingInterval = reachAndFrequencyPair.second } InternalMetricSpec.TypeCase.IMPRESSION_COUNT -> { - impression = metricSpec.impressionCount.toImpression() + val impressionPair = metricSpec.impressionCount.toImpression() + impression = impressionPair.first + vidSamplingInterval = impressionPair.second } InternalMetricSpec.TypeCase.WATCH_DURATION -> { - duration = metricSpec.watchDuration.toDuration() + val durationPair = metricSpec.watchDuration.toDuration() + duration = durationPair.first + vidSamplingInterval = durationPair.second } InternalMetricSpec.TypeCase.POPULATION_COUNT -> { population = MeasurementSpec.Population.getDefaultInstance() } InternalMetricSpec.TypeCase.TYPE_NOT_SET -> - failGrpc(status = Status.FAILED_PRECONDITION, cause = IllegalStateException()) { - "Unset metric type should've already raised error." - } + failGrpc(status = Status.FAILED_PRECONDITION, cause = IllegalStateException()) { + "Unset metric type should've already raised error." + } } - vidSamplingInterval = metricSpec.vidSamplingInterval.toCmmsVidSamplingInterval() // TODO(@jojijac0b): Add modelLine } } /** Builds a [Map] of [DataProvider] name to [DataProviderInfo]. */ private suspend fun buildDataProviderInfoMap( - apiAuthenticationKey: String, - dataProviderNames: Collection, + apiAuthenticationKey: String, + dataProviderNames: Collection, ): Map { val dataProviderInfoMap = mutableMapOf() @@ -585,55 +546,48 @@ class MetricsService( coroutineScope { for (dataProviderName in dataProviderNames) { deferredDataProviderInfoList.add( - async { - val dataProvider: DataProvider = - dataProviderCache.getValue( - ResourceNameApiAuthenticationKey( - name = dataProviderName, - apiAuthenticationKey = apiAuthenticationKey, - ) - ) - - val certificate = - certificateCache.getValue( - ResourceNameApiAuthenticationKey( - name = dataProvider.certificate, - apiAuthenticationKey = apiAuthenticationKey, - ) - ) - - if ( - certificate.revocationState != - Certificate.RevocationState.REVOCATION_STATE_UNSPECIFIED - ) { - throw Status.FAILED_PRECONDITION.withDescription( - "${certificate.name} revocation state is ${certificate.revocationState}" - ) - .asRuntimeException() - } + async { + val dataProvider: DataProvider = + dataProviderCache.getValue( + ResourceNameApiAuthenticationKey( + name = dataProviderName, + apiAuthenticationKey = apiAuthenticationKey, + )) + + val certificate = + certificateCache.getValue( + ResourceNameApiAuthenticationKey( + name = dataProvider.certificate, + apiAuthenticationKey = apiAuthenticationKey, + )) + + if (certificate.revocationState != + Certificate.RevocationState.REVOCATION_STATE_UNSPECIFIED) { + throw Status.FAILED_PRECONDITION.withDescription( + "${certificate.name} revocation state is ${certificate.revocationState}") + .asRuntimeException() + } - val x509Certificate: X509Certificate = readCertificate(certificate.x509Der) - val trustedIssuer: X509Certificate = - trustedCertificates[checkNotNull(x509Certificate.authorityKeyIdentifier)] - ?: throw Status.FAILED_PRECONDITION.withDescription( - "${certificate.name} not issued by trusted CA" - ) - .asRuntimeException() - try { - verifyEncryptionPublicKey(dataProvider.publicKey, x509Certificate, trustedIssuer) - } catch (e: CertPathValidatorException) { - throw Status.FAILED_PRECONDITION.withCause(e) - .withDescription("Certificate path for ${certificate.name} is invalid") - .asRuntimeException() - } catch (e: SignatureException) { - throw Status.FAILED_PRECONDITION.withCause(e) - .withDescription("DataProvider public key signature is invalid") - .asRuntimeException() - } + val x509Certificate: X509Certificate = readCertificate(certificate.x509Der) + val trustedIssuer: X509Certificate = + trustedCertificates[checkNotNull(x509Certificate.authorityKeyIdentifier)] + ?: throw Status.FAILED_PRECONDITION.withDescription( + "${certificate.name} not issued by trusted CA") + .asRuntimeException() + try { + verifyEncryptionPublicKey(dataProvider.publicKey, x509Certificate, trustedIssuer) + } catch (e: CertPathValidatorException) { + throw Status.FAILED_PRECONDITION.withCause(e) + .withDescription("Certificate path for ${certificate.name} is invalid") + .asRuntimeException() + } catch (e: SignatureException) { + throw Status.FAILED_PRECONDITION.withCause(e) + .withDescription("DataProvider public key signature is invalid") + .asRuntimeException() + } - DataProviderInfo(dataProvider.name, dataProvider.publicKey, certificate.name) - } - ) + DataProviderInfo(dataProvider.name, dataProvider.publicKey, certificate.name) + }) } for (deferredDataProviderInfo in deferredDataProviderInfoList.awaitAll()) { @@ -649,10 +603,10 @@ class MetricsService( * [eventGroupEntriesByDataProvider]. */ private fun buildDataProviderEntries( - eventGroupEntriesByDataProvider: Map>, - packedMeasurementEncryptionPublicKey: ProtoAny, - measurementConsumerSigningKey: SigningKeyHandle, - dataProviderInfoMap: Map, + eventGroupEntriesByDataProvider: Map>, + packedMeasurementEncryptionPublicKey: ProtoAny, + measurementConsumerSigningKey: SigningKeyHandle, + dataProviderInfoMap: Map, ): List { return eventGroupEntriesByDataProvider.map { (dataProviderKey, eventGroupEntriesList) -> val dataProviderName: String = dataProviderKey.toName() @@ -667,20 +621,20 @@ class MetricsService( nonce = secureRandom.nextLong() } val encryptRequisitionSpec = - encryptRequisitionSpec( - signRequisitionSpec(requisitionSpec, measurementConsumerSigningKey), - dataProviderInfo.publicKey.unpack(), - ) + encryptRequisitionSpec( + signRequisitionSpec(requisitionSpec, measurementConsumerSigningKey), + dataProviderInfo.publicKey.unpack(), + ) dataProviderEntry { key = dataProviderName value = - MeasurementKt.DataProviderEntryKt.value { - dataProviderCertificate = dataProviderInfo.certificateName - dataProviderPublicKey = dataProviderInfo.publicKey.message - this.encryptedRequisitionSpec = encryptRequisitionSpec - nonceHash = Hashing.hashSha256(requisitionSpec.nonce) - } + MeasurementKt.DataProviderEntryKt.value { + dataProviderCertificate = dataProviderInfo.certificateName + dataProviderPublicKey = dataProviderInfo.publicKey.message + this.encryptedRequisitionSpec = encryptRequisitionSpec + nonceHash = Hashing.hashSha256(requisitionSpec.nonce) + } } } } @@ -690,42 +644,44 @@ class MetricsService( * grouping them by DataProvider. */ private fun groupEventGroupEntriesByDataProvider( - measurement: InternalMeasurement, - internalPrimitiveReportingSetMap: Map, + measurement: InternalMeasurement, + internalPrimitiveReportingSetMap: Map, ): Map> { return measurement.primitiveReportingSetBasesList - .flatMap { primitiveReportingSetBasis -> - val internalPrimitiveReportingSet = - internalPrimitiveReportingSetMap.getValue( - primitiveReportingSetBasis.externalReportingSetId - ) - - internalPrimitiveReportingSet.primitive.eventGroupKeysList.map { internalEventGroupKey -> - val cmmsEventGroupKey = - CmmsEventGroupKey( - internalEventGroupKey.cmmsDataProviderId, - internalEventGroupKey.cmmsEventGroupId, - ) - val filtersList = primitiveReportingSetBasis.filtersList.filter { !it.isNullOrBlank() } - val filter: String? = if (filtersList.isEmpty()) null else buildConjunction(filtersList) - - cmmsEventGroupKey to - RequisitionSpecKt.eventGroupEntry { - key = cmmsEventGroupKey.toName() - value = - RequisitionSpecKt.EventGroupEntryKt.value { - collectionInterval = measurement.timeInterval - if (filter != null) { - this.filter = RequisitionSpecKt.eventFilter { expression = filter } - } + .flatMap { primitiveReportingSetBasis -> + val internalPrimitiveReportingSet = + internalPrimitiveReportingSetMap.getValue( + primitiveReportingSetBasis.externalReportingSetId) + + internalPrimitiveReportingSet.primitive.eventGroupKeysList.map { internalEventGroupKey + -> + val cmmsEventGroupKey = + CmmsEventGroupKey( + internalEventGroupKey.cmmsDataProviderId, + internalEventGroupKey.cmmsEventGroupId, + ) + val filtersList = + primitiveReportingSetBasis.filtersList.filter { !it.isNullOrBlank() } + val filter: String? = + if (filtersList.isEmpty()) null else buildConjunction(filtersList) + + cmmsEventGroupKey to + RequisitionSpecKt.eventGroupEntry { + key = cmmsEventGroupKey.toName() + value = + RequisitionSpecKt.EventGroupEntryKt.value { + collectionInterval = measurement.timeInterval + if (filter != null) { + this.filter = RequisitionSpecKt.eventFilter { expression = filter } + } + } } - } + } } - } - .groupBy( - { (cmmsEventGroupKey, _) -> DataProviderKey(cmmsEventGroupKey.dataProviderId) }, - { (_, eventGroupEntry) -> eventGroupEntry }, - ) + .groupBy( + { (cmmsEventGroupKey, _) -> DataProviderKey(cmmsEventGroupKey.dataProviderId) }, + { (_, eventGroupEntry) -> eventGroupEntry }, + ) } /** Combines event group filters. */ @@ -735,64 +691,36 @@ class MetricsService( /** Gets a [MeasurementConsumer] based on a CMMS ID. */ private suspend fun getMeasurementConsumer( - principal: MeasurementConsumerPrincipal + principal: MeasurementConsumerPrincipal ): MeasurementConsumer { return try { measurementConsumersStub - .withAuthenticationKey(principal.config.apiKey) - .getMeasurementConsumer( - getMeasurementConsumerRequest { name = principal.resourceKey.toName() } - ) - } catch (e: StatusException) { - throw when (e.status.code) { - Status.Code.NOT_FOUND -> - Status.NOT_FOUND.withDescription("${principal.resourceKey.toName()} not found.") - else -> - Status.UNKNOWN.withDescription( - "Unable to retrieve the measurement consumer [${principal.resourceKey.toName()}]." - ) - } - .withCause(e) - .asRuntimeException() - } - } - - /** Gets a batch of [InternalReportingSet]s. */ - private suspend fun batchGetInternalReportingSets( - cmmsMeasurementConsumerId: String, - externalReportingSetIds: List, - ): BatchGetReportingSetsResponse { - return try { - internalReportingSetsStub.batchGetReportingSets( - batchGetReportingSetsRequest { - this.cmmsMeasurementConsumerId = cmmsMeasurementConsumerId - this.externalReportingSetIds += externalReportingSetIds - } - ) + .withAuthenticationKey(principal.config.apiKey) + .getMeasurementConsumer( + getMeasurementConsumerRequest { name = principal.resourceKey.toName() }) } catch (e: StatusException) { throw when (e.status.code) { - Status.Code.NOT_FOUND -> Status.NOT_FOUND.withDescription("Reporting Set not found.") - else -> - Status.UNKNOWN.withDescription( - "Unable to retrieve ReportingSets used in the requesting metric." - ) - } - .withCause(e) - .asRuntimeException() + Status.Code.NOT_FOUND -> + Status.NOT_FOUND.withDescription("${principal.resourceKey.toName()} not found.") + else -> + Status.UNKNOWN.withDescription( + "Unable to retrieve the measurement consumer [${principal.resourceKey.toName()}].") + } + .withCause(e) + .asRuntimeException() } } /** Gets a signing certificate x509Der in ByteString. */ private suspend fun getSigningCertificateDer( - principal: MeasurementConsumerPrincipal + principal: MeasurementConsumerPrincipal ): ByteString { val certificate = - certificateCache.getValue( - ResourceNameApiAuthenticationKey( - name = principal.config.signingCertificateName, - apiAuthenticationKey = principal.config.apiKey, - ) - ) + certificateCache.getValue( + ResourceNameApiAuthenticationKey( + name = principal.config.signingCertificateName, + apiAuthenticationKey = principal.config.apiKey, + )) return certificate.x509Der } @@ -803,52 +731,52 @@ class MetricsService( * @return a boolean to indicate whether any [InternalMeasurement] was updated. */ suspend fun syncInternalMeasurements( - internalMeasurements: List, - apiAuthenticationKey: String, - principal: MeasurementConsumerPrincipal, + internalMeasurements: List, + apiAuthenticationKey: String, + principal: MeasurementConsumerPrincipal, ): Boolean { val failedMeasurements: MutableList = mutableListOf() // Most Measurements are expected to be SUCCEEDED so SUCCEEDED Measurements will be collected // via a Flow. val succeededMeasurements: Flow = - getCmmsMeasurements(internalMeasurements, principal).transform { measurements -> - for (measurement in measurements) { - @Suppress("WHEN_ENUM_CAN_BE_NULL_IN_JAVA") // Protobuf enum fields cannot be null. - when (measurement.state) { - Measurement.State.SUCCEEDED -> emit(measurement) - Measurement.State.CANCELLED, - Measurement.State.FAILED -> failedMeasurements.add(measurement) - Measurement.State.COMPUTING, - Measurement.State.AWAITING_REQUISITION_FULFILLMENT -> {} - Measurement.State.STATE_UNSPECIFIED -> - failGrpc(status = Status.FAILED_PRECONDITION, cause = IllegalStateException()) { - "The CMMS measurement state should've been set." - } - Measurement.State.UNRECOGNIZED -> { - failGrpc(status = Status.FAILED_PRECONDITION, cause = IllegalStateException()) { - "Unrecognized CMMS measurement state." + getCmmsMeasurements(internalMeasurements, principal).transform { measurements -> + for (measurement in measurements) { + @Suppress("WHEN_ENUM_CAN_BE_NULL_IN_JAVA") // Protobuf enum fields cannot be null. + when (measurement.state) { + Measurement.State.SUCCEEDED -> emit(measurement) + Measurement.State.CANCELLED, + Measurement.State.FAILED -> failedMeasurements.add(measurement) + Measurement.State.COMPUTING, + Measurement.State.AWAITING_REQUISITION_FULFILLMENT -> {} + Measurement.State.STATE_UNSPECIFIED -> + failGrpc(status = Status.FAILED_PRECONDITION, cause = IllegalStateException()) { + "The CMMS measurement state should've been set." + } + Measurement.State.UNRECOGNIZED -> { + failGrpc(status = Status.FAILED_PRECONDITION, cause = IllegalStateException()) { + "Unrecognized CMMS measurement state." + } } } } } - } var anyUpdate = false val callBatchSetInternalMeasurementResultsRpc: suspend (List) -> Unit = - { items -> - batchSetInternalMeasurementResults(items, apiAuthenticationKey, principal) - } - val count = - submitBatchRequests( - succeededMeasurements, - BATCH_SET_MEASUREMENT_RESULTS_LIMIT, - callBatchSetInternalMeasurementResultsRpc, - ) { _: Unit -> - emptyList() + { items -> + batchSetInternalMeasurementResults(items, apiAuthenticationKey, principal) } - .count() + val count = + submitBatchRequests( + succeededMeasurements, + BATCH_SET_MEASUREMENT_RESULTS_LIMIT, + callBatchSetInternalMeasurementResultsRpc, + ) { _: Unit -> + emptyList() + } + .count() if (count > 0) { anyUpdate = true @@ -856,17 +784,18 @@ class MetricsService( if (failedMeasurements.isNotEmpty()) { val callBatchSetInternalMeasurementFailuresRpc: suspend (List) -> Unit = - { items -> - batchSetInternalMeasurementFailures(items, principal.resourceKey.measurementConsumerId) - } + { items -> + batchSetInternalMeasurementFailures( + items, principal.resourceKey.measurementConsumerId) + } submitBatchRequests( - failedMeasurements.asFlow(), - BATCH_SET_MEASUREMENT_FAILURES_LIMIT, - callBatchSetInternalMeasurementFailuresRpc, - ) { _: Unit -> - emptyList() - } - .collect {} + failedMeasurements.asFlow(), + BATCH_SET_MEASUREMENT_FAILURES_LIMIT, + callBatchSetInternalMeasurementFailuresRpc, + ) { _: Unit -> + emptyList() + } + .collect {} anyUpdate = true } @@ -879,24 +808,23 @@ class MetricsService( * failed or canceled CMMS [Measurement]s. */ private suspend fun batchSetInternalMeasurementFailures( - failedMeasurementsList: List, - cmmsMeasurementConsumerId: String, + failedMeasurementsList: List, + cmmsMeasurementConsumerId: String, ) { val batchSetInternalMeasurementFailuresRequest = batchSetMeasurementFailuresRequest { this.cmmsMeasurementConsumerId = cmmsMeasurementConsumerId measurementFailures += - failedMeasurementsList.map { measurement -> - measurementFailure { - cmmsMeasurementId = MeasurementKey.fromName(measurement.name)!!.measurementId - failure = measurement.failure.toInternal() + failedMeasurementsList.map { measurement -> + measurementFailure { + cmmsMeasurementId = MeasurementKey.fromName(measurement.name)!!.measurementId + failure = measurement.failure.toInternal() + } } - } } try { internalMeasurementsStub.batchSetMeasurementFailures( - batchSetInternalMeasurementFailuresRequest - ) + batchSetInternalMeasurementFailuresRequest) } catch (e: StatusException) { throw Exception("Unable to set measurement failures for Measurements.", e) } @@ -907,20 +835,20 @@ class MetricsService( * given succeeded CMMS [Measurement]s. */ private suspend fun batchSetInternalMeasurementResults( - succeededMeasurementsList: List, - apiAuthenticationKey: String, - principal: MeasurementConsumerPrincipal, + succeededMeasurementsList: List, + apiAuthenticationKey: String, + principal: MeasurementConsumerPrincipal, ) { val batchSetMeasurementResultsRequest = batchSetMeasurementResultsRequest { cmmsMeasurementConsumerId = principal.resourceKey.measurementConsumerId measurementResults += - succeededMeasurementsList.map { measurement -> - buildInternalMeasurementResult( - measurement, - apiAuthenticationKey, - principal.resourceKey.toName(), - ) - } + succeededMeasurementsList.map { measurement -> + buildInternalMeasurementResult( + measurement, + apiAuthenticationKey, + principal.resourceKey.toName(), + ) + } } try { @@ -932,18 +860,18 @@ class MetricsService( /** Retrieves [Measurement]s from the CMMS. */ private suspend fun getCmmsMeasurements( - internalMeasurements: List, - principal: MeasurementConsumerPrincipal, + internalMeasurements: List, + principal: MeasurementConsumerPrincipal, ): Flow> { val measurementNames: Flow = flow { buildSet { for (internalMeasurement in internalMeasurements) { val name = - MeasurementKey( - principal.resourceKey.measurementConsumerId, - internalMeasurement.cmmsMeasurementId, - ) - .toName() + MeasurementKey( + principal.resourceKey.measurementConsumerId, + internalMeasurement.cmmsMeasurementId, + ) + .toName() // Checks if the set already contains the name if (!contains(name)) { // If the set doesn't contain the name, emit it and add it to the set so it won't @@ -956,14 +884,14 @@ class MetricsService( } val callBatchGetMeasurementsRpc: suspend (List) -> BatchGetMeasurementsResponse = - { items -> - batchGetCmmsMeasurements(principal, items) - } + { items -> + batchGetCmmsMeasurements(principal, items) + } return submitBatchRequests( - measurementNames, - BATCH_KINGDOM_MEASUREMENTS_LIMIT, - callBatchGetMeasurementsRpc, + measurementNames, + BATCH_KINGDOM_MEASUREMENTS_LIMIT, + callBatchGetMeasurementsRpc, ) { response: BatchGetMeasurementsResponse -> response.measurementsList } @@ -971,103 +899,99 @@ class MetricsService( /** Batch get CMMS measurements. */ private suspend fun batchGetCmmsMeasurements( - principal: MeasurementConsumerPrincipal, - measurementNames: List, + principal: MeasurementConsumerPrincipal, + measurementNames: List, ): BatchGetMeasurementsResponse { try { return measurementsStub - .withAuthenticationKey(principal.config.apiKey) - .batchGetMeasurements( - batchGetMeasurementsRequest { - parent = principal.resourceKey.toName() - names += measurementNames - } - ) + .withAuthenticationKey(principal.config.apiKey) + .batchGetMeasurements( + batchGetMeasurementsRequest { + parent = principal.resourceKey.toName() + names += measurementNames + }) } catch (e: StatusException) { throw when (e.status.code) { - Status.Code.NOT_FOUND -> Status.NOT_FOUND.withDescription("Measurements not found.") - Status.Code.PERMISSION_DENIED -> - Status.PERMISSION_DENIED.withDescription( - "Doesn't have permission to get measurements." - ) - else -> Status.UNKNOWN.withDescription("Unable to retrieve Measurements.") - } - .withCause(e) - .asRuntimeException() + Status.Code.NOT_FOUND -> Status.NOT_FOUND.withDescription("Measurements not found.") + Status.Code.PERMISSION_DENIED -> + Status.PERMISSION_DENIED.withDescription( + "Doesn't have permission to get measurements.") + else -> Status.UNKNOWN.withDescription("Unable to retrieve Measurements.") + } + .withCause(e) + .asRuntimeException() } } /** Builds an [InternalMeasurement.Result]. */ private suspend fun buildInternalMeasurementResult( - measurement: Measurement, - apiAuthenticationKey: String, - principalName: String, + measurement: Measurement, + apiAuthenticationKey: String, + principalName: String, ): BatchSetMeasurementResultsRequest.MeasurementResult { val measurementSpec: MeasurementSpec = measurement.measurementSpec.unpack() val encryptionPrivateKeyHandle = - encryptionKeyPairStore.getPrivateKeyHandle( - principalName, - measurementSpec.measurementPublicKey.unpack().data, - ) - ?: failGrpc(Status.FAILED_PRECONDITION) { - "Encryption private key not found for the measurement ${measurement.name}." - } + encryptionKeyPairStore.getPrivateKeyHandle( + principalName, + measurementSpec.measurementPublicKey.unpack().data, + ) + ?: failGrpc(Status.FAILED_PRECONDITION) { + "Encryption private key not found for the measurement ${measurement.name}." + } val decryptedMeasurementResults: List = - measurement.resultsList.map { - decryptMeasurementResultOutput(it, encryptionPrivateKeyHandle, apiAuthenticationKey) - } + measurement.resultsList.map { + decryptMeasurementResultOutput(it, encryptionPrivateKeyHandle, apiAuthenticationKey) + } return measurementResult { cmmsMeasurementId = MeasurementKey.fromName(measurement.name)!!.measurementId results += - decryptedMeasurementResults.map { - try { - it.toInternal(measurement.protocolConfig) - } catch (e: NoiseMechanismUnrecognizedException) { - failGrpc(Status.UNKNOWN) { - listOfNotNull("Unrecognized noise mechanism.", e.message, e.cause?.message) - .joinToString(separator = "\n") - } - } catch (e: Throwable) { - failGrpc(Status.UNKNOWN) { - listOfNotNull("Unable to read measurement result.", e.message, e.cause?.message) - .joinToString(separator = "\n") + decryptedMeasurementResults.map { + try { + it.toInternal(measurement.protocolConfig) + } catch (e: NoiseMechanismUnrecognizedException) { + failGrpc(Status.UNKNOWN) { + listOfNotNull("Unrecognized noise mechanism.", e.message, e.cause?.message) + .joinToString(separator = "\n") + } + } catch (e: Throwable) { + failGrpc(Status.UNKNOWN) { + listOfNotNull("Unable to read measurement result.", e.message, e.cause?.message) + .joinToString(separator = "\n") + } } } - } } } /** Decrypts a [Measurement.ResultOutput] to [Measurement.Result] */ private suspend fun decryptMeasurementResultOutput( - measurementResultOutput: Measurement.ResultOutput, - encryptionPrivateKeyHandle: PrivateKeyHandle, - apiAuthenticationKey: String, + measurementResultOutput: Measurement.ResultOutput, + encryptionPrivateKeyHandle: PrivateKeyHandle, + apiAuthenticationKey: String, ): Measurement.Result { val certificate = - certificateCache.getValue( - ResourceNameApiAuthenticationKey( - name = measurementResultOutput.certificate, - apiAuthenticationKey = apiAuthenticationKey, - ) - ) + certificateCache.getValue( + ResourceNameApiAuthenticationKey( + name = measurementResultOutput.certificate, + apiAuthenticationKey = apiAuthenticationKey, + )) val signedResult = - decryptResult(measurementResultOutput.encryptedResult, encryptionPrivateKeyHandle) + decryptResult(measurementResultOutput.encryptedResult, encryptionPrivateKeyHandle) if (certificate.revocationState != Certificate.RevocationState.REVOCATION_STATE_UNSPECIFIED) { throw Status.FAILED_PRECONDITION.withDescription( - "${certificate.name} revocation state is ${certificate.revocationState}" - ) - .asRuntimeException() + "${certificate.name} revocation state is ${certificate.revocationState}") + .asRuntimeException() } val x509Certificate: X509Certificate = readCertificate(certificate.x509Der) val trustedIssuer: X509Certificate = - checkNotNull(trustedCertificates[checkNotNull(x509Certificate.authorityKeyIdentifier)]) { - "${certificate.name} not issued by trusted CA" - } + checkNotNull(trustedCertificates[checkNotNull(x509Certificate.authorityKeyIdentifier)]) { + "${certificate.name} not issued by trusted CA" + } // TODO: Record verification failure in internal Measurement rather than having the RPC fail. try { @@ -1091,16 +1015,16 @@ class MetricsService( private suspend fun getCertificate(name: String, apiAuthenticationKey: String): Certificate { return try { certificatesStub - .withAuthenticationKey(apiAuthenticationKey) - .getCertificate(getCertificateRequest { this.name = name }) + .withAuthenticationKey(apiAuthenticationKey) + .getCertificate(getCertificateRequest { this.name = name }) } catch (e: StatusException) { throw when (e.status.code) { - Status.Code.NOT_FOUND -> - Status.FAILED_PRECONDITION.withDescription("Certificate $name not found.") - else -> Status.UNKNOWN.withDescription("Unable to retrieve Certificate $name.") - } - .withCause(e) - .asRuntimeException() + Status.Code.NOT_FOUND -> + Status.FAILED_PRECONDITION.withDescription("Certificate $name not found.") + else -> Status.UNKNOWN.withDescription("Unable to retrieve Certificate $name.") + } + .withCause(e) + .asRuntimeException() } } @@ -1115,24 +1039,24 @@ class MetricsService( private suspend fun getDataProvider(name: String, apiAuthenticationKey: String): DataProvider { return try { dataProvidersStub - .withAuthenticationKey(apiAuthenticationKey) - .getDataProvider(getDataProviderRequest { this.name = name }) + .withAuthenticationKey(apiAuthenticationKey) + .getDataProvider(getDataProviderRequest { this.name = name }) } catch (e: StatusException) { throw when (e.status.code) { - Status.Code.NOT_FOUND -> Status.FAILED_PRECONDITION.withDescription("$name not found") - else -> Status.UNKNOWN.withDescription("Unable to retrieve $name") - } - .withCause(e) - .asRuntimeException() + Status.Code.NOT_FOUND -> Status.FAILED_PRECONDITION.withDescription("$name not found") + else -> Status.UNKNOWN.withDescription("Unable to retrieve $name") + } + .withCause(e) + .asRuntimeException() } } } override suspend fun getMetric(request: GetMetricRequest): Metric { val metricKey = - grpcRequireNotNull(MetricKey.fromName(request.name)) { - "Metric name is either unspecified or invalid." - } + grpcRequireNotNull(MetricKey.fromName(request.name)) { + "Metric name is either unspecified or invalid." + } val principal: ReportingPrincipal = principalFromCurrentContext when (principal) { @@ -1146,20 +1070,20 @@ class MetricsService( } val internalMetric: InternalMetric = - getInternalMetric(metricKey.cmmsMeasurementConsumerId, metricKey.metricId) + getInternalMetric(metricKey.cmmsMeasurementConsumerId, metricKey.metricId) return syncAndConvertInternalMetricsToPublicMetrics( - mapOf(internalMetric.state to listOf(internalMetric)), - principal, - ) - .single() + mapOf(internalMetric.state to listOf(internalMetric)), + principal, + ) + .single() } override suspend fun batchGetMetrics(request: BatchGetMetricsRequest): BatchGetMetricsResponse { val parentKey = - grpcRequireNotNull(MeasurementConsumerKey.fromName(request.parent)) { - "Parent is either unspecified or invalid." - } + grpcRequireNotNull(MeasurementConsumerKey.fromName(request.parent)) { + "Parent is either unspecified or invalid." + } val principal: ReportingPrincipal = principalFromCurrentContext @@ -1179,18 +1103,18 @@ class MetricsService( } val metricIds: List = - request.namesList.map { metricName -> - val metricKey = - grpcRequireNotNull(MetricKey.fromName(metricName)) { - "Metric name is either unspecified or invalid." - } - metricKey.metricId - } + request.namesList.map { metricName -> + val metricKey = + grpcRequireNotNull(MetricKey.fromName(metricName)) { + "Metric name is either unspecified or invalid." + } + metricKey.metricId + } val internalMetricsByState: Map> = - batchGetInternalMetrics(principal.resourceKey.measurementConsumerId, metricIds).groupBy { - it.state - } + batchGetInternalMetrics(principal.resourceKey.measurementConsumerId, metricIds).groupBy { + it.state + } return batchGetMetricsResponse { metrics += syncAndConvertInternalMetricsToPublicMetrics(internalMetricsByState, principal) @@ -1199,9 +1123,9 @@ class MetricsService( override suspend fun listMetrics(request: ListMetricsRequest): ListMetricsResponse { val parentKey = - grpcRequireNotNull(MeasurementConsumerKey.fromName(request.parent)) { - "Parent is either unspecified or invalid." - } + grpcRequireNotNull(MeasurementConsumerKey.fromName(request.parent)) { + "Parent is either unspecified or invalid." + } val principal: ReportingPrincipal = principalFromCurrentContext when (principal) { @@ -1215,36 +1139,34 @@ class MetricsService( } val listMetricsPageToken: ListMetricsPageToken = request.toListMetricsPageToken() - val apiAuthenticationKey: String = principal.config.apiKey - val streamInternalMetricRequest: StreamMetricsRequest = - listMetricsPageToken.toStreamMetricsRequest() + listMetricsPageToken.toStreamMetricsRequest() val results: List = - try { - internalMetricsStub.streamMetrics(streamInternalMetricRequest).toList() - } catch (e: StatusException) { - throw Exception("Unable to list Metrics.", e) - } + try { + internalMetricsStub.streamMetrics(streamInternalMetricRequest).toList() + } catch (e: StatusException) { + throw Exception("Unable to list Metrics.", e) + } if (results.isEmpty()) { return ListMetricsResponse.getDefaultInstance() } val nextPageToken: ListMetricsPageToken? = - if (results.size > listMetricsPageToken.pageSize) { - listMetricsPageToken.copy { - lastMetric = previousPageEnd { - cmmsMeasurementConsumerId = results[results.lastIndex - 1].cmmsMeasurementConsumerId - externalMetricId = results[results.lastIndex - 1].externalMetricId + if (results.size > listMetricsPageToken.pageSize) { + listMetricsPageToken.copy { + lastMetric = previousPageEnd { + cmmsMeasurementConsumerId = results[results.lastIndex - 1].cmmsMeasurementConsumerId + externalMetricId = results[results.lastIndex - 1].externalMetricId + } } + } else { + null } - } else { - null - } val subResultsByState: Map> = - results.subList(0, min(results.size, listMetricsPageToken.pageSize)).groupBy { it.state } + results.subList(0, min(results.size, listMetricsPageToken.pageSize)).groupBy { it.state } return listMetricsResponse { metrics += syncAndConvertInternalMetricsToPublicMetrics(subResultsByState, principal) @@ -1257,8 +1179,8 @@ class MetricsService( /** Gets a batch of [InternalMetric]s. */ private suspend fun batchGetInternalMetrics( - cmmsMeasurementConsumerId: String, - metricIds: List, + cmmsMeasurementConsumerId: String, + metricIds: List, ): List { val batchGetMetricsRequest = batchGetMetricsRequest { this.cmmsMeasurementConsumerId = cmmsMeasurementConsumerId @@ -1274,8 +1196,8 @@ class MetricsService( /** Gets an [InternalMetric]. */ private suspend fun getInternalMetric( - cmmsMeasurementConsumerId: String, - metricId: String, + cmmsMeasurementConsumerId: String, + metricId: String, ): InternalMetric { return try { batchGetInternalMetrics(cmmsMeasurementConsumerId, listOf(metricId)).first() @@ -1287,9 +1209,9 @@ class MetricsService( override suspend fun createMetric(request: CreateMetricRequest): Metric { val parentKey = - grpcRequireNotNull(MeasurementConsumerKey.fromName(request.parent)) { - "Parent is either unspecified or invalid." - } + grpcRequireNotNull(MeasurementConsumerKey.fromName(request.parent)) { + "Parent is either unspecified or invalid." + } val principal: ReportingPrincipal = principalFromCurrentContext @@ -1305,42 +1227,54 @@ class MetricsService( grpcRequire(request.hasMetric()) { "Metric is not specified." } + val reportingSetKey = grpcRequireNotNull(ReportingSetKey.fromName(request.metric.reportingSet)) { + "Invalid reporting set name ${request.metric.reportingSet}." + } + + if (reportingSetKey.cmmsMeasurementConsumerId != parentKey.measurementConsumerId) { + failGrpc(Status.PERMISSION_DENIED) { "No access to the reporting set [${request.metric.reportingSet}]." } + } + val batchGetReportingSetsResponse = - batchGetInternalReportingSets( - parentKey.measurementConsumerId, - listOf(request.metric.reportingSet), - ) + batchGetInternalReportingSets( + parentKey.measurementConsumerId, + listOf(reportingSetKey.reportingSetId), + ) + + val internalPrimitiveReportingSetMap: Map = + buildInternalPrimitiveReportingSetMap( + parentKey.measurementConsumerId, batchGetReportingSetsResponse.reportingSetsList) val internalCreateMetricRequest: InternalCreateMetricRequest = - buildInternalCreateMetricRequest( - principal.resourceKey.measurementConsumerId, - request, - batchGetReportingSetsResponse.reportingSetsList.single(), - ) + buildInternalCreateMetricRequest( + principal.resourceKey.measurementConsumerId, + request, + batchGetReportingSetsResponse.reportingSetsList.single(), + internalPrimitiveReportingSetMap, + ) val internalMetric = - try { - internalMetricsStub.createMetric(internalCreateMetricRequest) - } catch (e: StatusException) { - throw when (e.status.code) { - Status.Code.ALREADY_EXISTS -> - Status.ALREADY_EXISTS.withDescription( - "Metric with ID ${request.metricId} already exists under ${request.parent}" - ) - Status.Code.NOT_FOUND -> - Status.NOT_FOUND.withDescription("Reporting set used in the metric not found.") - Status.Code.FAILED_PRECONDITION -> - Status.FAILED_PRECONDITION.withDescription( - "Unable to create the metric. The measurement consumer not found." - ) - else -> Status.UNKNOWN.withDescription("Unable to create Metric.") - } - .withCause(e) - .asRuntimeException() - } + try { + internalMetricsStub.createMetric(internalCreateMetricRequest) + } catch (e: StatusException) { + throw when (e.status.code) { + Status.Code.ALREADY_EXISTS -> + Status.ALREADY_EXISTS.withDescription( + "Metric with ID ${request.metricId} already exists under ${request.parent}") + Status.Code.NOT_FOUND -> + Status.NOT_FOUND.withDescription("Reporting set used in the metric not found.") + Status.Code.FAILED_PRECONDITION -> + Status.FAILED_PRECONDITION.withDescription( + "Unable to create the metric. The measurement consumer not found.") + else -> Status.UNKNOWN.withDescription("Unable to create Metric.") + } + .withCause(e) + .asRuntimeException() + } if (internalMetric.state == InternalMetric.State.RUNNING) { - measurementSupplier.createCmmsMeasurements(listOf(internalMetric), principal) + measurementSupplier.createCmmsMeasurements( + listOf(internalMetric), internalPrimitiveReportingSetMap, principal) } // Convert the internal metric to public and return it. @@ -1348,12 +1282,12 @@ class MetricsService( } override suspend fun batchCreateMetrics( - request: BatchCreateMetricsRequest + request: BatchCreateMetricsRequest ): BatchCreateMetricsResponse { val parentKey = - grpcRequireNotNull(MeasurementConsumerKey.fromName(request.parent)) { - "Parent is either unspecified or invalid." - } + grpcRequireNotNull(MeasurementConsumerKey.fromName(request.parent)) { + "Parent is either unspecified or invalid." + } val principal: ReportingPrincipal = principalFromCurrentContext @@ -1378,79 +1312,93 @@ class MetricsService( } val reportingSetNames = - request.requestsList - .map { - grpcRequire(it.hasMetric()) { "Metric is not specified." } + request.requestsList + .map { + grpcRequire(it.hasMetric()) { "Metric is not specified." } - it.metric.reportingSet - } - .distinct() + it.metric.reportingSet + } + .distinct() val callRpc: suspend (List) -> BatchGetReportingSetsResponse = { items -> - batchGetInternalReportingSets(parentKey.measurementConsumerId, items) - } + val externalReportingSetIds: List = + items.map { + val reportingSetKey = + grpcRequireNotNull(ReportingSetKey.fromName(it)) { "Invalid reporting set name $it." } - val reportingSetNameToInternalReportingSetMap: Map = buildMap { - submitBatchRequests(reportingSetNames.asFlow(), BATCH_GET_REPORTING_SETS_LIMIT, callRpc) { - response -> - response.reportingSetsList - } - .collect { reportingSetsList -> - for (reportingSet in reportingSetsList) { - putIfAbsent( - ReportingSetKey(parentKey.measurementConsumerId, reportingSet.externalReportingSetId) - .toName(), - reportingSet, - ) + if (reportingSetKey.cmmsMeasurementConsumerId != parentKey.measurementConsumerId) { + failGrpc(Status.PERMISSION_DENIED) { "No access to the reporting set [$it]." } } + + reportingSetKey.reportingSetId } + batchGetInternalReportingSets(parentKey.measurementConsumerId, externalReportingSetIds) + } + + val reportingSetNameToInternalReportingSetMap: Map = buildMap { + submitBatchRequests(reportingSetNames.asFlow(), BATCH_GET_REPORTING_SETS_LIMIT, callRpc) { + response -> + response.reportingSetsList + } + .collect { reportingSetsList -> + for (reportingSet in reportingSetsList) { + putIfAbsent( + ReportingSetKey( + parentKey.measurementConsumerId, reportingSet.externalReportingSetId) + .toName(), + reportingSet, + ) + } + } } + val internalPrimitiveReportingSetMap: Map = + buildInternalPrimitiveReportingSetMap( + parentKey.measurementConsumerId, reportingSetNameToInternalReportingSetMap.values) + val internalCreateMetricRequestsList: List> = - coroutineScope { - request.requestsList.map { createMetricRequest -> - async { - buildInternalCreateMetricRequest( - parentKey.measurementConsumerId, - createMetricRequest, - reportingSetNameToInternalReportingSetMap.getValue( - createMetricRequest.metric.reportingSet - ), - ) + coroutineScope { + request.requestsList.map { createMetricRequest -> + async { + buildInternalCreateMetricRequest( + parentKey.measurementConsumerId, + createMetricRequest, + reportingSetNameToInternalReportingSetMap.getValue( + createMetricRequest.metric.reportingSet), + internalPrimitiveReportingSetMap) + } } } - } val internalMetrics = - try { - internalMetricsStub - .batchCreateMetrics( - internalBatchCreateMetricsRequest { - cmmsMeasurementConsumerId = parentKey.measurementConsumerId - requests += internalCreateMetricRequestsList.awaitAll() - } - ) - .metricsList - } catch (e: StatusException) { - throw when (e.status.code) { - Status.Code.NOT_FOUND -> - Status.NOT_FOUND.withDescription("Reporting set used in metrics not found.") - Status.Code.FAILED_PRECONDITION -> - Status.FAILED_PRECONDITION.withDescription( - "Unable to create the metrics. The measurement consumer not found." - ) - else -> Status.UNKNOWN.withDescription("Unable to create Metrics.") - } - .withCause(e) - .asRuntimeException() - } + try { + internalMetricsStub + .batchCreateMetrics( + internalBatchCreateMetricsRequest { + cmmsMeasurementConsumerId = parentKey.measurementConsumerId + requests += internalCreateMetricRequestsList.awaitAll() + }) + .metricsList + } catch (e: StatusException) { + throw when (e.status.code) { + Status.Code.NOT_FOUND -> + Status.NOT_FOUND.withDescription("Reporting set used in metrics not found.") + Status.Code.FAILED_PRECONDITION -> + Status.FAILED_PRECONDITION.withDescription( + "Unable to create the metrics. The measurement consumer not found.") + else -> Status.UNKNOWN.withDescription("Unable to create Metrics.") + } + .withCause(e) + .asRuntimeException() + } val internalRunningMetrics = - internalMetrics.filter { internalMetric -> - internalMetric.state == InternalMetric.State.RUNNING - } + internalMetrics.filter { internalMetric -> + internalMetric.state == InternalMetric.State.RUNNING + } if (internalRunningMetrics.isNotEmpty()) { - measurementSupplier.createCmmsMeasurements(internalRunningMetrics, principal) + measurementSupplier.createCmmsMeasurements( + internalRunningMetrics, internalPrimitiveReportingSetMap, principal) } // Convert the internal metric to public and return it. @@ -1459,9 +1407,10 @@ class MetricsService( /** Builds an [InternalCreateMetricRequest]. */ private fun buildInternalCreateMetricRequest( - cmmsMeasurementConsumerId: String, - request: CreateMetricRequest, - internalReportingSet: InternalReportingSet, + cmmsMeasurementConsumerId: String, + request: CreateMetricRequest, + internalReportingSet: InternalReportingSet, + internalPrimitiveReportingSetMap: Map, ): InternalCreateMetricRequest { grpcRequire(request.metricId.matches(RESOURCE_ID_REGEX)) { "Metric ID is invalid." } grpcRequire(request.metric.reportingSet.isNotEmpty()) { @@ -1469,31 +1418,28 @@ class MetricsService( } grpcRequire(request.metric.hasTimeInterval()) { "Time interval in metric is not specified." } grpcRequire( - request.metric.timeInterval.startTime.seconds > 0 || - request.metric.timeInterval.startTime.nanos > 0 - ) { - "TimeInterval startTime is unspecified." - } + request.metric.timeInterval.startTime.seconds > 0 || + request.metric.timeInterval.startTime.nanos > 0) { + "TimeInterval startTime is unspecified." + } grpcRequire( - request.metric.timeInterval.endTime.seconds > 0 || - request.metric.timeInterval.endTime.nanos > 0 - ) { - "TimeInterval endTime is unspecified." - } + request.metric.timeInterval.endTime.seconds > 0 || + request.metric.timeInterval.endTime.nanos > 0) { + "TimeInterval endTime is unspecified." + } grpcRequire( - request.metric.timeInterval.endTime.seconds > request.metric.timeInterval.startTime.seconds || - request.metric.timeInterval.endTime.nanos > request.metric.timeInterval.startTime.nanos - ) { - "TimeInterval endTime is not later than startTime." - } + request.metric.timeInterval.endTime.seconds > + request.metric.timeInterval.startTime.seconds || + request.metric.timeInterval.endTime.nanos > + request.metric.timeInterval.startTime.nanos) { + "TimeInterval endTime is not later than startTime." + } grpcRequire(request.metric.hasMetricSpec()) { "Metric spec in metric is not specified." } // Utilizes the property of the set expression compilation result -- If the set expression // contains only union operators, the compilation result has to be a single component. - if ( - request.metric.metricSpec.hasReachAndFrequency() && - internalReportingSet.weightedSubsetUnionsList.size != 1 - ) { + if (request.metric.metricSpec.hasReachAndFrequency() && + internalReportingSet.weightedSubsetUnionsList.size != 1) { failGrpc(Status.INVALID_ARGUMENT) { "Reach-and-frequency metrics can only be computed on union-only set expressions." } @@ -1507,32 +1453,79 @@ class MetricsService( externalReportingSetId = internalReportingSet.externalReportingSetId timeInterval = request.metric.timeInterval metricSpec = - try { - request.metric.metricSpec.withDefaults(metricSpecConfig).toInternal() - } catch (e: MetricSpecDefaultsException) { - failGrpc(Status.INVALID_ARGUMENT) { - listOfNotNull("Invalid metric spec.", e.message, e.cause?.message) - .joinToString(separator = "\n") + try { + request.metric.metricSpec.withDefaults(metricSpecConfig, secureRandom).toInternal() + } catch (e: MetricSpecDefaultsException) { + failGrpc(Status.INVALID_ARGUMENT) { + listOfNotNull("Invalid metric spec.", e.message, e.cause?.message) + .joinToString(separator = "\n") + } + } catch (e: Exception) { + failGrpc(Status.UNKNOWN) { "Failed to read the metric spec." } } - } catch (e: Exception) { - failGrpc(Status.UNKNOWN) { "Failed to read the metric spec." } - } weightedMeasurements += - buildInitialInternalMeasurements( - cmmsMeasurementConsumerId, - request.metric, - internalReportingSet, - ) + buildInitialInternalMeasurements( + cmmsMeasurementConsumerId, + request.metric, + internalReportingSet, + internalPrimitiveReportingSetMap) details = InternalMetricKt.details { filters += request.metric.filtersList } } } } + /** Build a map of external ReportingSetId to Primitive [InternalReportingSet]. */ + private suspend fun buildInternalPrimitiveReportingSetMap( + measurementConsumerId: String, + internalReportingSetsList: Collection, + ): Map { + // Gets all external IDs of primitive reporting sets from the metric list. + val externalPrimitiveReportingSetIds: Flow = flow { + buildSet { + for (internalReportingSet in internalReportingSetsList) { + for (weightedSubsetUnion in internalReportingSet.weightedSubsetUnionsList) { + for (primitiveReportingSetBasis in weightedSubsetUnion.primitiveReportingSetBasesList) { + // Checks if the set already contains the ID + if (!contains(primitiveReportingSetBasis.externalReportingSetId)) { + // If the set doesn't contain the ID, emit it and add it to the set so it won't + // get emitted again. + emit(primitiveReportingSetBasis.externalReportingSetId) + add(primitiveReportingSetBasis.externalReportingSetId) + } + } + } + } + } + } + + val callBatchGetInternalReportingSetsRpc: + suspend (List) -> BatchGetReportingSetsResponse = + { items -> + batchGetInternalReportingSets(measurementConsumerId, items) + } + + return buildMap { + submitBatchRequests( + externalPrimitiveReportingSetIds, + BATCH_GET_REPORTING_SETS_LIMIT, + callBatchGetInternalReportingSetsRpc, + ) { response: BatchGetReportingSetsResponse -> + response.reportingSetsList + } + .collect { reportingSets: List -> + for (reportingSet in reportingSets) { + computeIfAbsent(reportingSet.externalReportingSetId) { reportingSet } + } + } + } + } + /** Builds [InternalMeasurement]s for a [Metric] over an [InternalReportingSet]. */ private fun buildInitialInternalMeasurements( - cmmsMeasurementConsumerId: String, - metric: Metric, - internalReportingSet: InternalReportingSet, + cmmsMeasurementConsumerId: String, + metric: Metric, + internalReportingSet: InternalReportingSet, + internalPrimitiveReportingSetMap: Map, ): List { return internalReportingSet.weightedSubsetUnionsList.map { weightedSubsetUnion -> weightedMeasurement { @@ -1541,39 +1534,49 @@ class MetricsService( measurement = internalMeasurement { this.cmmsMeasurementConsumerId = cmmsMeasurementConsumerId timeInterval = metric.timeInterval + val firstCmmsDataProviderId = + internalPrimitiveReportingSetMap + .getValue( + weightedSubsetUnion.primitiveReportingSetBasesList + .first() + .externalReportingSetId) + .primitive + .eventGroupKeysList + .first() + .cmmsDataProviderId + isSingleDataProvider = true this.primitiveReportingSetBases += - weightedSubsetUnion.primitiveReportingSetBasesList.map { primitiveReportingSetBasis -> - primitiveReportingSetBasis.copy { filters += metric.filtersList } - } + weightedSubsetUnion.primitiveReportingSetBasesList.map { primitiveReportingSetBasis -> + if (isSingleDataProvider) { + internalPrimitiveReportingSetMap + .getValue(primitiveReportingSetBasis.externalReportingSetId) + .primitive + .eventGroupKeysList + .forEach { + if (firstCmmsDataProviderId != it.cmmsDataProviderId) { + isSingleDataProvider = false + return@forEach + } + } + } + primitiveReportingSetBasis.copy { filters += metric.filtersList } + } } } } } - /** Batch get [InternalReportingSet]s based on [ReportingSet] names. */ + /** Batch get [InternalReportingSet]s based on external IDs. */ private suspend fun batchGetInternalReportingSets( - cmmsMeasurementConsumerId: String, - reportingSetNames: List, + cmmsMeasurementConsumerId: String, + externalReportingSetIds: List, ): BatchGetReportingSetsResponse { - val externalReportingSetIds: List = - reportingSetNames.map { - val reportingSetKey = - grpcRequireNotNull(ReportingSetKey.fromName(it)) { "Invalid reporting set name $it." } - - if (reportingSetKey.cmmsMeasurementConsumerId != cmmsMeasurementConsumerId) { - failGrpc(Status.PERMISSION_DENIED) { "No access to the reporting set [$it]." } - } - - reportingSetKey.reportingSetId - } - return try { internalReportingSetsStub.batchGetReportingSets( - batchGetReportingSetsRequest { - this.cmmsMeasurementConsumerId = cmmsMeasurementConsumerId - this.externalReportingSetIds += externalReportingSetIds - } - ) + batchGetReportingSetsRequest { + this.cmmsMeasurementConsumerId = cmmsMeasurementConsumerId + this.externalReportingSetIds += externalReportingSetIds + }) } catch (e: StatusException) { throw Exception("Unable to retrieve ReportingSets using the provided names.", e) } @@ -1581,43 +1584,45 @@ class MetricsService( /** Converts [InternalMetric]s to public [Metric]s after syncing [Measurement]s. */ private suspend fun syncAndConvertInternalMetricsToPublicMetrics( - metricsByState: Map>, - principal: MeasurementConsumerPrincipal, + metricsByState: Map>, + principal: MeasurementConsumerPrincipal, ): List { // Only syncs pending measurements which can only be in metrics that are still running. val toBeSyncedInternalMeasurements: List = - if (metricsByState.containsKey(InternalMetric.State.RUNNING)) { - metricsByState - .getValue(InternalMetric.State.RUNNING) - .flatMap { internalMetric -> internalMetric.weightedMeasurementsList } - .map { weightedMeasurement -> weightedMeasurement.measurement } - .filter { internalMeasurement -> - internalMeasurement.state == InternalMeasurement.State.PENDING - } - } else { - emptyList() - } + if (metricsByState.containsKey(InternalMetric.State.RUNNING)) { + metricsByState + .getValue(InternalMetric.State.RUNNING) + .flatMap { internalMetric -> internalMetric.weightedMeasurementsList } + .map { weightedMeasurement -> weightedMeasurement.measurement } + .filter { internalMeasurement -> + internalMeasurement.state == InternalMeasurement.State.PENDING + } + } else { + emptyList() + } val anyMeasurementUpdated: Boolean = - measurementSupplier.syncInternalMeasurements( - toBeSyncedInternalMeasurements, - principal.config.apiKey, - principal, - ) + measurementSupplier.syncInternalMeasurements( + toBeSyncedInternalMeasurements, + principal.config.apiKey, + principal, + ) return buildList { for (state in metricsByState.keys) { when (state) { InternalMetric.State.SUCCEEDED, InternalMetric.State.FAILED -> - addAll(metricsByState.getValue(state).map { it.toMetric(variances) }) + addAll(metricsByState.getValue(state).map { it.toMetric(variances) }) InternalMetric.State.RUNNING -> { if (anyMeasurementUpdated) { val updatedInternalMetrics = - batchGetInternalMetrics( - principal.resourceKey.measurementConsumerId, - metricsByState.getValue(InternalMetric.State.RUNNING).map { it.externalMetricId }, - ) + batchGetInternalMetrics( + principal.resourceKey.measurementConsumerId, + metricsByState.getValue(InternalMetric.State.RUNNING).map { + it.externalMetricId + }, + ) addAll(updatedInternalMetrics.map { it.toMetric(variances) }) } else { addAll(metricsByState.getValue(state).map { it.toMetric(variances) }) @@ -1627,12 +1632,11 @@ class MetricsService( // Metrics created before state was tracked in the database will have the state be // unspecified. This calculates the correct state for those metrics. addAll( - metricsByState.getValue(state).map { internalMetric -> - internalMetric - .copy { this.state = internalMetric.calculateState() } - .toMetric(variances) - } - ) + metricsByState.getValue(state).map { internalMetric -> + internalMetric + .copy { this.state = internalMetric.calculateState() } + .toMetric(variances) + }) } InternalMetric.State.UNRECOGNIZED -> error("Invalid Metric State") } @@ -1652,9 +1656,9 @@ fun ListMetricsRequest.toListMetricsPageToken(): ListMetricsPageToken { grpcRequire(source.pageSize >= 0) { "Page size cannot be less than 0." } val parentKey: MeasurementConsumerKey = - grpcRequireNotNull(MeasurementConsumerKey.fromName(source.parent)) { - "Parent is either unspecified or invalid." - } + grpcRequireNotNull(MeasurementConsumerKey.fromName(source.parent)) { + "Parent is either unspecified or invalid." + } val cmmsMeasurementConsumerId = parentKey.measurementConsumerId return if (pageToken.isNotBlank()) { @@ -1670,11 +1674,11 @@ fun ListMetricsRequest.toListMetricsPageToken(): ListMetricsPageToken { } else { listMetricsPageToken { pageSize = - when { - source.pageSize < MIN_PAGE_SIZE -> DEFAULT_PAGE_SIZE - source.pageSize > MAX_PAGE_SIZE -> MAX_PAGE_SIZE - else -> source.pageSize - } + when { + source.pageSize < MIN_PAGE_SIZE -> DEFAULT_PAGE_SIZE + source.pageSize > MAX_PAGE_SIZE -> MAX_PAGE_SIZE + else -> source.pageSize + } this.cmmsMeasurementConsumerId = cmmsMeasurementConsumerId } } @@ -1685,13 +1689,13 @@ private fun InternalMetric.toMetric(variances: Variances): Metric { val source = this return metric { name = - MetricKey( - cmmsMeasurementConsumerId = source.cmmsMeasurementConsumerId, - metricId = source.externalMetricId, - ) - .toName() + MetricKey( + cmmsMeasurementConsumerId = source.cmmsMeasurementConsumerId, + metricId = source.externalMetricId, + ) + .toName() reportingSet = - ReportingSetKey(source.cmmsMeasurementConsumerId, source.externalReportingSetId).toName() + ReportingSetKey(source.cmmsMeasurementConsumerId, source.externalReportingSetId).toName() timeInterval = source.timeInterval metricSpec = source.metricSpec.toMetricSpec() filters += source.details.filtersList @@ -1707,49 +1711,49 @@ private fun InternalMetric.toMetric(variances: Variances): Metric { private fun buildMetricResult(metric: InternalMetric, variances: Variances): MetricResult { return metricResult { cmmsMeasurements += - metric.weightedMeasurementsList.map { - MeasurementKey(metric.cmmsMeasurementConsumerId, it.measurement.cmmsMeasurementId).toName() - } + metric.weightedMeasurementsList.map { + MeasurementKey(metric.cmmsMeasurementConsumerId, it.measurement.cmmsMeasurementId) + .toName() + } @Suppress("WHEN_ENUM_CAN_BE_NULL_IN_JAVA") // Proto enum fields are never null. when (metric.metricSpec.typeCase) { InternalMetricSpec.TypeCase.REACH -> { reach = - calculateReachResult( - metric.weightedMeasurementsList, - metric.metricSpec.vidSamplingInterval, - metric.metricSpec.reach.privacyParams, - variances, - ) + calculateReachResult( + metric.weightedMeasurementsList, + metric.metricSpec.reach, + variances, + ) } InternalMetricSpec.TypeCase.REACH_AND_FREQUENCY -> { reachAndFrequency = reachAndFrequencyResult { reach = - calculateReachResult( - metric.weightedMeasurementsList, - metric.metricSpec.vidSamplingInterval, - metric.metricSpec.reachAndFrequency.reachPrivacyParams, - variances, - ) + calculateReachResult( + metric.weightedMeasurementsList, + metric.metricSpec.reachAndFrequency, + variances, + ) frequencyHistogram = - calculateFrequencyHistogramResults( - metric.weightedMeasurementsList, - metric.metricSpec, - variances, - ) + calculateFrequencyHistogramResults( + metric.weightedMeasurementsList, + metric.metricSpec.reachAndFrequency, + variances, + ) } } InternalMetricSpec.TypeCase.IMPRESSION_COUNT -> { impressionCount = - calculateImpressionResult(metric.weightedMeasurementsList, metric.metricSpec, variances) + calculateImpressionResult( + metric.weightedMeasurementsList, metric.metricSpec.impressionCount, variances) } InternalMetricSpec.TypeCase.WATCH_DURATION -> { watchDuration = - calculateWatchDurationResult( - metric.weightedMeasurementsList, - metric.metricSpec, - variances, - ) + calculateWatchDurationResult( + metric.weightedMeasurementsList, + metric.metricSpec.watchDuration, + variances, + ) } InternalMetricSpec.TypeCase.POPULATION_COUNT -> { populationCount = calculatePopulationResult(metric.weightedMeasurementsList) @@ -1765,7 +1769,7 @@ private fun buildMetricResult(metric: InternalMetric, variances: Variances): Met /** Aggregates a list of [InternalMeasurement.Result]s to a [InternalMeasurement.Result] */ private fun aggregateResults( - internalMeasurementResults: List + internalMeasurementResults: List ): InternalMeasurement.Result { if (internalMeasurementResults.isEmpty()) { failGrpc(status = Status.FAILED_PRECONDITION, cause = IllegalStateException()) { @@ -1791,10 +1795,10 @@ private fun aggregateResults( } for ((frequency, percentage) in result.frequency.relativeFrequencyDistributionMap) { val previousTotalReachCount = - frequencyDistribution.getOrDefault(frequency, 0.0) * reachValue + frequencyDistribution.getOrDefault(frequency, 0.0) * reachValue val currentReachCount = percentage * result.reach.value frequencyDistribution[frequency] = - (previousTotalReachCount + currentReachCount) / (reachValue + result.reach.value) + (previousTotalReachCount + currentReachCount) / (reachValue + result.reach.value) } } if (result.hasReach()) { @@ -1817,16 +1821,16 @@ private fun aggregateResults( } if (internalMeasurementResults.first().hasFrequency()) { this.frequency = - InternalMeasurementKt.ResultKt.frequency { - relativeFrequencyDistribution.putAll(frequencyDistribution) - } + InternalMeasurementKt.ResultKt.frequency { + relativeFrequencyDistribution.putAll(frequencyDistribution) + } } if (internalMeasurementResults.first().hasImpression()) { this.impression = InternalMeasurementKt.ResultKt.impression { value = impressionValue } } if (internalMeasurementResults.first().hasWatchDuration()) { this.watchDuration = - InternalMeasurementKt.ResultKt.watchDuration { value = watchDurationValue } + InternalMeasurementKt.ResultKt.watchDuration { value = watchDurationValue } } if (internalMeasurementResults.first().hasPopulation()) { this.population = InternalMeasurementKt.ResultKt.population { value = populationValue } @@ -1836,9 +1840,9 @@ private fun aggregateResults( /** Calculates the watch duration result from [WeightedMeasurement]s. */ private fun calculateWatchDurationResult( - weightedMeasurements: List, - metricSpec: InternalMetricSpec, - variances: Variances, + weightedMeasurements: List, + watchDurationParams: InternalMetricSpec.WatchDurationParams, + variances: Variances, ): MetricResult.WatchDurationResult { for (weightedMeasurement in weightedMeasurements) { if (weightedMeasurement.measurement.details.resultsList.any { !it.hasWatchDuration() }) { @@ -1849,24 +1853,24 @@ private fun calculateWatchDurationResult( } return watchDurationResult { val watchDuration: ProtoDuration = - weightedMeasurements - .map { weightedMeasurement -> - aggregateResults(weightedMeasurement.measurement.details.resultsList) - .watchDuration - .value * weightedMeasurement.weight - } - .reduce { sum, element -> sum + element } + weightedMeasurements + .map { weightedMeasurement -> + aggregateResults(weightedMeasurement.measurement.details.resultsList) + .watchDuration + .value * weightedMeasurement.weight + } + .reduce { sum, element -> sum + element } value = watchDuration.toDoubleSecond() // Only compute univariate statistics for union-only operations, i.e. single source measurement. if (weightedMeasurements.size == 1) { val weightedMeasurement = weightedMeasurements.single() val weightedMeasurementVarianceParamsList: - List = - buildWeightedWatchDurationMeasurementVarianceParamsPerResult( - weightedMeasurement, - metricSpec, - ) + List = + buildWeightedWatchDurationMeasurementVarianceParamsPerResult( + weightedMeasurement, + watchDurationParams, + ) // If any measurement result contains insufficient data for variance calculation, univariate // statistics won't be computed. @@ -1875,26 +1879,23 @@ private fun calculateWatchDurationResult( // Watch duration results in a measurement are independent to each other. The variance is // the sum of the variances of each result. standardDeviation = - sqrt( - weightedMeasurementVarianceParamsList.sumOf { weightedMeasurementVarianceParams -> - try { - variances.computeMetricVariance( - WatchDurationMetricVarianceParams( - listOf(requireNotNull(weightedMeasurementVarianceParams)) - ) - ) - } catch (e: Throwable) { - failGrpc(Status.UNKNOWN) { - listOfNotNull( - "Unable to compute variance of watch duration metric.", - e.message, - e.cause?.message, - ) - .joinToString(separator = "\n") - } - } - } - ) + sqrt( + weightedMeasurementVarianceParamsList.sumOf { weightedMeasurementVarianceParams -> + try { + variances.computeMetricVariance( + WatchDurationMetricVarianceParams( + listOf(requireNotNull(weightedMeasurementVarianceParams)))) + } catch (e: Throwable) { + failGrpc(Status.UNKNOWN) { + listOfNotNull( + "Unable to compute variance of watch duration metric.", + e.message, + e.cause?.message, + ) + .joinToString(separator = "\n") + } + } + }) } } } @@ -1903,11 +1904,11 @@ private fun calculateWatchDurationResult( /** Calculates the population result from [WeightedMeasurement]s. */ private fun calculatePopulationResult( - weightedMeasurements: List + weightedMeasurements: List ): MetricResult.PopulationCountResult { // Only take the first measurement because Population measurements will only have one element. val populationResult = - aggregateResults(weightedMeasurements.single().measurement.details.resultsList) + aggregateResults(weightedMeasurements.single().measurement.details.resultsList) return populationCountResult { value = populationResult.population.value } } @@ -1923,11 +1924,11 @@ private fun ProtoDuration.toDoubleSecond(): Double { * @throws io.grpc.StatusRuntimeException when measurement noise mechanism is unrecognized. */ fun buildWeightedWatchDurationMeasurementVarianceParamsPerResult( - weightedMeasurement: WeightedMeasurement, - metricSpec: MetricSpec, + weightedMeasurement: WeightedMeasurement, + watchDurationParams: InternalMetricSpec.WatchDurationParams, ): List { val watchDurationResults: List = - weightedMeasurement.measurement.details.resultsList.map { it.watchDuration } + weightedMeasurement.measurement.details.resultsList.map { it.watchDuration } if (watchDurationResults.isEmpty()) { failGrpc(status = Status.FAILED_PRECONDITION, cause = IllegalStateException()) { @@ -1937,51 +1938,53 @@ fun buildWeightedWatchDurationMeasurementVarianceParamsPerResult( return watchDurationResults.map { watchDurationResult -> val statsNoiseMechanism: StatsNoiseMechanism = - try { - watchDurationResult.noiseMechanism.toStatsNoiseMechanism() - } catch (e: NoiseMechanismUnspecifiedException) { - return@map null - } catch (e: NoiseMechanismUnrecognizedException) { - failGrpc(Status.UNKNOWN) { - listOfNotNull( - "Unrecognized noise mechanism should've been caught earlier.", - e.message, - e.cause?.message, - ) - .joinToString(separator = "\n") + try { + watchDurationResult.noiseMechanism.toStatsNoiseMechanism() + } catch (e: NoiseMechanismUnspecifiedException) { + return@map null + } catch (e: NoiseMechanismUnrecognizedException) { + failGrpc(Status.UNKNOWN) { + listOfNotNull( + "Unrecognized noise mechanism should've been caught earlier.", + e.message, + e.cause?.message, + ) + .joinToString(separator = "\n") + } } - } val methodology: Methodology = - try { - buildStatsMethodology(watchDurationResult) - } catch (e: MeasurementVarianceNotComputableException) { - return@map null - } + try { + buildStatsMethodology(watchDurationResult) + } catch (e: MeasurementVarianceNotComputableException) { + return@map null + } WeightedWatchDurationMeasurementVarianceParams( - binaryRepresentation = weightedMeasurement.binaryRepresentation, - weight = weightedMeasurement.weight, - measurementVarianceParams = - WatchDurationMeasurementVarianceParams( - duration = max(0.0, watchDurationResult.value.toDoubleSecond()), - measurementParams = - WatchDurationMeasurementParams( - vidSamplingInterval = metricSpec.vidSamplingInterval.toStatsVidSamplingInterval(), - dpParams = metricSpec.watchDuration.privacyParams.toNoiserDpParams(), - maximumDurationPerUser = - metricSpec.watchDuration.maximumWatchDurationPerUser.toDoubleSecond(), - noiseMechanism = statsNoiseMechanism, + binaryRepresentation = weightedMeasurement.binaryRepresentation, + weight = weightedMeasurement.weight, + measurementVarianceParams = + WatchDurationMeasurementVarianceParams( + duration = max(0.0, watchDurationResult.value.toDoubleSecond()), + measurementParams = + WatchDurationMeasurementParams( + vidSamplingInterval = + watchDurationParams.params.vidSamplingInterval + .toStatsVidSamplingInterval(), + dpParams = watchDurationParams.params.privacyParams.toNoiserDpParams(), + maximumDurationPerUser = + watchDurationParams.maximumWatchDurationPerUser.toDoubleSecond(), + noiseMechanism = statsNoiseMechanism, + ), ), - ), - methodology = methodology, + methodology = methodology, ) } } /** Builds a [Methodology] from an [InternalMeasurement.Result.WatchDuration]. */ fun buildStatsMethodology( - watchDurationResult: InternalMeasurement.Result.WatchDuration + watchDurationResult: InternalMeasurement.Result.WatchDuration ): Methodology { @Suppress("WHEN_ENUM_CAN_BE_NULL_IN_JAVA") return when (watchDurationResult.methodologyCase) { @@ -1998,8 +2001,7 @@ fun buildStatsMethodology( } CustomDirectMethodology.Variance.TypeCase.UNAVAILABLE -> { throw MeasurementVarianceNotComputableException( - "Watch duration computed from a custom methodology doesn't have variance." - ) + "Watch duration computed from a custom methodology doesn't have variance.") } CustomDirectMethodology.Variance.TypeCase.TYPE_NOT_SET -> { failGrpc(status = Status.FAILED_PRECONDITION, cause = IllegalStateException()) { @@ -2019,9 +2021,9 @@ fun buildStatsMethodology( /** Calculates the impression result from [WeightedMeasurement]s. */ private fun calculateImpressionResult( - weightedMeasurements: List, - metricSpec: InternalMetricSpec, - variances: Variances, + weightedMeasurements: List, + impressionParams: InternalMetricSpec.ImpressionCountParams, + variances: Variances, ): MetricResult.ImpressionCountResult { for (weightedMeasurement in weightedMeasurements) { if (weightedMeasurement.measurement.details.resultsList.any { !it.hasImpression() }) { @@ -2033,17 +2035,18 @@ private fun calculateImpressionResult( return impressionCountResult { value = - weightedMeasurements.sumOf { weightedMeasurement -> - aggregateResults(weightedMeasurement.measurement.details.resultsList).impression.value * - weightedMeasurement.weight - } + weightedMeasurements.sumOf { weightedMeasurement -> + aggregateResults(weightedMeasurement.measurement.details.resultsList).impression.value * + weightedMeasurement.weight + } // Only compute univariate statistics for union-only operations, i.e. single source measurement. if (weightedMeasurements.size == 1) { val weightedMeasurement = weightedMeasurements.single() val weightedMeasurementVarianceParamsList: - List = - buildWeightedImpressionMeasurementVarianceParamsPerResult(weightedMeasurement, metricSpec) + List = + buildWeightedImpressionMeasurementVarianceParamsPerResult( + weightedMeasurement, impressionParams) // If any measurement result contains insufficient data for variance calculation, univariate // statistics won't be computed. @@ -2052,26 +2055,23 @@ private fun calculateImpressionResult( // Impression results in a measurement are independent to each other. The variance is the // sum of the variances of each result. standardDeviation = - sqrt( - weightedMeasurementVarianceParamsList.sumOf { weightedMeasurementVarianceParams -> - try { - variances.computeMetricVariance( - ImpressionMetricVarianceParams( - listOf(requireNotNull(weightedMeasurementVarianceParams)) - ) - ) - } catch (e: Throwable) { - failGrpc(Status.UNKNOWN) { - listOfNotNull( - "Unable to compute variance of impression metric.", - e.message, - e.cause?.message, - ) - .joinToString(separator = "\n") - } - } - } - ) + sqrt( + weightedMeasurementVarianceParamsList.sumOf { weightedMeasurementVarianceParams -> + try { + variances.computeMetricVariance( + ImpressionMetricVarianceParams( + listOf(requireNotNull(weightedMeasurementVarianceParams)))) + } catch (e: Throwable) { + failGrpc(Status.UNKNOWN) { + listOfNotNull( + "Unable to compute variance of impression metric.", + e.message, + e.cause?.message, + ) + .joinToString(separator = "\n") + } + } + }) } } } @@ -2084,11 +2084,11 @@ private fun calculateImpressionResult( * @throws io.grpc.StatusRuntimeException when measurement noise mechanism is unrecognized. */ fun buildWeightedImpressionMeasurementVarianceParamsPerResult( - weightedMeasurement: WeightedMeasurement, - metricSpec: MetricSpec, + weightedMeasurement: WeightedMeasurement, + impressionParams: InternalMetricSpec.ImpressionCountParams, ): List { val impressionResults: List = - weightedMeasurement.measurement.details.resultsList.map { it.impression } + weightedMeasurement.measurement.details.resultsList.map { it.impression } if (impressionResults.isEmpty()) { failGrpc(status = Status.FAILED_PRECONDITION, cause = IllegalStateException()) { @@ -2098,50 +2098,52 @@ fun buildWeightedImpressionMeasurementVarianceParamsPerResult( return impressionResults.map { impressionResult -> val statsNoiseMechanism: StatsNoiseMechanism = - try { - impressionResult.noiseMechanism.toStatsNoiseMechanism() - } catch (e: NoiseMechanismUnspecifiedException) { - return@map null - } catch (e: NoiseMechanismUnrecognizedException) { - failGrpc(Status.UNKNOWN) { - listOfNotNull( - "Unrecognized noise mechanism should've been caught earlier.", - e.message, - e.cause?.message, - ) - .joinToString(separator = "\n") + try { + impressionResult.noiseMechanism.toStatsNoiseMechanism() + } catch (e: NoiseMechanismUnspecifiedException) { + return@map null + } catch (e: NoiseMechanismUnrecognizedException) { + failGrpc(Status.UNKNOWN) { + listOfNotNull( + "Unrecognized noise mechanism should've been caught earlier.", + e.message, + e.cause?.message, + ) + .joinToString(separator = "\n") + } } - } val methodology: Methodology = - try { - buildStatsMethodology(impressionResult) - } catch (e: MeasurementVarianceNotComputableException) { - return@map null - } + try { + buildStatsMethodology(impressionResult) + } catch (e: MeasurementVarianceNotComputableException) { + return@map null + } val maxFrequencyPerUser = - if (impressionResult.deterministicCount.customMaximumFrequencyPerUser != 0) { - impressionResult.deterministicCount.customMaximumFrequencyPerUser - } else { - metricSpec.impressionCount.maximumFrequencyPerUser - } + if (impressionResult.deterministicCount.customMaximumFrequencyPerUser != 0) { + impressionResult.deterministicCount.customMaximumFrequencyPerUser + } else { + impressionParams.maximumFrequencyPerUser + } WeightedImpressionMeasurementVarianceParams( - binaryRepresentation = weightedMeasurement.binaryRepresentation, - weight = weightedMeasurement.weight, - measurementVarianceParams = - ImpressionMeasurementVarianceParams( - impression = max(0L, impressionResult.value), - measurementParams = - ImpressionMeasurementParams( - vidSamplingInterval = metricSpec.vidSamplingInterval.toStatsVidSamplingInterval(), - dpParams = metricSpec.impressionCount.privacyParams.toNoiserDpParams(), - maximumFrequencyPerUser = maxFrequencyPerUser, - noiseMechanism = statsNoiseMechanism, + binaryRepresentation = weightedMeasurement.binaryRepresentation, + weight = weightedMeasurement.weight, + measurementVarianceParams = + ImpressionMeasurementVarianceParams( + impression = max(0L, impressionResult.value), + measurementParams = + ImpressionMeasurementParams( + vidSamplingInterval = + impressionParams.params.vidSamplingInterval + .toStatsVidSamplingInterval(), + dpParams = impressionParams.params.privacyParams.toNoiserDpParams(), + maximumFrequencyPerUser = maxFrequencyPerUser, + noiseMechanism = statsNoiseMechanism, + ), ), - ), - methodology = methodology, + methodology = methodology, ) } } @@ -2163,8 +2165,7 @@ fun buildStatsMethodology(impressionResult: InternalMeasurement.Result.Impressio } CustomDirectMethodology.Variance.TypeCase.UNAVAILABLE -> { throw MeasurementVarianceNotComputableException( - "Impression computed from a custom methodology doesn't have variance." - ) + "Impression computed from a custom methodology doesn't have variance.") } CustomDirectMethodology.Variance.TypeCase.TYPE_NOT_SET -> { failGrpc(status = Status.FAILED_PRECONDITION, cause = IllegalStateException()) { @@ -2184,96 +2185,113 @@ fun buildStatsMethodology(impressionResult: InternalMeasurement.Result.Impressio /** Calculates the frequency histogram result from [WeightedMeasurement]s. */ private fun calculateFrequencyHistogramResults( - weightedMeasurements: List, - metricSpec: InternalMetricSpec, - variances: Variances, + weightedMeasurements: List, + reachAndFrequencyParams: InternalMetricSpec.ReachAndFrequencyParams, + variances: Variances, ): MetricResult.HistogramResult { val aggregatedFrequencyHistogramMap: MutableMap = - weightedMeasurements - .map { weightedMeasurement -> - if ( - weightedMeasurement.measurement.details.resultsList.any { - !it.hasReach() || !it.hasFrequency() + weightedMeasurements + .map { weightedMeasurement -> + if (weightedMeasurement.measurement.details.resultsList.any { + !it.hasReach() || !it.hasFrequency() + }) { + failGrpc(status = Status.FAILED_PRECONDITION, cause = IllegalStateException()) { + "Reach-Frequency measurement is missing." + } + } + val result = aggregateResults(weightedMeasurement.measurement.details.resultsList) + val reach = result.reach.value + result.frequency.relativeFrequencyDistributionMap.mapValues { (_, rate) -> + rate * weightedMeasurement.weight * reach + } } - ) { - failGrpc(status = Status.FAILED_PRECONDITION, cause = IllegalStateException()) { - "Reach-Frequency measurement is missing." + .fold(mutableMapOf().withDefault { 0.0 }) { + aggregatedFrequencyHistogramMap: MutableMap, + weightedFrequencyHistogramMap -> + for ((frequency, count) in weightedFrequencyHistogramMap) { + aggregatedFrequencyHistogramMap[frequency] = + aggregatedFrequencyHistogramMap.getValue(frequency) + count + } + aggregatedFrequencyHistogramMap } - } - val result = aggregateResults(weightedMeasurement.measurement.details.resultsList) - val reach = result.reach.value - result.frequency.relativeFrequencyDistributionMap.mapValues { (_, rate) -> - rate * weightedMeasurement.weight * reach - } - } - .fold(mutableMapOf().withDefault { 0.0 }) { - aggregatedFrequencyHistogramMap: MutableMap, - weightedFrequencyHistogramMap -> - for ((frequency, count) in weightedFrequencyHistogramMap) { - aggregatedFrequencyHistogramMap[frequency] = - aggregatedFrequencyHistogramMap.getValue(frequency) + count - } - aggregatedFrequencyHistogramMap - } // Fill the buckets that don't have any count with zeros. - for (frequency in (1L..metricSpec.reachAndFrequency.maximumFrequency)) { + for (frequency in (1L..reachAndFrequencyParams.maximumFrequency)) { if (!aggregatedFrequencyHistogramMap.containsKey(frequency)) { aggregatedFrequencyHistogramMap[frequency] = 0.0 } } val weightedMeasurementVarianceParamsList: List = - weightedMeasurements.mapNotNull { weightedMeasurement -> - buildWeightedFrequencyMeasurementVarianceParams(weightedMeasurement, metricSpec, variances) - } + weightedMeasurements.mapNotNull { weightedMeasurement -> + if (weightedMeasurement.measurement.isSingleDataProvider && reachAndFrequencyParams.hasSingleDataProviderParams()) { + buildWeightedFrequencyMeasurementVarianceParams( + weightedMeasurement = weightedMeasurement, + vidSamplingInterval = + reachAndFrequencyParams.singleDataProviderParams.vidSamplingInterval, + reachPrivacyParams = reachAndFrequencyParams.singleDataProviderParams.privacyParams, + frequencyPrivacyParams = + reachAndFrequencyParams.singleDataProviderParams.frequencyPrivacyParams, + maximumFrequency = reachAndFrequencyParams.maximumFrequency, + variances = variances) + } else { + buildWeightedFrequencyMeasurementVarianceParams( + weightedMeasurement = weightedMeasurement, + vidSamplingInterval = + reachAndFrequencyParams.multipleDataProviderParams.vidSamplingInterval, + reachPrivacyParams = reachAndFrequencyParams.multipleDataProviderParams.privacyParams, + frequencyPrivacyParams = + reachAndFrequencyParams.multipleDataProviderParams.frequencyPrivacyParams, + maximumFrequency = reachAndFrequencyParams.maximumFrequency, + variances = variances) + } + } val frequencyVariances: FrequencyVariances? = - if (weightedMeasurementVarianceParamsList.size == weightedMeasurements.size) { - try { - variances.computeMetricVariance( - FrequencyMetricVarianceParams(weightedMeasurementVarianceParamsList) - ) - } catch (e: Throwable) { - failGrpc(Status.UNKNOWN) { - listOfNotNull( - "Unable to compute variance of reach-frequency metric.", - e.message, - e.cause?.message, - ) - .joinToString(separator = "\n") + if (weightedMeasurementVarianceParamsList.size == weightedMeasurements.size) { + try { + variances.computeMetricVariance( + FrequencyMetricVarianceParams(weightedMeasurementVarianceParamsList)) + } catch (e: Throwable) { + failGrpc(Status.UNKNOWN) { + listOfNotNull( + "Unable to compute variance of reach-frequency metric.", + e.message, + e.cause?.message, + ) + .joinToString(separator = "\n") + } } + } else { + null } - } else { - null - } return histogramResult { bins += - aggregatedFrequencyHistogramMap.map { (frequency, count) -> - bin { - label = frequency.toString() - binResult = binResult { value = count } - if (frequencyVariances != null) { - resultUnivariateStatistics = univariateStatistics { - standardDeviation = - sqrt(frequencyVariances.countVariances.getValue(frequency.toInt())) - } - relativeUnivariateStatistics = univariateStatistics { - standardDeviation = - sqrt(frequencyVariances.relativeVariances.getValue(frequency.toInt())) - } - kPlusUnivariateStatistics = univariateStatistics { - standardDeviation = - sqrt(frequencyVariances.kPlusCountVariances.getValue(frequency.toInt())) - } - relativeKPlusUnivariateStatistics = univariateStatistics { - standardDeviation = - sqrt(frequencyVariances.kPlusRelativeVariances.getValue(frequency.toInt())) + aggregatedFrequencyHistogramMap.map { (frequency, count) -> + bin { + label = frequency.toString() + binResult = binResult { value = count } + if (frequencyVariances != null) { + resultUnivariateStatistics = univariateStatistics { + standardDeviation = + sqrt(frequencyVariances.countVariances.getValue(frequency.toInt())) + } + relativeUnivariateStatistics = univariateStatistics { + standardDeviation = + sqrt(frequencyVariances.relativeVariances.getValue(frequency.toInt())) + } + kPlusUnivariateStatistics = univariateStatistics { + standardDeviation = + sqrt(frequencyVariances.kPlusCountVariances.getValue(frequency.toInt())) + } + relativeKPlusUnivariateStatistics = univariateStatistics { + standardDeviation = + sqrt(frequencyVariances.kPlusRelativeVariances.getValue(frequency.toInt())) + } } } } - } } } @@ -2285,81 +2303,84 @@ private fun calculateFrequencyHistogramResults( * @throws io.grpc.StatusRuntimeException when measurement noise mechanism is unrecognized. */ fun buildWeightedFrequencyMeasurementVarianceParams( - weightedMeasurement: WeightedMeasurement, - metricSpec: MetricSpec, - variances: Variances, + weightedMeasurement: WeightedMeasurement, + vidSamplingInterval: InternalMetricSpec.VidSamplingInterval, + reachPrivacyParams: InternalMetricSpec.DifferentialPrivacyParams, + frequencyPrivacyParams: InternalMetricSpec.DifferentialPrivacyParams, + maximumFrequency: Int, + variances: Variances, ): WeightedFrequencyMeasurementVarianceParams? { // Get reach measurement variance params val weightedReachMeasurementVarianceParams: WeightedReachMeasurementVarianceParams = - buildWeightedReachMeasurementVarianceParams( - weightedMeasurement, - metricSpec.vidSamplingInterval, - metricSpec.reachAndFrequency.reachPrivacyParams, - ) ?: return null + buildWeightedReachMeasurementVarianceParams( + weightedMeasurement, + vidSamplingInterval, + reachPrivacyParams, + ) ?: return null val reachMeasurementVariance: Double = - variances.computeMeasurementVariance( - weightedReachMeasurementVarianceParams.methodology, - ReachMeasurementVarianceParams( - weightedReachMeasurementVarianceParams.measurementVarianceParams.reach, - weightedReachMeasurementVarianceParams.measurementVarianceParams.measurementParams, - ), - ) + variances.computeMeasurementVariance( + weightedReachMeasurementVarianceParams.methodology, + ReachMeasurementVarianceParams( + weightedReachMeasurementVarianceParams.measurementVarianceParams.reach, + weightedReachMeasurementVarianceParams.measurementVarianceParams.measurementParams, + ), + ) val frequencyResult: InternalMeasurement.Result.Frequency = - if (weightedMeasurement.measurement.details.resultsList.size == 1) { - weightedMeasurement.measurement.details.resultsList.single().frequency - } else if (weightedMeasurement.measurement.details.resultsList.size > 1) { - failGrpc(status = Status.FAILED_PRECONDITION, cause = IllegalStateException()) { - "No supported methodology generates more than one frequency result." - } - } else { - failGrpc(status = Status.FAILED_PRECONDITION, cause = IllegalStateException()) { - "Frequency measurement should've had frequency results." + if (weightedMeasurement.measurement.details.resultsList.size == 1) { + weightedMeasurement.measurement.details.resultsList.single().frequency + } else if (weightedMeasurement.measurement.details.resultsList.size > 1) { + failGrpc(status = Status.FAILED_PRECONDITION, cause = IllegalStateException()) { + "No supported methodology generates more than one frequency result." + } + } else { + failGrpc(status = Status.FAILED_PRECONDITION, cause = IllegalStateException()) { + "Frequency measurement should've had frequency results." + } } - } val frequencyStatsNoiseMechanism: StatsNoiseMechanism = - try { - frequencyResult.noiseMechanism.toStatsNoiseMechanism() - } catch (e: NoiseMechanismUnspecifiedException) { - return null - } catch (e: NoiseMechanismUnrecognizedException) { - failGrpc(Status.UNKNOWN) { - listOfNotNull( - "Unrecognized noise mechanism should've been caught earlier.", - e.message, - e.cause?.message, - ) - .joinToString(separator = "\n") + try { + frequencyResult.noiseMechanism.toStatsNoiseMechanism() + } catch (e: NoiseMechanismUnspecifiedException) { + return null + } catch (e: NoiseMechanismUnrecognizedException) { + failGrpc(Status.UNKNOWN) { + listOfNotNull( + "Unrecognized noise mechanism should've been caught earlier.", + e.message, + e.cause?.message, + ) + .joinToString(separator = "\n") + } } - } val frequencyMethodology: Methodology = - try { - buildStatsMethodology(frequencyResult) - } catch (e: MeasurementVarianceNotComputableException) { - return null - } + try { + buildStatsMethodology(frequencyResult) + } catch (e: MeasurementVarianceNotComputableException) { + return null + } return WeightedFrequencyMeasurementVarianceParams( - binaryRepresentation = weightedMeasurement.binaryRepresentation, - weight = weightedMeasurement.weight, - measurementVarianceParams = - FrequencyMeasurementVarianceParams( - totalReach = weightedReachMeasurementVarianceParams.measurementVarianceParams.reach, - reachMeasurementVariance = reachMeasurementVariance, - relativeFrequencyDistribution = - frequencyResult.relativeFrequencyDistributionMap.mapKeys { it.key.toInt() }, - measurementParams = - FrequencyMeasurementParams( - vidSamplingInterval = metricSpec.vidSamplingInterval.toStatsVidSamplingInterval(), - dpParams = metricSpec.reachAndFrequency.frequencyPrivacyParams.toNoiserDpParams(), - noiseMechanism = frequencyStatsNoiseMechanism, - maximumFrequency = metricSpec.reachAndFrequency.maximumFrequency, + binaryRepresentation = weightedMeasurement.binaryRepresentation, + weight = weightedMeasurement.weight, + measurementVarianceParams = + FrequencyMeasurementVarianceParams( + totalReach = weightedReachMeasurementVarianceParams.measurementVarianceParams.reach, + reachMeasurementVariance = reachMeasurementVariance, + relativeFrequencyDistribution = + frequencyResult.relativeFrequencyDistributionMap.mapKeys { it.key.toInt() }, + measurementParams = + FrequencyMeasurementParams( + vidSamplingInterval = vidSamplingInterval.toStatsVidSamplingInterval(), + dpParams = frequencyPrivacyParams.toNoiserDpParams(), + noiseMechanism = frequencyStatsNoiseMechanism, + maximumFrequency = maximumFrequency, + ), ), - ), - methodology = frequencyMethodology, + methodology = frequencyMethodology, ) } @@ -2377,18 +2398,17 @@ fun buildStatsMethodology(frequencyResult: InternalMeasurement.Result.Frequency) } CustomDirectMethodology.Variance.TypeCase.FREQUENCY -> { CustomDirectFrequencyMethodology( - frequencyResult.customDirectMethodology.variance.frequency.variancesMap.mapKeys { - it.key.toInt() - }, - frequencyResult.customDirectMethodology.variance.frequency.kPlusVariancesMap.mapKeys { - it.key.toInt() - }, + frequencyResult.customDirectMethodology.variance.frequency.variancesMap.mapKeys { + it.key.toInt() + }, + frequencyResult.customDirectMethodology.variance.frequency.kPlusVariancesMap.mapKeys { + it.key.toInt() + }, ) } CustomDirectMethodology.Variance.TypeCase.UNAVAILABLE -> { throw MeasurementVarianceNotComputableException( - "Frequency computed from a custom methodology doesn't have variance." - ) + "Frequency computed from a custom methodology doesn't have variance.") } CustomDirectMethodology.Variance.TypeCase.TYPE_NOT_SET -> { failGrpc(status = Status.FAILED_PRECONDITION, cause = IllegalStateException()) { @@ -2402,15 +2422,16 @@ fun buildStatsMethodology(frequencyResult: InternalMeasurement.Result.Frequency) } InternalMeasurement.Result.Frequency.MethodologyCase.LIQUID_LEGIONS_DISTRIBUTION -> { LiquidLegionsSketchMethodology( - decayRate = frequencyResult.liquidLegionsDistribution.decayRate, - sketchSize = frequencyResult.liquidLegionsDistribution.maxSize, + decayRate = frequencyResult.liquidLegionsDistribution.decayRate, + sketchSize = frequencyResult.liquidLegionsDistribution.maxSize, ) } InternalMeasurement.Result.Frequency.MethodologyCase.LIQUID_LEGIONS_V2 -> { LiquidLegionsV2Methodology( - decayRate = frequencyResult.liquidLegionsV2.sketchParams.decayRate, - sketchSize = frequencyResult.liquidLegionsV2.sketchParams.maxSize, - samplingIndicatorSize = frequencyResult.liquidLegionsV2.sketchParams.samplingIndicatorSize, + decayRate = frequencyResult.liquidLegionsV2.sketchParams.decayRate, + sketchSize = frequencyResult.liquidLegionsV2.sketchParams.maxSize, + samplingIndicatorSize = + frequencyResult.liquidLegionsV2.sketchParams.samplingIndicatorSize, ) } InternalMeasurement.Result.Frequency.MethodologyCase.METHODOLOGY_NOT_SET -> { @@ -2421,10 +2442,9 @@ fun buildStatsMethodology(frequencyResult: InternalMeasurement.Result.Frequency) /** Calculates the reach result from [WeightedMeasurement]s. */ private fun calculateReachResult( - weightedMeasurements: List, - vidSamplingInterval: InternalMetricSpec.VidSamplingInterval, - privacyParams: InternalMetricSpec.DifferentialPrivacyParams, - variances: Variances, + weightedMeasurements: List, + reachParams: InternalMetricSpec.ReachParams, + variances: Variances, ): MetricResult.ReachResult { for (weightedMeasurement in weightedMeasurements) { if (weightedMeasurement.measurement.details.resultsList.any { !it.hasReach() }) { @@ -2436,41 +2456,109 @@ private fun calculateReachResult( return reachResult { value = - weightedMeasurements.sumOf { weightedMeasurement -> - aggregateResults(weightedMeasurement.measurement.details.resultsList).reach.value * - weightedMeasurement.weight - } + weightedMeasurements.sumOf { weightedMeasurement -> + aggregateResults(weightedMeasurement.measurement.details.resultsList).reach.value * + weightedMeasurement.weight + } val weightedMeasurementVarianceParamsList: List = - weightedMeasurements.mapNotNull { weightedMeasurement -> - buildWeightedReachMeasurementVarianceParams( - weightedMeasurement, - vidSamplingInterval, - privacyParams, - ) + weightedMeasurements.mapNotNull { weightedMeasurement -> + if (weightedMeasurement.measurement.isSingleDataProvider && reachParams.hasSingleDataProviderParams()) { + buildWeightedReachMeasurementVarianceParams( + weightedMeasurement, + reachParams.singleDataProviderParams.vidSamplingInterval, + reachParams.singleDataProviderParams.privacyParams, + ) + } else { + buildWeightedReachMeasurementVarianceParams( + weightedMeasurement, + reachParams.multipleDataProviderParams.vidSamplingInterval, + reachParams.multipleDataProviderParams.privacyParams, + ) + } + } + + // If any measurement contains insufficient data for variance calculation, univariate statistics + // won't be computed. + if (weightedMeasurementVarianceParamsList.size == weightedMeasurements.size) { + univariateStatistics = univariateStatistics { + standardDeviation = + sqrt( + try { + variances.computeMetricVariance( + ReachMetricVarianceParams(weightedMeasurementVarianceParamsList)) + } catch (e: Throwable) { + failGrpc(Status.UNKNOWN) { + listOfNotNull( + "Unable to compute variance of reach metric.", + e.message, + e.cause?.message, + ) + .joinToString(separator = "\n") + } + }) + } + } + } +} + +/** Calculates the reach result from [WeightedMeasurement]s. */ +private fun calculateReachResult( + weightedMeasurements: List, + reachAndFrequencyParams: InternalMetricSpec.ReachAndFrequencyParams, + variances: Variances, +): MetricResult.ReachResult { + for (weightedMeasurement in weightedMeasurements) { + if (weightedMeasurement.measurement.details.resultsList.any { !it.hasReach() }) { + failGrpc(status = Status.FAILED_PRECONDITION, cause = IllegalStateException()) { + "Reach measurement result is missing." } + } + } + + return reachResult { + value = + weightedMeasurements.sumOf { weightedMeasurement -> + aggregateResults(weightedMeasurement.measurement.details.resultsList).reach.value * + weightedMeasurement.weight + } + + val weightedMeasurementVarianceParamsList: List = + weightedMeasurements.mapNotNull { weightedMeasurement -> + if (weightedMeasurement.measurement.isSingleDataProvider && reachAndFrequencyParams.hasSingleDataProviderParams()) { + buildWeightedReachMeasurementVarianceParams( + weightedMeasurement, + reachAndFrequencyParams.singleDataProviderParams.vidSamplingInterval, + reachAndFrequencyParams.singleDataProviderParams.privacyParams, + ) + } else { + buildWeightedReachMeasurementVarianceParams( + weightedMeasurement, + reachAndFrequencyParams.multipleDataProviderParams.vidSamplingInterval, + reachAndFrequencyParams.multipleDataProviderParams.privacyParams, + ) + } + } // If any measurement contains insufficient data for variance calculation, univariate statistics // won't be computed. if (weightedMeasurementVarianceParamsList.size == weightedMeasurements.size) { univariateStatistics = univariateStatistics { standardDeviation = - sqrt( - try { - variances.computeMetricVariance( - ReachMetricVarianceParams(weightedMeasurementVarianceParamsList) - ) - } catch (e: Throwable) { - failGrpc(Status.UNKNOWN) { - listOfNotNull( - "Unable to compute variance of reach metric.", - e.message, - e.cause?.message, - ) - .joinToString(separator = "\n") - } - } - ) + sqrt( + try { + variances.computeMetricVariance( + ReachMetricVarianceParams(weightedMeasurementVarianceParamsList)) + } catch (e: Throwable) { + failGrpc(Status.UNKNOWN) { + listOfNotNull( + "Unable to compute variance of reach metric.", + e.message, + e.cause?.message, + ) + .joinToString(separator = "\n") + } + }) } } } @@ -2484,60 +2572,60 @@ private fun calculateReachResult( * @throws io.grpc.StatusRuntimeException when measurement noise mechanism is unrecognized. */ private fun buildWeightedReachMeasurementVarianceParams( - weightedMeasurement: WeightedMeasurement, - vidSamplingInterval: InternalMetricSpec.VidSamplingInterval, - privacyParams: InternalMetricSpec.DifferentialPrivacyParams, + weightedMeasurement: WeightedMeasurement, + vidSamplingInterval: InternalMetricSpec.VidSamplingInterval, + privacyParams: InternalMetricSpec.DifferentialPrivacyParams, ): WeightedReachMeasurementVarianceParams? { val reachResult = - if (weightedMeasurement.measurement.details.resultsList.size == 1) { - weightedMeasurement.measurement.details.resultsList.first().reach - } else if (weightedMeasurement.measurement.details.resultsList.size > 1) { - failGrpc(status = Status.FAILED_PRECONDITION, cause = IllegalStateException()) { - "No supported methodology generates more than one reach result." - } - } else { - failGrpc(status = Status.FAILED_PRECONDITION, cause = IllegalStateException()) { - "Reach measurement should've had reach results." + if (weightedMeasurement.measurement.details.resultsList.size == 1) { + weightedMeasurement.measurement.details.resultsList.first().reach + } else if (weightedMeasurement.measurement.details.resultsList.size > 1) { + failGrpc(status = Status.FAILED_PRECONDITION, cause = IllegalStateException()) { + "No supported methodology generates more than one reach result." + } + } else { + failGrpc(status = Status.FAILED_PRECONDITION, cause = IllegalStateException()) { + "Reach measurement should've had reach results." + } } - } val statsNoiseMechanism: StatsNoiseMechanism = - try { - reachResult.noiseMechanism.toStatsNoiseMechanism() - } catch (e: NoiseMechanismUnspecifiedException) { - return null - } catch (e: NoiseMechanismUnrecognizedException) { - failGrpc(Status.UNKNOWN) { - listOfNotNull( - "Unrecognized noise mechanism should've been caught earlier.", - e.message, - e.cause?.message, - ) - .joinToString(separator = "\n") + try { + reachResult.noiseMechanism.toStatsNoiseMechanism() + } catch (e: NoiseMechanismUnspecifiedException) { + return null + } catch (e: NoiseMechanismUnrecognizedException) { + failGrpc(Status.UNKNOWN) { + listOfNotNull( + "Unrecognized noise mechanism should've been caught earlier.", + e.message, + e.cause?.message, + ) + .joinToString(separator = "\n") + } } - } val methodology: Methodology = - try { - buildStatsMethodology(reachResult) - } catch (e: MeasurementVarianceNotComputableException) { - return null - } + try { + buildStatsMethodology(reachResult) + } catch (e: MeasurementVarianceNotComputableException) { + return null + } return WeightedReachMeasurementVarianceParams( - binaryRepresentation = weightedMeasurement.binaryRepresentation, - weight = weightedMeasurement.weight, - measurementVarianceParams = - ReachMeasurementVarianceParams( - reach = max(0L, reachResult.value), - measurementParams = - ReachMeasurementParams( - vidSamplingInterval = vidSamplingInterval.toStatsVidSamplingInterval(), - dpParams = privacyParams.toNoiserDpParams(), - noiseMechanism = statsNoiseMechanism, + binaryRepresentation = weightedMeasurement.binaryRepresentation, + weight = weightedMeasurement.weight, + measurementVarianceParams = + ReachMeasurementVarianceParams( + reach = max(0L, reachResult.value), + measurementParams = + ReachMeasurementParams( + vidSamplingInterval = vidSamplingInterval.toStatsVidSamplingInterval(), + dpParams = privacyParams.toNoiserDpParams(), + noiseMechanism = statsNoiseMechanism, + ), ), - ), - methodology = methodology, + methodology = methodology, ) } @@ -2558,8 +2646,7 @@ fun buildStatsMethodology(reachResult: InternalMeasurement.Result.Reach): Method } CustomDirectMethodology.Variance.TypeCase.UNAVAILABLE -> { throw MeasurementVarianceNotComputableException( - "Reach computed from a custom methodology doesn't have variance." - ) + "Reach computed from a custom methodology doesn't have variance.") } CustomDirectMethodology.Variance.TypeCase.TYPE_NOT_SET -> { failGrpc(status = Status.FAILED_PRECONDITION, cause = IllegalStateException()) { @@ -2573,22 +2660,22 @@ fun buildStatsMethodology(reachResult: InternalMeasurement.Result.Reach): Method } InternalMeasurement.Result.Reach.MethodologyCase.LIQUID_LEGIONS_COUNT_DISTINCT -> { LiquidLegionsSketchMethodology( - decayRate = reachResult.liquidLegionsCountDistinct.decayRate, - sketchSize = reachResult.liquidLegionsCountDistinct.maxSize, + decayRate = reachResult.liquidLegionsCountDistinct.decayRate, + sketchSize = reachResult.liquidLegionsCountDistinct.maxSize, ) } InternalMeasurement.Result.Reach.MethodologyCase.LIQUID_LEGIONS_V2 -> { LiquidLegionsV2Methodology( - decayRate = reachResult.liquidLegionsV2.sketchParams.decayRate, - sketchSize = reachResult.liquidLegionsV2.sketchParams.maxSize, - samplingIndicatorSize = reachResult.liquidLegionsV2.sketchParams.samplingIndicatorSize, + decayRate = reachResult.liquidLegionsV2.sketchParams.decayRate, + sketchSize = reachResult.liquidLegionsV2.sketchParams.maxSize, + samplingIndicatorSize = reachResult.liquidLegionsV2.sketchParams.samplingIndicatorSize, ) } InternalMeasurement.Result.Reach.MethodologyCase.REACH_ONLY_LIQUID_LEGIONS_V2 -> { LiquidLegionsV2Methodology( - decayRate = reachResult.reachOnlyLiquidLegionsV2.sketchParams.decayRate, - sketchSize = reachResult.reachOnlyLiquidLegionsV2.sketchParams.maxSize, - samplingIndicatorSize = 0L, + decayRate = reachResult.reachOnlyLiquidLegionsV2.sketchParams.decayRate, + sketchSize = reachResult.reachOnlyLiquidLegionsV2.sketchParams.maxSize, + samplingIndicatorSize = 0L, ) } InternalMeasurement.Result.Reach.MethodologyCase.METHODOLOGY_NOT_SET -> { @@ -2601,7 +2688,7 @@ private operator fun ProtoDuration.times(weight: Int): ProtoDuration { val source = this return duration { val weightedTotalNanos: Long = - (TimeUnit.SECONDS.toNanos(source.seconds) + source.nanos) * weight + (TimeUnit.SECONDS.toNanos(source.seconds) + source.nanos) * weight seconds = TimeUnit.NANOSECONDS.toSeconds(weightedTotalNanos) nanos = (weightedTotalNanos % NANOS_PER_SECOND).toInt() } diff --git a/src/main/kotlin/org/wfanet/measurement/reporting/service/api/v2alpha/ProtoConversions.kt b/src/main/kotlin/org/wfanet/measurement/reporting/service/api/v2alpha/ProtoConversions.kt index db80785e7ff..bcb3c094efc 100644 --- a/src/main/kotlin/org/wfanet/measurement/reporting/service/api/v2alpha/ProtoConversions.kt +++ b/src/main/kotlin/org/wfanet/measurement/reporting/service/api/v2alpha/ProtoConversions.kt @@ -105,18 +105,6 @@ import org.wfanet.measurement.reporting.v2alpha.reportSchedule import org.wfanet.measurement.reporting.v2alpha.reportingSet import org.wfanet.measurement.reporting.v2alpha.timeIntervals -/** - * Converts an [MetricSpecConfig.VidSamplingInterval] to an - * [InternalMetricSpec.VidSamplingInterval]. - */ -fun MetricSpecConfig.VidSamplingInterval.toInternal(): InternalMetricSpec.VidSamplingInterval { - val source = this - return InternalMetricSpecKt.vidSamplingInterval { - start = source.start - width = source.width - } -} - /** Converts an [InternalMetricSpec.VidSamplingInterval] to a CMMS [VidSamplingInterval]. */ fun InternalMetricSpec.VidSamplingInterval.toCmmsVidSamplingInterval(): VidSamplingInterval { val source = this @@ -149,19 +137,26 @@ fun MetricSpec.toInternal(): InternalMetricSpec { val source = this return internalMetricSpec { + val vidSamplingInterval: InternalMetricSpec.VidSamplingInterval? = + if (source.hasVidSamplingInterval()) { + source.vidSamplingInterval.toInternal() + } else { + null + } + @Suppress("WHEN_ENUM_CAN_BE_NULL_IN_JAVA") when (source.typeCase) { MetricSpec.TypeCase.REACH -> { - reach = source.reach.toInternal() + reach = source.reach.toInternal(vidSamplingInterval) } MetricSpec.TypeCase.REACH_AND_FREQUENCY -> { - reachAndFrequency = source.reachAndFrequency.toInternal() + reachAndFrequency = source.reachAndFrequency.toInternal(vidSamplingInterval) } MetricSpec.TypeCase.IMPRESSION_COUNT -> { - impressionCount = source.impressionCount.toInternal() + impressionCount = source.impressionCount.toInternal(vidSamplingInterval) } MetricSpec.TypeCase.WATCH_DURATION -> { - watchDuration = source.watchDuration.toInternal() + watchDuration = source.watchDuration.toInternal(vidSamplingInterval) } MetricSpec.TypeCase.POPULATION_COUNT -> { populationCount = InternalMetricSpec.PopulationCountParams.getDefaultInstance() @@ -172,46 +167,56 @@ fun MetricSpec.toInternal(): InternalMetricSpec { IllegalArgumentException("The metric type in Metric is not specified."), ) } - - if (source.hasVidSamplingInterval()) { - vidSamplingInterval = source.vidSamplingInterval.toInternal() - } } } /** Converts a [MetricSpec.WatchDurationParams] to an [InternalMetricSpec.WatchDurationParams]. */ -fun MetricSpec.WatchDurationParams.toInternal(): InternalMetricSpec.WatchDurationParams { +fun MetricSpec.WatchDurationParams.toInternal(vidSamplingInterval: InternalMetricSpec.VidSamplingInterval?): InternalMetricSpec.WatchDurationParams { val source = this - if (!source.hasPrivacyParams()) { - throw MetricSpecDefaultsException( - "Invalid privacy params", - IllegalArgumentException("privacyParams in watch duration is not set."), - ) - } return InternalMetricSpecKt.watchDurationParams { - privacyParams = source.privacyParams.toInternal() if (source.hasMaximumWatchDurationPerUser()) { maximumWatchDurationPerUser = source.maximumWatchDurationPerUser } + + params = InternalMetricSpecKt.params { + privacyParams = if (source.hasPrivacyParams()) { + source.privacyParams.toInternal() + } else { + source.params.privacyParams.toInternal() + } + + if (vidSamplingInterval != null) { + this.vidSamplingInterval = vidSamplingInterval + } else { + this.vidSamplingInterval = source.params.vidSamplingInterval.toInternal() + } + } } } /** * Converts a [MetricSpec.ImpressionCountParams] to an [InternalMetricSpec.ImpressionCountParams]. */ -fun MetricSpec.ImpressionCountParams.toInternal(): InternalMetricSpec.ImpressionCountParams { +fun MetricSpec.ImpressionCountParams.toInternal(vidSamplingInterval: InternalMetricSpec.VidSamplingInterval?): InternalMetricSpec.ImpressionCountParams { val source = this - if (!source.hasPrivacyParams()) { - throw MetricSpecDefaultsException( - "Invalid privacy params", - IllegalArgumentException("privacyParams in impression count is not set."), - ) - } return InternalMetricSpecKt.impressionCountParams { - privacyParams = source.privacyParams.toInternal() if (source.hasMaximumFrequencyPerUser()) { maximumFrequencyPerUser = source.maximumFrequencyPerUser } + + params = InternalMetricSpecKt.params { + privacyParams = if (source.hasPrivacyParams()) { + source.privacyParams.toInternal() + } else { + source.params.privacyParams.toInternal() + } + + if (vidSamplingInterval != null) { + this.vidSamplingInterval = vidSamplingInterval + } else { + this.vidSamplingInterval = source.params.vidSamplingInterval.toInternal() + } + } } } @@ -219,37 +224,64 @@ fun MetricSpec.ImpressionCountParams.toInternal(): InternalMetricSpec.Impression * Converts a [MetricSpec.ReachAndFrequencyParams] to an * [InternalMetricSpec.ReachAndFrequencyParams]. */ -fun MetricSpec.ReachAndFrequencyParams.toInternal(): InternalMetricSpec.ReachAndFrequencyParams { +fun MetricSpec.ReachAndFrequencyParams.toInternal(vidSamplingInterval: InternalMetricSpec.VidSamplingInterval?): InternalMetricSpec.ReachAndFrequencyParams { val source = this - if (!source.hasReachPrivacyParams()) { - throw MetricSpecDefaultsException( - "Invalid privacy params", - IllegalArgumentException("reachPrivacyParams in reach-and-frequency is not set."), - ) - } - if (!source.hasFrequencyPrivacyParams()) { - throw MetricSpecDefaultsException( - "Invalid privacy params", - IllegalArgumentException("frequencyPrivacyParams in reach-and-frequency is not set."), - ) - } return InternalMetricSpecKt.reachAndFrequencyParams { - reachPrivacyParams = source.reachPrivacyParams.toInternal() - frequencyPrivacyParams = source.frequencyPrivacyParams.toInternal() maximumFrequency = source.maximumFrequency + + multipleDataProviderParams = InternalMetricSpecKt.params { + if (source.hasReachPrivacyParams()) { + privacyParams = source.reachPrivacyParams.toInternal() + frequencyPrivacyParams = source.frequencyPrivacyParams.toInternal() + } else { + privacyParams = source.multipleDataProviderParams.privacyParams.toInternal() + frequencyPrivacyParams = source.multipleDataProviderParams.frequencyPrivacyParams.toInternal() + } + + if (vidSamplingInterval != null) { + this.vidSamplingInterval = vidSamplingInterval + } else { + this.vidSamplingInterval = source.multipleDataProviderParams.vidSamplingInterval.toInternal() + } + } + + if (source.hasSingleDataProviderParams()) { + singleDataProviderParams = InternalMetricSpecKt.params { + privacyParams = source.singleDataProviderParams.privacyParams.toInternal() + frequencyPrivacyParams = source.singleDataProviderParams.frequencyPrivacyParams.toInternal() + this.vidSamplingInterval = + source.singleDataProviderParams.vidSamplingInterval.toInternal() + } + } } } /** Converts a [MetricSpec.ReachParams] to an [InternalMetricSpec.ReachParams]. */ -fun MetricSpec.ReachParams.toInternal(): InternalMetricSpec.ReachParams { +fun MetricSpec.ReachParams.toInternal(vidSamplingInterval: InternalMetricSpec.VidSamplingInterval?): InternalMetricSpec.ReachParams { val source = this - if (!source.hasPrivacyParams()) { - throw MetricSpecDefaultsException( - "Invalid privacy params", - IllegalArgumentException("privacyParams in reach is not set."), - ) + return InternalMetricSpecKt.reachParams { + multipleDataProviderParams = InternalMetricSpecKt.params { + privacyParams = if (source.hasPrivacyParams()) { + source.privacyParams.toInternal() + } else { + source.multipleDataProviderParams.privacyParams.toInternal() + } + + if (vidSamplingInterval != null) { + this.vidSamplingInterval = vidSamplingInterval + } else { + this.vidSamplingInterval = source.multipleDataProviderParams.vidSamplingInterval.toInternal() + } + } + + if (source.hasSingleDataProviderParams()) { + singleDataProviderParams = InternalMetricSpecKt.params { + privacyParams = source.singleDataProviderParams.privacyParams.toInternal() + this.vidSamplingInterval = + source.singleDataProviderParams.vidSamplingInterval.toInternal() + } + } } - return InternalMetricSpecKt.reachParams { privacyParams = source.privacyParams.toInternal() } } /** @@ -273,40 +305,82 @@ fun MetricSpec.DifferentialPrivacyParams.toInternal(): fun InternalMetricSpec.toMetricSpec(): MetricSpec { val source = this return metricSpec { + val spec = this @Suppress("WHEN_ENUM_CAN_BE_NULL_IN_JAVA") // Proto enum fields are never null. when (source.typeCase) { InternalMetricSpec.TypeCase.REACH -> reach = - MetricSpecKt.reachParams { privacyParams = source.reach.privacyParams.toPrivacyParams() } + MetricSpecKt.reachParams { + if (source.reach.hasSingleDataProviderParams()) { + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = source.reach.multipleDataProviderParams.privacyParams.toPrivacyParams() + vidSamplingInterval = source.reach.multipleDataProviderParams.vidSamplingInterval.toVidSamplingInterval() + } + + singleDataProviderParams = MetricSpecKt.params { + privacyParams = source.reach.singleDataProviderParams.privacyParams.toPrivacyParams() + vidSamplingInterval = source.reach.singleDataProviderParams.vidSamplingInterval.toVidSamplingInterval() + } + } else { + privacyParams = source.reach.multipleDataProviderParams.privacyParams.toPrivacyParams() + spec.vidSamplingInterval = source.reach.multipleDataProviderParams.vidSamplingInterval.toVidSamplingInterval() + } + } InternalMetricSpec.TypeCase.REACH_AND_FREQUENCY -> reachAndFrequency = MetricSpecKt.reachAndFrequencyParams { maximumFrequency = source.reachAndFrequency.maximumFrequency - reachPrivacyParams = source.reachAndFrequency.reachPrivacyParams.toPrivacyParams() - frequencyPrivacyParams = - source.reachAndFrequency.frequencyPrivacyParams.toPrivacyParams() + if (source.reachAndFrequency.hasSingleDataProviderParams()) { + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = source.reachAndFrequency.multipleDataProviderParams.privacyParams.toPrivacyParams() + frequencyPrivacyParams = source.reachAndFrequency.multipleDataProviderParams.frequencyPrivacyParams.toPrivacyParams() + vidSamplingInterval = source.reachAndFrequency.multipleDataProviderParams.vidSamplingInterval.toVidSamplingInterval() + } + + singleDataProviderParams = MetricSpecKt.params { + privacyParams = source.reachAndFrequency.singleDataProviderParams.privacyParams.toPrivacyParams() + frequencyPrivacyParams = source.reachAndFrequency.singleDataProviderParams.frequencyPrivacyParams.toPrivacyParams() + vidSamplingInterval = source.reachAndFrequency.singleDataProviderParams.vidSamplingInterval.toVidSamplingInterval() + } + } else { + reachPrivacyParams = source.reachAndFrequency.multipleDataProviderParams.privacyParams.toPrivacyParams() + frequencyPrivacyParams = source.reachAndFrequency.multipleDataProviderParams.frequencyPrivacyParams.toPrivacyParams() + spec.vidSamplingInterval = source.reachAndFrequency.multipleDataProviderParams.vidSamplingInterval.toVidSamplingInterval() + } } - InternalMetricSpec.TypeCase.IMPRESSION_COUNT -> + InternalMetricSpec.TypeCase.IMPRESSION_COUNT -> { impressionCount = MetricSpecKt.impressionCountParams { maximumFrequencyPerUser = source.impressionCount.maximumFrequencyPerUser - privacyParams = source.impressionCount.privacyParams.toPrivacyParams() + params = MetricSpecKt.params { + privacyParams = source.impressionCount.params.privacyParams.toPrivacyParams() + vidSamplingInterval = + source.impressionCount.params.vidSamplingInterval.toVidSamplingInterval() + } + privacyParams = source.impressionCount.params.privacyParams.toPrivacyParams() } - InternalMetricSpec.TypeCase.WATCH_DURATION -> + vidSamplingInterval = + source.impressionCount.params.vidSamplingInterval.toVidSamplingInterval() + } + InternalMetricSpec.TypeCase.WATCH_DURATION -> { watchDuration = MetricSpecKt.watchDurationParams { maximumWatchDurationPerUser = source.watchDuration.maximumWatchDurationPerUser - privacyParams = source.watchDuration.privacyParams.toPrivacyParams() + params = MetricSpecKt.params { + privacyParams = source.watchDuration.params.privacyParams.toPrivacyParams() + vidSamplingInterval = + source.watchDuration.params.vidSamplingInterval.toVidSamplingInterval() + } + privacyParams = source.watchDuration.params.privacyParams.toPrivacyParams() } + vidSamplingInterval = source.watchDuration.params.vidSamplingInterval.toVidSamplingInterval() + } InternalMetricSpec.TypeCase.POPULATION_COUNT -> { populationCount = MetricSpec.PopulationCountParams.getDefaultInstance() } InternalMetricSpec.TypeCase.TYPE_NOT_SET -> throw IllegalArgumentException("The metric type in Metric is not specified.") } - if (source.hasVidSamplingInterval()) { - vidSamplingInterval = source.vidSamplingInterval.toVidSamplingInterval() - } // TODO(@jojijac0b): To add model line check and assignment } } @@ -334,41 +408,57 @@ fun InternalMetricSpec.DifferentialPrivacyParams.toCmmsPrivacyParams(): Differen } /** Converts an [InternalMetricSpec.ReachParams] to a [MeasurementSpec.Reach]. */ -fun InternalMetricSpec.ReachParams.toReach(): MeasurementSpec.Reach { +fun InternalMetricSpec.ReachParams.toReach(isSingleDataProvider: Boolean): Pair { val source = this - return MeasurementSpecKt.reach { privacyParams = source.privacyParams.toCmmsPrivacyParams() } + return if (isSingleDataProvider && source.hasSingleDataProviderParams()) { + Pair(MeasurementSpecKt.reach { + privacyParams = source.singleDataProviderParams.privacyParams.toCmmsPrivacyParams() + }, source.singleDataProviderParams.vidSamplingInterval.toCmmsVidSamplingInterval()) + } else { + Pair(MeasurementSpecKt.reach { + privacyParams = source.multipleDataProviderParams.privacyParams.toCmmsPrivacyParams() + }, source.multipleDataProviderParams.vidSamplingInterval.toCmmsVidSamplingInterval()) + } } /** * Converts an [InternalMetricSpec.ReachAndFrequencyParams] to a * [MeasurementSpec.ReachAndFrequency]. */ -fun InternalMetricSpec.ReachAndFrequencyParams.toReachAndFrequency(): - MeasurementSpec.ReachAndFrequency { +fun InternalMetricSpec.ReachAndFrequencyParams.toReachAndFrequency(isSingleDataProvider: Boolean): + Pair { val source = this - return MeasurementSpecKt.reachAndFrequency { - reachPrivacyParams = source.reachPrivacyParams.toCmmsPrivacyParams() - frequencyPrivacyParams = source.frequencyPrivacyParams.toCmmsPrivacyParams() - maximumFrequency = source.maximumFrequency + return if (isSingleDataProvider && source.hasSingleDataProviderParams()) { + Pair(MeasurementSpecKt.reachAndFrequency { + reachPrivacyParams = source.singleDataProviderParams.privacyParams.toCmmsPrivacyParams() + frequencyPrivacyParams = source.singleDataProviderParams.frequencyPrivacyParams.toCmmsPrivacyParams() + maximumFrequency = source.maximumFrequency + }, source.singleDataProviderParams.vidSamplingInterval.toCmmsVidSamplingInterval()) + } else { + Pair(MeasurementSpecKt.reachAndFrequency { + reachPrivacyParams = source.multipleDataProviderParams.privacyParams.toCmmsPrivacyParams() + frequencyPrivacyParams = source.multipleDataProviderParams.frequencyPrivacyParams.toCmmsPrivacyParams() + maximumFrequency = source.maximumFrequency + }, source.multipleDataProviderParams.vidSamplingInterval.toCmmsVidSamplingInterval()) } } /** Builds a [MeasurementSpec.ReachAndFrequency] for impression count. */ -fun InternalMetricSpec.ImpressionCountParams.toImpression(): MeasurementSpec.Impression { +fun InternalMetricSpec.ImpressionCountParams.toImpression(): Pair { val source = this - return MeasurementSpecKt.impression { - privacyParams = source.privacyParams.toCmmsPrivacyParams() + return Pair(MeasurementSpecKt.impression { + privacyParams = source.params.privacyParams.toCmmsPrivacyParams() maximumFrequencyPerUser = source.maximumFrequencyPerUser - } + }, source.params.vidSamplingInterval.toCmmsVidSamplingInterval()) } /** Builds a [MeasurementSpec.ReachAndFrequency] for watch duration. */ -fun InternalMetricSpec.WatchDurationParams.toDuration(): MeasurementSpec.Duration { +fun InternalMetricSpec.WatchDurationParams.toDuration(): Pair { val source = this - return MeasurementSpecKt.duration { - privacyParams = source.privacyParams.toCmmsPrivacyParams() + return Pair(MeasurementSpecKt.duration { + privacyParams = source.params.privacyParams.toCmmsPrivacyParams() maximumWatchDurationPerUser = source.maximumWatchDurationPerUser - } + }, source.params.vidSamplingInterval.toCmmsVidSamplingInterval()) } /** Converts an internal [InternalMetric.State] to a public [Metric.State]. */ diff --git a/src/main/kotlin/org/wfanet/measurement/reporting/service/api/v2alpha/ReportsService.kt b/src/main/kotlin/org/wfanet/measurement/reporting/service/api/v2alpha/ReportsService.kt index 2639b0f8f34..2b07f28ba5a 100644 --- a/src/main/kotlin/org/wfanet/measurement/reporting/service/api/v2alpha/ReportsService.kt +++ b/src/main/kotlin/org/wfanet/measurement/reporting/service/api/v2alpha/ReportsService.kt @@ -64,6 +64,7 @@ import org.wfanet.measurement.internal.reporting.v2.batchGetMetricCalculationSpe import org.wfanet.measurement.internal.reporting.v2.createReportRequest as internalCreateReportRequest import org.wfanet.measurement.internal.reporting.v2.getReportRequest as internalGetReportRequest import org.wfanet.measurement.internal.reporting.v2.report as internalReport +import kotlin.random.Random import org.wfanet.measurement.reporting.service.api.submitBatchRequests import org.wfanet.measurement.reporting.service.api.v2alpha.MetadataPrincipalServerInterceptor.Companion.withPrincipalName import org.wfanet.measurement.reporting.service.api.v2alpha.ReportScheduleInfoServerInterceptor.Companion.reportScheduleInfoFromCurrentContext @@ -103,6 +104,7 @@ class ReportsService( private val internalMetricCalculationSpecsStub: MetricCalculationSpecsCoroutineStub, private val metricsStub: MetricsCoroutineStub, private val metricSpecConfig: MetricSpecConfig, + private val secureRandom: Random, ) : ReportsCoroutineImplBase() { private data class CreateReportInfo( val parent: String, @@ -813,7 +815,7 @@ class ReportsService( InternalReportKt.ReportingMetricKt.details { this.metricSpec = try { - metricSpec.toMetricSpec().withDefaults(metricSpecConfig).toInternal() + metricSpec.toMetricSpec().withDefaults(metricSpecConfig, secureRandom).toInternal() } catch (e: MetricSpecDefaultsException) { failGrpc(Status.INVALID_ARGUMENT) { listOfNotNull("Invalid metric spec.", e.message, e.cause?.message) diff --git a/src/main/kotlin/org/wfanet/measurement/reporting/service/internal/testing/v2/MeasurementsServiceTest.kt b/src/main/kotlin/org/wfanet/measurement/reporting/service/internal/testing/v2/MeasurementsServiceTest.kt index fb2d827af98..34cdfcf7fb2 100644 --- a/src/main/kotlin/org/wfanet/measurement/reporting/service/internal/testing/v2/MeasurementsServiceTest.kt +++ b/src/main/kotlin/org/wfanet/measurement/reporting/service/internal/testing/v2/MeasurementsServiceTest.kt @@ -1365,16 +1365,18 @@ abstract class MeasurementsServiceTest { metricSpec = metricSpec { reach = MetricSpecKt.reachParams { - privacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } - } - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = 0.1f - width = 0.5f + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = 0.1f + width = 0.5f + } + } } } weightedMeasurements += @@ -211,23 +213,25 @@ abstract class MetricsServiceTest { metricSpec = metricSpec { reachAndFrequency = MetricSpecKt.reachAndFrequencyParams { - reachPrivacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } - frequencyPrivacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + frequencyPrivacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = 0.1f + width = 0.5f + } + } maximumFrequency = 5 } - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = 0.1f - width = 0.5f - } } weightedMeasurements += MetricKt.weightedMeasurement { @@ -315,18 +319,20 @@ abstract class MetricsServiceTest { metricSpec = metricSpec { impressionCount = MetricSpecKt.impressionCountParams { - privacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } + params = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = 0.1f + width = 0.5f + } + } maximumFrequencyPerUser = 5 } - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = 0.1f - width = 0.5f - } } weightedMeasurements += MetricKt.weightedMeasurement { @@ -414,18 +420,20 @@ abstract class MetricsServiceTest { metricSpec = metricSpec { watchDuration = MetricSpecKt.watchDurationParams { - privacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } + params = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = 0.1f + width = 0.5f + } + } maximumWatchDurationPerUser = Durations.fromSeconds(100) } - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = 0.1f - width = 0.5f - } } weightedMeasurements += MetricKt.weightedMeasurement { @@ -565,18 +573,20 @@ abstract class MetricsServiceTest { metricSpec = metricSpec { watchDuration = MetricSpecKt.watchDurationParams { - privacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } + params = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = 0.1f + width = 0.5f + } + } maximumWatchDurationPerUser = Durations.fromSeconds(100) } - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = 0.1f - width = 0.5f - } } weightedMeasurements += MetricKt.weightedMeasurement { @@ -632,23 +642,25 @@ abstract class MetricsServiceTest { metricSpec = metricSpec { reachAndFrequency = MetricSpecKt.reachAndFrequencyParams { - reachPrivacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } - frequencyPrivacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + frequencyPrivacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = 0.1f + width = 0.5f + } + } maximumFrequency = 5 } - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = 0.1f - width = 0.5f - } } weightedMeasurements += MetricKt.weightedMeasurement { @@ -746,23 +758,25 @@ abstract class MetricsServiceTest { metricSpec = metricSpec { reachAndFrequency = MetricSpecKt.reachAndFrequencyParams { - reachPrivacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } - frequencyPrivacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + frequencyPrivacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = 0.1f + width = 0.5f + } + } maximumFrequency = 5 } - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = 0.1f - width = 0.5f - } } weightedMeasurements += MetricKt.weightedMeasurement { @@ -858,23 +872,25 @@ abstract class MetricsServiceTest { metricSpec = metricSpec { reachAndFrequency = MetricSpecKt.reachAndFrequencyParams { - reachPrivacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } - frequencyPrivacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 3.0 - delta = 4.0 - } + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + frequencyPrivacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = 0.1f + width = 0.5f + } + } maximumFrequency = 5 } - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = 0.1f - width = 0.5f - } } weightedMeasurements += MetricKt.weightedMeasurement { @@ -930,23 +946,25 @@ abstract class MetricsServiceTest { metricSpec = metricSpec { reachAndFrequency = MetricSpecKt.reachAndFrequencyParams { - reachPrivacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } - frequencyPrivacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + frequencyPrivacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = 0.1f + width = 0.5f + } + } maximumFrequency = 5 } - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = 0.1f - width = 0.5f - } } weightedMeasurements += MetricKt.weightedMeasurement { @@ -1011,16 +1029,18 @@ abstract class MetricsServiceTest { metricSpec = metricSpec { reach = MetricSpecKt.reachParams { - privacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } - } - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = 0.1f - width = 0.5f + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = 0.1f + width = 0.5f + } + } } } weightedMeasurements += @@ -1075,11 +1095,6 @@ abstract class MetricsServiceTest { endTime = timestamp { seconds = 100 } } metricSpec = metricSpec { - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = 0.1f - width = 0.5f - } } weightedMeasurements += MetricKt.weightedMeasurement { @@ -1120,69 +1135,6 @@ abstract class MetricsServiceTest { assertThat(exception.message).contains("type") } - @Test - fun `createMetric throws INVALID_ARGUMENT when metric spec missing vid sampling interval`() = - runBlocking { - createMeasurementConsumer(CMMS_MEASUREMENT_CONSUMER_ID, measurementConsumersService) - val createdReportingSet = - createReportingSet(CMMS_MEASUREMENT_CONSUMER_ID, reportingSetsService) - - val metric = metric { - cmmsMeasurementConsumerId = CMMS_MEASUREMENT_CONSUMER_ID - externalReportingSetId = createdReportingSet.externalReportingSetId - timeInterval = interval { - startTime = timestamp { seconds = 10 } - endTime = timestamp { seconds = 100 } - } - metricSpec = metricSpec { - reach = - MetricSpecKt.reachParams { - privacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } - } - } - weightedMeasurements += - MetricKt.weightedMeasurement { - weight = 2 - binaryRepresentation = 1 - measurement = measurement { - cmmsMeasurementConsumerId = CMMS_MEASUREMENT_CONSUMER_ID - timeInterval = interval { - startTime = timestamp { seconds = 10 } - endTime = timestamp { seconds = 100 } - } - primitiveReportingSetBases += - ReportingSetKt.primitiveReportingSetBasis { - externalReportingSetId = createdReportingSet.externalReportingSetId - filters += "filter1" - filters += "filter2" - } - } - } - details = - MetricKt.details { - filters += "filter1" - filters += "filter2" - } - } - - val exception = - assertFailsWith { - service.createMetric( - createMetricRequest { - this.metric = metric - externalMetricId = "external-metric-id" - } - ) - } - - assertThat(exception.status.code).isEqualTo(Status.Code.INVALID_ARGUMENT) - assertThat(exception.message).contains("vid") - } - @Test fun `createMetric throws INVALID_ARGUMENT when metric missing weighted measurements`() = runBlocking { @@ -1200,16 +1152,18 @@ abstract class MetricsServiceTest { metricSpec = metricSpec { reach = MetricSpecKt.reachParams { - privacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } - } - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = 0.1f - width = 0.5f + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = 0.1f + width = 0.5f + } + } } } details = @@ -1248,23 +1202,25 @@ abstract class MetricsServiceTest { metricSpec = metricSpec { reachAndFrequency = MetricSpecKt.reachAndFrequencyParams { - reachPrivacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } - frequencyPrivacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + frequencyPrivacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = 0.1f + width = 0.5f + } + } maximumFrequency = 5 } - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = 0.1f - width = 0.5f - } } weightedMeasurements += MetricKt.weightedMeasurement { @@ -1320,23 +1276,25 @@ abstract class MetricsServiceTest { metricSpec = metricSpec { reachAndFrequency = MetricSpecKt.reachAndFrequencyParams { - reachPrivacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } - frequencyPrivacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + frequencyPrivacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = 0.1f + width = 0.5f + } + } maximumFrequency = 5 } - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = 0.1f - width = 0.5f - } } weightedMeasurements += MetricKt.weightedMeasurement { @@ -1399,23 +1357,25 @@ abstract class MetricsServiceTest { metricSpec = metricSpec { reachAndFrequency = MetricSpecKt.reachAndFrequencyParams { - reachPrivacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } - frequencyPrivacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + frequencyPrivacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = 0.1f + width = 0.5f + } + } maximumFrequency = 5 } - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = 0.1f - width = 0.5f - } } weightedMeasurements += MetricKt.weightedMeasurement { @@ -1490,23 +1450,25 @@ abstract class MetricsServiceTest { metricSpec = metricSpec { reachAndFrequency = MetricSpecKt.reachAndFrequencyParams { - reachPrivacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } - frequencyPrivacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + frequencyPrivacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = 0.1f + width = 0.5f + } + } maximumFrequency = 5 } - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = 0.1f - width = 0.5f - } } weightedMeasurements += MetricKt.weightedMeasurement { @@ -1589,23 +1551,25 @@ abstract class MetricsServiceTest { metricSpec = metricSpec { reachAndFrequency = MetricSpecKt.reachAndFrequencyParams { - reachPrivacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } - frequencyPrivacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + frequencyPrivacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = 0.1f + width = 0.5f + } + } maximumFrequency = 5 } - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = 0.1f - width = 0.5f - } } weightedMeasurements += MetricKt.weightedMeasurement { @@ -1680,23 +1644,25 @@ abstract class MetricsServiceTest { metricSpec = metricSpec { reachAndFrequency = MetricSpecKt.reachAndFrequencyParams { - reachPrivacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } - frequencyPrivacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + frequencyPrivacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = 0.1f + width = 0.5f + } + } maximumFrequency = 5 } - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = 0.1f - width = 0.5f - } } weightedMeasurements += MetricKt.weightedMeasurement { @@ -1760,23 +1726,25 @@ abstract class MetricsServiceTest { metricSpec = metricSpec { reachAndFrequency = MetricSpecKt.reachAndFrequencyParams { - reachPrivacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } - frequencyPrivacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + frequencyPrivacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = 0.1f + width = 0.5f + } + } maximumFrequency = 5 } - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = 0.1f - width = 0.5f - } } weightedMeasurements += MetricKt.weightedMeasurement { @@ -1839,88 +1807,8 @@ abstract class MetricsServiceTest { metricSpec = metricSpec { reachAndFrequency = MetricSpecKt.reachAndFrequencyParams { - reachPrivacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } - frequencyPrivacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } - maximumFrequency = 5 - } - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = 0.1f - width = 0.5f - } - } - weightedMeasurements += - MetricKt.weightedMeasurement { - weight = 2 - binaryRepresentation = 1 - measurement = measurement { - cmmsMeasurementConsumerId = CMMS_MEASUREMENT_CONSUMER_ID - timeInterval = interval { - startTime = timestamp { seconds = 10 } - endTime = timestamp { seconds = 100 } - } - primitiveReportingSetBases += - ReportingSetKt.primitiveReportingSetBasis { - externalReportingSetId = createdReportingSet.externalReportingSetId - filters += "filter1" - filters += "filter2" - } - } - } - details = - MetricKt.details { - filters += "filter1" - filters += "filter2" - } - } - - val exception = - assertFailsWith { - service.batchCreateMetrics( - batchCreateMetricsRequest { - cmmsMeasurementConsumerId = CMMS_MEASUREMENT_CONSUMER_ID - requests += createMetricRequest { - this.metric = metric - externalMetricId = "externalMetricId1" - } - requests += createMetricRequest { - this.metric = metric.copy { metricSpec = metricSpec.copy { clearType() } } - externalMetricId = "externalMetricId2" - } - } - ) - } - - assertThat(exception.status.code).isEqualTo(Status.Code.INVALID_ARGUMENT) - assertThat(exception.message).contains("type") - } - - @Test - fun `batchCreateMetrics throws INVALID_ARGUMENT when metric spec missing vid sampling`() = - runBlocking { - createMeasurementConsumer(CMMS_MEASUREMENT_CONSUMER_ID, measurementConsumersService) - val createdReportingSet = - createReportingSet(CMMS_MEASUREMENT_CONSUMER_ID, reportingSetsService) - - val metric = metric { - cmmsMeasurementConsumerId = CMMS_MEASUREMENT_CONSUMER_ID - externalReportingSetId = createdReportingSet.externalReportingSetId - timeInterval = interval { - startTime = timestamp { seconds = 10 } - endTime = timestamp { seconds = 100 } - } - metricSpec = metricSpec { - reachAndFrequency = - MetricSpecKt.reachAndFrequencyParams { - reachPrivacyParams = + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = MetricSpecKt.differentialPrivacyParams { epsilon = 1.0 delta = 2.0 @@ -1930,61 +1818,61 @@ abstract class MetricsServiceTest { epsilon = 1.0 delta = 2.0 } - maximumFrequency = 5 - } - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = 0.1f - width = 0.5f - } - } - weightedMeasurements += - MetricKt.weightedMeasurement { - weight = 2 - binaryRepresentation = 1 - measurement = measurement { - cmmsMeasurementConsumerId = CMMS_MEASUREMENT_CONSUMER_ID - timeInterval = interval { - startTime = timestamp { seconds = 10 } - endTime = timestamp { seconds = 100 } - } - primitiveReportingSetBases += - ReportingSetKt.primitiveReportingSetBasis { - externalReportingSetId = createdReportingSet.externalReportingSetId - filters += "filter1" - filters += "filter2" - } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = 0.1f + width = 0.5f + } } - } - details = - MetricKt.details { - filters += "filter1" - filters += "filter2" + maximumFrequency = 5 } } - - val exception = - assertFailsWith { - service.batchCreateMetrics( - batchCreateMetricsRequest { - cmmsMeasurementConsumerId = CMMS_MEASUREMENT_CONSUMER_ID - requests += createMetricRequest { - this.metric = metric - externalMetricId = "externalMetricId1" - } - requests += createMetricRequest { - this.metric = - metric.copy { metricSpec = metricSpec.copy { clearVidSamplingInterval() } } - externalMetricId = "externalMetricId2" - } + weightedMeasurements += + MetricKt.weightedMeasurement { + weight = 2 + binaryRepresentation = 1 + measurement = measurement { + cmmsMeasurementConsumerId = CMMS_MEASUREMENT_CONSUMER_ID + timeInterval = interval { + startTime = timestamp { seconds = 10 } + endTime = timestamp { seconds = 100 } } - ) + primitiveReportingSetBases += + ReportingSetKt.primitiveReportingSetBasis { + externalReportingSetId = createdReportingSet.externalReportingSetId + filters += "filter1" + filters += "filter2" + } + } + } + details = + MetricKt.details { + filters += "filter1" + filters += "filter2" } - - assertThat(exception.status.code).isEqualTo(Status.Code.INVALID_ARGUMENT) - assertThat(exception.message).contains("vid") } + val exception = + assertFailsWith { + service.batchCreateMetrics( + batchCreateMetricsRequest { + cmmsMeasurementConsumerId = CMMS_MEASUREMENT_CONSUMER_ID + requests += createMetricRequest { + this.metric = metric + externalMetricId = "externalMetricId1" + } + requests += createMetricRequest { + this.metric = metric.copy { metricSpec = metricSpec.copy { clearType() } } + externalMetricId = "externalMetricId2" + } + } + ) + } + + assertThat(exception.status.code).isEqualTo(Status.Code.INVALID_ARGUMENT) + assertThat(exception.message).contains("type") + } + @Test fun `batchCreateMetrics throws INVALID_ARGUMENT when metric missing weighted measurements`() = runBlocking { @@ -2002,23 +1890,25 @@ abstract class MetricsServiceTest { metricSpec = metricSpec { reachAndFrequency = MetricSpecKt.reachAndFrequencyParams { - reachPrivacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } - frequencyPrivacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + frequencyPrivacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = 0.1f + width = 0.5f + } + } maximumFrequency = 5 } - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = 0.1f - width = 0.5f - } } weightedMeasurements += MetricKt.weightedMeasurement { @@ -2083,23 +1973,25 @@ abstract class MetricsServiceTest { metricSpec = metricSpec { reachAndFrequency = MetricSpecKt.reachAndFrequencyParams { - reachPrivacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } - frequencyPrivacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + frequencyPrivacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = 0.1f + width = 0.5f + } + } maximumFrequency = 5 } - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = 0.1f - width = 0.5f - } } weightedMeasurements += MetricKt.weightedMeasurement { @@ -2163,23 +2055,25 @@ abstract class MetricsServiceTest { metricSpec = metricSpec { reachAndFrequency = MetricSpecKt.reachAndFrequencyParams { - reachPrivacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } - frequencyPrivacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + frequencyPrivacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = 0.1f + width = 0.5f + } + } maximumFrequency = 5 } - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = 0.1f - width = 0.5f - } } weightedMeasurements += MetricKt.weightedMeasurement { @@ -2236,16 +2130,18 @@ abstract class MetricsServiceTest { metricSpec = metricSpec { reach = MetricSpecKt.reachParams { - privacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } - } - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = 0.1f - width = 0.5f + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = 0.1f + width = 0.5f + } + } } } } @@ -2266,7 +2162,111 @@ abstract class MetricsServiceTest { } @Test - fun `batchGetMetrics succeeds when metric spec type is frequency duration`(): Unit = runBlocking { + fun `batchGetMetrics succeeds when metric spec type is reach and single params set`(): Unit = + runBlocking { + createMeasurementConsumer(CMMS_MEASUREMENT_CONSUMER_ID, measurementConsumersService) + + val createMetricRequest = + createCreateMetricRequest(CMMS_MEASUREMENT_CONSUMER_ID, reportingSetsService).copy { + metric = + metric.copy { + metricSpec = metricSpec { + reach = + MetricSpecKt.reachParams { + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = 0.1f + width = 0.5f + } + } + singleDataProviderParams = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 2.0 + delta = 4.0 + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = 0.2f + width = 0.6f + } + } + } + } + } + } + val createdMetric = service.createMetric(createMetricRequest) + + assertThat(createdMetric.metricSpec.reach).isEqualTo(createMetricRequest.metric.metricSpec.reach) + + val retrievedMetrics = + service.batchGetMetrics( + batchGetMetricsRequest { + cmmsMeasurementConsumerId = CMMS_MEASUREMENT_CONSUMER_ID + externalMetricIds += createdMetric.externalMetricId + } + ) + + assertThat(retrievedMetrics.metricsList) + .ignoringRepeatedFieldOrder() + .containsExactly(createdMetric) + } + + @Test + fun `batchGetMetrics succeeds when measurement has isSingleDataProvider set to true`(): Unit = + runBlocking { + createMeasurementConsumer(CMMS_MEASUREMENT_CONSUMER_ID, measurementConsumersService) + + val createMetricRequest = + createCreateMetricRequest(CMMS_MEASUREMENT_CONSUMER_ID, reportingSetsService).copy { + val source = this + metric = + metric.copy { + weightedMeasurements.clear() + weightedMeasurements += + MetricKt.weightedMeasurement { + weight = 2 + binaryRepresentation = 1 + measurement = measurement { + this.cmmsMeasurementConsumerId = source.metric.cmmsMeasurementConsumerId + timeInterval = interval { + startTime = timestamp { seconds = 10 } + endTime = timestamp { seconds = 100 } + } + primitiveReportingSetBases += + ReportingSetKt.primitiveReportingSetBasis { + externalReportingSetId = source.metric.externalReportingSetId + filters += "filter1" + } + isSingleDataProvider = true + } + } + } + } + val createdMetric = service.createMetric(createMetricRequest) + + val retrievedMetrics = + service.batchGetMetrics( + batchGetMetricsRequest { + cmmsMeasurementConsumerId = CMMS_MEASUREMENT_CONSUMER_ID + externalMetricIds += createdMetric.externalMetricId + } + ) + + assertThat(createdMetric.weightedMeasurementsList.first().measurement.isSingleDataProvider).isTrue() + assertThat(retrievedMetrics.metricsList) + .ignoringRepeatedFieldOrder() + .containsExactly(createdMetric) + } + + @Test + fun `batchGetMetrics succeeds when metric spec type is reach and frequency`(): Unit = runBlocking { createMeasurementConsumer(CMMS_MEASUREMENT_CONSUMER_ID, measurementConsumersService) val createMetricRequest = @@ -2276,23 +2276,25 @@ abstract class MetricsServiceTest { metricSpec = metricSpec { reachAndFrequency = MetricSpecKt.reachAndFrequencyParams { - reachPrivacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } - frequencyPrivacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + frequencyPrivacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = 0.1f + width = 0.5f + } + } maximumFrequency = 5 } - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = 0.1f - width = 0.5f - } } } } @@ -2311,6 +2313,74 @@ abstract class MetricsServiceTest { .containsExactly(createdMetric) } + @Test + fun `batchGetMetrics succeeds when metric spec type is rf and single params set`(): Unit = + runBlocking { + createMeasurementConsumer(CMMS_MEASUREMENT_CONSUMER_ID, measurementConsumersService) + + val createMetricRequest = + createCreateMetricRequest(CMMS_MEASUREMENT_CONSUMER_ID, reportingSetsService).copy { + metric = + metric.copy { + metricSpec = metricSpec { + reachAndFrequency = + MetricSpecKt.reachAndFrequencyParams { + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + frequencyPrivacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = 0.1f + width = 0.5f + } + } + singleDataProviderParams = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 2.0 + delta = 4.0 + } + frequencyPrivacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 3.0 + delta = 5.0 + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = 0.3f + width = 0.6f + } + } + maximumFrequency = 5 + } + } + } + } + val createdMetric = service.createMetric(createMetricRequest) + + assertThat(createdMetric.metricSpec.reachAndFrequency).isEqualTo(createMetricRequest.metric.metricSpec.reachAndFrequency) + + val retrievedMetrics = + service.batchGetMetrics( + batchGetMetricsRequest { + cmmsMeasurementConsumerId = CMMS_MEASUREMENT_CONSUMER_ID + externalMetricIds += createdMetric.externalMetricId + } + ) + + assertThat(retrievedMetrics.metricsList) + .ignoringRepeatedFieldOrder() + .containsExactly(createdMetric) + } + @Test fun `batchGetMetrics succeeds when metric spec type is impression count`(): Unit = runBlocking { createMeasurementConsumer(CMMS_MEASUREMENT_CONSUMER_ID, measurementConsumersService) @@ -2322,18 +2392,20 @@ abstract class MetricsServiceTest { metricSpec = metricSpec { impressionCount = MetricSpecKt.impressionCountParams { - privacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } + params = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = 0.1f + width = 0.5f + } + } maximumFrequencyPerUser = 5 } - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = 0.1f - width = 0.5f - } } } } @@ -2363,18 +2435,20 @@ abstract class MetricsServiceTest { metricSpec = metricSpec { watchDuration = MetricSpecKt.watchDurationParams { - privacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } + params = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = 0.1f + width = 0.5f + } + } maximumWatchDurationPerUser = Durations.fromSeconds(100) } - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = 0.1f - width = 0.5f - } } } } @@ -2886,7 +2960,7 @@ abstract class MetricsServiceTest { } @Test - fun `streamReportingSets filters when id after filter is set`(): Unit = runBlocking { + fun `streamMetrics filters when id after filter is set`(): Unit = runBlocking { createMeasurementConsumer(CMMS_MEASUREMENT_CONSUMER_ID, measurementConsumersService) val createMetricRequest = @@ -3068,16 +3142,18 @@ abstract class MetricsServiceTest { metricSpec = metricSpec { reach = MetricSpecKt.reachParams { - privacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } - } - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = 0.1f - width = 0.5f + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = 0.1f + width = 0.5f + } + } } } weightedMeasurements += diff --git a/src/main/kotlin/org/wfanet/measurement/reporting/service/internal/testing/v2/ReportsServiceTest.kt b/src/main/kotlin/org/wfanet/measurement/reporting/service/internal/testing/v2/ReportsServiceTest.kt index be3d583db74..e2fa6b0af21 100644 --- a/src/main/kotlin/org/wfanet/measurement/reporting/service/internal/testing/v2/ReportsServiceTest.kt +++ b/src/main/kotlin/org/wfanet/measurement/reporting/service/internal/testing/v2/ReportsServiceTest.kt @@ -210,16 +210,18 @@ abstract class ReportsServiceTest { metricSpec = metricSpec { reach = MetricSpecKt.reachParams { - privacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } - } - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = 0.1f - width = 0.5f + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = 0.1f + width = 0.5f + } + } } } timeInterval = interval { @@ -245,16 +247,18 @@ abstract class ReportsServiceTest { metricSpec = metricSpec { reach = MetricSpecKt.reachParams { - privacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } - } - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = 0.1f - width = 0.5f + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = 0.1f + width = 0.5f + } + } } } timeInterval = interval { @@ -341,16 +345,18 @@ abstract class ReportsServiceTest { metricSpec = metricSpec { reach = MetricSpecKt.reachParams { - privacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } - } - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = 0.1f - width = 0.5f + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = 0.1f + width = 0.5f + } + } } } timeInterval = interval { @@ -367,16 +373,18 @@ abstract class ReportsServiceTest { metricSpec = metricSpec { reach = MetricSpecKt.reachParams { - privacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } - } - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = 0.1f - width = 0.5f + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = 0.1f + width = 0.5f + } + } } } timeInterval = interval { @@ -594,16 +602,18 @@ abstract class ReportsServiceTest { metricSpec = metricSpec { reach = MetricSpecKt.reachParams { - privacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } - } - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = 0.1f - width = 0.5f + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = 0.1f + width = 0.5f + } + } } } this.timeInterval = interval @@ -765,16 +775,18 @@ abstract class ReportsServiceTest { metricSpec = metricSpec { reach = MetricSpecKt.reachParams { - privacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } - } - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = 0.1f - width = 0.5f + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = 0.1f + width = 0.5f + } + } } } this.timeInterval = interval @@ -941,16 +953,18 @@ abstract class ReportsServiceTest { metricSpec = metricSpec { reach = MetricSpecKt.reachParams { - privacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } - } - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = 0.1f - width = 0.5f + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = 0.1f + width = 0.5f + } + } } } timeInterval = interval { @@ -967,16 +981,18 @@ abstract class ReportsServiceTest { metricSpec = metricSpec { reach = MetricSpecKt.reachParams { - privacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } - } - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = 0.1f - width = 0.5f + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = 0.1f + width = 0.5f + } + } } } timeInterval = interval { @@ -1034,16 +1050,18 @@ abstract class ReportsServiceTest { metricSpec = metricSpec { reach = MetricSpecKt.reachParams { - privacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } - } - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = 0.1f - width = 0.5f + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = 0.1f + width = 0.5f + } + } } } timeInterval = interval { @@ -1060,16 +1078,18 @@ abstract class ReportsServiceTest { metricSpec = metricSpec { reach = MetricSpecKt.reachParams { - privacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } - } - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = 0.1f - width = 0.5f + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = 0.1f + width = 0.5f + } + } } } timeInterval = interval { @@ -1132,16 +1152,18 @@ abstract class ReportsServiceTest { metricSpec = metricSpec { reach = MetricSpecKt.reachParams { - privacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } - } - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = 0.1f - width = 0.5f + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = 0.1f + width = 0.5f + } + } } } timeInterval = interval { @@ -1418,16 +1440,18 @@ abstract class ReportsServiceTest { metricSpec = metricSpec { reach = MetricSpecKt.reachParams { - privacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } - } - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = 0.1f - width = 0.5f + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = 0.1f + width = 0.5f + } + } } } timeInterval = interval { @@ -1453,16 +1477,18 @@ abstract class ReportsServiceTest { metricSpec = metricSpec { reach = MetricSpecKt.reachParams { - privacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } - } - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = 0.1f - width = 0.5f + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = 0.1f + width = 0.5f + } + } } } timeInterval = interval { @@ -1540,16 +1566,18 @@ abstract class ReportsServiceTest { metricSpec = metricSpec { reach = MetricSpecKt.reachParams { - privacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } - } - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = 0.1f - width = 0.5f + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = 0.1f + width = 0.5f + } + } } } timeInterval = interval { @@ -1935,16 +1963,18 @@ abstract class ReportsServiceTest { metricSpec = metricSpec { reach = MetricSpecKt.reachParams { - privacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } - } - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = 0.1f - width = 0.5f + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = 0.1f + width = 0.5f + } + } } } timeInterval = interval { @@ -1998,16 +2028,18 @@ abstract class ReportsServiceTest { metricSpec = metricSpec { reach = MetricSpecKt.reachParams { - privacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } - } - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = 0.1f - width = 0.5f + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = 0.1f + width = 0.5f + } + } } } timeInterval = interval { diff --git a/src/main/kotlin/org/wfanet/measurement/reporting/service/internal/testing/v2/ResourceCreationMethods.kt b/src/main/kotlin/org/wfanet/measurement/reporting/service/internal/testing/v2/ResourceCreationMethods.kt index 105c3ef68f2..5829fd6c5cc 100644 --- a/src/main/kotlin/org/wfanet/measurement/reporting/service/internal/testing/v2/ResourceCreationMethods.kt +++ b/src/main/kotlin/org/wfanet/measurement/reporting/service/internal/testing/v2/ResourceCreationMethods.kt @@ -90,16 +90,18 @@ suspend fun createMetricCalculationSpec( metricSpecs += metricSpec { reach = MetricSpecKt.reachParams { - privacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = 1.0 - delta = 2.0 - } - } - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = 0.1f - width = 0.5f + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = 1.0 + delta = 2.0 + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = 0.1f + width = 0.5f + } + } } } groupings += MetricCalculationSpecKt.grouping { predicates += "age > 10" } diff --git a/src/main/proto/wfa/measurement/config/reporting/metric_spec_config.proto b/src/main/proto/wfa/measurement/config/reporting/metric_spec_config.proto index be77c12d096..8a73db20f0b 100644 --- a/src/main/proto/wfa/measurement/config/reporting/metric_spec_config.proto +++ b/src/main/proto/wfa/measurement/config/reporting/metric_spec_config.proto @@ -35,27 +35,60 @@ message MetricSpecConfig { double delta = 2; } - // Parameters that are used to generate `Reach` metric. - message ReachParams { - // Differential privacy parameters for reach. + // Specifies a range of VIDs to be sampled. + message VidSamplingInterval { + message FixedStart { + // The start of the sampling interval in [0, 1) + float start = 1; + // The width of the sampling interval. + float width = 2; + } + + message RandomStart { + float width = 1; + } + + oneof start { + FixedStart fixed_start = 1; + RandomStart random_start = 2; + } + } + + message Params { DifferentialPrivacyParams privacy_params = 1; + DifferentialPrivacyParams frequency_privacy_params = 2; + VidSamplingInterval vid_sampling_interval = 3; + } + + // Parameters that are used to generate `Reach` metrics. Applied on a + // per-Measurement basis. + message ReachParams { + // Parameters for multiple DataProviders. + Params multiple_data_provider_params = 1; + + // Parameters for a single DataProvider. + Params single_data_provider_params = 2; } - // Parameters that are used to generate `ReachAndFrequency` metric. + // Parameters that are used to generate `ReachAndFrequency` metrics. Applied + // on a per-Measurement basis. message ReachAndFrequencyParams { - // Differential privacy parameters for reach. - DifferentialPrivacyParams reach_privacy_params = 1; - // Differential privacy parameters for frequency. - DifferentialPrivacyParams frequency_privacy_params = 2; + // Parameters for multiple DataProviders. + Params multiple_data_provider_params = 1; + + // Parameters for a single DataProvider. + Params single_data_provider_params = 2; + // Maximum frequency cut-off value in frequency histogram. // // Counts with frequency higher than `maximum_frequency` will be aggregated // together. - int32 maximum_frequency = 3; + int32 maximum_frequency = 5; } - // Parameters that are used to generate `Impression Count` metric. + // Parameters that are used to generate `Impression Count` metrics. Applied on + // a per-Measurement basis. message ImpressionCountParams { - // Differential privacy parameters. - DifferentialPrivacyParams privacy_params = 1; + Params params = 1; + // Maximum frequency per user that will be included in this metric. Enforced // on a per EDP basis. // @@ -66,16 +99,17 @@ message MetricSpecConfig { // too small, there's a truncation bias. int32 maximum_frequency_per_user = 2; } - // Parameters that are used to generate `Watch Duration` metric. + // Parameters that are used to generate `Watch Duration` metrics. Applied on a + // per-Measurement basis. message WatchDurationParams { - // Differential privacy parameters. - DifferentialPrivacyParams privacy_params = 1; + Params params = 1; + // Maximum watch duration per user that will be included in this metric. // Enforced on a per EDP basis. google.protobuf.Duration maximum_watch_duration_per_user = 2; } - // Parameters that are used to generate `Population` metric. + // Parameters that are used to generate `Population` metrics. message PopulationCountParams {} // Parameters for generating the count of unique audiences reached given a set @@ -89,20 +123,4 @@ message MetricSpecConfig { WatchDurationParams watch_duration_params = 4; // Parameters for generating the population. PopulationCountParams population_count_params = 5; - - // Specifies a range of VIDs to be sampled. - message VidSamplingInterval { - // The start of the sampling interval in [0, 1) - float start = 1; - // The width of the sampling interval. - float width = 2; - } - // Range of VIDs that will be included for reach measurements. - VidSamplingInterval reach_vid_sampling_interval = 6; - // Range of VIDs that will be included for reach-and-frequency measurements. - VidSamplingInterval reach_and_frequency_vid_sampling_interval = 7; - // Range of VIDs that will be included for impression count measurements. - VidSamplingInterval impression_count_vid_sampling_interval = 8; - // Range of VIDs that will be included for watch duration measurements. - VidSamplingInterval watch_duration_vid_sampling_interval = 9; } diff --git a/src/main/proto/wfa/measurement/internal/reporting/v2/measurement.proto b/src/main/proto/wfa/measurement/internal/reporting/v2/measurement.proto index 3ca87a0bd9a..0c7fd54e2bc 100644 --- a/src/main/proto/wfa/measurement/internal/reporting/v2/measurement.proto +++ b/src/main/proto/wfa/measurement/internal/reporting/v2/measurement.proto @@ -135,4 +135,6 @@ message Measurement { repeated Result results = 2; } Details details = 7; + + bool is_single_data_provider = 8; } diff --git a/src/main/proto/wfa/measurement/internal/reporting/v2/metric.proto b/src/main/proto/wfa/measurement/internal/reporting/v2/metric.proto index 1dd42589e5c..6d96bfa23d2 100644 --- a/src/main/proto/wfa/measurement/internal/reporting/v2/metric.proto +++ b/src/main/proto/wfa/measurement/internal/reporting/v2/metric.proto @@ -29,21 +29,31 @@ message MetricSpec { double epsilon = 1; double delta = 2; } + message VidSamplingInterval { + float start = 1; + float width = 2; + } + message Params { + DifferentialPrivacyParams privacy_params = 1; + DifferentialPrivacyParams frequency_privacy_params = 2; + VidSamplingInterval vid_sampling_interval = 3; + } message ReachParams { - DifferentialPrivacyParams privacy_params = 1; + Params multiple_data_provider_params = 2; + Params single_data_provider_params = 3; } message ReachAndFrequencyParams { - DifferentialPrivacyParams reach_privacy_params = 1; - DifferentialPrivacyParams frequency_privacy_params = 2; + Params multiple_data_provider_params = 4; + Params single_data_provider_params = 5; int32 maximum_frequency = 3; } message ImpressionCountParams { - DifferentialPrivacyParams privacy_params = 1; + Params params = 3; int32 maximum_frequency_per_user = 2; } message WatchDurationParams { - DifferentialPrivacyParams privacy_params = 1; + Params params = 3; google.protobuf.Duration maximum_watch_duration_per_user = 2; } message PopulationCountParams {} @@ -55,12 +65,6 @@ message MetricSpec { WatchDurationParams watch_duration = 4; PopulationCountParams population_count = 5; } - - message VidSamplingInterval { - float start = 1; - float width = 2; - } - VidSamplingInterval vid_sampling_interval = 6; } // Proto representation of the internal Metric entity type. diff --git a/src/main/proto/wfa/measurement/reporting/v2alpha/metric.proto b/src/main/proto/wfa/measurement/reporting/v2alpha/metric.proto index 236e80e9ea8..4f825c72ec1 100644 --- a/src/main/proto/wfa/measurement/reporting/v2alpha/metric.proto +++ b/src/main/proto/wfa/measurement/reporting/v2alpha/metric.proto @@ -41,31 +41,70 @@ message MetricSpec { optional double delta = 2; } - // Parameters that are used to generate `Reach` metric. + // Specifies a range of VIDs to be sampled. + message VidSamplingInterval { + // The start of the sampling interval in [0, 1) + float start = 1; + // The width of the sampling interval. + // + // start + width cannot be larger than 1 + float width = 2 [(google.api.field_behavior) = REQUIRED]; + } + + message Params { + DifferentialPrivacyParams privacy_params = 1; + // Only used in `ReachAndFrequencyParams`. + DifferentialPrivacyParams frequency_privacy_params = 2; + VidSamplingInterval vid_sampling_interval = 3; + } + + // Parameters that are used to generate `Reach` metrics. Applied on a + // per-Measurement basis. message ReachParams { // Differential privacy parameters for reach. - DifferentialPrivacyParams privacy_params = 1 - [(google.api.field_behavior) = REQUIRED]; + DifferentialPrivacyParams privacy_params = 1 [deprecated = true]; + + // Parameters for multiple DataProviders. Will be required in a future + // release. If set, `single_data_provider_params` needs to be set. Takes + // precedence over `privacy_params`. + Params multiple_data_provider_params = 2; + // Parameters for a single DataProvider. Will be required in a future + // release. If set, `multiple_data_provider_params` needs to be set. + Params single_data_provider_params = 3; } - // Parameters that are used to generate `ReachAndFrequency` metric. + + // Parameters that are used to generate `ReachAndFrequency` metrics. Applied + // on a per-Measurement basis. message ReachAndFrequencyParams { - // Differential privacy parameters for reach. - DifferentialPrivacyParams reach_privacy_params = 1 - [(google.api.field_behavior) = REQUIRED]; + // Differential privacy parameters for reach and multiple DataProviders. + DifferentialPrivacyParams reach_privacy_params = 1 [deprecated = true]; // Differential privacy parameters for frequency distribution. - DifferentialPrivacyParams frequency_privacy_params = 2 - [(google.api.field_behavior) = REQUIRED]; + DifferentialPrivacyParams frequency_privacy_params = 2 [deprecated = true]; + + // Parameters for multiple DataProviders. Will be required in a future + // release. If set, `single_data_provider_params` needs to be set. Takes + // precedence over `reach_privacy_params` and `frequency_privacy_params`. + Params multiple_data_provider_params = 4; + // Parameters for a single DataProvider. Will be required in a future + // release. If set, `multiple_data_provider_params` needs to be set. + Params single_data_provider_params = 5; + // Maximum frequency cut-off value in frequency histogram. // // Counts with frequency higher than `maximum_frequency` will be aggregated // together. If not set, the default value will be used and outputted here. int32 maximum_frequency = 3; } - // Parameters that are used to generate `Impression Count` metric. + // Parameters that are used to generate `Impression Count` metrics. Applied on + // a per-Measurement basis. message ImpressionCountParams { // Differential privacy parameters. - DifferentialPrivacyParams privacy_params = 1 - [(google.api.field_behavior) = REQUIRED]; + DifferentialPrivacyParams privacy_params = 1 [deprecated = true]; + + // Parameters for impression. Will be required in a future release. Takes + // precedence over `privacy_params`. + Params params = 3; + // Maximum frequency per user that will be included in this metric. Enforced // on a per EDP basis. // @@ -79,11 +118,16 @@ message MetricSpec { // If not set, the default value will be used and outputted here. optional int32 maximum_frequency_per_user = 2; } - // Parameters that are used to generate `Watch Duration` metric. + // Parameters that are used to generate `Watch Duration` metrics. Applied on a + // per-Measurement basis. message WatchDurationParams { // Differential privacy parameters. - DifferentialPrivacyParams privacy_params = 1 - [(google.api.field_behavior) = REQUIRED]; + DifferentialPrivacyParams privacy_params = 1 [deprecated = true]; + + // Parameters for watch duration. Will be required in a future release. Takes + // precedence over `privacy_params`. + Params params = 3; + // Maximum watch duration per user that will be included in this metric. // // Recommended maximum_watch_duration_per_user = cap on the total watch @@ -93,7 +137,7 @@ message MetricSpec { google.protobuf.Duration maximum_watch_duration_per_user = 2; } - // Parameters that are used to generate `Population` metric. + // Parameters that are used to generate `Population` metrics. message PopulationCountParams {} // Types of metric with parameters. @@ -117,20 +161,11 @@ message MetricSpec { [(google.api.field_behavior) = IMMUTABLE]; } - // Specifies a range of VIDs to be sampled. - message VidSamplingInterval { - // The start of the sampling interval in [0, 1) - float start = 1 [(google.api.field_behavior) = REQUIRED]; - // The width of the sampling interval. - // - // start + width cannot be larger than 1 - float width = 2 [(google.api.field_behavior) = REQUIRED]; - } // Range of VIDs that will be included in this measurement // // If this field is unspecified in a request message, the service // implementation will use the value specified in the system defaults. - VidSamplingInterval vid_sampling_interval = 6; + VidSamplingInterval vid_sampling_interval = 6 [deprecated = true]; } // Statistics of a scalar value diff --git a/src/main/resources/reporting/postgres/add-additional-metric-spec-columns-to-metrics.sql b/src/main/resources/reporting/postgres/add-additional-metric-spec-columns-to-metrics.sql new file mode 100644 index 00000000000..4d23ffb0ff5 --- /dev/null +++ b/src/main/resources/reporting/postgres/add-additional-metric-spec-columns-to-metrics.sql @@ -0,0 +1,63 @@ +-- liquibase formatted sql + +-- Copyright 2024 The Cross-Media Measurement Authors +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +-- Postgres database schema for the Reporting server. +-- +-- Table hierarchy: +-- Root +-- └── MeasurementConsumers +-- ├── EventGroups +-- ├── ReportingSets +-- │ ├── ReportingSetEventGroups +-- │ ├── PrimitiveReportingSetBases +-- │ │ └── PrimitiveReportingSetBasisFilters +-- │ ├── SetExpressions +-- │ └── WeightedSubsetUnions +-- │ └── WeightedSubsetUnionPrimitiveReportingSetBases +-- ├── Metrics +-- │ └── MetricMeasurements +-- ├── Measurements +-- │ └── MeasurementPrimitiveReportingSetBases +-- ├── MetricCalculationSpecs +-- └── Reports +-- ├── ReportTimeIntervals +-- └── MetricCalculationSpecReportingMetrics + + + DifferentialPrivacyEpsilon DOUBLE PRECISION NOT NULL, + DifferentialPrivacyDelta DOUBLE PRECISION NOT NULL, + + -- Frequency has a second set of differential privacy params. + FrequencyDifferentialPrivacyEpsilon DOUBLE PRECISION, + FrequencyDifferentialPrivacyDelta DOUBLE PRECISION, + + -- Must not be NULL if MetricType is REACH_AND_FREQUENCY + MaximumFrequency bigint, + -- Must not be NULL if MetricType is IMPRESSION_COUNT + MaximumFrequencyPerUser bigint, + -- Must not be NULL if MetricType is WATCH_DURATION + MaximumWatchDurationPerUser interval, + + VidSamplingIntervalStart DOUBLE PRECISION NOT NULL, + VidSamplingIntervalWidth DOUBLE PRECISION NOT NULL, +-- changeset tristanvuong2021:add-spec-columns-metrics-table dbms:postgresql +ALTER TABLE Metrics + ADD COLUMN SingleDataProviderDifferentialPrivacyEpsilon DOUBLE PRECISION, + ADD COLUMN SingleDataProviderDifferentialPrivacyDelta DOUBLE PRECISION, + ADD COLUMN SingleDataProviderFrequencyDifferentialPrivacyEpsilon DOUBLE PRECISION, + ADD COLUMN SingleDataProviderFrequencyDifferentialPrivacyDelta DOUBLE PRECISION, + ADD COLUMN SingleDataProviderVidSamplingIntervalStart DOUBLE PRECISION, + ADD COLUMN SingleDataProviderVidSamplingIntervalWidth DOUBLE PRECISION; diff --git a/src/main/resources/reporting/postgres/add-is-single-data-provider-column-to-measurements.sql b/src/main/resources/reporting/postgres/add-is-single-data-provider-column-to-measurements.sql new file mode 100644 index 00000000000..4bb44ac38b4 --- /dev/null +++ b/src/main/resources/reporting/postgres/add-is-single-data-provider-column-to-measurements.sql @@ -0,0 +1,40 @@ +-- liquibase formatted sql + +-- Copyright 2024 The Cross-Media Measurement Authors +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +-- Postgres database schema for the Reporting server. +-- +-- Table hierarchy: +-- Root +-- └── MeasurementConsumers +-- ├── EventGroups +-- ├── ReportingSets +-- │ ├── ReportingSetEventGroups +-- │ ├── PrimitiveReportingSetBases +-- │ │ └── PrimitiveReportingSetBasisFilters +-- │ ├── SetExpressions +-- │ └── WeightedSubsetUnions +-- │ └── WeightedSubsetUnionPrimitiveReportingSetBases +-- ├── Metrics +-- │ └── MetricMeasurements +-- ├── Measurements +-- │ └── MeasurementPrimitiveReportingSetBases +-- ├── MetricCalculationSpecs +-- └── Reports +-- ├── ReportTimeIntervals +-- └── MetricCalculationSpecReportingMetrics + +-- changeset tristanvuong2021:add-is-single-data-provider-col-measurements-table dbms:postgresql +ALTER TABLE Measurements ADD COLUMN IsSingleDataProvider boolean NOT NULL DEFAULT FALSE; diff --git a/src/main/resources/reporting/postgres/changelog-v2.yaml b/src/main/resources/reporting/postgres/changelog-v2.yaml index b41518d5e96..252b32f1058 100644 --- a/src/main/resources/reporting/postgres/changelog-v2.yaml +++ b/src/main/resources/reporting/postgres/changelog-v2.yaml @@ -52,3 +52,9 @@ databaseChangeLog: - include: file: add-indexes-for-metric-reuse.sql relativeToChangeLogFile: true +- include: + file: add-additional-metric-spec-columns-to-metrics.sql + relativeToChangeLogFile: true +- include: + file: add-is-single-data-provider-column-to-measurements.sql + relativeToChangeLogFile: true diff --git a/src/test/kotlin/org/wfanet/measurement/reporting/service/api/v2alpha/BUILD.bazel b/src/test/kotlin/org/wfanet/measurement/reporting/service/api/v2alpha/BUILD.bazel index 5ef31c1517f..aa420898214 100644 --- a/src/test/kotlin/org/wfanet/measurement/reporting/service/api/v2alpha/BUILD.bazel +++ b/src/test/kotlin/org/wfanet/measurement/reporting/service/api/v2alpha/BUILD.bazel @@ -199,6 +199,9 @@ kt_jvm_test( associates = [ "//src/main/kotlin/org/wfanet/measurement/reporting/service/api/v2alpha:metric_calculation_specs_service", ], + data = [ + "//src/main/k8s/testing/secretfiles:all_configs", + ], test_class = "org.wfanet.measurement.reporting.service.api.v2alpha.MetricCalculationSpecsServiceTest", deps = [ "//src/main/kotlin/org/wfanet/measurement/api/v2alpha:principal_server_interceptor", @@ -206,6 +209,7 @@ kt_jvm_test( "//src/main/proto/wfa/measurement/config/reporting:measurement_consumer_config_kt_jvm_proto", "@wfa_common_jvm//imports/java/com/google/common/truth", "@wfa_common_jvm//imports/java/com/google/common/truth/extensions/proto", + "@wfa_common_jvm//imports/java/com/google/devtools/build/runfiles", "@wfa_common_jvm//imports/java/com/google/protobuf", "@wfa_common_jvm//imports/kotlin/kotlin/test", "@wfa_common_jvm//imports/kotlin/kotlinx/coroutines:core", @@ -221,6 +225,9 @@ kt_jvm_test( associates = [ "//src/main/kotlin/org/wfanet/measurement/reporting/service/api/v2alpha:reports_service", ], + data = [ + "//src/main/k8s/testing/secretfiles:all_configs", + ], test_class = "org.wfanet.measurement.reporting.service.api.v2alpha.ReportsServiceTest", deps = [ "//src/main/kotlin/org/wfanet/measurement/api/v2alpha:principal_server_interceptor", @@ -229,6 +236,7 @@ kt_jvm_test( "//src/main/proto/wfa/measurement/config/reporting:measurement_consumer_config_kt_jvm_proto", "@wfa_common_jvm//imports/java/com/google/common/truth", "@wfa_common_jvm//imports/java/com/google/common/truth/extensions/proto", + "@wfa_common_jvm//imports/java/com/google/devtools/build/runfiles", "@wfa_common_jvm//imports/java/com/google/protobuf", "@wfa_common_jvm//imports/kotlin/kotlin/test", "@wfa_common_jvm//imports/kotlin/kotlinx/coroutines:core", diff --git a/src/test/kotlin/org/wfanet/measurement/reporting/service/api/v2alpha/MetricCalculationSpecsServiceTest.kt b/src/test/kotlin/org/wfanet/measurement/reporting/service/api/v2alpha/MetricCalculationSpecsServiceTest.kt index af7bf029d9c..8ccb8fa40d4 100644 --- a/src/test/kotlin/org/wfanet/measurement/reporting/service/api/v2alpha/MetricCalculationSpecsServiceTest.kt +++ b/src/test/kotlin/org/wfanet/measurement/reporting/service/api/v2alpha/MetricCalculationSpecsServiceTest.kt @@ -18,7 +18,7 @@ package org.wfanet.measurement.reporting.service.api.v2alpha import com.google.common.truth.Truth.assertThat import com.google.common.truth.extensions.proto.ProtoTruth.assertThat -import com.google.protobuf.util.Durations +import com.google.devtools.build.runfiles.Runfiles import com.google.type.DayOfWeek import io.grpc.Status import io.grpc.StatusRuntimeException @@ -39,9 +39,7 @@ import org.wfanet.measurement.common.grpc.testing.GrpcTestServerRule import org.wfanet.measurement.common.grpc.testing.mockService import org.wfanet.measurement.common.identity.ExternalId import org.wfanet.measurement.common.testing.verifyProtoArgument -import org.wfanet.measurement.config.reporting.MetricSpecConfigKt import org.wfanet.measurement.config.reporting.measurementConsumerConfig -import org.wfanet.measurement.config.reporting.metricSpecConfig import org.wfanet.measurement.internal.reporting.v2.MetricCalculationSpec as InternalMetricCalculationSpec import org.wfanet.measurement.internal.reporting.v2.MetricCalculationSpecKt as InternalMetricCalculationSpecKt import org.wfanet.measurement.internal.reporting.v2.MetricCalculationSpecsGrpcKt.MetricCalculationSpecsCoroutineImplBase @@ -55,6 +53,15 @@ import org.wfanet.measurement.internal.reporting.v2.listMetricCalculationSpecsRe import org.wfanet.measurement.internal.reporting.v2.listMetricCalculationSpecsResponse as internalListMetricCalculationSpecsResponse import org.wfanet.measurement.internal.reporting.v2.metricCalculationSpec as internalMetricCalculationSpec import org.wfanet.measurement.internal.reporting.v2.metricSpec as internalMetricSpec +import java.nio.file.Path +import java.nio.file.Paths +import kotlin.random.Random +import org.mockito.kotlin.doReturn +import org.mockito.kotlin.mock +import org.mockito.kotlin.stub +import org.wfanet.measurement.common.getRuntimePath +import org.wfanet.measurement.common.parseTextProto +import org.wfanet.measurement.config.reporting.MetricSpecConfig import org.wfanet.measurement.reporting.v2alpha.ListMetricCalculationSpecsPageTokenKt import org.wfanet.measurement.reporting.v2alpha.MetricCalculationSpec import org.wfanet.measurement.reporting.v2alpha.MetricCalculationSpecKt @@ -69,6 +76,9 @@ import org.wfanet.measurement.reporting.v2alpha.listMetricCalculationSpecsRespon import org.wfanet.measurement.reporting.v2alpha.metricCalculationSpec import org.wfanet.measurement.reporting.v2alpha.metricSpec +private const val RANDOM_OUTPUT_INT = 0 +private const val RANDOM_OUTPUT_LONG = 0L + @RunWith(JUnit4::class) class MetricCalculationSpecsServiceTest { private val internalMetricCalculationSpecsMock: MetricCalculationSpecsCoroutineImplBase = @@ -95,6 +105,8 @@ class MetricCalculationSpecsServiceTest { ) } + private val randomMock: Random = mock() + @get:Rule val grpcTestServerRule = GrpcTestServerRule { addService(internalMetricCalculationSpecsMock) } @@ -102,10 +114,16 @@ class MetricCalculationSpecsServiceTest { @Before fun initService() { + randomMock.stub { + on { nextInt(any()) } doReturn RANDOM_OUTPUT_INT + on { nextLong() } doReturn RANDOM_OUTPUT_LONG + } + service = MetricCalculationSpecsService( MetricCalculationSpecsCoroutineStub(grpcTestServerRule.channel), METRIC_SPEC_CONFIG, + randomMock, ) } @@ -114,16 +132,30 @@ class MetricCalculationSpecsServiceTest { val reachMetricSpec = metricSpec { reach = MetricSpecKt.reachParams { - privacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = REACH_ONLY_REACH_EPSILON - delta = DIFFERENTIAL_PRIVACY_DELTA - } - } - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = REACH_ONLY_VID_SAMPLING_START - width = REACH_ONLY_VID_SAMPLING_WIDTH + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = METRIC_SPEC_CONFIG.reachParams.multipleDataProviderParams.privacyParams.epsilon + delta = METRIC_SPEC_CONFIG.reachParams.multipleDataProviderParams.privacyParams.delta + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = METRIC_SPEC_CONFIG.reachParams.multipleDataProviderParams.vidSamplingInterval.fixedStart.start + width = METRIC_SPEC_CONFIG.reachParams.multipleDataProviderParams.vidSamplingInterval.fixedStart.width + } + } + singleDataProviderParams = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = METRIC_SPEC_CONFIG.reachParams.singleDataProviderParams.privacyParams.epsilon + delta = METRIC_SPEC_CONFIG.reachParams.singleDataProviderParams.privacyParams.delta + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = METRIC_SPEC_CONFIG.reachParams.singleDataProviderParams.vidSamplingInterval.fixedStart.start + width = METRIC_SPEC_CONFIG.reachParams.singleDataProviderParams.vidSamplingInterval.fixedStart.width + } + } } } @@ -161,16 +193,30 @@ class MetricCalculationSpecsServiceTest { val internalReachMetricSpec = internalMetricSpec { reach = InternalMetricSpecKt.reachParams { - privacyParams = - InternalMetricSpecKt.differentialPrivacyParams { - epsilon = REACH_ONLY_REACH_EPSILON - delta = DIFFERENTIAL_PRIVACY_DELTA - } - } - vidSamplingInterval = - InternalMetricSpecKt.vidSamplingInterval { - start = REACH_ONLY_VID_SAMPLING_START - width = REACH_ONLY_VID_SAMPLING_WIDTH + multipleDataProviderParams = InternalMetricSpecKt.params { + privacyParams = + InternalMetricSpecKt.differentialPrivacyParams { + epsilon = METRIC_SPEC_CONFIG.reachParams.multipleDataProviderParams.privacyParams.epsilon + delta = METRIC_SPEC_CONFIG.reachParams.multipleDataProviderParams.privacyParams.delta + } + vidSamplingInterval = + InternalMetricSpecKt.vidSamplingInterval { + start = METRIC_SPEC_CONFIG.reachParams.multipleDataProviderParams.vidSamplingInterval.fixedStart.start + width = METRIC_SPEC_CONFIG.reachParams.multipleDataProviderParams.vidSamplingInterval.fixedStart.width + } + } + singleDataProviderParams = InternalMetricSpecKt.params { + privacyParams = + InternalMetricSpecKt.differentialPrivacyParams { + epsilon = METRIC_SPEC_CONFIG.reachParams.singleDataProviderParams.privacyParams.epsilon + delta = METRIC_SPEC_CONFIG.reachParams.singleDataProviderParams.privacyParams.delta + } + vidSamplingInterval = + InternalMetricSpecKt.vidSamplingInterval { + start = METRIC_SPEC_CONFIG.reachParams.singleDataProviderParams.vidSamplingInterval.fixedStart.start + width = METRIC_SPEC_CONFIG.reachParams.singleDataProviderParams.vidSamplingInterval.fixedStart.width + } + } } } @@ -601,7 +647,7 @@ class MetricCalculationSpecsServiceTest { } @Test - fun `createMetricCalculationSpec throws INVALID_ARGUMENT when metric_spec bad`() { + fun `createMetricCalculationSpec throws INVALID_ARGUMENT when metric_spec vid width too big`() { val request = createMetricCalculationSpecRequest { parent = MEASUREMENT_CONSUMER_NAME metricCalculationSpec = @@ -609,7 +655,11 @@ class MetricCalculationSpecsServiceTest { metricSpecs.clear() metricSpecs += REACH_METRIC_SPEC.copy { - vidSamplingInterval = MetricSpecKt.vidSamplingInterval { width = 2.0f } + reach = reach.copy { + multipleDataProviderParams = multipleDataProviderParams.copy { + vidSamplingInterval = MetricSpecKt.vidSamplingInterval { width = 2.0f } + } + } } } metricCalculationSpecId = METRIC_CALCULATION_SPEC_ID @@ -1348,42 +1398,44 @@ class MetricCalculationSpecsServiceTest { private const val METRIC_CALCULATION_SPEC_NAME = "$MEASUREMENT_CONSUMER_NAME/metricCalculationSpecs/$METRIC_CALCULATION_SPEC_ID" - private const val NUMBER_VID_BUCKETS = 300 - private const val REACH_ONLY_VID_SAMPLING_WIDTH = 3.0f / NUMBER_VID_BUCKETS - private const val REACH_ONLY_VID_SAMPLING_START = 0.0f - private const val REACH_ONLY_REACH_EPSILON = 0.0041 - - private const val REACH_FREQUENCY_VID_SAMPLING_WIDTH = 5.0f / NUMBER_VID_BUCKETS - private const val REACH_FREQUENCY_VID_SAMPLING_START = 48.0f / NUMBER_VID_BUCKETS - private const val REACH_FREQUENCY_REACH_EPSILON = 0.0033 - private const val REACH_FREQUENCY_FREQUENCY_EPSILON = 0.115 - private const val REACH_FREQUENCY_MAXIMUM_FREQUENCY = 10 - - private const val IMPRESSION_VID_SAMPLING_WIDTH = 62.0f / NUMBER_VID_BUCKETS - private const val IMPRESSION_VID_SAMPLING_START = 143.0f / NUMBER_VID_BUCKETS - private const val IMPRESSION_EPSILON = 0.0011 - private const val IMPRESSION_MAXIMUM_FREQUENCY_PER_USER = 60 - - private const val WATCH_DURATION_VID_SAMPLING_WIDTH = 95.0f / NUMBER_VID_BUCKETS - private const val WATCH_DURATION_VID_SAMPLING_START = 205.0f / NUMBER_VID_BUCKETS - private const val WATCH_DURATION_EPSILON = 0.001 - private val MAXIMUM_WATCH_DURATION_PER_USER = Durations.fromSeconds(4000) + private val SECRET_FILES_PATH: Path = run { + val runfiles = Runfiles.preload(buildMap { + put("RUNFILES_DIR", "src/main/k8s/testing/") + put("metric_spec_config.textproto", "metric_spec_config.textproto") + }).unmapped() + checkNotNull(runfiles.getRuntimePath(Paths.get("secretfiles"))) + } - private const val DIFFERENTIAL_PRIVACY_DELTA = 1e-12 + private val METRIC_SPEC_CONFIG: MetricSpecConfig = + parseTextProto(SECRET_FILES_PATH.resolve("metric_spec_config.textproto").toFile(), MetricSpecConfig.getDefaultInstance()) private val REACH_METRIC_SPEC: MetricSpec = metricSpec { reach = MetricSpecKt.reachParams { - privacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = REACH_ONLY_REACH_EPSILON - delta = DIFFERENTIAL_PRIVACY_DELTA - } - } - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = REACH_ONLY_VID_SAMPLING_START - width = REACH_ONLY_VID_SAMPLING_WIDTH + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = METRIC_SPEC_CONFIG.reachParams.multipleDataProviderParams.privacyParams.epsilon + delta = METRIC_SPEC_CONFIG.reachParams.multipleDataProviderParams.privacyParams.delta + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = METRIC_SPEC_CONFIG.reachParams.multipleDataProviderParams.vidSamplingInterval.fixedStart.start + width = METRIC_SPEC_CONFIG.reachParams.multipleDataProviderParams.vidSamplingInterval.fixedStart.width + } + } + singleDataProviderParams = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = METRIC_SPEC_CONFIG.reachParams.singleDataProviderParams.privacyParams.epsilon + delta = METRIC_SPEC_CONFIG.reachParams.singleDataProviderParams.privacyParams.delta + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = METRIC_SPEC_CONFIG.reachParams.singleDataProviderParams.vidSamplingInterval.fixedStart.start + width = METRIC_SPEC_CONFIG.reachParams.singleDataProviderParams.vidSamplingInterval.fixedStart.width + } + } } } @@ -1418,16 +1470,30 @@ class MetricCalculationSpecsServiceTest { private val INTERNAL_REACH_METRIC_SPEC: InternalMetricSpec = internalMetricSpec { reach = InternalMetricSpecKt.reachParams { - privacyParams = - InternalMetricSpecKt.differentialPrivacyParams { - epsilon = REACH_ONLY_REACH_EPSILON - delta = DIFFERENTIAL_PRIVACY_DELTA - } - } - vidSamplingInterval = - InternalMetricSpecKt.vidSamplingInterval { - start = REACH_ONLY_VID_SAMPLING_START - width = REACH_ONLY_VID_SAMPLING_WIDTH + multipleDataProviderParams = InternalMetricSpecKt.params { + privacyParams = + InternalMetricSpecKt.differentialPrivacyParams { + epsilon = METRIC_SPEC_CONFIG.reachParams.multipleDataProviderParams.privacyParams.epsilon + delta = METRIC_SPEC_CONFIG.reachParams.multipleDataProviderParams.privacyParams.delta + } + vidSamplingInterval = + InternalMetricSpecKt.vidSamplingInterval { + start = METRIC_SPEC_CONFIG.reachParams.multipleDataProviderParams.vidSamplingInterval.fixedStart.start + width = METRIC_SPEC_CONFIG.reachParams.multipleDataProviderParams.vidSamplingInterval.fixedStart.width + } + } + singleDataProviderParams = InternalMetricSpecKt.params { + privacyParams = + InternalMetricSpecKt.differentialPrivacyParams { + epsilon = METRIC_SPEC_CONFIG.reachParams.singleDataProviderParams.privacyParams.epsilon + delta = METRIC_SPEC_CONFIG.reachParams.singleDataProviderParams.privacyParams.delta + } + vidSamplingInterval = + InternalMetricSpecKt.vidSamplingInterval { + start = METRIC_SPEC_CONFIG.reachParams.singleDataProviderParams.vidSamplingInterval.fixedStart.start + width = METRIC_SPEC_CONFIG.reachParams.singleDataProviderParams.vidSamplingInterval.fixedStart.width + } + } } } @@ -1458,71 +1524,5 @@ class MetricCalculationSpecsServiceTest { private val INTERNAL_METRIC_CALCULATION_SPEC_WITH_GREATER_ID = INTERNAL_METRIC_CALCULATION_SPEC.copy { externalMetricCalculationSpecId += "2" } - - private val METRIC_SPEC_CONFIG = metricSpecConfig { - reachParams = - MetricSpecConfigKt.reachParams { - privacyParams = - MetricSpecConfigKt.differentialPrivacyParams { - epsilon = REACH_ONLY_REACH_EPSILON - delta = DIFFERENTIAL_PRIVACY_DELTA - } - } - reachVidSamplingInterval = - MetricSpecConfigKt.vidSamplingInterval { - start = REACH_ONLY_VID_SAMPLING_START - width = REACH_ONLY_VID_SAMPLING_WIDTH - } - - reachAndFrequencyParams = - MetricSpecConfigKt.reachAndFrequencyParams { - reachPrivacyParams = - MetricSpecConfigKt.differentialPrivacyParams { - epsilon = REACH_FREQUENCY_REACH_EPSILON - delta = DIFFERENTIAL_PRIVACY_DELTA - } - frequencyPrivacyParams = - MetricSpecConfigKt.differentialPrivacyParams { - epsilon = REACH_FREQUENCY_FREQUENCY_EPSILON - delta = DIFFERENTIAL_PRIVACY_DELTA - } - maximumFrequency = REACH_FREQUENCY_MAXIMUM_FREQUENCY - } - reachAndFrequencyVidSamplingInterval = - MetricSpecConfigKt.vidSamplingInterval { - start = REACH_FREQUENCY_VID_SAMPLING_START - width = REACH_FREQUENCY_VID_SAMPLING_WIDTH - } - - impressionCountParams = - MetricSpecConfigKt.impressionCountParams { - privacyParams = - MetricSpecConfigKt.differentialPrivacyParams { - epsilon = IMPRESSION_EPSILON - delta = DIFFERENTIAL_PRIVACY_DELTA - } - maximumFrequencyPerUser = IMPRESSION_MAXIMUM_FREQUENCY_PER_USER - } - impressionCountVidSamplingInterval = - MetricSpecConfigKt.vidSamplingInterval { - start = IMPRESSION_VID_SAMPLING_START - width = IMPRESSION_VID_SAMPLING_WIDTH - } - - watchDurationParams = - MetricSpecConfigKt.watchDurationParams { - privacyParams = - MetricSpecConfigKt.differentialPrivacyParams { - epsilon = WATCH_DURATION_EPSILON - delta = DIFFERENTIAL_PRIVACY_DELTA - } - maximumWatchDurationPerUser = MAXIMUM_WATCH_DURATION_PER_USER - } - watchDurationVidSamplingInterval = - MetricSpecConfigKt.vidSamplingInterval { - start = WATCH_DURATION_VID_SAMPLING_START - width = WATCH_DURATION_VID_SAMPLING_WIDTH - } - } } } diff --git a/src/test/kotlin/org/wfanet/measurement/reporting/service/api/v2alpha/MetricSpecDefaultsTest.kt b/src/test/kotlin/org/wfanet/measurement/reporting/service/api/v2alpha/MetricSpecDefaultsTest.kt index 8cd9e2f151c..936d0a230ec 100644 --- a/src/test/kotlin/org/wfanet/measurement/reporting/service/api/v2alpha/MetricSpecDefaultsTest.kt +++ b/src/test/kotlin/org/wfanet/measurement/reporting/service/api/v2alpha/MetricSpecDefaultsTest.kt @@ -18,11 +18,17 @@ package org.wfanet.measurement.reporting.service.api.v2alpha import com.google.common.truth.Truth.assertThat import com.google.protobuf.util.Durations +import kotlin.random.Random import org.junit.Assert.assertThrows import org.junit.Test import org.junit.runner.RunWith import org.junit.runners.JUnit4 +import org.mockito.kotlin.any +import org.mockito.kotlin.doReturn +import org.mockito.kotlin.mock +import org.mockito.kotlin.stub import org.wfanet.measurement.config.reporting.MetricSpecConfigKt +import org.wfanet.measurement.config.reporting.copy import org.wfanet.measurement.config.reporting.metricSpecConfig import org.wfanet.measurement.reporting.v2alpha.MetricSpec import org.wfanet.measurement.reporting.v2alpha.MetricSpecKt @@ -37,12 +43,19 @@ private const val NUMBER_VID_BUCKETS = 300 private const val REACH_ONLY_VID_SAMPLING_WIDTH = 3.0f / NUMBER_VID_BUCKETS private const val REACH_ONLY_VID_SAMPLING_START = 0.0f private const val REACH_ONLY_REACH_EPSILON = 0.0041 +private const val SINGLE_DATA_PROVIDER_REACH_ONLY_VID_SAMPLING_WIDTH = 3.1f / NUMBER_VID_BUCKETS +private const val SINGLE_DATA_PROVIDER_REACH_ONLY_VID_SAMPLING_START = 0.01f +private const val SINGLE_DATA_PROVIDER_REACH_ONLY_REACH_EPSILON = 0.0042 private const val REACH_FREQUENCY_VID_SAMPLING_WIDTH = 5.0f / NUMBER_VID_BUCKETS private const val REACH_FREQUENCY_VID_SAMPLING_START = 48.0f / NUMBER_VID_BUCKETS private const val REACH_FREQUENCY_REACH_EPSILON = 0.0033 private const val REACH_FREQUENCY_FREQUENCY_EPSILON = 0.115 private const val REACH_FREQUENCY_MAXIMUM_FREQUENCY = 10 +private const val SINGLE_DATA_PROVIDER_REACH_FREQUENCY_VID_SAMPLING_WIDTH = 5.1f / NUMBER_VID_BUCKETS +private const val SINGLE_DATA_PROVIDER_REACH_FREQUENCY_VID_SAMPLING_START = 48.1f / NUMBER_VID_BUCKETS +private const val SINGLE_DATA_PROVIDER_REACH_FREQUENCY_REACH_EPSILON = 0.0034 +private const val SINGLE_DATA_PROVIDER_REACH_FREQUENCY_FREQUENCY_EPSILON = 0.116 private const val IMPRESSION_VID_SAMPLING_WIDTH = 62.0f / NUMBER_VID_BUCKETS private const val IMPRESSION_VID_SAMPLING_START = 143.0f / NUMBER_VID_BUCKETS @@ -58,189 +71,708 @@ private const val DIFFERENTIAL_PRIVACY_DELTA = 1e-12 @RunWith(JUnit4::class) class MetricSpecDefaultsTest { + private val randomMock: Random = mock() @Test - fun `buildMetricSpec builds a reach metric spec when no field is filled`() { - val result = REACH_METRIC_SPEC.withDefaults(METRIC_SPEC_CONFIG) - val expect = metricSpec { + fun `buildMetricSpec builds a reach metric spec when no field is filled in privacy_params`() { + val result = OLD_EMPTY_REACH_METRIC_SPEC.withDefaults(METRIC_SPEC_CONFIG, randomMock) + val expected = metricSpec { reach = reachParams { - privacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = METRIC_SPEC_CONFIG.reachParams.privacyParams.epsilon - delta = METRIC_SPEC_CONFIG.reachParams.privacyParams.delta + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = MetricSpecKt.differentialPrivacyParams { + epsilon = METRIC_SPEC_CONFIG.reachParams.multipleDataProviderParams.privacyParams.epsilon + delta = METRIC_SPEC_CONFIG.reachParams.multipleDataProviderParams.privacyParams.delta + } + vidSamplingInterval = MetricSpecKt.vidSamplingInterval { + start = METRIC_SPEC_CONFIG.reachParams.multipleDataProviderParams.vidSamplingInterval.fixedStart.start + width = METRIC_SPEC_CONFIG.reachParams.multipleDataProviderParams.vidSamplingInterval.fixedStart.width } + } } - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = METRIC_SPEC_CONFIG.reachVidSamplingInterval.start - width = METRIC_SPEC_CONFIG.reachVidSamplingInterval.width + } + assertThat(result).isEqualTo(expected) + } + + @Test + fun `buildMetricSpec builds a reach metric spec when all fields are filled in privacy_params`() { + val initial = metricSpec { + reach = reachParams { + privacyParams = MetricSpecKt.differentialPrivacyParams { + epsilon = METRIC_SPEC_CONFIG.reachParams.multipleDataProviderParams.privacyParams.epsilon * 2 + delta = METRIC_SPEC_CONFIG.reachParams.multipleDataProviderParams.privacyParams.delta * 2 } + } + vidSamplingInterval = MetricSpecKt.vidSamplingInterval { + start = METRIC_SPEC_CONFIG.reachParams.multipleDataProviderParams.vidSamplingInterval.fixedStart.start + 0.001f + width = METRIC_SPEC_CONFIG.reachParams.multipleDataProviderParams.vidSamplingInterval.fixedStart.width / 2 + } + } + val result = initial.withDefaults(METRIC_SPEC_CONFIG, randomMock) + + val expected = metricSpec { + reach = reachParams { + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = MetricSpecKt.differentialPrivacyParams { + epsilon = METRIC_SPEC_CONFIG.reachParams.multipleDataProviderParams.privacyParams.epsilon * 2 + delta = METRIC_SPEC_CONFIG.reachParams.multipleDataProviderParams.privacyParams.delta * 2 + } + vidSamplingInterval = MetricSpecKt.vidSamplingInterval { + start = METRIC_SPEC_CONFIG.reachParams.multipleDataProviderParams.vidSamplingInterval.fixedStart.start + 0.001f + width = METRIC_SPEC_CONFIG.reachParams.multipleDataProviderParams.vidSamplingInterval.fixedStart.width / 2 + } + } + } } - assertThat(result).isEqualTo(expect) + + assertThat(result).isEqualTo(expected) } @Test - fun `buildMetricSpec builds a reach metric spec when all fields are filled`() { - val expect = metricSpec { + fun `buildMetricSpec builds reach spec when no field filled in multiple and single edp fields`() { + val result = NEW_EMPTY_REACH_METRIC_SPEC.withDefaults(METRIC_SPEC_CONFIG, randomMock) + val expected = metricSpec { reach = reachParams { - privacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = METRIC_SPEC_CONFIG.reachParams.privacyParams.epsilon * 2 - delta = METRIC_SPEC_CONFIG.reachParams.privacyParams.delta * 2 + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = MetricSpecKt.differentialPrivacyParams { + epsilon = METRIC_SPEC_CONFIG.reachParams.multipleDataProviderParams.privacyParams.epsilon + delta = METRIC_SPEC_CONFIG.reachParams.multipleDataProviderParams.privacyParams.delta + } + vidSamplingInterval = MetricSpecKt.vidSamplingInterval { + start = METRIC_SPEC_CONFIG.reachParams.multipleDataProviderParams.vidSamplingInterval.fixedStart.start + width = METRIC_SPEC_CONFIG.reachParams.multipleDataProviderParams.vidSamplingInterval.fixedStart.width + } + } + singleDataProviderParams = MetricSpecKt.params { + privacyParams = MetricSpecKt.differentialPrivacyParams { + epsilon = METRIC_SPEC_CONFIG.reachParams.singleDataProviderParams.privacyParams.epsilon + delta = METRIC_SPEC_CONFIG.reachParams.singleDataProviderParams.privacyParams.delta + } + vidSamplingInterval = MetricSpecKt.vidSamplingInterval { + start = METRIC_SPEC_CONFIG.reachParams.singleDataProviderParams.vidSamplingInterval.fixedStart.start + width = METRIC_SPEC_CONFIG.reachParams.singleDataProviderParams.vidSamplingInterval.fixedStart.width } + } } - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = METRIC_SPEC_CONFIG.reachVidSamplingInterval.start + 0.001f - width = METRIC_SPEC_CONFIG.reachVidSamplingInterval.width / 2 + } + assertThat(result).isEqualTo(expected) + } + + @Test + fun `buildMetricSpec builds reach spec when fields filled in multiple and single edp fields`() { + val expected = metricSpec { + reach = reachParams { + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = MetricSpecKt.differentialPrivacyParams { + epsilon = METRIC_SPEC_CONFIG.reachParams.multipleDataProviderParams.privacyParams.epsilon * 2 + delta = METRIC_SPEC_CONFIG.reachParams.multipleDataProviderParams.privacyParams.delta * 2 + } + vidSamplingInterval = MetricSpecKt.vidSamplingInterval { + start = METRIC_SPEC_CONFIG.reachParams.multipleDataProviderParams.vidSamplingInterval.fixedStart.start + 0.001f + width = METRIC_SPEC_CONFIG.reachParams.multipleDataProviderParams.vidSamplingInterval.fixedStart.width / 2 + } } + singleDataProviderParams = MetricSpecKt.params { + privacyParams = MetricSpecKt.differentialPrivacyParams { + epsilon = METRIC_SPEC_CONFIG.reachParams.singleDataProviderParams.privacyParams.epsilon * 2 + delta = METRIC_SPEC_CONFIG.reachParams.singleDataProviderParams.privacyParams.delta * 2 + } + vidSamplingInterval = MetricSpecKt.vidSamplingInterval { + start = METRIC_SPEC_CONFIG.reachParams.singleDataProviderParams.vidSamplingInterval.fixedStart.start + 0.001f + width = METRIC_SPEC_CONFIG.reachParams.singleDataProviderParams.vidSamplingInterval.fixedStart.width / 2 + } + } + } } - val result = expect.withDefaults(METRIC_SPEC_CONFIG) + val result = expected.withDefaults(METRIC_SPEC_CONFIG, randomMock) - assertThat(result).isEqualTo(expect) + assertThat(result).isEqualTo(expected) } @Test - fun `buildMetricSpec builds a reach and frequency metric spec when no field is filled`() { - val result = REACH_AND_FREQUENCY_METRIC_SPEC.withDefaults(METRIC_SPEC_CONFIG) - val expect = metricSpec { - reachAndFrequency = reachAndFrequencyParams { - reachPrivacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = METRIC_SPEC_CONFIG.reachAndFrequencyParams.reachPrivacyParams.epsilon - delta = METRIC_SPEC_CONFIG.reachAndFrequencyParams.reachPrivacyParams.delta + fun `buildMetricSpec builds reach spec when edp fields set and privacy_params also set`() { + val expected = metricSpec { + reach = reachParams { + privacyParams = MetricSpec.DifferentialPrivacyParams.getDefaultInstance() + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = MetricSpecKt.differentialPrivacyParams { + epsilon = METRIC_SPEC_CONFIG.reachParams.multipleDataProviderParams.privacyParams.epsilon * 2 + delta = METRIC_SPEC_CONFIG.reachParams.multipleDataProviderParams.privacyParams.delta * 2 } - frequencyPrivacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = METRIC_SPEC_CONFIG.reachAndFrequencyParams.frequencyPrivacyParams.epsilon - delta = METRIC_SPEC_CONFIG.reachAndFrequencyParams.frequencyPrivacyParams.delta + vidSamplingInterval = MetricSpecKt.vidSamplingInterval { + start = METRIC_SPEC_CONFIG.reachParams.multipleDataProviderParams.vidSamplingInterval.fixedStart.start + 0.001f + width = METRIC_SPEC_CONFIG.reachParams.multipleDataProviderParams.vidSamplingInterval.fixedStart.width / 2 } - maximumFrequency = METRIC_SPEC_CONFIG.reachAndFrequencyParams.maximumFrequency + } + singleDataProviderParams = MetricSpecKt.params { + privacyParams = MetricSpecKt.differentialPrivacyParams { + epsilon = METRIC_SPEC_CONFIG.reachParams.singleDataProviderParams.privacyParams.epsilon * 2 + delta = METRIC_SPEC_CONFIG.reachParams.singleDataProviderParams.privacyParams.delta * 2 + } + vidSamplingInterval = MetricSpecKt.vidSamplingInterval { + start = METRIC_SPEC_CONFIG.reachParams.singleDataProviderParams.vidSamplingInterval.fixedStart.start + 0.001f + width = METRIC_SPEC_CONFIG.reachParams.singleDataProviderParams.vidSamplingInterval.fixedStart.width / 2 + } + } } - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = METRIC_SPEC_CONFIG.reachAndFrequencyVidSamplingInterval.start - width = METRIC_SPEC_CONFIG.reachAndFrequencyVidSamplingInterval.width + } + + val result = expected.withDefaults(METRIC_SPEC_CONFIG, randomMock) + + assertThat(result).isEqualTo(expected.copy { + reach = reach.copy { + clearPrivacyParams() + } + }) + } + + @Test + fun `buildMetricSpec builds reach spec when fields not set and config has random start`() { + val chosenStart = 5000 + randomMock.stub { + on { nextInt(any()) } doReturn chosenStart + } + + val configWithRandomStart = METRIC_SPEC_CONFIG.copy { + reachParams = reachParams.copy { + singleDataProviderParams = singleDataProviderParams.copy { + vidSamplingInterval = MetricSpecConfigKt.vidSamplingInterval { + randomStart = MetricSpecConfigKt.VidSamplingIntervalKt.randomStart { + width = METRIC_SPEC_CONFIG.reachParams.singleDataProviderParams.vidSamplingInterval.fixedStart.width + } + } } + } } - assertThat(result).isEqualTo(expect) + + val result = NEW_EMPTY_REACH_METRIC_SPEC.withDefaults(configWithRandomStart, randomMock) + val expected = metricSpec { + reach = reachParams { + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = MetricSpecKt.differentialPrivacyParams { + epsilon = METRIC_SPEC_CONFIG.reachParams.multipleDataProviderParams.privacyParams.epsilon + delta = METRIC_SPEC_CONFIG.reachParams.multipleDataProviderParams.privacyParams.delta + } + vidSamplingInterval = MetricSpecKt.vidSamplingInterval { + start = METRIC_SPEC_CONFIG.reachParams.multipleDataProviderParams.vidSamplingInterval.fixedStart.start + width = METRIC_SPEC_CONFIG.reachParams.multipleDataProviderParams.vidSamplingInterval.fixedStart.width + } + } + singleDataProviderParams = MetricSpecKt.params { + privacyParams = MetricSpecKt.differentialPrivacyParams { + epsilon = METRIC_SPEC_CONFIG.reachParams.singleDataProviderParams.privacyParams.epsilon + delta = METRIC_SPEC_CONFIG.reachParams.singleDataProviderParams.privacyParams.delta + } + vidSamplingInterval = MetricSpecKt.vidSamplingInterval { + start = chosenStart.toFloat() / 10000 + width = METRIC_SPEC_CONFIG.reachParams.singleDataProviderParams.vidSamplingInterval.fixedStart.width + } + } + } + } + assertThat(result).isEqualTo(expected) } @Test - fun `buildMetricSpec builds a reach and frequency metric spec when all fields are filled`() { - val expect = metricSpec { + fun `buildMetricSpec builds rf spec when no field filled in reach and freq privacy_params`() { + val result = OLD_EMPTY_REACH_AND_FREQUENCY_METRIC_SPEC.withDefaults(METRIC_SPEC_CONFIG, randomMock) + val expected = metricSpec { + reachAndFrequency = reachAndFrequencyParams { + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = METRIC_SPEC_CONFIG.reachAndFrequencyParams.multipleDataProviderParams.privacyParams.epsilon + delta = METRIC_SPEC_CONFIG.reachAndFrequencyParams.multipleDataProviderParams.privacyParams.delta + } + frequencyPrivacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = METRIC_SPEC_CONFIG.reachAndFrequencyParams.multipleDataProviderParams.frequencyPrivacyParams.epsilon + delta = METRIC_SPEC_CONFIG.reachAndFrequencyParams.multipleDataProviderParams.frequencyPrivacyParams.delta + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = METRIC_SPEC_CONFIG.reachAndFrequencyParams.multipleDataProviderParams.vidSamplingInterval.fixedStart.start + width = METRIC_SPEC_CONFIG.reachAndFrequencyParams.multipleDataProviderParams.vidSamplingInterval.fixedStart.width + } + } + maximumFrequency = METRIC_SPEC_CONFIG.reachAndFrequencyParams.maximumFrequency + } + } + assertThat(result).isEqualTo(expected) + } + + @Test + fun `buildMetricSpec builds rf spec when fields filled in reach and freq privacy_params`() { + val initial = metricSpec { reachAndFrequency = reachAndFrequencyParams { reachPrivacyParams = MetricSpecKt.differentialPrivacyParams { - epsilon = METRIC_SPEC_CONFIG.reachAndFrequencyParams.reachPrivacyParams.epsilon * 2 - delta = METRIC_SPEC_CONFIG.reachAndFrequencyParams.reachPrivacyParams.delta * 2 + epsilon = METRIC_SPEC_CONFIG.reachAndFrequencyParams.multipleDataProviderParams.privacyParams.epsilon * 2 + delta = METRIC_SPEC_CONFIG.reachAndFrequencyParams.multipleDataProviderParams.privacyParams.delta * 2 } frequencyPrivacyParams = MetricSpecKt.differentialPrivacyParams { - epsilon = METRIC_SPEC_CONFIG.reachAndFrequencyParams.frequencyPrivacyParams.epsilon * 2 - delta = METRIC_SPEC_CONFIG.reachAndFrequencyParams.frequencyPrivacyParams.delta * 2 + epsilon = METRIC_SPEC_CONFIG.reachAndFrequencyParams.multipleDataProviderParams.frequencyPrivacyParams.epsilon * 2 + delta = METRIC_SPEC_CONFIG.reachAndFrequencyParams.multipleDataProviderParams.frequencyPrivacyParams.delta * 2 } maximumFrequency = METRIC_SPEC_CONFIG.reachAndFrequencyParams.maximumFrequency * 2 } vidSamplingInterval = MetricSpecKt.vidSamplingInterval { - start = METRIC_SPEC_CONFIG.reachAndFrequencyVidSamplingInterval.start + 0.001f - width = METRIC_SPEC_CONFIG.reachAndFrequencyVidSamplingInterval.width / 2 + start = METRIC_SPEC_CONFIG.reachAndFrequencyParams.multipleDataProviderParams.vidSamplingInterval.fixedStart.start + 0.001f + width = METRIC_SPEC_CONFIG.reachAndFrequencyParams.multipleDataProviderParams.vidSamplingInterval.fixedStart.width / 2 + } + } + val result = initial.withDefaults(METRIC_SPEC_CONFIG, randomMock) + val expected = metricSpec { + reachAndFrequency = reachAndFrequencyParams { + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = METRIC_SPEC_CONFIG.reachAndFrequencyParams.multipleDataProviderParams.privacyParams.epsilon * 2 + delta = METRIC_SPEC_CONFIG.reachAndFrequencyParams.multipleDataProviderParams.privacyParams.delta * 2 + } + frequencyPrivacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = METRIC_SPEC_CONFIG.reachAndFrequencyParams.multipleDataProviderParams.frequencyPrivacyParams.epsilon * 2 + delta = METRIC_SPEC_CONFIG.reachAndFrequencyParams.multipleDataProviderParams.frequencyPrivacyParams.delta * 2 + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = METRIC_SPEC_CONFIG.reachAndFrequencyParams.multipleDataProviderParams.vidSamplingInterval.fixedStart.start + 0.001f + width = METRIC_SPEC_CONFIG.reachAndFrequencyParams.multipleDataProviderParams.vidSamplingInterval.fixedStart.width / 2 + } + } + maximumFrequency = METRIC_SPEC_CONFIG.reachAndFrequencyParams.maximumFrequency * 2 + } + } + assertThat(result).isEqualTo(expected) + } + + @Test + fun `buildMetricSpec builds rf spec when no fields filled in multiple and single edp fields`() { + val result = NEW_EMPTY_REACH_AND_FREQUENCY_METRIC_SPEC.withDefaults(METRIC_SPEC_CONFIG, randomMock) + val expected = metricSpec { + reachAndFrequency = reachAndFrequencyParams { + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = METRIC_SPEC_CONFIG.reachAndFrequencyParams.multipleDataProviderParams.privacyParams.epsilon + delta = METRIC_SPEC_CONFIG.reachAndFrequencyParams.multipleDataProviderParams.privacyParams.delta + } + frequencyPrivacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = METRIC_SPEC_CONFIG.reachAndFrequencyParams.multipleDataProviderParams.frequencyPrivacyParams.epsilon + delta = METRIC_SPEC_CONFIG.reachAndFrequencyParams.multipleDataProviderParams.frequencyPrivacyParams.delta + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = METRIC_SPEC_CONFIG.reachAndFrequencyParams.multipleDataProviderParams.vidSamplingInterval.fixedStart.start + width = METRIC_SPEC_CONFIG.reachAndFrequencyParams.multipleDataProviderParams.vidSamplingInterval.fixedStart.width + } } + singleDataProviderParams = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = METRIC_SPEC_CONFIG.reachAndFrequencyParams.singleDataProviderParams.privacyParams.epsilon + delta = METRIC_SPEC_CONFIG.reachAndFrequencyParams.singleDataProviderParams.privacyParams.delta + } + frequencyPrivacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = METRIC_SPEC_CONFIG.reachAndFrequencyParams.singleDataProviderParams.frequencyPrivacyParams.epsilon + delta = METRIC_SPEC_CONFIG.reachAndFrequencyParams.singleDataProviderParams.frequencyPrivacyParams.delta + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = METRIC_SPEC_CONFIG.reachAndFrequencyParams.singleDataProviderParams.vidSamplingInterval.fixedStart.start + width = METRIC_SPEC_CONFIG.reachAndFrequencyParams.singleDataProviderParams.vidSamplingInterval.fixedStart.width + } + } + maximumFrequency = METRIC_SPEC_CONFIG.reachAndFrequencyParams.maximumFrequency + } } - val result = expect.withDefaults(METRIC_SPEC_CONFIG) - assertThat(result).isEqualTo(expect) + assertThat(result).isEqualTo(expected) } @Test - fun `buildMetricSpec builds an impression count metric spec when no field is filled`() { - val result = IMPRESSION_COUNT_METRIC_SPEC.withDefaults(METRIC_SPEC_CONFIG) - val expect = metricSpec { + fun `buildMetricSpec builds rf spec when fields filled in multiple and single edp fields`() { + val expected = metricSpec { + reachAndFrequency = reachAndFrequencyParams { + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = METRIC_SPEC_CONFIG.reachAndFrequencyParams.multipleDataProviderParams.privacyParams.epsilon * 2 + delta = METRIC_SPEC_CONFIG.reachAndFrequencyParams.multipleDataProviderParams.privacyParams.delta * 2 + } + frequencyPrivacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = METRIC_SPEC_CONFIG.reachAndFrequencyParams.multipleDataProviderParams.frequencyPrivacyParams.epsilon * 2 + delta = METRIC_SPEC_CONFIG.reachAndFrequencyParams.multipleDataProviderParams.frequencyPrivacyParams.delta * 2 + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = METRIC_SPEC_CONFIG.reachAndFrequencyParams.multipleDataProviderParams.vidSamplingInterval.fixedStart.start + 0.001f + width = METRIC_SPEC_CONFIG.reachAndFrequencyParams.multipleDataProviderParams.vidSamplingInterval.fixedStart.width / 2 + } + } + singleDataProviderParams = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = METRIC_SPEC_CONFIG.reachAndFrequencyParams.singleDataProviderParams.privacyParams.epsilon * 2 + delta = METRIC_SPEC_CONFIG.reachAndFrequencyParams.singleDataProviderParams.privacyParams.delta * 2 + } + frequencyPrivacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = METRIC_SPEC_CONFIG.reachAndFrequencyParams.singleDataProviderParams.frequencyPrivacyParams.epsilon * 2 + delta = METRIC_SPEC_CONFIG.reachAndFrequencyParams.singleDataProviderParams.frequencyPrivacyParams.delta * 2 + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = METRIC_SPEC_CONFIG.reachAndFrequencyParams.singleDataProviderParams.vidSamplingInterval.fixedStart.start + 0.001f + width = METRIC_SPEC_CONFIG.reachAndFrequencyParams.singleDataProviderParams.vidSamplingInterval.fixedStart.width / 2 + } + } + maximumFrequency = METRIC_SPEC_CONFIG.reachAndFrequencyParams.maximumFrequency + } + } + val result = expected.withDefaults(METRIC_SPEC_CONFIG, randomMock) + assertThat(result).isEqualTo(expected) + } + + @Test + fun `buildMetricSpec builds rf spec when edp fields set and reach_privacy_params also set`() { + val expected = metricSpec { + reachAndFrequency = reachAndFrequencyParams { + reachPrivacyParams = MetricSpec.DifferentialPrivacyParams.getDefaultInstance() + frequencyPrivacyParams = MetricSpec.DifferentialPrivacyParams.getDefaultInstance() + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = METRIC_SPEC_CONFIG.reachAndFrequencyParams.multipleDataProviderParams.privacyParams.epsilon * 2 + delta = METRIC_SPEC_CONFIG.reachAndFrequencyParams.multipleDataProviderParams.privacyParams.delta * 2 + } + frequencyPrivacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = METRIC_SPEC_CONFIG.reachAndFrequencyParams.multipleDataProviderParams.frequencyPrivacyParams.epsilon * 2 + delta = METRIC_SPEC_CONFIG.reachAndFrequencyParams.multipleDataProviderParams.frequencyPrivacyParams.delta * 2 + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = METRIC_SPEC_CONFIG.reachAndFrequencyParams.multipleDataProviderParams.vidSamplingInterval.fixedStart.start + 0.001f + width = METRIC_SPEC_CONFIG.reachAndFrequencyParams.multipleDataProviderParams.vidSamplingInterval.fixedStart.width / 2 + } + } + singleDataProviderParams = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = METRIC_SPEC_CONFIG.reachAndFrequencyParams.singleDataProviderParams.privacyParams.epsilon * 2 + delta = METRIC_SPEC_CONFIG.reachAndFrequencyParams.singleDataProviderParams.privacyParams.delta * 2 + } + frequencyPrivacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = METRIC_SPEC_CONFIG.reachAndFrequencyParams.singleDataProviderParams.frequencyPrivacyParams.epsilon * 2 + delta = METRIC_SPEC_CONFIG.reachAndFrequencyParams.singleDataProviderParams.frequencyPrivacyParams.delta * 2 + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = METRIC_SPEC_CONFIG.reachAndFrequencyParams.singleDataProviderParams.vidSamplingInterval.fixedStart.start + 0.001f + width = METRIC_SPEC_CONFIG.reachAndFrequencyParams.singleDataProviderParams.vidSamplingInterval.fixedStart.width / 2 + } + } + maximumFrequency = METRIC_SPEC_CONFIG.reachAndFrequencyParams.maximumFrequency + } + } + val result = expected.withDefaults(METRIC_SPEC_CONFIG, randomMock) + assertThat(result).isEqualTo(expected.copy { + reachAndFrequency = reachAndFrequency.copy { + clearReachPrivacyParams() + clearFrequencyPrivacyParams() + } + }) + } + + @Test + fun `buildMetricSpec builds impression count spec when no field filled in privacy_params`() { + val result = OLD_EMPTY_IMPRESSION_COUNT_METRIC_SPEC.withDefaults(METRIC_SPEC_CONFIG, randomMock) + val expected = metricSpec { impressionCount = impressionCountParams { - privacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = METRIC_SPEC_CONFIG.impressionCountParams.privacyParams.epsilon - delta = METRIC_SPEC_CONFIG.impressionCountParams.privacyParams.delta - } + params = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = METRIC_SPEC_CONFIG.impressionCountParams.params.privacyParams.epsilon + delta = METRIC_SPEC_CONFIG.impressionCountParams.params.privacyParams.delta + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = METRIC_SPEC_CONFIG.impressionCountParams.params.vidSamplingInterval.fixedStart.start + width = METRIC_SPEC_CONFIG.impressionCountParams.params.vidSamplingInterval.fixedStart.width + } + } maximumFrequencyPerUser = METRIC_SPEC_CONFIG.impressionCountParams.maximumFrequencyPerUser } - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = METRIC_SPEC_CONFIG.impressionCountVidSamplingInterval.start - width = METRIC_SPEC_CONFIG.impressionCountVidSamplingInterval.width - } } - assertThat(result).isEqualTo(expect) + assertThat(result).isEqualTo(expected) } @Test - fun `buildMetricSpec builds an impression count metric spec when all fields are filled`() { - val expect = metricSpec { + fun `buildMetricSpec builds impression count spec when all fields filled in privacy_params`() { + val initial = metricSpec { impressionCount = impressionCountParams { privacyParams = MetricSpecKt.differentialPrivacyParams { - epsilon = METRIC_SPEC_CONFIG.impressionCountParams.privacyParams.epsilon * 2 - delta = METRIC_SPEC_CONFIG.impressionCountParams.privacyParams.delta * 2 + epsilon = METRIC_SPEC_CONFIG.impressionCountParams.params.privacyParams.epsilon * 2 + delta = METRIC_SPEC_CONFIG.impressionCountParams.params.privacyParams.delta * 2 } maximumFrequencyPerUser = METRIC_SPEC_CONFIG.impressionCountParams.maximumFrequencyPerUser * 2 } vidSamplingInterval = MetricSpecKt.vidSamplingInterval { - start = METRIC_SPEC_CONFIG.impressionCountVidSamplingInterval.start + 0.001f - width = METRIC_SPEC_CONFIG.impressionCountVidSamplingInterval.width / 2 + start = METRIC_SPEC_CONFIG.impressionCountParams.params.vidSamplingInterval.fixedStart.start + 0.001f + width = METRIC_SPEC_CONFIG.impressionCountParams.params.vidSamplingInterval.fixedStart.width / 2 + } + } + val result = initial.withDefaults(METRIC_SPEC_CONFIG, randomMock) + val expected = metricSpec { + impressionCount = impressionCountParams { + params = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = METRIC_SPEC_CONFIG.impressionCountParams.params.privacyParams.epsilon * 2 + delta = METRIC_SPEC_CONFIG.impressionCountParams.params.privacyParams.delta * 2 + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = METRIC_SPEC_CONFIG.impressionCountParams.params.vidSamplingInterval.fixedStart.start + 0.001f + width = METRIC_SPEC_CONFIG.impressionCountParams.params.vidSamplingInterval.fixedStart.width / 2 + } + } + maximumFrequencyPerUser = METRIC_SPEC_CONFIG.impressionCountParams.maximumFrequencyPerUser * 2 + } + } + assertThat(result).isEqualTo(expected) + } + + @Test + fun `buildMetricSpec builds impression count spec when no field filled in params`() { + val result = NEW_EMPTY_IMPRESSION_COUNT_METRIC_SPEC.withDefaults(METRIC_SPEC_CONFIG, randomMock) + val expected = metricSpec { + impressionCount = impressionCountParams { + params = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = METRIC_SPEC_CONFIG.impressionCountParams.params.privacyParams.epsilon + delta = METRIC_SPEC_CONFIG.impressionCountParams.params.privacyParams.delta + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = METRIC_SPEC_CONFIG.impressionCountParams.params.vidSamplingInterval.fixedStart.start + width = METRIC_SPEC_CONFIG.impressionCountParams.params.vidSamplingInterval.fixedStart.width + } + } + maximumFrequencyPerUser = METRIC_SPEC_CONFIG.impressionCountParams.maximumFrequencyPerUser + } + } + assertThat(result).isEqualTo(expected) + } + + @Test + fun `buildMetricSpec builds impression count spec when all fields filled in params`() { + val expected = metricSpec { + impressionCount = impressionCountParams { + params = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = METRIC_SPEC_CONFIG.impressionCountParams.params.privacyParams.epsilon * 2 + delta = METRIC_SPEC_CONFIG.impressionCountParams.params.privacyParams.delta * 2 + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = METRIC_SPEC_CONFIG.impressionCountParams.params.vidSamplingInterval.fixedStart.start + 0.001f + width = METRIC_SPEC_CONFIG.impressionCountParams.params.vidSamplingInterval.fixedStart.width / 2 + } } + maximumFrequencyPerUser = METRIC_SPEC_CONFIG.impressionCountParams.maximumFrequencyPerUser * 2 + } } - val result = expect.withDefaults(METRIC_SPEC_CONFIG) - assertThat(result).isEqualTo(expect) + val result = expected.withDefaults(METRIC_SPEC_CONFIG, randomMock) + assertThat(result).isEqualTo(expected) } @Test - fun `buildMetricSpec builds a watch duration metric spec when no field is filled`() { - val result = WATCH_DURATION_METRIC_SPEC.withDefaults(METRIC_SPEC_CONFIG) - val expect = metricSpec { + fun `buildMetricSpec builds impression count spec when params and privacy_params set`() { + val expected = metricSpec { + impressionCount = impressionCountParams { + privacyParams = MetricSpec.DifferentialPrivacyParams.getDefaultInstance() + params = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = METRIC_SPEC_CONFIG.impressionCountParams.params.privacyParams.epsilon * 2 + delta = METRIC_SPEC_CONFIG.impressionCountParams.params.privacyParams.delta * 2 + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = METRIC_SPEC_CONFIG.impressionCountParams.params.vidSamplingInterval.fixedStart.start + 0.001f + width = METRIC_SPEC_CONFIG.impressionCountParams.params.vidSamplingInterval.fixedStart.width / 2 + } + } + maximumFrequencyPerUser = METRIC_SPEC_CONFIG.impressionCountParams.maximumFrequencyPerUser * 2 + } + } + val result = expected.withDefaults(METRIC_SPEC_CONFIG, randomMock) + assertThat(result).isEqualTo(expected.copy { + impressionCount = impressionCount.copy { + clearPrivacyParams() + } + }) + } + + @Test + fun `buildMetricSpec builds watch duration metric spec when no field filled in privacy_params`() { + val result = OLD_EMPTY_WATCH_DURATION_METRIC_SPEC.withDefaults(METRIC_SPEC_CONFIG, randomMock) + val expected = metricSpec { watchDuration = watchDurationParams { - privacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = METRIC_SPEC_CONFIG.watchDurationParams.privacyParams.epsilon - delta = METRIC_SPEC_CONFIG.watchDurationParams.privacyParams.delta - } + params = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = METRIC_SPEC_CONFIG.watchDurationParams.params.privacyParams.epsilon + delta = METRIC_SPEC_CONFIG.watchDurationParams.params.privacyParams.delta + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = METRIC_SPEC_CONFIG.watchDurationParams.params.vidSamplingInterval.fixedStart.start + width = METRIC_SPEC_CONFIG.watchDurationParams.params.vidSamplingInterval.fixedStart.width + } + } maximumWatchDurationPerUser = METRIC_SPEC_CONFIG.watchDurationParams.maximumWatchDurationPerUser } - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = METRIC_SPEC_CONFIG.watchDurationVidSamplingInterval.start - width = METRIC_SPEC_CONFIG.watchDurationVidSamplingInterval.width - } } - assertThat(result).isEqualTo(expect) + assertThat(result).isEqualTo(expected) } @Test - fun `buildMetricSpec builds a watch duration metric spec when all fields are filled`() { - val expect = metricSpec { + fun `buildMetricSpec builds watch duration metric spec when fields filled in privacy_params`() { + val initial = metricSpec { watchDuration = watchDurationParams { privacyParams = MetricSpecKt.differentialPrivacyParams { - epsilon = METRIC_SPEC_CONFIG.watchDurationParams.privacyParams.epsilon * 2 - delta = METRIC_SPEC_CONFIG.watchDurationParams.privacyParams.delta * 2 + epsilon = METRIC_SPEC_CONFIG.watchDurationParams.params.privacyParams.epsilon * 2 + delta = METRIC_SPEC_CONFIG.watchDurationParams.params.privacyParams.delta * 2 } + maximumWatchDurationPerUser = Durations.add( + METRIC_SPEC_CONFIG.watchDurationParams.maximumWatchDurationPerUser, + METRIC_SPEC_CONFIG.watchDurationParams.maximumWatchDurationPerUser) + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = METRIC_SPEC_CONFIG.watchDurationParams.params.vidSamplingInterval.fixedStart.start + 0.001f + width = METRIC_SPEC_CONFIG.watchDurationParams.params.vidSamplingInterval.fixedStart.width / 2 + } + } + val result = initial.withDefaults(METRIC_SPEC_CONFIG, randomMock) + val expected = metricSpec { + watchDuration = watchDurationParams { + params = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = METRIC_SPEC_CONFIG.watchDurationParams.params.privacyParams.epsilon * 2 + delta = METRIC_SPEC_CONFIG.watchDurationParams.params.privacyParams.delta * 2 + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = METRIC_SPEC_CONFIG.watchDurationParams.params.vidSamplingInterval.fixedStart.start + 0.001f + width = METRIC_SPEC_CONFIG.watchDurationParams.params.vidSamplingInterval.fixedStart.width / 2 + } + } maximumWatchDurationPerUser = Durations.add( METRIC_SPEC_CONFIG.watchDurationParams.maximumWatchDurationPerUser, + METRIC_SPEC_CONFIG.watchDurationParams.maximumWatchDurationPerUser) + } + } + assertThat(result).isEqualTo(expected) + } + + @Test + fun `buildMetricSpec builds watch duration metric spec when no field filled in params`() { + val result = NEW_EMPTY_WATCH_DURATION_METRIC_SPEC.withDefaults(METRIC_SPEC_CONFIG, randomMock) + val expected = metricSpec { + watchDuration = watchDurationParams { + params = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = METRIC_SPEC_CONFIG.watchDurationParams.params.privacyParams.epsilon + delta = METRIC_SPEC_CONFIG.watchDurationParams.params.privacyParams.delta + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = METRIC_SPEC_CONFIG.watchDurationParams.params.vidSamplingInterval.fixedStart.start + width = METRIC_SPEC_CONFIG.watchDurationParams.params.vidSamplingInterval.fixedStart.width + } + } + maximumWatchDurationPerUser = + METRIC_SPEC_CONFIG.watchDurationParams.maximumWatchDurationPerUser + } + } + assertThat(result).isEqualTo(expected) + } + + @Test + fun `buildMetricSpec builds watch duration metric spec when fields filled in params`() { + val expected = metricSpec { + watchDuration = watchDurationParams { + params = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = METRIC_SPEC_CONFIG.watchDurationParams.params.privacyParams.epsilon * 2 + delta = METRIC_SPEC_CONFIG.watchDurationParams.params.privacyParams.delta * 2 + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = METRIC_SPEC_CONFIG.watchDurationParams.params.vidSamplingInterval.fixedStart.start + 0.001f + width = METRIC_SPEC_CONFIG.watchDurationParams.params.vidSamplingInterval.fixedStart.width / 2 + } + } + maximumWatchDurationPerUser = + Durations.add( METRIC_SPEC_CONFIG.watchDurationParams.maximumWatchDurationPerUser, - ) + METRIC_SPEC_CONFIG.watchDurationParams.maximumWatchDurationPerUser) } - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = METRIC_SPEC_CONFIG.watchDurationVidSamplingInterval.start + 0.001f - width = METRIC_SPEC_CONFIG.watchDurationVidSamplingInterval.width / 2 + } + val result = expected.withDefaults(METRIC_SPEC_CONFIG, randomMock) + assertThat(result).isEqualTo(expected) + } + + @Test + fun `buildMetricSpec builds watch duration metric spec when params and privacy_params set`() { + val expected = metricSpec { + watchDuration = watchDurationParams { + privacyParams = MetricSpec.DifferentialPrivacyParams.getDefaultInstance() + params = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = METRIC_SPEC_CONFIG.watchDurationParams.params.privacyParams.epsilon * 2 + delta = METRIC_SPEC_CONFIG.watchDurationParams.params.privacyParams.delta * 2 + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = METRIC_SPEC_CONFIG.watchDurationParams.params.vidSamplingInterval.fixedStart.start + 0.001f + width = METRIC_SPEC_CONFIG.watchDurationParams.params.vidSamplingInterval.fixedStart.width / 2 + } } + maximumWatchDurationPerUser = + Durations.add( + METRIC_SPEC_CONFIG.watchDurationParams.maximumWatchDurationPerUser, + METRIC_SPEC_CONFIG.watchDurationParams.maximumWatchDurationPerUser) + } } - val result = expect.withDefaults(METRIC_SPEC_CONFIG) - assertThat(result).isEqualTo(expect) + val result = expected.withDefaults(METRIC_SPEC_CONFIG, randomMock) + assertThat(result).isEqualTo(expected.copy { + watchDuration = watchDuration.copy { + clearPrivacyParams() + } + }) } @Test @@ -249,21 +781,51 @@ class MetricSpecDefaultsTest { val exception = assertThrows(MetricSpecDefaultsException::class.java) { - metricSpecWithoutType.withDefaults(METRIC_SPEC_CONFIG) + metricSpecWithoutType.withDefaults(METRIC_SPEC_CONFIG, randomMock) } assertThat(exception).hasCauseThat().isInstanceOf(IllegalArgumentException::class.java) assertThat(exception).hasMessageThat().contains("metric spec type") } @Test - fun `buildMetricSpec throw MetricSpecBuildingException when reach privacy params is not set`() { + fun `buildMetricSpec throw MetricSpecBuildingException when no reach params set set`() { val metricSpecWithoutPrivacyParams = metricSpec { reach = reachParams {} } val exception = assertThrows(MetricSpecDefaultsException::class.java) { - metricSpecWithoutPrivacyParams.withDefaults(METRIC_SPEC_CONFIG) + metricSpecWithoutPrivacyParams.withDefaults(METRIC_SPEC_CONFIG, randomMock) + } + assertThat(exception).hasMessageThat().contains("privacy_params") + assertThat(exception).hasCauseThat().isInstanceOf(IllegalArgumentException::class.java) + assertThat(exception.cause).hasMessageThat().contains("reach") + } + + @Test + fun `buildMetricSpec throw MetricSpecBuildingException when single reach params not set`() { + val metricSpec = metricSpec { reach = reachParams { + multipleDataProviderParams = MetricSpec.Params.getDefaultInstance() + } } + + val exception = + assertThrows(MetricSpecDefaultsException::class.java) { + metricSpec.withDefaults(METRIC_SPEC_CONFIG, randomMock) } - assertThat(exception).hasMessageThat().contains("privacy params") + assertThat(exception).hasMessageThat().contains("privacy_params") + assertThat(exception).hasCauseThat().isInstanceOf(IllegalArgumentException::class.java) + assertThat(exception.cause).hasMessageThat().contains("reach") + } + + @Test + fun `buildMetricSpec throw MetricSpecBuildingException when multiple reach params not set`() { + val metricSpec = metricSpec { reach = reachParams { + singleDataProviderParams = MetricSpec.Params.getDefaultInstance() + } } + + val exception = + assertThrows(MetricSpecDefaultsException::class.java) { + metricSpec.withDefaults(METRIC_SPEC_CONFIG, randomMock) + } + assertThat(exception).hasMessageThat().contains("privacy_params") assertThat(exception).hasCauseThat().isInstanceOf(IllegalArgumentException::class.java) assertThat(exception.cause).hasMessageThat().contains("reach") } @@ -271,71 +833,101 @@ class MetricSpecDefaultsTest { @Test fun `buildMetricSpec throw MetricSpecBuildingException when reach privacy params in reach and frequency is not set`() { val metricSpecWithoutPrivacyParams = - REACH_AND_FREQUENCY_METRIC_SPEC.copy { + OLD_EMPTY_REACH_AND_FREQUENCY_METRIC_SPEC.copy { reachAndFrequency = reachAndFrequency.copy { clearReachPrivacyParams() } } val exception = assertThrows(MetricSpecDefaultsException::class.java) { - metricSpecWithoutPrivacyParams.withDefaults(METRIC_SPEC_CONFIG) + metricSpecWithoutPrivacyParams.withDefaults(METRIC_SPEC_CONFIG, randomMock) } - assertThat(exception).hasMessageThat().contains("privacy params") + assertThat(exception).hasMessageThat().contains("privacy_params") assertThat(exception).hasCauseThat().isInstanceOf(IllegalArgumentException::class.java) - assertThat(exception.cause).hasMessageThat().contains("reachPrivacyParams") + assertThat(exception.cause).hasMessageThat().contains("frequency") } @Test fun `buildMetricSpec throw MetricSpecBuildingException when frequency privacy params in reach and frequency is not set`() { val metricSpecWithoutPrivacyParams = - REACH_AND_FREQUENCY_METRIC_SPEC.copy { + OLD_EMPTY_REACH_AND_FREQUENCY_METRIC_SPEC.copy { reachAndFrequency = reachAndFrequency.copy { clearFrequencyPrivacyParams() } } val exception = assertThrows(MetricSpecDefaultsException::class.java) { - metricSpecWithoutPrivacyParams.withDefaults(METRIC_SPEC_CONFIG) + metricSpecWithoutPrivacyParams.withDefaults(METRIC_SPEC_CONFIG, randomMock) } - assertThat(exception).hasMessageThat().contains("privacy params") + assertThat(exception).hasMessageThat().contains("privacy_params") assertThat(exception).hasCauseThat().isInstanceOf(IllegalArgumentException::class.java) - assertThat(exception.cause).hasMessageThat().contains("frequencyPrivacyParams") + assertThat(exception.cause).hasMessageThat().contains("frequency") } @Test - fun `buildMetricSpec throw MetricSpecBuildingException when impression privacy params is not set`() { + fun `buildMetricSpec throw MetricSpecBuildingException when single rf params not set`() { + val metricSpec = metricSpec { reachAndFrequency = reachAndFrequencyParams { + multipleDataProviderParams = MetricSpec.Params.getDefaultInstance() + } } + + val exception = + assertThrows(MetricSpecDefaultsException::class.java) { + metricSpec.withDefaults(METRIC_SPEC_CONFIG, randomMock) + } + assertThat(exception).hasMessageThat().contains("privacy_params") + assertThat(exception).hasCauseThat().isInstanceOf(IllegalArgumentException::class.java) + assertThat(exception.cause).hasMessageThat().contains("frequency") + } + + @Test + fun `buildMetricSpec throw MetricSpecBuildingException when multiple rf params not set`() { + val metricSpec = metricSpec { reachAndFrequency = reachAndFrequencyParams { + singleDataProviderParams = MetricSpec.Params.getDefaultInstance() + } } + + val exception = + assertThrows(MetricSpecDefaultsException::class.java) { + metricSpec.withDefaults(METRIC_SPEC_CONFIG, randomMock) + } + assertThat(exception).hasMessageThat().contains("privacy_params") + assertThat(exception).hasCauseThat().isInstanceOf(IllegalArgumentException::class.java) + assertThat(exception.cause).hasMessageThat().contains("frequency") + } + + @Test + fun `buildMetricSpec throw MetricSpecBuildingException when impression params not set`() { val metricSpecWithoutPrivacyParams = metricSpec { impressionCount = impressionCountParams {} } val exception = assertThrows(MetricSpecDefaultsException::class.java) { - metricSpecWithoutPrivacyParams.withDefaults(METRIC_SPEC_CONFIG) + metricSpecWithoutPrivacyParams.withDefaults(METRIC_SPEC_CONFIG, randomMock) } - assertThat(exception).hasMessageThat().contains("privacy params") + assertThat(exception).hasMessageThat().contains("privacy_params") assertThat(exception).hasCauseThat().isInstanceOf(IllegalArgumentException::class.java) - assertThat(exception.cause).hasMessageThat().contains("impression count") + assertThat(exception.cause).hasMessageThat().contains("impression") } @Test - fun `buildMetricSpec throw MetricSpecBuildingException when watch duration privacy params is not set`() { + fun `buildMetricSpec throw MetricSpecBuildingException when watch duration params not set`() { val metricSpecWithoutPrivacyParams = metricSpec { watchDuration = watchDurationParams {} } val exception = assertThrows(MetricSpecDefaultsException::class.java) { - metricSpecWithoutPrivacyParams.withDefaults(METRIC_SPEC_CONFIG) + metricSpecWithoutPrivacyParams.withDefaults(METRIC_SPEC_CONFIG, randomMock) } - assertThat(exception).hasMessageThat().contains("privacy params") + assertThat(exception).hasMessageThat().contains("privacy_params") assertThat(exception).hasCauseThat().isInstanceOf(IllegalArgumentException::class.java) - assertThat(exception.cause).hasMessageThat().contains("watch duration") + assertThat(exception.cause).hasMessageThat().contains("duration") } @Test fun `buildMetricSpec throw MetricSpecBuildingException when vidSamplingInterval start is less than 0`() { val metricSpec = - REACH_METRIC_SPEC.copy { + OLD_EMPTY_REACH_METRIC_SPEC.copy { vidSamplingInterval = MetricSpecKt.vidSamplingInterval { start = -1.0f } } val exception = assertThrows(MetricSpecDefaultsException::class.java) { - metricSpec.withDefaults(METRIC_SPEC_CONFIG) + metricSpec.withDefaults(METRIC_SPEC_CONFIG, randomMock) } assertThat(exception).hasMessageThat().contains("vidSamplingInterval") assertThat(exception).hasCauseThat().isInstanceOf(IllegalArgumentException::class.java) @@ -345,13 +937,13 @@ class MetricSpecDefaultsTest { @Test fun `buildMetricSpec throw MetricSpecBuildingException when vidSamplingInterval start is not less than 1`() { val metricSpec = - REACH_METRIC_SPEC.copy { + OLD_EMPTY_REACH_METRIC_SPEC.copy { vidSamplingInterval = MetricSpecKt.vidSamplingInterval { start = 1.0f } } val exception = assertThrows(MetricSpecDefaultsException::class.java) { - metricSpec.withDefaults(METRIC_SPEC_CONFIG) + metricSpec.withDefaults(METRIC_SPEC_CONFIG, randomMock) } assertThat(exception).hasMessageThat().contains("vidSamplingInterval") assertThat(exception).hasCauseThat().isInstanceOf(IllegalArgumentException::class.java) @@ -361,13 +953,13 @@ class MetricSpecDefaultsTest { @Test fun `buildMetricSpec throw MetricSpecBuildingException when vidSamplingInterval width is not larger than 0`() { val metricSpec = - REACH_METRIC_SPEC.copy { + OLD_EMPTY_REACH_METRIC_SPEC.copy { vidSamplingInterval = MetricSpecKt.vidSamplingInterval { width = 0f } } val exception = assertThrows(MetricSpecDefaultsException::class.java) { - metricSpec.withDefaults(METRIC_SPEC_CONFIG) + metricSpec.withDefaults(METRIC_SPEC_CONFIG, randomMock) } assertThat(exception).hasMessageThat().contains("vidSamplingInterval") assertThat(exception).hasCauseThat().isInstanceOf(IllegalArgumentException::class.java) @@ -377,7 +969,7 @@ class MetricSpecDefaultsTest { @Test fun `buildMetricSpec throw MetricSpecBuildingException when vidSamplingInterval is too long`() { val metricSpec = - REACH_METRIC_SPEC.copy { + OLD_EMPTY_REACH_METRIC_SPEC.copy { vidSamplingInterval = MetricSpecKt.vidSamplingInterval { start = 0.5f @@ -387,7 +979,7 @@ class MetricSpecDefaultsTest { val exception = assertThrows(MetricSpecDefaultsException::class.java) { - metricSpec.withDefaults(METRIC_SPEC_CONFIG) + metricSpec.withDefaults(METRIC_SPEC_CONFIG, randomMock) } assertThat(exception).hasMessageThat().contains("vidSamplingInterval") assertThat(exception).hasCauseThat().isInstanceOf(IllegalArgumentException::class.java) @@ -398,91 +990,158 @@ class MetricSpecDefaultsTest { private val METRIC_SPEC_CONFIG = metricSpecConfig { reachParams = MetricSpecConfigKt.reachParams { - privacyParams = - MetricSpecConfigKt.differentialPrivacyParams { - epsilon = REACH_ONLY_REACH_EPSILON - delta = DIFFERENTIAL_PRIVACY_DELTA + multipleDataProviderParams = MetricSpecConfigKt.params { + privacyParams = + MetricSpecConfigKt.differentialPrivacyParams { + epsilon = REACH_ONLY_REACH_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + vidSamplingInterval = MetricSpecConfigKt.vidSamplingInterval { + fixedStart = MetricSpecConfigKt.VidSamplingIntervalKt.fixedStart { + start = REACH_ONLY_VID_SAMPLING_START + width = REACH_ONLY_VID_SAMPLING_WIDTH + } } - } - reachVidSamplingInterval = - MetricSpecConfigKt.vidSamplingInterval { - start = REACH_ONLY_VID_SAMPLING_START - width = REACH_ONLY_VID_SAMPLING_WIDTH + } + + singleDataProviderParams = MetricSpecConfigKt.params { + privacyParams = + MetricSpecConfigKt.differentialPrivacyParams { + epsilon = SINGLE_DATA_PROVIDER_REACH_ONLY_REACH_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + vidSamplingInterval = MetricSpecConfigKt.vidSamplingInterval { + fixedStart = MetricSpecConfigKt.VidSamplingIntervalKt.fixedStart { + start = SINGLE_DATA_PROVIDER_REACH_ONLY_VID_SAMPLING_START + width = SINGLE_DATA_PROVIDER_REACH_ONLY_VID_SAMPLING_WIDTH + } + } + } } reachAndFrequencyParams = MetricSpecConfigKt.reachAndFrequencyParams { - reachPrivacyParams = - MetricSpecConfigKt.differentialPrivacyParams { - epsilon = REACH_FREQUENCY_REACH_EPSILON - delta = DIFFERENTIAL_PRIVACY_DELTA + multipleDataProviderParams = MetricSpecConfigKt.params { + privacyParams = + MetricSpecConfigKt.differentialPrivacyParams { + epsilon = REACH_FREQUENCY_REACH_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + frequencyPrivacyParams = + MetricSpecConfigKt.differentialPrivacyParams { + epsilon = REACH_FREQUENCY_FREQUENCY_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + vidSamplingInterval = MetricSpecConfigKt.vidSamplingInterval { + fixedStart = MetricSpecConfigKt.VidSamplingIntervalKt.fixedStart { + start = REACH_FREQUENCY_VID_SAMPLING_START + width = REACH_FREQUENCY_VID_SAMPLING_WIDTH + } } - frequencyPrivacyParams = - MetricSpecConfigKt.differentialPrivacyParams { - epsilon = REACH_FREQUENCY_FREQUENCY_EPSILON - delta = DIFFERENTIAL_PRIVACY_DELTA + } + + singleDataProviderParams = MetricSpecConfigKt.params { + privacyParams = + MetricSpecConfigKt.differentialPrivacyParams { + epsilon = SINGLE_DATA_PROVIDER_REACH_FREQUENCY_REACH_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + frequencyPrivacyParams = + MetricSpecConfigKt.differentialPrivacyParams { + epsilon = SINGLE_DATA_PROVIDER_REACH_FREQUENCY_FREQUENCY_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + vidSamplingInterval = MetricSpecConfigKt.vidSamplingInterval { + fixedStart = MetricSpecConfigKt.VidSamplingIntervalKt.fixedStart { + start = SINGLE_DATA_PROVIDER_REACH_FREQUENCY_VID_SAMPLING_START + width = SINGLE_DATA_PROVIDER_REACH_FREQUENCY_VID_SAMPLING_WIDTH + } } + } maximumFrequency = REACH_FREQUENCY_MAXIMUM_FREQUENCY } - reachAndFrequencyVidSamplingInterval = - MetricSpecConfigKt.vidSamplingInterval { - start = REACH_FREQUENCY_VID_SAMPLING_START - width = REACH_FREQUENCY_VID_SAMPLING_WIDTH - } impressionCountParams = MetricSpecConfigKt.impressionCountParams { - privacyParams = - MetricSpecConfigKt.differentialPrivacyParams { - epsilon = IMPRESSION_EPSILON - delta = DIFFERENTIAL_PRIVACY_DELTA + params = MetricSpecConfigKt.params { + privacyParams = + MetricSpecConfigKt.differentialPrivacyParams { + epsilon = IMPRESSION_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + vidSamplingInterval = MetricSpecConfigKt.vidSamplingInterval { + fixedStart = MetricSpecConfigKt.VidSamplingIntervalKt.fixedStart { + start = IMPRESSION_VID_SAMPLING_START + width = IMPRESSION_VID_SAMPLING_WIDTH + } } + } maximumFrequencyPerUser = IMPRESSION_MAXIMUM_FREQUENCY_PER_USER } - impressionCountVidSamplingInterval = - MetricSpecConfigKt.vidSamplingInterval { - start = IMPRESSION_VID_SAMPLING_START - width = IMPRESSION_VID_SAMPLING_WIDTH - } watchDurationParams = MetricSpecConfigKt.watchDurationParams { - privacyParams = - MetricSpecConfigKt.differentialPrivacyParams { - epsilon = WATCH_DURATION_EPSILON - delta = DIFFERENTIAL_PRIVACY_DELTA + params = MetricSpecConfigKt.params { + privacyParams = + MetricSpecConfigKt.differentialPrivacyParams { + epsilon = WATCH_DURATION_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + vidSamplingInterval = MetricSpecConfigKt.vidSamplingInterval { + fixedStart = MetricSpecConfigKt.VidSamplingIntervalKt.fixedStart { + start = WATCH_DURATION_VID_SAMPLING_START + width = WATCH_DURATION_VID_SAMPLING_WIDTH + } } + } maximumWatchDurationPerUser = MAXIMUM_WATCH_DURATION_PER_USER } - watchDurationVidSamplingInterval = - MetricSpecConfigKt.vidSamplingInterval { - start = WATCH_DURATION_VID_SAMPLING_START - width = WATCH_DURATION_VID_SAMPLING_WIDTH - } } // Metric Specs - private val REACH_METRIC_SPEC: MetricSpec = metricSpec { + private val OLD_EMPTY_REACH_METRIC_SPEC: MetricSpec = metricSpec { reach = reachParams { privacyParams = MetricSpec.DifferentialPrivacyParams.getDefaultInstance() } } - private val REACH_AND_FREQUENCY_METRIC_SPEC: MetricSpec = metricSpec { + private val NEW_EMPTY_REACH_METRIC_SPEC: MetricSpec = metricSpec { + reach = reachParams { + multipleDataProviderParams = MetricSpec.Params.getDefaultInstance() + singleDataProviderParams = MetricSpec.Params.getDefaultInstance() + } + } + private val OLD_EMPTY_REACH_AND_FREQUENCY_METRIC_SPEC: MetricSpec = metricSpec { reachAndFrequency = reachAndFrequencyParams { reachPrivacyParams = MetricSpec.DifferentialPrivacyParams.getDefaultInstance() frequencyPrivacyParams = MetricSpec.DifferentialPrivacyParams.getDefaultInstance() } } - private val IMPRESSION_COUNT_METRIC_SPEC: MetricSpec = metricSpec { + private val NEW_EMPTY_REACH_AND_FREQUENCY_METRIC_SPEC: MetricSpec = metricSpec { + reachAndFrequency = reachAndFrequencyParams { + multipleDataProviderParams = MetricSpec.Params.getDefaultInstance() + singleDataProviderParams = MetricSpec.Params.getDefaultInstance() + } + } + private val OLD_EMPTY_IMPRESSION_COUNT_METRIC_SPEC: MetricSpec = metricSpec { impressionCount = impressionCountParams { privacyParams = MetricSpec.DifferentialPrivacyParams.getDefaultInstance() } } - private val WATCH_DURATION_METRIC_SPEC: MetricSpec = metricSpec { + private val NEW_EMPTY_IMPRESSION_COUNT_METRIC_SPEC: MetricSpec = metricSpec { + impressionCount = impressionCountParams { + params = MetricSpec.Params.getDefaultInstance() + } + } + private val OLD_EMPTY_WATCH_DURATION_METRIC_SPEC: MetricSpec = metricSpec { watchDuration = watchDurationParams { privacyParams = MetricSpec.DifferentialPrivacyParams.getDefaultInstance() } } + private val NEW_EMPTY_WATCH_DURATION_METRIC_SPEC: MetricSpec = metricSpec { + watchDuration = watchDurationParams { + params = MetricSpec.Params.getDefaultInstance() + } + } } } diff --git a/src/test/kotlin/org/wfanet/measurement/reporting/service/api/v2alpha/MetricsServiceTest.kt b/src/test/kotlin/org/wfanet/measurement/reporting/service/api/v2alpha/MetricsServiceTest.kt index 1c422e0c544..2184e32a474 100644 --- a/src/test/kotlin/org/wfanet/measurement/reporting/service/api/v2alpha/MetricsServiceTest.kt +++ b/src/test/kotlin/org/wfanet/measurement/reporting/service/api/v2alpha/MetricsServiceTest.kt @@ -197,6 +197,8 @@ import org.wfanet.measurement.internal.reporting.v2.metricSpec as internalMetric import org.wfanet.measurement.internal.reporting.v2.reachOnlyLiquidLegionsSketchParams as internalReachOnlyLiquidLegionsSketchParams import org.wfanet.measurement.internal.reporting.v2.reachOnlyLiquidLegionsV2 import org.wfanet.measurement.internal.reporting.v2.reportingSet as internalReportingSet +import com.google.devtools.build.runfiles.Runfiles +import java.io.File import org.wfanet.measurement.internal.reporting.v2.streamMetricsRequest import org.wfanet.measurement.measurementconsumer.stats.FrequencyMeasurementVarianceParams import org.wfanet.measurement.measurementconsumer.stats.FrequencyMetricVarianceParams @@ -244,2099 +246,1093 @@ private const val BATCH_SET_MEASUREMENT_RESULTS_LIMIT = 1000 private const val BATCH_SET_MEASUREMENT_FAILURES_LIMIT = 1000 private const val BATCH_KINGDOM_MEASUREMENTS_LIMIT = 50 -private const val NUMBER_VID_BUCKETS = 300 -private const val REACH_ONLY_VID_SAMPLING_WIDTH = 3.0f / NUMBER_VID_BUCKETS -private const val REACH_ONLY_VID_SAMPLING_START = 0.0f -private const val REACH_ONLY_REACH_EPSILON = 0.0041 - -private const val REACH_FREQUENCY_VID_SAMPLING_WIDTH = 5.0f / NUMBER_VID_BUCKETS -private const val REACH_FREQUENCY_VID_SAMPLING_START = 48.0f / NUMBER_VID_BUCKETS -private const val REACH_FREQUENCY_REACH_EPSILON = 0.0033 -private const val REACH_FREQUENCY_FREQUENCY_EPSILON = 0.115 -private const val REACH_FREQUENCY_MAXIMUM_FREQUENCY = 5 - -private const val IMPRESSION_VID_SAMPLING_WIDTH = 62.0f / NUMBER_VID_BUCKETS -private const val IMPRESSION_VID_SAMPLING_START = 143.0f / NUMBER_VID_BUCKETS -private const val IMPRESSION_EPSILON = 0.0011 -private const val IMPRESSION_MAXIMUM_FREQUENCY_PER_USER = 60 -private const val IMPRESSION_CUSTOM_MAXIMUM_FREQUENCY_PER_USER = 100 - -private const val WATCH_DURATION_VID_SAMPLING_WIDTH = 95.0f / NUMBER_VID_BUCKETS -private const val WATCH_DURATION_VID_SAMPLING_START = 205.0f / NUMBER_VID_BUCKETS -private const val WATCH_DURATION_EPSILON = 0.001 -private val MAXIMUM_WATCH_DURATION_PER_USER = Durations.fromSeconds(4000) - -private const val DIFFERENTIAL_PRIVACY_DELTA = 1e-12 - -private const val RANDOM_OUTPUT_INT = 0 -private const val RANDOM_OUTPUT_LONG = 0L - -private val METRIC_SPEC_CONFIG = metricSpecConfig { - reachParams = - MetricSpecConfigKt.reachParams { - privacyParams = - MetricSpecConfigKt.differentialPrivacyParams { - epsilon = REACH_ONLY_REACH_EPSILON - delta = DIFFERENTIAL_PRIVACY_DELTA - } - } - reachVidSamplingInterval = - MetricSpecConfigKt.vidSamplingInterval { - start = REACH_ONLY_VID_SAMPLING_START - width = REACH_ONLY_VID_SAMPLING_WIDTH - } - - reachAndFrequencyParams = - MetricSpecConfigKt.reachAndFrequencyParams { - reachPrivacyParams = - MetricSpecConfigKt.differentialPrivacyParams { - epsilon = REACH_FREQUENCY_REACH_EPSILON - delta = DIFFERENTIAL_PRIVACY_DELTA - } - frequencyPrivacyParams = - MetricSpecConfigKt.differentialPrivacyParams { - epsilon = REACH_FREQUENCY_FREQUENCY_EPSILON - delta = DIFFERENTIAL_PRIVACY_DELTA +@RunWith(JUnit4::class) +class MetricsServiceTest { + private val internalMetricsMock: MetricsCoroutineImplBase = mockService { + onBlocking { createMetric(any()) }.thenReturn(INTERNAL_PENDING_INITIAL_INCREMENTAL_REACH_METRIC) + onBlocking { batchCreateMetrics(any()) } + .thenReturn( + internalBatchCreateMetricsResponse { + metrics += INTERNAL_PENDING_INITIAL_INCREMENTAL_REACH_METRIC + metrics += INTERNAL_PENDING_INITIAL_SINGLE_PUBLISHER_IMPRESSION_METRIC } - maximumFrequency = REACH_FREQUENCY_MAXIMUM_FREQUENCY - } - reachAndFrequencyVidSamplingInterval = - MetricSpecConfigKt.vidSamplingInterval { - start = REACH_FREQUENCY_VID_SAMPLING_START - width = REACH_FREQUENCY_VID_SAMPLING_WIDTH - } - - impressionCountParams = - MetricSpecConfigKt.impressionCountParams { - privacyParams = - MetricSpecConfigKt.differentialPrivacyParams { - epsilon = IMPRESSION_EPSILON - delta = DIFFERENTIAL_PRIVACY_DELTA + ) + onBlocking { streamMetrics(any()) } + .thenReturn( + flowOf( + INTERNAL_PENDING_INCREMENTAL_REACH_METRIC, + INTERNAL_PENDING_SINGLE_PUBLISHER_IMPRESSION_METRIC, + ) + ) + onBlocking { batchGetMetrics(any()) } + .thenReturn( + internalBatchGetMetricsResponse { + metrics += INTERNAL_PENDING_INCREMENTAL_REACH_METRIC + metrics += INTERNAL_PENDING_SINGLE_PUBLISHER_IMPRESSION_METRIC } - maximumFrequencyPerUser = IMPRESSION_MAXIMUM_FREQUENCY_PER_USER - } - impressionCountVidSamplingInterval = - MetricSpecConfigKt.vidSamplingInterval { - start = IMPRESSION_VID_SAMPLING_START - width = IMPRESSION_VID_SAMPLING_WIDTH - } + ) + } - watchDurationParams = - MetricSpecConfigKt.watchDurationParams { - privacyParams = - MetricSpecConfigKt.differentialPrivacyParams { - epsilon = WATCH_DURATION_EPSILON - delta = DIFFERENTIAL_PRIVACY_DELTA + private val internalReportingSetsMock: + InternalReportingSetsGrpcKt.ReportingSetsCoroutineImplBase = + mockService { + onBlocking { batchGetReportingSets(any()) } + .thenAnswer { + val request = it.arguments[0] as BatchGetReportingSetsRequest + val internalReportingSetsMap = + mapOf( + INTERNAL_INCREMENTAL_REPORTING_SET.externalReportingSetId to + INTERNAL_INCREMENTAL_REPORTING_SET, + INTERNAL_UNION_ALL_REPORTING_SET.externalReportingSetId to + INTERNAL_UNION_ALL_REPORTING_SET, + INTERNAL_UNION_ALL_BUT_LAST_PUBLISHER_REPORTING_SET.externalReportingSetId to + INTERNAL_UNION_ALL_BUT_LAST_PUBLISHER_REPORTING_SET, + INTERNAL_SINGLE_PUBLISHER_REPORTING_SET.externalReportingSetId to + INTERNAL_SINGLE_PUBLISHER_REPORTING_SET, + INTERNAL_POPULATION_REPORTING_SET.externalReportingSetId to + INTERNAL_POPULATION_REPORTING_SET, + ) + batchGetReportingSetsResponse { + reportingSets += + request.externalReportingSetIdsList.map { externalReportingSetId -> + internalReportingSetsMap.getValue(externalReportingSetId) + } + } } - maximumWatchDurationPerUser = MAXIMUM_WATCH_DURATION_PER_USER - } - watchDurationVidSamplingInterval = - MetricSpecConfigKt.vidSamplingInterval { - start = WATCH_DURATION_VID_SAMPLING_START - width = WATCH_DURATION_VID_SAMPLING_WIDTH } - populationCountParams = MetricSpecConfig.PopulationCountParams.getDefaultInstance() -} - -private val SECRETS_DIR = - getRuntimePath( - Paths.get("wfa_measurement_system", "src", "main", "k8s", "testing", "secretfiles") - )!! - .toFile() - -// Authentication key -private const val API_AUTHENTICATION_KEY = "nR5QPN7ptx" - -// Aggregator certificate - -private val AGGREGATOR_SIGNING_KEY: SigningKeyHandle by lazy { - loadSigningKey( - SECRETS_DIR.resolve("aggregator_cs_cert.der"), - SECRETS_DIR.resolve("aggregator_cs_private.der"), - ) -} -private val AGGREGATOR_CERTIFICATE = certificate { - name = "duchies/aggregator/certificates/abc123" - x509Der = AGGREGATOR_SIGNING_KEY.certificate.encoded.toByteString() -} -private val AGGREGATOR_ROOT_CERTIFICATE: X509Certificate = - readCertificate(SECRETS_DIR.resolve("aggregator_root.pem")) - -// Measurement consumer crypto - -private val TRUSTED_MEASUREMENT_CONSUMER_ISSUER: X509Certificate = - readCertificate(SECRETS_DIR.resolve("mc_root.pem")) -private val MEASUREMENT_CONSUMER_SIGNING_KEY_HANDLE = - loadSigningKey(SECRETS_DIR.resolve("mc_cs_cert.der"), SECRETS_DIR.resolve("mc_cs_private.der")) -private val MEASUREMENT_CONSUMER_CERTIFICATE = MEASUREMENT_CONSUMER_SIGNING_KEY_HANDLE.certificate -private val MEASUREMENT_CONSUMER_PRIVATE_KEY_HANDLE: PrivateKeyHandle = - loadPrivateKey(SECRETS_DIR.resolve("mc_enc_private.tink")) -private val MEASUREMENT_CONSUMER_PUBLIC_KEY = encryptionPublicKey { - format = EncryptionPublicKey.Format.TINK_KEYSET - data = SECRETS_DIR.resolve("mc_enc_public.tink").readByteString() -} + private val internalMeasurementsMock: InternalMeasurementsCoroutineImplBase = mockService { + onBlocking { batchSetCmmsMeasurementIds(any()) }.thenReturn(Empty.getDefaultInstance()) + onBlocking { batchSetMeasurementResults(any()) }.thenReturn(Empty.getDefaultInstance()) + onBlocking { batchSetMeasurementFailures(any()) }.thenReturn(Empty.getDefaultInstance()) + } -private val MEASUREMENT_CONSUMERS: Map = - (1L..2L).associate { - val measurementConsumerKey = MeasurementConsumerKey(ExternalId(it + 110L).apiId.value) - val certificateKey = - MeasurementConsumerCertificateKey( - measurementConsumerKey.measurementConsumerId, - ExternalId(it + 120L).apiId.value, - ) - measurementConsumerKey to - measurementConsumer { - name = measurementConsumerKey.toName() - certificate = certificateKey.toName() - certificateDer = MEASUREMENT_CONSUMER_SIGNING_KEY_HANDLE.certificate.encoded.toByteString() - publicKey = - signEncryptionPublicKey( - MEASUREMENT_CONSUMER_PUBLIC_KEY, - MEASUREMENT_CONSUMER_SIGNING_KEY_HANDLE, + private val measurementsMock: MeasurementsCoroutineImplBase = mockService { + onBlocking { batchGetMeasurements(any()) } + .thenAnswer { + val batchGetMeasurementsRequest = it.arguments[0] as BatchGetMeasurementsRequest + val measurementsMap = + mapOf( + PENDING_UNION_ALL_REACH_MEASUREMENT.name to PENDING_UNION_ALL_REACH_MEASUREMENT, + PENDING_UNION_ALL_BUT_LAST_PUBLISHER_REACH_MEASUREMENT.name to + PENDING_UNION_ALL_BUT_LAST_PUBLISHER_REACH_MEASUREMENT, + PENDING_SINGLE_PUBLISHER_IMPRESSION_MEASUREMENT.name to + PENDING_SINGLE_PUBLISHER_IMPRESSION_MEASUREMENT, + PENDING_UNION_ALL_WATCH_DURATION_MEASUREMENT.name to + PENDING_UNION_ALL_WATCH_DURATION_MEASUREMENT, + PENDING_POPULATION_MEASUREMENT.name to PENDING_POPULATION_MEASUREMENT, ) + batchGetMeasurementsResponse { + measurements += + batchGetMeasurementsRequest.namesList.map { name -> measurementsMap.getValue(name) } + } } - } - -private val CONFIG = measurementConsumerConfig { - apiKey = API_AUTHENTICATION_KEY - signingCertificateName = MEASUREMENT_CONSUMERS.values.first().certificate - signingPrivateKeyPath = "mc_cs_private.der" -} - -// InMemoryEncryptionKeyPairStore -private val ENCRYPTION_KEY_PAIR_STORE = - InMemoryEncryptionKeyPairStore( - MEASUREMENT_CONSUMERS.values.associateBy( - { it.name }, - { - listOf( - it.publicKey.unpack().data to MEASUREMENT_CONSUMER_PRIVATE_KEY_HANDLE - ) - }, - ) - ) -private val DATA_PROVIDER_PUBLIC_KEY = encryptionPublicKey { - format = EncryptionPublicKey.Format.TINK_KEYSET - data = SECRETS_DIR.resolve("edp1_enc_public.tink").readByteString() -} -private val DATA_PROVIDER_PRIVATE_KEY_HANDLE = - loadPrivateKey(SECRETS_DIR.resolve("edp1_enc_private.tink")) -private val DATA_PROVIDER_SIGNING_KEY = - loadSigningKey( - SECRETS_DIR.resolve("edp1_cs_cert.der"), - SECRETS_DIR.resolve("edp1_cs_private.der"), - ) -private val DATA_PROVIDER_ROOT_CERTIFICATE = readCertificate(SECRETS_DIR.resolve("edp1_root.pem")) - -// Data providers - -private val DATA_PROVIDERS = - (1L..3L).associate { - val dataProviderKey = DataProviderKey(ExternalId(it + 550L).apiId.value) - val certificateKey = - DataProviderCertificateKey(dataProviderKey.dataProviderId, ExternalId(it + 560L).apiId.value) - dataProviderKey to - dataProvider { - name = dataProviderKey.toName() - certificate = certificateKey.toName() - publicKey = signEncryptionPublicKey(DATA_PROVIDER_PUBLIC_KEY, DATA_PROVIDER_SIGNING_KEY) + onBlocking { batchCreateMeasurements(any()) } + .thenAnswer { + val batchCreateMeasurementsRequest = it.arguments[0] as BatchCreateMeasurementsRequest + val measurementsMap = + mapOf( + INTERNAL_PENDING_UNION_ALL_REACH_MEASUREMENT.cmmsCreateMeasurementRequestId to + PENDING_UNION_ALL_REACH_MEASUREMENT, + INTERNAL_PENDING_UNION_ALL_BUT_LAST_PUBLISHER_REACH_MEASUREMENT + .cmmsCreateMeasurementRequestId to + PENDING_UNION_ALL_BUT_LAST_PUBLISHER_REACH_MEASUREMENT, + INTERNAL_PENDING_SINGLE_PUBLISHER_IMPRESSION_MEASUREMENT + .cmmsCreateMeasurementRequestId to PENDING_SINGLE_PUBLISHER_IMPRESSION_MEASUREMENT, + INTERNAL_PENDING_POPULATION_MEASUREMENT.cmmsCreateMeasurementRequestId to + PENDING_POPULATION_MEASUREMENT, + ) + batchCreateMeasurementsResponse { + measurements += + batchCreateMeasurementsRequest.requestsList.map { createMeasurementRequest -> + measurementsMap.getValue(createMeasurementRequest.requestId) + } + } } } -private val DATA_PROVIDERS_LIST = DATA_PROVIDERS.values.toList() -// Event group keys + private val measurementConsumersMock: + MeasurementConsumersGrpcKt.MeasurementConsumersCoroutineImplBase = + mockService { + onBlocking { + getMeasurementConsumer( + eq(getMeasurementConsumerRequest { name = MEASUREMENT_CONSUMERS.values.first().name }) + ) + } + .thenReturn(MEASUREMENT_CONSUMERS.values.first()) + } -private val CMMS_EVENT_GROUP_KEYS = - DATA_PROVIDERS.keys.mapIndexed { index, dataProviderKey -> - CmmsEventGroupKey(dataProviderKey.dataProviderId, ExternalId(index + 660L).apiId.value) + private val dataProvidersMock: DataProvidersGrpcKt.DataProvidersCoroutineImplBase = mockService { + for (dataProvider in DATA_PROVIDERS.values) { + onBlocking { getDataProvider(eq(getDataProviderRequest { name = dataProvider.name })) } + .thenReturn(dataProvider) + } } -// Event filters -private const val INCREMENTAL_REPORTING_SET_FILTER = "AGE>18" -private const val METRIC_FILTER = "media_type==video" -private const val PRIMITIVE_REPORTING_SET_FILTER = "gender==male" -private val ALL_FILTERS = - listOf(INCREMENTAL_REPORTING_SET_FILTER, METRIC_FILTER, PRIMITIVE_REPORTING_SET_FILTER) - -// Internal reporting sets - -private val INTERNAL_UNION_ALL_REPORTING_SET = internalReportingSet { - cmmsMeasurementConsumerId = MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId - externalReportingSetId = "220L" - this.primitive = - InternalReportingSetKt.primitive { - eventGroupKeys += CMMS_EVENT_GROUP_KEYS.map { it.toInternal() } - } - filter = PRIMITIVE_REPORTING_SET_FILTER - displayName = "$cmmsMeasurementConsumerId-$externalReportingSetId-$filter" - weightedSubsetUnions += weightedSubsetUnion { - primitiveReportingSetBases += primitiveReportingSetBasis { - externalReportingSetId = this@internalReportingSet.externalReportingSetId - filters += this@internalReportingSet.filter - } - weight = 1 - binaryRepresentation = 1 - } -} -private val INTERNAL_UNION_ALL_BUT_LAST_PUBLISHER_REPORTING_SET = internalReportingSet { - cmmsMeasurementConsumerId = MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId - externalReportingSetId = INTERNAL_UNION_ALL_REPORTING_SET.externalReportingSetId + "1" - this.primitive = - InternalReportingSetKt.primitive { - (0 until CMMS_EVENT_GROUP_KEYS.size - 1).map { i -> - eventGroupKeys += CMMS_EVENT_GROUP_KEYS[i].toInternal() - } - } - filter = PRIMITIVE_REPORTING_SET_FILTER - displayName = "$cmmsMeasurementConsumerId-$externalReportingSetId-$filter" - weightedSubsetUnions += weightedSubsetUnion { - primitiveReportingSetBases += primitiveReportingSetBasis { - externalReportingSetId = this@internalReportingSet.externalReportingSetId - filters += this@internalReportingSet.filter - } - weight = 1 - binaryRepresentation = 1 - } -} -private val INTERNAL_SINGLE_PUBLISHER_REPORTING_SET = internalReportingSet { - cmmsMeasurementConsumerId = MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId - externalReportingSetId = - INTERNAL_UNION_ALL_BUT_LAST_PUBLISHER_REPORTING_SET.externalReportingSetId + "1" - this.primitive = - InternalReportingSetKt.primitive { - eventGroupKeys += - (0L until 3L) - .map { index -> - CmmsEventGroupKey( - DATA_PROVIDERS.keys.first().dataProviderId, - ExternalId(index + 670L).apiId.value, - ) + private val certificatesMock: CertificatesGrpcKt.CertificatesCoroutineImplBase = mockService { + onBlocking { getCertificate(eq(getCertificateRequest { name = AGGREGATOR_CERTIFICATE.name })) } + .thenReturn(AGGREGATOR_CERTIFICATE) + for (dataProvider in DATA_PROVIDERS.values) { + onBlocking { getCertificate(eq(getCertificateRequest { name = dataProvider.certificate })) } + .thenReturn( + certificate { + name = dataProvider.certificate + x509Der = DATA_PROVIDER_SIGNING_KEY.certificate.encoded.toByteString() } - .map { it.toInternal() } - } - filter = PRIMITIVE_REPORTING_SET_FILTER - displayName = "$cmmsMeasurementConsumerId-$externalReportingSetId-$filter" - weightedSubsetUnions += weightedSubsetUnion { - primitiveReportingSetBases += primitiveReportingSetBasis { - externalReportingSetId = this@internalReportingSet.externalReportingSetId - filters += this@internalReportingSet.filter + ) } - weight = 1 - binaryRepresentation = 1 - } -} - -private val INTERNAL_INCREMENTAL_REPORTING_SET = internalReportingSet { - cmmsMeasurementConsumerId = MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId - externalReportingSetId = INTERNAL_SINGLE_PUBLISHER_REPORTING_SET.externalReportingSetId + "1" - this.composite = - InternalReportingSetKt.setExpression { - operation = InternalSetExpression.Operation.DIFFERENCE - lhs = - InternalReportingSetKt.SetExpressionKt.operand { - externalReportingSetId = INTERNAL_UNION_ALL_REPORTING_SET.externalReportingSetId - } - rhs = - InternalReportingSetKt.SetExpressionKt.operand { - externalReportingSetId = - INTERNAL_UNION_ALL_BUT_LAST_PUBLISHER_REPORTING_SET.externalReportingSetId + for (measurementConsumer in MEASUREMENT_CONSUMERS.values) { + onBlocking { + getCertificate(eq(getCertificateRequest { name = measurementConsumer.certificate })) } + .thenReturn( + certificate { + name = measurementConsumer.certificate + x509Der = measurementConsumer.certificateDer + } + ) } - filter = INCREMENTAL_REPORTING_SET_FILTER - displayName = "$cmmsMeasurementConsumerId-$externalReportingSetId-$filter" - weightedSubsetUnions += weightedSubsetUnion { - primitiveReportingSetBases += primitiveReportingSetBasis { - externalReportingSetId = INTERNAL_UNION_ALL_REPORTING_SET.externalReportingSetId - filters += INCREMENTAL_REPORTING_SET_FILTER - filters += INTERNAL_UNION_ALL_REPORTING_SET.filter - } - primitiveReportingSetBases += primitiveReportingSetBasis { - externalReportingSetId = - INTERNAL_UNION_ALL_BUT_LAST_PUBLISHER_REPORTING_SET.externalReportingSetId - filters += INCREMENTAL_REPORTING_SET_FILTER - filters += INTERNAL_UNION_ALL_BUT_LAST_PUBLISHER_REPORTING_SET.filter - } - weight = 1 - binaryRepresentation = 3 - } - weightedSubsetUnions += weightedSubsetUnion { - primitiveReportingSetBases += primitiveReportingSetBasis { - externalReportingSetId = - INTERNAL_UNION_ALL_BUT_LAST_PUBLISHER_REPORTING_SET.externalReportingSetId - filters += INCREMENTAL_REPORTING_SET_FILTER - filters += INTERNAL_UNION_ALL_BUT_LAST_PUBLISHER_REPORTING_SET.filter - } - weight = -1 - binaryRepresentation = 2 - } -} -private val INTERNAL_POPULATION_REPORTING_SET = internalReportingSet { - cmmsMeasurementConsumerId = MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId - externalReportingSetId = INTERNAL_INCREMENTAL_REPORTING_SET.externalReportingSetId + "1" - this.primitive = - InternalReportingSetKt.primitive { - eventGroupKeys += CMMS_EVENT_GROUP_KEYS.first().toInternal() - } - filter = INCREMENTAL_REPORTING_SET_FILTER - displayName = "$cmmsMeasurementConsumerId-$externalReportingSetId-$filter" - weightedSubsetUnions += weightedSubsetUnion { - primitiveReportingSetBases += primitiveReportingSetBasis { - externalReportingSetId = this@internalReportingSet.externalReportingSetId - } - weight = 1 - binaryRepresentation = 1 } -} - -// Time intervals -private val START_INSTANT = Instant.now() -private val TIME_RANGE = OpenEndTimeRange(START_INSTANT, START_INSTANT.plus(Duration.ofDays(1))) -private val TIME_INTERVAL: Interval = TIME_RANGE.toInterval() + private val randomMock: Random = mock() -// Requisition specs -private val REQUISITION_SPECS: Map = - CMMS_EVENT_GROUP_KEYS.groupBy( - { it.parentKey }, - { - RequisitionSpecKt.eventGroupEntry { - key = it.toName() - value = - RequisitionSpecKt.EventGroupEntryKt.value { - collectionInterval = TIME_INTERVAL - filter = - RequisitionSpecKt.eventFilter { - expression = - "($INCREMENTAL_REPORTING_SET_FILTER) AND ($METRIC_FILTER) AND ($PRIMITIVE_REPORTING_SET_FILTER)" - } - } - } - }, - ) - .mapValues { - requisitionSpec { - events = RequisitionSpecKt.events { eventGroups += it.value } - measurementPublicKey = MEASUREMENT_CONSUMERS.values.first().publicKey.message - nonce = RANDOM_OUTPUT_LONG - } - } - -// Data provider entries -private val DATA_PROVIDER_ENTRIES = - REQUISITION_SPECS.mapValues { (dataProviderKey, requisitionSpec) -> - val dataProvider = DATA_PROVIDERS.getValue(dataProviderKey) - MeasurementKt.dataProviderEntry { - key = dataProvider.name - value = - MeasurementKt.DataProviderEntryKt.value { - dataProviderCertificate = dataProvider.certificate - dataProviderPublicKey = dataProvider.publicKey.message - encryptedRequisitionSpec = - encryptRequisitionSpec( - signRequisitionSpec(requisitionSpec, MEASUREMENT_CONSUMER_SIGNING_KEY_HANDLE), - dataProvider.publicKey.unpack(), - ) - nonceHash = Hashing.hashSha256(requisitionSpec.nonce) - } + private object VariancesMock : Variances { + override fun computeMetricVariance(params: ReachMetricVarianceParams): Double { + val epsilon = params.weightedMeasurementVarianceParamsList.first().measurementVarianceParams.measurementParams.dpParams.epsilon + return if (epsilon == REACH_ONLY_REACH_EPSILON || epsilon == REACH_FREQUENCY_REACH_EPSILON) { + VARIANCE_VALUE + } else { + SINGLE_DATA_PROVIDER_VARIANCE_VALUE + } } - } -// Measurements + override fun computeMeasurementVariance( + methodology: Methodology, + measurementVarianceParams: ReachMeasurementVarianceParams, + ): Double = VARIANCE_VALUE -private val BASE_MEASUREMENT = measurement { - measurementConsumerCertificate = MEASUREMENT_CONSUMERS.values.first().certificate -} + override fun computeMetricVariance(params: FrequencyMetricVarianceParams): FrequencyVariances { + val epsilon = params.weightedMeasurementVarianceParamsList.first().measurementVarianceParams.measurementParams.dpParams.epsilon + return if (epsilon == REACH_FREQUENCY_FREQUENCY_EPSILON) { + FREQUENCY_VARIANCES + } else { + SINGLE_DATA_PROVIDER_FREQUENCY_VARIANCES + } + } -private const val LL_DISTRIBUTION_DECAY_RATE = 2e-2 -private const val LL_DISTRIBUTION_SKETCH_SIZE = 20000L -private const val REACH_ONLY_LLV2_DECAY_RATE = 1e-2 -private const val REACH_ONLY_LLV2_SKETCH_SIZE = 10000L - -// Measurement values -private const val UNION_ALL_REACH_VALUE = 100_000L -private const val UNION_ALL_BUT_LAST_PUBLISHER_REACH_VALUE = 70_000L -private const val INCREMENTAL_REACH_VALUE = - UNION_ALL_REACH_VALUE - UNION_ALL_BUT_LAST_PUBLISHER_REACH_VALUE -private const val REACH_FREQUENCY_REACH_VALUE = 100_000L -private val REACH_FREQUENCY_FREQUENCY_VALUE = mapOf(1L to 0.1, 2L to 0.2, 3L to 0.3, 4L to 0.4) -private const val IMPRESSION_VALUE = 1_000_000L -private val WATCH_DURATION_SECOND_LIST = listOf(100L, 200L, 300L) -private val WATCH_DURATION_LIST = WATCH_DURATION_SECOND_LIST.map { duration { seconds = it } } -private val TOTAL_WATCH_DURATION = duration { seconds = WATCH_DURATION_SECOND_LIST.sum() } -private val TOTAL_POPULATION_VALUE = 1000L - -// Internal incremental reach measurements - -private val INTERNAL_PENDING_UNION_ALL_REACH_MEASUREMENT = internalMeasurement { - cmmsMeasurementConsumerId = MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId - cmmsCreateMeasurementRequestId = "UNION_ALL_REACH_MEASUREMENT" - cmmsMeasurementId = externalIdToApiId(401L) - timeInterval = TIME_INTERVAL - primitiveReportingSetBases += primitiveReportingSetBasis { - externalReportingSetId = INTERNAL_UNION_ALL_REPORTING_SET.externalReportingSetId - filters += ALL_FILTERS - } - primitiveReportingSetBases += primitiveReportingSetBasis { - externalReportingSetId = - INTERNAL_UNION_ALL_BUT_LAST_PUBLISHER_REPORTING_SET.externalReportingSetId - filters += ALL_FILTERS - } - state = InternalMeasurement.State.PENDING -} + override fun computeMeasurementVariance( + methodology: Methodology, + measurementVarianceParams: FrequencyMeasurementVarianceParams, + ): FrequencyVariances = FrequencyVariances(mapOf(), mapOf(), mapOf(), mapOf()) -private val INTERNAL_PENDING_UNION_ALL_BUT_LAST_PUBLISHER_REACH_MEASUREMENT = internalMeasurement { - cmmsMeasurementConsumerId = MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId - cmmsCreateMeasurementRequestId = "UNION_ALL_BUT_LAST_PUBLISHER_REACH_MEASUREMENT" - cmmsMeasurementId = externalIdToApiId(402L) - timeInterval = TIME_INTERVAL - primitiveReportingSetBases += primitiveReportingSetBasis { - externalReportingSetId = - INTERNAL_UNION_ALL_BUT_LAST_PUBLISHER_REPORTING_SET.externalReportingSetId - filters += ALL_FILTERS - } - state = InternalMeasurement.State.PENDING -} + override fun computeMetricVariance(params: ImpressionMetricVarianceParams): Double = + VARIANCE_VALUE -private val INTERNAL_SUCCEEDED_UNION_ALL_REACH_MEASUREMENT = - INTERNAL_PENDING_UNION_ALL_REACH_MEASUREMENT.copy { - state = InternalMeasurement.State.SUCCEEDED - details = - InternalMeasurementKt.details { - results += - InternalMeasurementKt.result { - reach = - InternalMeasurementKt.ResultKt.reach { - value = UNION_ALL_REACH_VALUE - noiseMechanism = NoiseMechanism.DISCRETE_GAUSSIAN - reachOnlyLiquidLegionsV2 = reachOnlyLiquidLegionsV2 { - sketchParams = internalReachOnlyLiquidLegionsSketchParams { - decayRate = REACH_ONLY_LLV2_DECAY_RATE - maxSize = REACH_ONLY_LLV2_SKETCH_SIZE - } - } - } - } - } - } + override fun computeMeasurementVariance( + methodology: Methodology, + measurementVarianceParams: ImpressionMeasurementVarianceParams, + ): Double = VARIANCE_VALUE -private val INTERNAL_SUCCEEDED_UNION_ALL_BUT_LAST_PUBLISHER_REACH_MEASUREMENT = - INTERNAL_PENDING_UNION_ALL_BUT_LAST_PUBLISHER_REACH_MEASUREMENT.copy { - state = InternalMeasurement.State.SUCCEEDED - details = - InternalMeasurementKt.details { - results += - InternalMeasurementKt.result { - reach = - InternalMeasurementKt.ResultKt.reach { - value = UNION_ALL_BUT_LAST_PUBLISHER_REACH_VALUE - noiseMechanism = NoiseMechanism.DISCRETE_GAUSSIAN - reachOnlyLiquidLegionsV2 = reachOnlyLiquidLegionsV2 { - sketchParams = internalReachOnlyLiquidLegionsSketchParams { - decayRate = REACH_ONLY_LLV2_DECAY_RATE - maxSize = REACH_ONLY_LLV2_SKETCH_SIZE - } - } - } - } - } - } + override fun computeMetricVariance(params: WatchDurationMetricVarianceParams): Double = + VARIANCE_VALUE -// Internal single publisher reach-frequency measurements - -private val INTERNAL_PENDING_SINGLE_PUBLISHER_REACH_FREQUENCY_MEASUREMENT = internalMeasurement { - cmmsMeasurementConsumerId = MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId - cmmsCreateMeasurementRequestId = "SINGLE_PUBLISHER_REACH_FREQUENCY_MEASUREMENT" - cmmsMeasurementId = externalIdToApiId(443L) - timeInterval = TIME_INTERVAL - primitiveReportingSetBases += primitiveReportingSetBasis { - externalReportingSetId = INTERNAL_SINGLE_PUBLISHER_REPORTING_SET.externalReportingSetId - filters += METRIC_FILTER - filters += PRIMITIVE_REPORTING_SET_FILTER + override fun computeMeasurementVariance( + methodology: Methodology, + measurementVarianceParams: WatchDurationMeasurementVarianceParams, + ): Double = VARIANCE_VALUE } - state = InternalMeasurement.State.PENDING -} -private val INTERNAL_SUCCEEDED_SINGLE_PUBLISHER_REACH_FREQUENCY_MEASUREMENT = - INTERNAL_PENDING_SINGLE_PUBLISHER_REACH_FREQUENCY_MEASUREMENT.copy { - state = InternalMeasurement.State.SUCCEEDED - details = - InternalMeasurementKt.details { - results += - InternalMeasurementKt.result { - reach = - InternalMeasurementKt.ResultKt.reach { - value = REACH_FREQUENCY_REACH_VALUE - noiseMechanism = NoiseMechanism.CONTINUOUS_LAPLACE - deterministicCountDistinct = InternalDeterministicCountDistinct.getDefaultInstance() - } - frequency = - InternalMeasurementKt.ResultKt.frequency { - relativeFrequencyDistribution.putAll(REACH_FREQUENCY_FREQUENCY_VALUE) - noiseMechanism = NoiseMechanism.CONTINUOUS_LAPLACE - liquidLegionsDistribution = internalLiquidLegionsDistribution { - decayRate = LL_DISTRIBUTION_DECAY_RATE - maxSize = LL_DISTRIBUTION_SKETCH_SIZE - } - } - } - } + @get:Rule + val grpcTestServerRule = GrpcTestServerRule { + addService(internalMetricsMock) + addService(internalReportingSetsMock) + addService(internalMeasurementsMock) + addService(measurementsMock) + addService(measurementConsumersMock) + addService(dataProvidersMock) + addService(certificatesMock) } -// Internal single publisher impression measurements - -private val INTERNAL_PENDING_SINGLE_PUBLISHER_IMPRESSION_MEASUREMENT = internalMeasurement { - cmmsMeasurementConsumerId = MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId - cmmsCreateMeasurementRequestId = "SINGLE_PUBLISHER_IMPRESSION_MEASUREMENT" - cmmsMeasurementId = externalIdToApiId(403L) - timeInterval = TIME_INTERVAL - primitiveReportingSetBases += primitiveReportingSetBasis { - externalReportingSetId = INTERNAL_SINGLE_PUBLISHER_REPORTING_SET.externalReportingSetId - filters += METRIC_FILTER - filters += PRIMITIVE_REPORTING_SET_FILTER - } - state = InternalMeasurement.State.PENDING -} + private lateinit var service: MetricsService -private val INTERNAL_FAILED_SINGLE_PUBLISHER_IMPRESSION_MEASUREMENT = - INTERNAL_PENDING_SINGLE_PUBLISHER_IMPRESSION_MEASUREMENT.copy { - state = InternalMeasurement.State.FAILED - details = - InternalMeasurementKt.details { - failure = - InternalMeasurementKt.failure { - reason = InternalMeasurement.Failure.Reason.REQUISITION_REFUSED - message = "Privacy budget exceeded." - } - } - } + @Before + fun initService() { + randomMock.stub { + on { nextInt(any()) } doReturn RANDOM_OUTPUT_INT + on { nextLong() } doReturn RANDOM_OUTPUT_LONG + } -private val INTERNAL_SUCCEEDED_SINGLE_PUBLISHER_IMPRESSION_MEASUREMENT = - INTERNAL_PENDING_SINGLE_PUBLISHER_IMPRESSION_MEASUREMENT.copy { - state = InternalMeasurement.State.SUCCEEDED - details = - InternalMeasurementKt.details { - results += - InternalMeasurementKt.result { - impression = - InternalMeasurementKt.ResultKt.impression { - value = IMPRESSION_VALUE - noiseMechanism = NoiseMechanism.CONTINUOUS_LAPLACE - deterministicCount = InternalDeterministicCount.getDefaultInstance() - } - } - } + service = + MetricsService( + METRIC_SPEC_CONFIG, + InternalReportingSetsGrpcKt.ReportingSetsCoroutineStub(grpcTestServerRule.channel), + InternalMetricsGrpcKt.MetricsCoroutineStub(grpcTestServerRule.channel), + VariancesMock, + InternalMeasurementsGrpcKt.MeasurementsCoroutineStub(grpcTestServerRule.channel), + DataProvidersGrpcKt.DataProvidersCoroutineStub(grpcTestServerRule.channel), + MeasurementsGrpcKt.MeasurementsCoroutineStub(grpcTestServerRule.channel), + CertificatesGrpcKt.CertificatesCoroutineStub(grpcTestServerRule.channel), + MeasurementConsumersGrpcKt.MeasurementConsumersCoroutineStub(grpcTestServerRule.channel), + ENCRYPTION_KEY_PAIR_STORE, + randomMock, + SECRETS_DIR, + listOf(AGGREGATOR_ROOT_CERTIFICATE, DATA_PROVIDER_ROOT_CERTIFICATE).associateBy { + it.subjectKeyIdentifier!! + }, + ) } -private val INTERNAL_SUCCEEDED_SINGLE_PUBLISHER_IMPRESSION_MEASUREMENT_CUSTOM_CAP = - INTERNAL_PENDING_SINGLE_PUBLISHER_IMPRESSION_MEASUREMENT.copy { - state = InternalMeasurement.State.SUCCEEDED - details = - InternalMeasurementKt.details { - results += - InternalMeasurementKt.result { - impression = - InternalMeasurementKt.ResultKt.impression { - value = IMPRESSION_VALUE - noiseMechanism = NoiseMechanism.CONTINUOUS_LAPLACE - deterministicCount = internalDeterministicCount { - customMaximumFrequencyPerUser = IMPRESSION_CUSTOM_MAXIMUM_FREQUENCY_PER_USER - } - } - } - } - } + @Test + fun `createMetric creates CMMS measurements for incremental reach`() { + val request = createMetricRequest { + parent = MEASUREMENT_CONSUMERS.values.first().name + metric = REQUESTING_INCREMENTAL_REACH_METRIC + metricId = "metric-id" + } -// Internal cross-publisher watch duration measurements -private val INTERNAL_REQUESTING_UNION_ALL_WATCH_DURATION_MEASUREMENT = internalMeasurement { - cmmsMeasurementConsumerId = MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId - timeInterval = TIME_INTERVAL - primitiveReportingSetBases += primitiveReportingSetBasis { - externalReportingSetId = INTERNAL_UNION_ALL_REPORTING_SET.externalReportingSetId - filters += listOf(METRIC_FILTER, PRIMITIVE_REPORTING_SET_FILTER) - } -} + val result = + withMeasurementConsumerPrincipal(request.parent, CONFIG) { + runBlocking { service.createMetric(request) } + } -private val INTERNAL_PENDING_NOT_CREATED_UNION_ALL_WATCH_DURATION_MEASUREMENT = - INTERNAL_REQUESTING_UNION_ALL_WATCH_DURATION_MEASUREMENT.copy { - cmmsMeasurementId = externalIdToApiId(414L) - cmmsCreateMeasurementRequestId = "UNION_ALL_WATCH_DURATION_MEASUREMENT" - state = InternalMeasurement.State.PENDING - } + val expected = PENDING_INCREMENTAL_REACH_METRIC -private val INTERNAL_PENDING_UNION_ALL_WATCH_DURATION_MEASUREMENT = - INTERNAL_PENDING_NOT_CREATED_UNION_ALL_WATCH_DURATION_MEASUREMENT.copy { - cmmsMeasurementId = externalIdToApiId(404L) - } + // Verify proto argument of the internal MetricsCoroutineImplBase::createMetric + verifyProtoArgument(internalMetricsMock, MetricsCoroutineImplBase::createMetric) + .ignoringRepeatedFieldOrder() + .isEqualTo( + internalCreateMetricRequest { + metric = INTERNAL_REQUESTING_INCREMENTAL_REACH_METRIC + externalMetricId = "metric-id" + } + ) -private val INTERNAL_SUCCEEDED_UNION_ALL_WATCH_DURATION_MEASUREMENT = - INTERNAL_PENDING_UNION_ALL_WATCH_DURATION_MEASUREMENT.copy { - state = InternalMeasurement.State.SUCCEEDED - details = - InternalMeasurementKt.details { - results += - WATCH_DURATION_LIST.map { duration -> - InternalMeasurementKt.result { - watchDuration = - InternalMeasurementKt.ResultKt.watchDuration { - value = duration - noiseMechanism = NoiseMechanism.CONTINUOUS_LAPLACE - deterministicSum = InternalDeterministicSum.getDefaultInstance() - } - } + // Verify proto argument of MeasurementsCoroutineImplBase::batchCreateMeasurements + val measurementsCaptor: KArgumentCaptor = argumentCaptor() + verifyBlocking(measurementsMock, times(1)) { + batchCreateMeasurements(measurementsCaptor.capture()) + } + val capturedMeasurementRequests = measurementsCaptor.allValues + assertThat(capturedMeasurementRequests) + .ignoringRepeatedFieldOrder() + .ignoringFieldDescriptors(MEASUREMENT_SPEC_FIELD, ENCRYPTED_REQUISITION_SPEC_FIELD) + .containsExactly( + batchCreateMeasurementsRequest { + parent = request.parent + requests += createMeasurementRequest { + parent = request.parent + measurement = REQUESTING_UNION_ALL_REACH_MEASUREMENT + requestId = INTERNAL_PENDING_UNION_ALL_REACH_MEASUREMENT.cmmsCreateMeasurementRequestId } - } - } + requests += createMeasurementRequest { + parent = request.parent + measurement = REQUESTING_UNION_ALL_BUT_LAST_PUBLISHER_REACH_MEASUREMENT + requestId = + INTERNAL_PENDING_UNION_ALL_BUT_LAST_PUBLISHER_REACH_MEASUREMENT + .cmmsCreateMeasurementRequestId + } + } + ) -// Internal population measurements + capturedMeasurementRequests.single().requestsList.forEach { createMeasurementRequest -> + verifyMeasurementSpec( + createMeasurementRequest.measurement.measurementSpec, + MEASUREMENT_CONSUMER_CERTIFICATE, + TRUSTED_MEASUREMENT_CONSUMER_ISSUER, + ) -val INTERNAL_PENDING_POPULATION_MEASUREMENT = internalMeasurement { - cmmsMeasurementConsumerId = MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId - cmmsCreateMeasurementRequestId = "POPULATION_MEASUREMENT" - cmmsMeasurementId = externalIdToApiId(443L) - timeInterval = TIME_INTERVAL - primitiveReportingSetBases += primitiveReportingSetBasis { - externalReportingSetId = INTERNAL_POPULATION_REPORTING_SET.externalReportingSetId - filters += INCREMENTAL_REPORTING_SET_FILTER - } - state = InternalMeasurement.State.PENDING -} + val dataProvidersList = + createMeasurementRequest.measurement.dataProvidersList.sortedBy { it.key } -val INTERNAL_SUCCEEDED_POPULATION_MEASUREMENT = - INTERNAL_PENDING_POPULATION_MEASUREMENT.copy { - state = InternalMeasurement.State.SUCCEEDED - details = - InternalMeasurementKt.details { - results += - InternalMeasurementKt.result { - population = - InternalMeasurementKt.ResultKt.population { value = TOTAL_POPULATION_VALUE } + val measurementSpec: MeasurementSpec = + createMeasurementRequest.measurement.measurementSpec.unpack() + assertThat(measurementSpec) + .isEqualTo( + UNION_ALL_BUT_LAST_PUBLISHER_REACH_MEASUREMENT_SPEC.copy { + nonceHashes.clear() + nonceHashes += List(dataProvidersList.size) { Hashing.hashSha256(RANDOM_OUTPUT_LONG) } } - } - } - -// CMMS measurements -private val BASE_MEASUREMENT_SPEC = measurementSpec { - measurementPublicKey = MEASUREMENT_CONSUMER_PUBLIC_KEY.pack() - // TODO(world-federation-of-advertisers/cross-media-measurement#1301): Stop setting this field. - serializedMeasurementPublicKey = measurementPublicKey.value -} - -// CMMS incremental reach measurements -private val UNION_ALL_BUT_LAST_PUBLISHER_REACH_MEASUREMENT_SPEC = - BASE_MEASUREMENT_SPEC.copy { - nonceHashes += - listOf(Hashing.hashSha256(RANDOM_OUTPUT_LONG), Hashing.hashSha256(RANDOM_OUTPUT_LONG)) + ) - reach = - MeasurementSpecKt.reach { - privacyParams = differentialPrivacyParams { - epsilon = REACH_ONLY_REACH_EPSILON - delta = DIFFERENTIAL_PRIVACY_DELTA - } - } - vidSamplingInterval = - MeasurementSpecKt.vidSamplingInterval { - start = REACH_ONLY_VID_SAMPLING_START - width = REACH_ONLY_VID_SAMPLING_WIDTH + dataProvidersList.map { dataProviderEntry -> + val signedRequisitionSpec = + decryptRequisitionSpec( + dataProviderEntry.value.encryptedRequisitionSpec, + DATA_PROVIDER_PRIVATE_KEY_HANDLE, + ) + val requisitionSpec: RequisitionSpec = signedRequisitionSpec.unpack() + verifyRequisitionSpec( + signedRequisitionSpec, + requisitionSpec, + measurementSpec, + MEASUREMENT_CONSUMER_CERTIFICATE, + TRUSTED_MEASUREMENT_CONSUMER_ISSUER, + ) } - } - -private val REACH_PROTOCOL_CONFIG: ProtocolConfig = protocolConfig { - measurementType = ProtocolConfig.MeasurementType.REACH - protocols += - ProtocolConfigKt.protocol { - reachOnlyLiquidLegionsV2 = - ProtocolConfigKt.reachOnlyLiquidLegionsV2 { - sketchParams = reachOnlyLiquidLegionsSketchParams { - decayRate = REACH_ONLY_LLV2_DECAY_RATE - maxSize = REACH_ONLY_LLV2_SKETCH_SIZE - } - noiseMechanism = ProtocolConfig.NoiseMechanism.DISCRETE_GAUSSIAN - } } -} - -private val REQUESTING_UNION_ALL_REACH_MEASUREMENT = - BASE_MEASUREMENT.copy { - dataProviders += DATA_PROVIDERS.keys.map { DATA_PROVIDER_ENTRIES.getValue(it) } - measurementSpec = - signMeasurementSpec( - UNION_ALL_BUT_LAST_PUBLISHER_REACH_MEASUREMENT_SPEC.copy { - nonceHashes += Hashing.hashSha256(RANDOM_OUTPUT_LONG) - }, - MEASUREMENT_CONSUMER_SIGNING_KEY_HANDLE, + // Verify proto argument of internal MeasurementsCoroutineImplBase::batchSetCmmsMeasurementId + verifyProtoArgument( + internalMeasurementsMock, + InternalMeasurementsGrpcKt.MeasurementsCoroutineImplBase::batchSetCmmsMeasurementIds, ) - measurementReferenceId = - INTERNAL_PENDING_UNION_ALL_REACH_MEASUREMENT.cmmsCreateMeasurementRequestId - } -private val REQUESTING_UNION_ALL_BUT_LAST_PUBLISHER_REACH_MEASUREMENT = - BASE_MEASUREMENT.copy { - dataProviders += DATA_PROVIDERS.keys.take(2).map { DATA_PROVIDER_ENTRIES.getValue(it) } - - measurementSpec = - signMeasurementSpec( - UNION_ALL_BUT_LAST_PUBLISHER_REACH_MEASUREMENT_SPEC, - MEASUREMENT_CONSUMER_SIGNING_KEY_HANDLE, + .ignoringRepeatedFieldOrder() + .isEqualTo( + batchSetCmmsMeasurementIdsRequest { + cmmsMeasurementConsumerId = MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId + this.measurementIds += measurementIds { + cmmsCreateMeasurementRequestId = + INTERNAL_PENDING_UNION_ALL_REACH_MEASUREMENT.cmmsCreateMeasurementRequestId + cmmsMeasurementId = INTERNAL_PENDING_UNION_ALL_REACH_MEASUREMENT.cmmsMeasurementId + } + this.measurementIds += measurementIds { + cmmsCreateMeasurementRequestId = + INTERNAL_PENDING_UNION_ALL_BUT_LAST_PUBLISHER_REACH_MEASUREMENT + .cmmsCreateMeasurementRequestId + cmmsMeasurementId = + INTERNAL_PENDING_UNION_ALL_BUT_LAST_PUBLISHER_REACH_MEASUREMENT.cmmsMeasurementId + } + } ) - measurementReferenceId = - INTERNAL_PENDING_UNION_ALL_BUT_LAST_PUBLISHER_REACH_MEASUREMENT.cmmsCreateMeasurementRequestId - } -private val PENDING_UNION_ALL_REACH_MEASUREMENT = - REQUESTING_UNION_ALL_REACH_MEASUREMENT.copy { - name = - MeasurementKey( - MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId, - INTERNAL_PENDING_UNION_ALL_REACH_MEASUREMENT.cmmsMeasurementId, - ) - .toName() - protocolConfig = REACH_PROTOCOL_CONFIG - state = Measurement.State.COMPUTING - } -private val PENDING_UNION_ALL_BUT_LAST_PUBLISHER_REACH_MEASUREMENT = - REQUESTING_UNION_ALL_BUT_LAST_PUBLISHER_REACH_MEASUREMENT.copy { - name = - MeasurementKey( - MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId, - INTERNAL_PENDING_UNION_ALL_BUT_LAST_PUBLISHER_REACH_MEASUREMENT.cmmsMeasurementId, - ) - .toName() - protocolConfig = REACH_PROTOCOL_CONFIG - state = Measurement.State.COMPUTING + assertThat(result).isEqualTo(expected) } -private val SUCCEEDED_UNION_ALL_REACH_MEASUREMENT = - PENDING_UNION_ALL_REACH_MEASUREMENT.copy { - state = Measurement.State.SUCCEEDED + @Test + fun `createMetric creates measurements for single pub reach when single edp params set`() { + val cmmsMeasurementSpec = + measurementSpec { + measurementPublicKey = MEASUREMENT_CONSUMER_PUBLIC_KEY.pack() - results += resultOutput { - val result = - MeasurementKt.result { - reach = MeasurementKt.ResultKt.reach { value = UNION_ALL_REACH_VALUE } - } - encryptedResult = - encryptResult(signResult(result, AGGREGATOR_SIGNING_KEY), MEASUREMENT_CONSUMER_PUBLIC_KEY) - certificate = AGGREGATOR_CERTIFICATE.name - } - } -private val SUCCEEDED_UNION_ALL_BUT_LAST_PUBLISHER_REACH_MEASUREMENT = - PENDING_UNION_ALL_BUT_LAST_PUBLISHER_REACH_MEASUREMENT.copy { - state = Measurement.State.SUCCEEDED + nonceHashes.add(Hashing.hashSha256(RANDOM_OUTPUT_LONG)) - results += resultOutput { - val result = - MeasurementKt.result { - reach = MeasurementKt.ResultKt.reach { value = UNION_ALL_BUT_LAST_PUBLISHER_REACH_VALUE } - } - encryptedResult = - encryptResult(signResult(result, AGGREGATOR_SIGNING_KEY), MEASUREMENT_CONSUMER_PUBLIC_KEY) - certificate = AGGREGATOR_CERTIFICATE.name + reach = + MeasurementSpecKt.reach { + privacyParams = differentialPrivacyParams { + epsilon = SINGLE_DATA_PROVIDER_REACH_ONLY_REACH_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + } + vidSamplingInterval = + MeasurementSpecKt.vidSamplingInterval { + start = SINGLE_DATA_PROVIDER_REACH_ONLY_VID_SAMPLING_START + width = SINGLE_DATA_PROVIDER_REACH_ONLY_VID_SAMPLING_WIDTH + } + } + + val internalMeasurement = internalMeasurement { + cmmsMeasurementConsumerId = MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId + cmmsMeasurementId = "cmms_id" + cmmsCreateMeasurementRequestId = "SINGLE_PUBLISHER_REACH_MEASUREMENT" + timeInterval = TIME_INTERVAL + primitiveReportingSetBases += primitiveReportingSetBasis { + externalReportingSetId = INTERNAL_SINGLE_PUBLISHER_REPORTING_SET.externalReportingSetId + filters += METRIC_FILTER + filters += PRIMITIVE_REPORTING_SET_FILTER + } + state = InternalMeasurement.State.PENDING + isSingleDataProvider = true } - } -// CMMS single publisher reach-frequency measurements -private val SINGLE_PUBLISHER_REACH_FREQUENCY_MEASUREMENT_SPEC = - BASE_MEASUREMENT_SPEC.copy { - nonceHashes.add(Hashing.hashSha256(RANDOM_OUTPUT_LONG)) + val pendingSingleDataProviderReachMeasurementWithSingleDataProviderParams = + BASE_MEASUREMENT.copy { + dataProviders += DATA_PROVIDER_ENTRIES.getValue(DATA_PROVIDERS.keys.first()) + + measurementSpec = + signMeasurementSpec( + cmmsMeasurementSpec, + MEASUREMENT_CONSUMER_SIGNING_KEY_HANDLE, + ) + measurementReferenceId = + internalMeasurement.cmmsCreateMeasurementRequestId + + name = + MeasurementKey( + MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId, + internalMeasurement.cmmsMeasurementId, + ) + .toName() + protocolConfig = REACH_PROTOCOL_CONFIG + state = Measurement.State.COMPUTING + } - reachAndFrequency = - MeasurementSpecKt.reachAndFrequency { - reachPrivacyParams = differentialPrivacyParams { - epsilon = REACH_FREQUENCY_REACH_EPSILON - delta = DIFFERENTIAL_PRIVACY_DELTA + val internalPendingReachMetricWithSingleDataProviderParams = internalMetric { + cmmsMeasurementConsumerId = MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId + externalMetricId = "metric-id" + externalReportingSetId = INTERNAL_SINGLE_PUBLISHER_REPORTING_SET.externalReportingSetId + timeInterval = TIME_INTERVAL + metricSpec = internalMetricSpec { + reach = InternalMetricSpecKt.reachParams { + multipleDataProviderParams = InternalMetricSpecKt.params { + privacyParams = InternalMetricSpecKt.differentialPrivacyParams { + epsilon = REACH_ONLY_REACH_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + vidSamplingInterval = InternalMetricSpecKt.vidSamplingInterval { + start = REACH_ONLY_VID_SAMPLING_START + width = REACH_ONLY_VID_SAMPLING_WIDTH + } + } + singleDataProviderParams = InternalMetricSpecKt.params { + privacyParams = InternalMetricSpecKt.differentialPrivacyParams { + epsilon = SINGLE_DATA_PROVIDER_REACH_ONLY_REACH_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + vidSamplingInterval = InternalMetricSpecKt.vidSamplingInterval { + start = SINGLE_DATA_PROVIDER_REACH_ONLY_VID_SAMPLING_START + width = SINGLE_DATA_PROVIDER_REACH_ONLY_VID_SAMPLING_WIDTH + } + } } - frequencyPrivacyParams = differentialPrivacyParams { - epsilon = REACH_FREQUENCY_FREQUENCY_EPSILON - delta = DIFFERENTIAL_PRIVACY_DELTA + } + weightedMeasurements += weightedMeasurement { + weight = 1 + binaryRepresentation = 1 + measurement = internalMeasurement.copy { + clearCmmsMeasurementId() } - maximumFrequency = REACH_FREQUENCY_MAXIMUM_FREQUENCY } - vidSamplingInterval = - MeasurementSpecKt.vidSamplingInterval { - start = REACH_FREQUENCY_VID_SAMPLING_START - width = REACH_FREQUENCY_VID_SAMPLING_WIDTH + details = InternalMetricKt.details { + filters += METRIC_FILTER } - } - -private val REACH_FREQUENCY_PROTOCOL_CONFIG: ProtocolConfig = protocolConfig { - measurementType = ProtocolConfig.MeasurementType.REACH_AND_FREQUENCY - protocols += - ProtocolConfigKt.protocol { - direct = - ProtocolConfigKt.direct { - noiseMechanisms += - listOf( - ProtocolConfig.NoiseMechanism.NONE, - ProtocolConfig.NoiseMechanism.CONTINUOUS_LAPLACE, - ProtocolConfig.NoiseMechanism.CONTINUOUS_GAUSSIAN, - ) - deterministicCount = ProtocolConfig.Direct.DeterministicCount.getDefaultInstance() - liquidLegionsDistribution = - ProtocolConfig.Direct.LiquidLegionsDistribution.getDefaultInstance() - } + createTime = INTERNAL_PENDING_INCREMENTAL_REACH_METRIC.createTime + state = InternalMetric.State.RUNNING } -} - -private val REQUESTING_SINGLE_PUBLISHER_REACH_FREQUENCY_MEASUREMENT = - BASE_MEASUREMENT.copy { - dataProviders += DATA_PROVIDER_ENTRIES.getValue(DATA_PROVIDERS.keys.first()) - - measurementSpec = - signMeasurementSpec( - SINGLE_PUBLISHER_REACH_FREQUENCY_MEASUREMENT_SPEC, - MEASUREMENT_CONSUMER_SIGNING_KEY_HANDLE, - ) - measurementReferenceId = - INTERNAL_PENDING_SINGLE_PUBLISHER_REACH_FREQUENCY_MEASUREMENT.cmmsCreateMeasurementRequestId - } -private val PENDING_SINGLE_PUBLISHER_REACH_FREQUENCY_MEASUREMENT = - REQUESTING_SINGLE_PUBLISHER_REACH_FREQUENCY_MEASUREMENT.copy { - name = - MeasurementKey( - MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId, - INTERNAL_PENDING_SINGLE_PUBLISHER_REACH_FREQUENCY_MEASUREMENT.cmmsMeasurementId, + runBlocking { + whenever(internalMetricsMock.createMetric(any())) + .thenReturn(internalPendingReachMetricWithSingleDataProviderParams) + whenever(measurementsMock.batchCreateMeasurements(any())) + .thenReturn( + batchCreateMeasurementsResponse { + measurements += pendingSingleDataProviderReachMeasurementWithSingleDataProviderParams + } ) - .toName() - protocolConfig = REACH_FREQUENCY_PROTOCOL_CONFIG - state = Measurement.State.COMPUTING - } - -private val SUCCEEDED_SINGLE_PUBLISHER_REACH_FREQUENCY_MEASUREMENT = - PENDING_SINGLE_PUBLISHER_REACH_FREQUENCY_MEASUREMENT.copy { - state = Measurement.State.SUCCEEDED + } - results += resultOutput { - val result = - MeasurementKt.result { - reach = - MeasurementKt.ResultKt.reach { - value = REACH_FREQUENCY_REACH_VALUE - noiseMechanism = ProtocolConfig.NoiseMechanism.CONTINUOUS_LAPLACE - deterministicCountDistinct = DeterministicCountDistinct.getDefaultInstance() - } - frequency = - MeasurementKt.ResultKt.frequency { - relativeFrequencyDistribution.putAll(REACH_FREQUENCY_FREQUENCY_VALUE) - noiseMechanism = ProtocolConfig.NoiseMechanism.CONTINUOUS_LAPLACE - liquidLegionsDistribution = liquidLegionsDistribution { - decayRate = LL_DISTRIBUTION_DECAY_RATE - maxSize = LL_DISTRIBUTION_SKETCH_SIZE - } + val requestingReachMetricWithSingleDataProviderParams = metric { + reportingSet = INTERNAL_SINGLE_PUBLISHER_REPORTING_SET.resourceName + timeInterval = TIME_INTERVAL + metricSpec = metricSpec { + reach = reachParams { + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = MetricSpecKt.differentialPrivacyParams { + epsilon = REACH_ONLY_REACH_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + vidSamplingInterval = MetricSpecKt.vidSamplingInterval { + start = REACH_ONLY_VID_SAMPLING_START + width = REACH_ONLY_VID_SAMPLING_WIDTH + } + } + singleDataProviderParams = MetricSpecKt.params { + privacyParams = MetricSpecKt.differentialPrivacyParams { + epsilon = SINGLE_DATA_PROVIDER_REACH_ONLY_REACH_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + vidSamplingInterval = MetricSpecKt.vidSamplingInterval { + start = SINGLE_DATA_PROVIDER_REACH_ONLY_VID_SAMPLING_START + width = SINGLE_DATA_PROVIDER_REACH_ONLY_VID_SAMPLING_WIDTH } + } } - encryptedResult = - encryptResult(signResult(result, AGGREGATOR_SIGNING_KEY), MEASUREMENT_CONSUMER_PUBLIC_KEY) - certificate = AGGREGATOR_CERTIFICATE.name + } + filters += METRIC_FILTER } - } -// CMMS single publisher impression measurements -private val SINGLE_PUBLISHER_IMPRESSION_MEASUREMENT_SPEC = - BASE_MEASUREMENT_SPEC.copy { - nonceHashes.add(Hashing.hashSha256(RANDOM_OUTPUT_LONG)) + val request = createMetricRequest { + parent = MEASUREMENT_CONSUMERS.values.first().name + metric = requestingReachMetricWithSingleDataProviderParams + metricId = "metric-id" + } - impression = - MeasurementSpecKt.impression { - privacyParams = differentialPrivacyParams { - epsilon = IMPRESSION_EPSILON - delta = DIFFERENTIAL_PRIVACY_DELTA - } - maximumFrequencyPerUser = IMPRESSION_MAXIMUM_FREQUENCY_PER_USER - } - vidSamplingInterval = - MeasurementSpecKt.vidSamplingInterval { - start = IMPRESSION_VID_SAMPLING_START - width = IMPRESSION_VID_SAMPLING_WIDTH + val result = + withMeasurementConsumerPrincipal(request.parent, CONFIG) { + runBlocking { service.createMetric(request) } } - } -private val IMPRESSION_PROTOCOL_CONFIG: ProtocolConfig = protocolConfig { - measurementType = ProtocolConfig.MeasurementType.IMPRESSION - protocols += - ProtocolConfigKt.protocol { - direct = - ProtocolConfigKt.direct { - noiseMechanisms += - listOf( - ProtocolConfig.NoiseMechanism.NONE, - ProtocolConfig.NoiseMechanism.CONTINUOUS_LAPLACE, - ProtocolConfig.NoiseMechanism.CONTINUOUS_GAUSSIAN, - ) - deterministicCount = ProtocolConfig.Direct.DeterministicCount.getDefaultInstance() - } + val pendingReachMetricWithSingleDataProviderParams = requestingReachMetricWithSingleDataProviderParams.copy { + name = + MetricKey( + MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId, + request.metricId, + ) + .toName() + state = Metric.State.RUNNING + createTime = INTERNAL_PENDING_INCREMENTAL_REACH_METRIC.createTime } -} - -private val REQUESTING_SINGLE_PUBLISHER_IMPRESSION_MEASUREMENT = - BASE_MEASUREMENT.copy { - dataProviders += DATA_PROVIDER_ENTRIES.getValue(DATA_PROVIDERS.keys.first()) - measurementSpec = - signMeasurementSpec( - SINGLE_PUBLISHER_IMPRESSION_MEASUREMENT_SPEC, - MEASUREMENT_CONSUMER_SIGNING_KEY_HANDLE, - ) - measurementReferenceId = - INTERNAL_PENDING_SINGLE_PUBLISHER_IMPRESSION_MEASUREMENT.cmmsCreateMeasurementRequestId - } + assertThat(result).isEqualTo(pendingReachMetricWithSingleDataProviderParams) -private val PENDING_SINGLE_PUBLISHER_IMPRESSION_MEASUREMENT = - REQUESTING_SINGLE_PUBLISHER_IMPRESSION_MEASUREMENT.copy { - name = - MeasurementKey( - MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId, - INTERNAL_PENDING_SINGLE_PUBLISHER_IMPRESSION_MEASUREMENT.cmmsMeasurementId, - ) - .toName() - protocolConfig = IMPRESSION_PROTOCOL_CONFIG - state = Measurement.State.COMPUTING - } - -private val SUCCEEDED_SINGLE_PUBLISHER_IMPRESSION_MEASUREMENT = - PENDING_SINGLE_PUBLISHER_IMPRESSION_MEASUREMENT.copy { - state = Measurement.State.SUCCEEDED - results += resultOutput { - val result = - MeasurementKt.result { - impression = - MeasurementKt.ResultKt.impression { - value = IMPRESSION_VALUE - noiseMechanism = ProtocolConfig.NoiseMechanism.CONTINUOUS_LAPLACE - deterministicCount = DeterministicCount.getDefaultInstance() - } - } - encryptedResult = - encryptResult(signResult(result, AGGREGATOR_SIGNING_KEY), MEASUREMENT_CONSUMER_PUBLIC_KEY) - certificate = AGGREGATOR_CERTIFICATE.name - } - } - -private val SUCCEEDED_SINGLE_PUBLISHER_IMPRESSION_MEASUREMENT_CUSTOM_CAP = - PENDING_SINGLE_PUBLISHER_IMPRESSION_MEASUREMENT.copy { - state = Measurement.State.SUCCEEDED - results += resultOutput { - val result = - MeasurementKt.result { - impression = - MeasurementKt.ResultKt.impression { - value = IMPRESSION_VALUE - noiseMechanism = ProtocolConfig.NoiseMechanism.CONTINUOUS_LAPLACE - deterministicCount = deterministicCount { - customMaximumFrequencyPerUser = IMPRESSION_CUSTOM_MAXIMUM_FREQUENCY_PER_USER + // Verify proto argument of the internal MetricsCoroutineImplBase::createMetric + verifyProtoArgument(internalMetricsMock, MetricsCoroutineImplBase::createMetric) + .ignoringRepeatedFieldOrder() + .isEqualTo( + internalCreateMetricRequest { + metric = internalPendingReachMetricWithSingleDataProviderParams.copy { + clearState() + clearCreateTime() + clearExternalMetricId() + weightedMeasurements.clear() + weightedMeasurements += internalPendingReachMetricWithSingleDataProviderParams.weightedMeasurementsList.first().copy { + measurement = measurement.copy { + clearCmmsMeasurementId() + clearState() + clearCmmsCreateMeasurementRequestId() } } + } + externalMetricId = request.metricId } - encryptedResult = - encryptResult(signResult(result, AGGREGATOR_SIGNING_KEY), MEASUREMENT_CONSUMER_PUBLIC_KEY) - certificate = AGGREGATOR_CERTIFICATE.name - } - } - -// CMMS cross publisher watch duration measurements -private val UNION_ALL_WATCH_DURATION_MEASUREMENT_SPEC = - BASE_MEASUREMENT_SPEC.copy { - nonceHashes += - listOf( - Hashing.hashSha256(RANDOM_OUTPUT_LONG), - Hashing.hashSha256(RANDOM_OUTPUT_LONG), - Hashing.hashSha256(RANDOM_OUTPUT_LONG), ) - duration = - MeasurementSpecKt.duration { - privacyParams = differentialPrivacyParams { - epsilon = WATCH_DURATION_EPSILON - delta = DIFFERENTIAL_PRIVACY_DELTA - } - privacyParams = differentialPrivacyParams { - epsilon = WATCH_DURATION_EPSILON - delta = DIFFERENTIAL_PRIVACY_DELTA - } - maximumWatchDurationPerUser = MAXIMUM_WATCH_DURATION_PER_USER - } - vidSamplingInterval = - MeasurementSpecKt.vidSamplingInterval { - start = WATCH_DURATION_VID_SAMPLING_START - width = WATCH_DURATION_VID_SAMPLING_WIDTH - } - } - -private val WATCH_DURATION_PROTOCOL_CONFIG: ProtocolConfig = protocolConfig { - measurementType = ProtocolConfig.MeasurementType.DURATION - protocols += - ProtocolConfigKt.protocol { - direct = - ProtocolConfigKt.direct { - noiseMechanisms += - listOf( - ProtocolConfig.NoiseMechanism.NONE, - ProtocolConfig.NoiseMechanism.CONTINUOUS_LAPLACE, - ProtocolConfig.NoiseMechanism.CONTINUOUS_GAUSSIAN, - ) - deterministicSum = ProtocolConfig.Direct.DeterministicSum.getDefaultInstance() - } + // Verify proto argument of MeasurementsCoroutineImplBase::batchCreateMeasurements + val measurementsCaptor: KArgumentCaptor = argumentCaptor() + verifyBlocking(measurementsMock, times(1)) { + batchCreateMeasurements(measurementsCaptor.capture()) } -} - -private val REQUESTING_UNION_ALL_WATCH_DURATION_MEASUREMENT = - BASE_MEASUREMENT.copy { - dataProviders += DATA_PROVIDERS.keys.map { DATA_PROVIDER_ENTRIES.getValue(it) } - - measurementSpec = - signMeasurementSpec( - UNION_ALL_WATCH_DURATION_MEASUREMENT_SPEC.copy { - nonceHashes += Hashing.hashSha256(RANDOM_OUTPUT_LONG) - }, - MEASUREMENT_CONSUMER_SIGNING_KEY_HANDLE, - ) - } - -private val PENDING_UNION_ALL_WATCH_DURATION_MEASUREMENT = - REQUESTING_UNION_ALL_WATCH_DURATION_MEASUREMENT.copy { - name = - MeasurementKey( - MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId, - INTERNAL_PENDING_UNION_ALL_WATCH_DURATION_MEASUREMENT.cmmsMeasurementId, - ) - .toName() - protocolConfig = WATCH_DURATION_PROTOCOL_CONFIG - state = Measurement.State.COMPUTING - } - -private val SUCCEEDED_UNION_ALL_WATCH_DURATION_MEASUREMENT = - PENDING_UNION_ALL_WATCH_DURATION_MEASUREMENT.copy { - state = Measurement.State.SUCCEEDED - - results += - DATA_PROVIDERS.keys.zip(WATCH_DURATION_LIST).map { (dataProviderKey, watchDuration) -> - val dataProvider = DATA_PROVIDERS.getValue(dataProviderKey) - resultOutput { - val result = - MeasurementKt.result { - this.watchDuration = - MeasurementKt.ResultKt.watchDuration { - value = watchDuration - noiseMechanism = ProtocolConfig.NoiseMechanism.CONTINUOUS_LAPLACE - deterministicSum = DeterministicSum.getDefaultInstance() - } + val capturedMeasurementRequests = measurementsCaptor.allValues + assertThat(capturedMeasurementRequests) + .ignoringRepeatedFieldOrder() + .ignoringFieldDescriptors(MEASUREMENT_SPEC_FIELD, ENCRYPTED_REQUISITION_SPEC_FIELD) + .containsExactly( + batchCreateMeasurementsRequest { + parent = request.parent + requests += createMeasurementRequest { + parent = request.parent + measurement = pendingSingleDataProviderReachMeasurementWithSingleDataProviderParams.copy { + clearState() + clearProtocolConfig() + clearName() } - encryptedResult = - encryptResult( - signResult(result, DATA_PROVIDER_SIGNING_KEY), - MEASUREMENT_CONSUMER_PUBLIC_KEY, - ) - certificate = dataProvider.certificate + requestId = internalMeasurement.cmmsCreateMeasurementRequestId + } } - } - } - -// CMMS population measurements -private val POPULATION_MEASUREMENT_SPEC = - BASE_MEASUREMENT_SPEC.copy { - nonceHashes += - listOf( - Hashing.hashSha256(RANDOM_OUTPUT_LONG), - Hashing.hashSha256(RANDOM_OUTPUT_LONG), - Hashing.hashSha256(RANDOM_OUTPUT_LONG), ) - population = MeasurementSpec.Population.getDefaultInstance() - - vidSamplingInterval = MeasurementSpec.VidSamplingInterval.getDefaultInstance() - } - -private val REQUESTING_POPULATION_MEASUREMENT = - BASE_MEASUREMENT.copy { - dataProviders += DATA_PROVIDER_ENTRIES.getValue(DATA_PROVIDERS.keys.first()) + capturedMeasurementRequests.single().requestsList.forEach { createMeasurementRequest -> + verifyMeasurementSpec( + createMeasurementRequest.measurement.measurementSpec, + MEASUREMENT_CONSUMER_CERTIFICATE, + TRUSTED_MEASUREMENT_CONSUMER_ISSUER, + ) - measurementSpec = - signMeasurementSpec(POPULATION_MEASUREMENT_SPEC, MEASUREMENT_CONSUMER_SIGNING_KEY_HANDLE) + val dataProvidersList = + createMeasurementRequest.measurement.dataProvidersList.sortedBy { it.key } - measurementReferenceId = INTERNAL_PENDING_POPULATION_MEASUREMENT.cmmsCreateMeasurementRequestId - } + val measurementSpec: MeasurementSpec = + createMeasurementRequest.measurement.measurementSpec.unpack() + assertThat(measurementSpec) + .isEqualTo(cmmsMeasurementSpec) -private val PENDING_POPULATION_MEASUREMENT = - REQUESTING_POPULATION_MEASUREMENT.copy { - name = - MeasurementKey( - MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId, - INTERNAL_PENDING_POPULATION_MEASUREMENT.cmmsMeasurementId, + dataProvidersList.map { dataProviderEntry -> + val signedRequisitionSpec = + decryptRequisitionSpec( + dataProviderEntry.value.encryptedRequisitionSpec, + DATA_PROVIDER_PRIVATE_KEY_HANDLE, + ) + val requisitionSpec: RequisitionSpec = signedRequisitionSpec.unpack() + verifyRequisitionSpec( + signedRequisitionSpec, + requisitionSpec, + measurementSpec, + MEASUREMENT_CONSUMER_CERTIFICATE, + TRUSTED_MEASUREMENT_CONSUMER_ISSUER, ) - .toName() - state = Measurement.State.COMPUTING - } - -// Metric Specs - -private val REACH_METRIC_SPEC: MetricSpec = metricSpec { - reach = reachParams { privacyParams = MetricSpec.DifferentialPrivacyParams.getDefaultInstance() } -} -private val REACH_FREQUENCY_METRIC_SPEC: MetricSpec = metricSpec { - reachAndFrequency = reachAndFrequencyParams { - reachPrivacyParams = MetricSpec.DifferentialPrivacyParams.getDefaultInstance() - frequencyPrivacyParams = MetricSpec.DifferentialPrivacyParams.getDefaultInstance() - } -} -private val IMPRESSION_COUNT_METRIC_SPEC: MetricSpec = metricSpec { - impressionCount = impressionCountParams { - privacyParams = MetricSpec.DifferentialPrivacyParams.getDefaultInstance() - } -} -private val WATCH_DURATION_METRIC_SPEC: MetricSpec = metricSpec { - watchDuration = watchDurationParams { - privacyParams = MetricSpec.DifferentialPrivacyParams.getDefaultInstance() - } -} - -private val POPULATION_METRIC_SPEC: MetricSpec = metricSpec { - populationCount = MetricSpec.PopulationCountParams.getDefaultInstance() -} - -// Metrics - -// Metric idempotency keys -private const val INCREMENTAL_REACH_METRIC_IDEMPOTENCY_KEY = "TEST_INCREMENTAL_REACH_METRIC" - -// Internal Incremental Metrics -private val INTERNAL_REQUESTING_INCREMENTAL_REACH_METRIC = internalMetric { - cmmsMeasurementConsumerId = MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId - externalReportingSetId = INTERNAL_INCREMENTAL_REPORTING_SET.externalReportingSetId - timeInterval = TIME_INTERVAL - metricSpec = internalMetricSpec { - reach = - InternalMetricSpecKt.reachParams { - privacyParams = - InternalMetricSpecKt.differentialPrivacyParams { - epsilon = REACH_ONLY_REACH_EPSILON - delta = DIFFERENTIAL_PRIVACY_DELTA - } - } - vidSamplingInterval = - InternalMetricSpecKt.vidSamplingInterval { - start = REACH_ONLY_VID_SAMPLING_START - width = REACH_ONLY_VID_SAMPLING_WIDTH - } - } - weightedMeasurements += weightedMeasurement { - weight = 1 - binaryRepresentation = 3 - measurement = - INTERNAL_PENDING_UNION_ALL_REACH_MEASUREMENT.copy { - clearCmmsCreateMeasurementRequestId() - clearCmmsMeasurementId() - clearState() } - } - weightedMeasurements += weightedMeasurement { - weight = -1 - binaryRepresentation = 2 - measurement = - INTERNAL_PENDING_UNION_ALL_BUT_LAST_PUBLISHER_REACH_MEASUREMENT.copy { - clearCmmsCreateMeasurementRequestId() - clearCmmsMeasurementId() - clearState() - } - } - details = InternalMetricKt.details { filters += listOf(METRIC_FILTER) } -} + } -private val INTERNAL_PENDING_INITIAL_INCREMENTAL_REACH_METRIC = - INTERNAL_REQUESTING_INCREMENTAL_REACH_METRIC.copy { - externalMetricId = "331L" - createTime = Instant.now().toProtoTime() - state = InternalMetric.State.RUNNING - weightedMeasurements.clear() - weightedMeasurements += weightedMeasurement { - weight = 1 - binaryRepresentation = 3 - measurement = INTERNAL_PENDING_UNION_ALL_REACH_MEASUREMENT.copy { clearCmmsMeasurementId() } - } - weightedMeasurements += weightedMeasurement { - weight = -1 - binaryRepresentation = 2 - measurement = - INTERNAL_PENDING_UNION_ALL_BUT_LAST_PUBLISHER_REACH_MEASUREMENT.copy { - clearCmmsMeasurementId() + // Verify proto argument of internal MeasurementsCoroutineImplBase::batchSetCmmsMeasurementIds + verifyProtoArgument( + internalMeasurementsMock, + InternalMeasurementsGrpcKt.MeasurementsCoroutineImplBase::batchSetCmmsMeasurementIds, + ) + .ignoringRepeatedFieldOrder() + .isEqualTo( + batchSetCmmsMeasurementIdsRequest { + cmmsMeasurementConsumerId = MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId + this.measurementIds += measurementIds { + cmmsCreateMeasurementRequestId = + internalMeasurement.cmmsCreateMeasurementRequestId + cmmsMeasurementId = internalMeasurement.cmmsMeasurementId + } } - } + ) } -private val INTERNAL_PENDING_INCREMENTAL_REACH_METRIC = - INTERNAL_PENDING_INITIAL_INCREMENTAL_REACH_METRIC.copy { - weightedMeasurements.clear() - weightedMeasurements += weightedMeasurement { - weight = 1 - binaryRepresentation = 3 - measurement = INTERNAL_PENDING_UNION_ALL_REACH_MEASUREMENT - } - weightedMeasurements += weightedMeasurement { - weight = -1 - binaryRepresentation = 2 - measurement = INTERNAL_PENDING_UNION_ALL_BUT_LAST_PUBLISHER_REACH_MEASUREMENT - } - } + @Test + fun `createMetric creates measurements for single pub reach when single edp params not set`() { + val cmmsMeasurementSpec = + measurementSpec { + measurementPublicKey = MEASUREMENT_CONSUMER_PUBLIC_KEY.pack() -private val INTERNAL_SUCCEEDED_INCREMENTAL_REACH_METRIC = - INTERNAL_PENDING_INCREMENTAL_REACH_METRIC.copy { - state = InternalMetric.State.SUCCEEDED - weightedMeasurements.clear() - weightedMeasurements += weightedMeasurement { - weight = 1 - binaryRepresentation = 3 - measurement = INTERNAL_SUCCEEDED_UNION_ALL_REACH_MEASUREMENT - } - weightedMeasurements += weightedMeasurement { - weight = -1 - binaryRepresentation = 2 - measurement = INTERNAL_SUCCEEDED_UNION_ALL_BUT_LAST_PUBLISHER_REACH_MEASUREMENT - } - } + nonceHashes.add(Hashing.hashSha256(RANDOM_OUTPUT_LONG)) -// Internal Single publisher reach-frequency metrics -private val INTERNAL_REQUESTING_SINGLE_PUBLISHER_REACH_FREQUENCY_METRIC = internalMetric { - cmmsMeasurementConsumerId = MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId - externalReportingSetId = INTERNAL_SINGLE_PUBLISHER_REPORTING_SET.externalReportingSetId - timeInterval = TIME_INTERVAL - metricSpec = internalMetricSpec { - reachAndFrequency = - InternalMetricSpecKt.reachAndFrequencyParams { - reachPrivacyParams = - InternalMetricSpecKt.differentialPrivacyParams { - epsilon = REACH_FREQUENCY_REACH_EPSILON - delta = DIFFERENTIAL_PRIVACY_DELTA + reach = + MeasurementSpecKt.reach { + privacyParams = differentialPrivacyParams { + epsilon = REACH_ONLY_REACH_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } } - frequencyPrivacyParams = - InternalMetricSpecKt.differentialPrivacyParams { - epsilon = REACH_FREQUENCY_FREQUENCY_EPSILON - delta = DIFFERENTIAL_PRIVACY_DELTA + vidSamplingInterval = + MeasurementSpecKt.vidSamplingInterval { + start = REACH_ONLY_VID_SAMPLING_START + width = REACH_ONLY_VID_SAMPLING_WIDTH } - maximumFrequency = REACH_FREQUENCY_MAXIMUM_FREQUENCY - } - vidSamplingInterval = - InternalMetricSpecKt.vidSamplingInterval { - start = REACH_FREQUENCY_VID_SAMPLING_START - width = REACH_FREQUENCY_VID_SAMPLING_WIDTH } - } - weightedMeasurements += weightedMeasurement { - weight = 1 - binaryRepresentation = 1 - measurement = - INTERNAL_PENDING_SINGLE_PUBLISHER_REACH_FREQUENCY_MEASUREMENT.copy { - clearCmmsCreateMeasurementRequestId() - clearCmmsMeasurementId() - clearState() - } - } - details = InternalMetricKt.details { filters += listOf(METRIC_FILTER) } -} -private val INTERNAL_PENDING_INITIAL_SINGLE_PUBLISHER_REACH_FREQUENCY_METRIC = - INTERNAL_REQUESTING_SINGLE_PUBLISHER_REACH_FREQUENCY_METRIC.copy { - externalMetricId = "332L" - createTime = Instant.now().toProtoTime() - state = InternalMetric.State.RUNNING - weightedMeasurements.clear() - weightedMeasurements += weightedMeasurement { - weight = 1 - binaryRepresentation = 1 - measurement = - INTERNAL_PENDING_SINGLE_PUBLISHER_REACH_FREQUENCY_MEASUREMENT.copy { - clearCmmsMeasurementId() - } + val internalMeasurement = internalMeasurement { + cmmsMeasurementConsumerId = MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId + cmmsMeasurementId = "cmms_id" + cmmsCreateMeasurementRequestId = "SINGLE_PUBLISHER_REACH_MEASUREMENT" + timeInterval = TIME_INTERVAL + primitiveReportingSetBases += primitiveReportingSetBasis { + externalReportingSetId = INTERNAL_SINGLE_PUBLISHER_REPORTING_SET.externalReportingSetId + filters += METRIC_FILTER + filters += PRIMITIVE_REPORTING_SET_FILTER + } + state = InternalMeasurement.State.PENDING + isSingleDataProvider = true } - } -private val INTERNAL_PENDING_SINGLE_PUBLISHER_REACH_FREQUENCY_METRIC = - INTERNAL_PENDING_INITIAL_SINGLE_PUBLISHER_REACH_FREQUENCY_METRIC.copy { - weightedMeasurements.clear() - weightedMeasurements += weightedMeasurement { - weight = 1 - binaryRepresentation = 1 - measurement = INTERNAL_PENDING_SINGLE_PUBLISHER_REACH_FREQUENCY_MEASUREMENT - } - } + val pendingSingleDataProviderReachMeasurement = + BASE_MEASUREMENT.copy { + dataProviders += DATA_PROVIDER_ENTRIES.getValue(DATA_PROVIDERS.keys.first()) -private val INTERNAL_SUCCEEDED_SINGLE_PUBLISHER_REACH_FREQUENCY_METRIC = - INTERNAL_PENDING_SINGLE_PUBLISHER_REACH_FREQUENCY_METRIC.copy { - state = InternalMetric.State.SUCCEEDED - weightedMeasurements.clear() - weightedMeasurements += weightedMeasurement { - weight = 1 - binaryRepresentation = 1 - measurement = INTERNAL_SUCCEEDED_SINGLE_PUBLISHER_REACH_FREQUENCY_MEASUREMENT - } - } + measurementSpec = + signMeasurementSpec( + cmmsMeasurementSpec, + MEASUREMENT_CONSUMER_SIGNING_KEY_HANDLE, + ) + measurementReferenceId = + internalMeasurement.cmmsCreateMeasurementRequestId -// Internal Single publisher impression metrics -private val INTERNAL_REQUESTING_SINGLE_PUBLISHER_IMPRESSION_METRIC = internalMetric { - cmmsMeasurementConsumerId = MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId - externalReportingSetId = INTERNAL_SINGLE_PUBLISHER_REPORTING_SET.externalReportingSetId - timeInterval = TIME_INTERVAL - metricSpec = internalMetricSpec { - impressionCount = - InternalMetricSpecKt.impressionCountParams { - privacyParams = - InternalMetricSpecKt.differentialPrivacyParams { - epsilon = IMPRESSION_EPSILON - delta = DIFFERENTIAL_PRIVACY_DELTA + name = + MeasurementKey( + MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId, + internalMeasurement.cmmsMeasurementId, + ) + .toName() + protocolConfig = REACH_PROTOCOL_CONFIG + state = Measurement.State.COMPUTING + } + + val internalPendingReachMetric = internalMetric { + cmmsMeasurementConsumerId = MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId + externalMetricId = "metric-id" + externalReportingSetId = INTERNAL_SINGLE_PUBLISHER_REPORTING_SET.externalReportingSetId + timeInterval = TIME_INTERVAL + metricSpec = internalMetricSpec { + reach = InternalMetricSpecKt.reachParams { + multipleDataProviderParams = InternalMetricSpecKt.params { + privacyParams = InternalMetricSpecKt.differentialPrivacyParams { + epsilon = REACH_ONLY_REACH_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + vidSamplingInterval = InternalMetricSpecKt.vidSamplingInterval { + start = REACH_ONLY_VID_SAMPLING_START + width = REACH_ONLY_VID_SAMPLING_WIDTH + } } - maximumFrequencyPerUser = IMPRESSION_MAXIMUM_FREQUENCY_PER_USER + } } - vidSamplingInterval = - InternalMetricSpecKt.vidSamplingInterval { - start = IMPRESSION_VID_SAMPLING_START - width = IMPRESSION_VID_SAMPLING_WIDTH + weightedMeasurements += weightedMeasurement { + weight = 1 + binaryRepresentation = 1 + measurement = internalMeasurement.copy { + clearCmmsMeasurementId() + } } - } - weightedMeasurements += weightedMeasurement { - weight = 1 - binaryRepresentation = 1 - measurement = - INTERNAL_PENDING_SINGLE_PUBLISHER_IMPRESSION_MEASUREMENT.copy { - clearCmmsCreateMeasurementRequestId() - clearCmmsMeasurementId() - clearState() + details = InternalMetricKt.details { + filters += METRIC_FILTER } - } - details = InternalMetricKt.details { filters += listOf(METRIC_FILTER) } -} - -private val INTERNAL_PENDING_INITIAL_SINGLE_PUBLISHER_IMPRESSION_METRIC = - INTERNAL_REQUESTING_SINGLE_PUBLISHER_IMPRESSION_METRIC.copy { - externalMetricId = "333L" - createTime = Instant.now().toProtoTime() - state = InternalMetric.State.RUNNING - weightedMeasurements.clear() - weightedMeasurements += weightedMeasurement { - weight = 1 - binaryRepresentation = 1 - measurement = - INTERNAL_PENDING_SINGLE_PUBLISHER_IMPRESSION_MEASUREMENT.copy { clearCmmsMeasurementId() } - } - } - -private val INTERNAL_PENDING_SINGLE_PUBLISHER_IMPRESSION_METRIC = - INTERNAL_PENDING_INITIAL_SINGLE_PUBLISHER_IMPRESSION_METRIC.copy { - weightedMeasurements.clear() - weightedMeasurements += weightedMeasurement { - weight = 1 - binaryRepresentation = 1 - measurement = INTERNAL_PENDING_SINGLE_PUBLISHER_IMPRESSION_MEASUREMENT - } - } - -private val INTERNAL_FAILED_SINGLE_PUBLISHER_IMPRESSION_METRIC = - INTERNAL_PENDING_SINGLE_PUBLISHER_IMPRESSION_METRIC.copy { - state = InternalMetric.State.FAILED - weightedMeasurements.clear() - weightedMeasurements += weightedMeasurement { - weight = 1 - binaryRepresentation = 1 - measurement = INTERNAL_FAILED_SINGLE_PUBLISHER_IMPRESSION_MEASUREMENT - } - } - -private val INTERNAL_SUCCEEDED_SINGLE_PUBLISHER_IMPRESSION_METRIC = - INTERNAL_PENDING_SINGLE_PUBLISHER_IMPRESSION_METRIC.copy { - state = InternalMetric.State.SUCCEEDED - weightedMeasurements.clear() - weightedMeasurements += weightedMeasurement { - weight = 1 - binaryRepresentation = 1 - measurement = INTERNAL_SUCCEEDED_SINGLE_PUBLISHER_IMPRESSION_MEASUREMENT + createTime = INTERNAL_PENDING_INCREMENTAL_REACH_METRIC.createTime + state = InternalMetric.State.RUNNING } - } -private val INTERNAL_SUCCEEDED_SINGLE_PUBLISHER_IMPRESSION_METRIC_CUSTOM_CAP = - INTERNAL_PENDING_SINGLE_PUBLISHER_IMPRESSION_METRIC.copy { - state = InternalMetric.State.SUCCEEDED - weightedMeasurements.clear() - weightedMeasurements += weightedMeasurement { - weight = 1 - binaryRepresentation = 1 - measurement = INTERNAL_SUCCEEDED_SINGLE_PUBLISHER_IMPRESSION_MEASUREMENT_CUSTOM_CAP + runBlocking { + whenever(internalMetricsMock.createMetric(any())) + .thenReturn(internalPendingReachMetric) + whenever(measurementsMock.batchCreateMeasurements(any())) + .thenReturn( + batchCreateMeasurementsResponse { + measurements += pendingSingleDataProviderReachMeasurement + } + ) } - } -// Internal Cross Publisher Watch Duration Metrics -private val INTERNAL_REQUESTING_CROSS_PUBLISHER_WATCH_DURATION_METRIC = internalMetric { - cmmsMeasurementConsumerId = MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId - externalReportingSetId = INTERNAL_UNION_ALL_REPORTING_SET.externalReportingSetId - timeInterval = TIME_INTERVAL - metricSpec = internalMetricSpec { - watchDuration = - InternalMetricSpecKt.watchDurationParams { - privacyParams = - InternalMetricSpecKt.differentialPrivacyParams { - epsilon = WATCH_DURATION_EPSILON + val requestingReachMetric = metric { + reportingSet = INTERNAL_SINGLE_PUBLISHER_REPORTING_SET.resourceName + timeInterval = TIME_INTERVAL + metricSpec = metricSpec { + reach = reachParams { + privacyParams = MetricSpecKt.differentialPrivacyParams { + epsilon = REACH_ONLY_REACH_EPSILON delta = DIFFERENTIAL_PRIVACY_DELTA } - maximumWatchDurationPerUser = MAXIMUM_WATCH_DURATION_PER_USER - } - vidSamplingInterval = - InternalMetricSpecKt.vidSamplingInterval { - start = WATCH_DURATION_VID_SAMPLING_START - width = WATCH_DURATION_VID_SAMPLING_WIDTH + } + vidSamplingInterval = MetricSpecKt.vidSamplingInterval { + start = REACH_ONLY_VID_SAMPLING_START + width = REACH_ONLY_VID_SAMPLING_WIDTH + } } - } - weightedMeasurements += weightedMeasurement { - weight = 1 - binaryRepresentation = 1 - measurement = INTERNAL_REQUESTING_UNION_ALL_WATCH_DURATION_MEASUREMENT - } - details = InternalMetricKt.details { filters += listOf(METRIC_FILTER) } -} - -private val INTERNAL_PENDING_INITIAL_CROSS_PUBLISHER_WATCH_DURATION_METRIC = - INTERNAL_REQUESTING_CROSS_PUBLISHER_WATCH_DURATION_METRIC.copy { - externalMetricId = "334L" - createTime = Instant.now().toProtoTime() - state = InternalMetric.State.RUNNING - weightedMeasurements.clear() - weightedMeasurements += weightedMeasurement { - weight = 1 - binaryRepresentation = 1 - measurement = INTERNAL_PENDING_NOT_CREATED_UNION_ALL_WATCH_DURATION_MEASUREMENT - } - } - -private val INTERNAL_PENDING_CROSS_PUBLISHER_WATCH_DURATION_METRIC = - INTERNAL_PENDING_INITIAL_CROSS_PUBLISHER_WATCH_DURATION_METRIC.copy { - weightedMeasurements.clear() - weightedMeasurements += weightedMeasurement { - weight = 1 - binaryRepresentation = 1 - measurement = INTERNAL_PENDING_UNION_ALL_WATCH_DURATION_MEASUREMENT + filters += METRIC_FILTER } - } -private val INTERNAL_SUCCEEDED_CROSS_PUBLISHER_WATCH_DURATION_METRIC = - INTERNAL_PENDING_CROSS_PUBLISHER_WATCH_DURATION_METRIC.copy { - state = InternalMetric.State.SUCCEEDED - weightedMeasurements.clear() - weightedMeasurements += weightedMeasurement { - weight = 1 - binaryRepresentation = 1 - measurement = INTERNAL_SUCCEEDED_UNION_ALL_WATCH_DURATION_MEASUREMENT + val request = createMetricRequest { + parent = MEASUREMENT_CONSUMERS.values.first().name + metric = requestingReachMetric + metricId = "metric-id" } - } - -// Internal population metric -val INTERNAL_REQUESTING_POPULATION_METRIC = internalMetric { - cmmsMeasurementConsumerId = MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId - externalReportingSetId = INTERNAL_POPULATION_REPORTING_SET.externalReportingSetId - timeInterval = TIME_INTERVAL - metricSpec = internalMetricSpec { - populationCount = InternalMetricSpec.PopulationCountParams.getDefaultInstance() - } - weightedMeasurements += weightedMeasurement { - weight = 1 - binaryRepresentation = 1 - measurement = - INTERNAL_PENDING_POPULATION_MEASUREMENT.copy { - clearCmmsCreateMeasurementRequestId() - clearCmmsMeasurementId() - clearState() + val result = + withMeasurementConsumerPrincipal(request.parent, CONFIG) { + runBlocking { service.createMetric(request) } } - } - details = InternalMetricKt.details { filters += listOf(INCREMENTAL_REPORTING_SET_FILTER) } -} - -private val INTERNAL_PENDING_INITIAL_POPULATION_METRIC = - INTERNAL_REQUESTING_POPULATION_METRIC.copy { - externalMetricId = "331L" - createTime = Instant.now().toProtoTime() - state = InternalMetric.State.RUNNING - weightedMeasurements.clear() - - weightedMeasurements += weightedMeasurement { - weight = 1 - binaryRepresentation = 1 - measurement = INTERNAL_PENDING_POPULATION_MEASUREMENT.copy { clearCmmsMeasurementId() } - } - } - -val INTERNAL_PENDING_POPULATION_METRIC = - INTERNAL_PENDING_INITIAL_POPULATION_METRIC.copy { - weightedMeasurements.clear() - weightedMeasurements += weightedMeasurement { - weight = 1 - binaryRepresentation = 1 - measurement = INTERNAL_PENDING_POPULATION_MEASUREMENT - } - } -val INTERNAL_SUCCEEDED_POPULATION_METRIC = - INTERNAL_PENDING_POPULATION_METRIC.copy { - state = InternalMetric.State.SUCCEEDED - weightedMeasurements.clear() - weightedMeasurements += weightedMeasurement { - weight = 1 - binaryRepresentation = 1 - measurement = INTERNAL_SUCCEEDED_POPULATION_MEASUREMENT - } - } - -// Public Metrics - -// Incremental reach metrics -private val REQUESTING_INCREMENTAL_REACH_METRIC = metric { - reportingSet = INTERNAL_INCREMENTAL_REPORTING_SET.resourceName - timeInterval = TIME_INTERVAL - metricSpec = REACH_METRIC_SPEC - filters += INTERNAL_PENDING_INCREMENTAL_REACH_METRIC.details.filtersList -} - -private val PENDING_INCREMENTAL_REACH_METRIC = - REQUESTING_INCREMENTAL_REACH_METRIC.copy { - name = - MetricKey( + val pendingReachMetric = requestingReachMetric.copy { + name = + MetricKey( MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId, - INTERNAL_PENDING_INCREMENTAL_REACH_METRIC.externalMetricId, + request.metricId, ) - .toName() - state = Metric.State.RUNNING - metricSpec = metricSpec { - reach = reachParams { - privacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = REACH_ONLY_REACH_EPSILON - delta = DIFFERENTIAL_PRIVACY_DELTA - } - } - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = REACH_ONLY_VID_SAMPLING_START - width = REACH_ONLY_VID_SAMPLING_WIDTH - } + .toName() + state = Metric.State.RUNNING + createTime = INTERNAL_PENDING_INCREMENTAL_REACH_METRIC.createTime } - createTime = INTERNAL_PENDING_INCREMENTAL_REACH_METRIC.createTime - } -private const val VARIANCE_VALUE = 4.0 + assertThat(result).isEqualTo(pendingReachMetric) -private val FREQUENCY_VARIANCE: Map = - (1..REACH_FREQUENCY_MAXIMUM_FREQUENCY).associateWith { it.toDouble().pow(2.0) } -private val FREQUENCY_VARIANCES = - FrequencyVariances(FREQUENCY_VARIANCE, FREQUENCY_VARIANCE, FREQUENCY_VARIANCE, FREQUENCY_VARIANCE) - -private val SUCCEEDED_INCREMENTAL_REACH_METRIC = - PENDING_INCREMENTAL_REACH_METRIC.copy { - state = Metric.State.SUCCEEDED - - result = metricResult { - reach = - MetricResultKt.reachResult { - value = INCREMENTAL_REACH_VALUE - univariateStatistics = univariateStatistics { standardDeviation = sqrt(VARIANCE_VALUE) } + // Verify proto argument of the internal MetricsCoroutineImplBase::createMetric + verifyProtoArgument(internalMetricsMock, MetricsCoroutineImplBase::createMetric) + .ignoringRepeatedFieldOrder() + .isEqualTo( + internalCreateMetricRequest { + metric = internalPendingReachMetric.copy { + clearState() + clearCreateTime() + clearExternalMetricId() + weightedMeasurements.clear() + weightedMeasurements += internalPendingReachMetric.weightedMeasurementsList.first().copy { + measurement = measurement.copy { + clearCmmsMeasurementId() + clearState() + clearCmmsCreateMeasurementRequestId() + } + } + } + externalMetricId = request.metricId } - cmmsMeasurements += PENDING_UNION_ALL_REACH_MEASUREMENT.name - cmmsMeasurements += PENDING_UNION_ALL_BUT_LAST_PUBLISHER_REACH_MEASUREMENT.name + ) + + // Verify proto argument of MeasurementsCoroutineImplBase::batchCreateMeasurements + val measurementsCaptor: KArgumentCaptor = argumentCaptor() + verifyBlocking(measurementsMock, times(1)) { + batchCreateMeasurements(measurementsCaptor.capture()) } - } + val capturedMeasurementRequests = measurementsCaptor.allValues + assertThat(capturedMeasurementRequests) + .ignoringRepeatedFieldOrder() + .ignoringFieldDescriptors(MEASUREMENT_SPEC_FIELD, ENCRYPTED_REQUISITION_SPEC_FIELD) + .containsExactly( + batchCreateMeasurementsRequest { + parent = request.parent + requests += createMeasurementRequest { + parent = request.parent + measurement = pendingSingleDataProviderReachMeasurement.copy { + clearState() + clearProtocolConfig() + clearName() + } + requestId = internalMeasurement.cmmsCreateMeasurementRequestId + } + } + ) -// Single publisher reach-frequency metrics -private val REQUESTING_SINGLE_PUBLISHER_REACH_FREQUENCY_METRIC = metric { - reportingSet = INTERNAL_SINGLE_PUBLISHER_REPORTING_SET.resourceName - timeInterval = TIME_INTERVAL - metricSpec = REACH_FREQUENCY_METRIC_SPEC - filters += INTERNAL_REQUESTING_SINGLE_PUBLISHER_REACH_FREQUENCY_METRIC.details.filtersList -} + capturedMeasurementRequests.single().requestsList.forEach { createMeasurementRequest -> + verifyMeasurementSpec( + createMeasurementRequest.measurement.measurementSpec, + MEASUREMENT_CONSUMER_CERTIFICATE, + TRUSTED_MEASUREMENT_CONSUMER_ISSUER, + ) -private val PENDING_SINGLE_PUBLISHER_REACH_FREQUENCY_METRIC = - REQUESTING_SINGLE_PUBLISHER_REACH_FREQUENCY_METRIC.copy { - name = - MetricKey( - MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId, - INTERNAL_PENDING_SINGLE_PUBLISHER_REACH_FREQUENCY_METRIC.externalMetricId, + val dataProvidersList = + createMeasurementRequest.measurement.dataProvidersList.sortedBy { it.key } + + val measurementSpec: MeasurementSpec = + createMeasurementRequest.measurement.measurementSpec.unpack() + assertThat(measurementSpec) + .isEqualTo(cmmsMeasurementSpec) + + dataProvidersList.map { dataProviderEntry -> + val signedRequisitionSpec = + decryptRequisitionSpec( + dataProviderEntry.value.encryptedRequisitionSpec, + DATA_PROVIDER_PRIVATE_KEY_HANDLE, + ) + val requisitionSpec: RequisitionSpec = signedRequisitionSpec.unpack() + verifyRequisitionSpec( + signedRequisitionSpec, + requisitionSpec, + measurementSpec, + MEASUREMENT_CONSUMER_CERTIFICATE, + TRUSTED_MEASUREMENT_CONSUMER_ISSUER, ) - .toName() - metricSpec = metricSpec { - reachAndFrequency = reachAndFrequencyParams { - reachPrivacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = REACH_FREQUENCY_REACH_EPSILON - delta = DIFFERENTIAL_PRIVACY_DELTA - } - frequencyPrivacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = REACH_FREQUENCY_FREQUENCY_EPSILON - delta = DIFFERENTIAL_PRIVACY_DELTA - } - maximumFrequency = REACH_FREQUENCY_MAXIMUM_FREQUENCY } - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = REACH_FREQUENCY_VID_SAMPLING_START - width = REACH_FREQUENCY_VID_SAMPLING_WIDTH - } } - state = Metric.State.RUNNING - createTime = INTERNAL_PENDING_SINGLE_PUBLISHER_REACH_FREQUENCY_METRIC.createTime - } -private val SUCCEEDED_SINGLE_PUBLISHER_REACH_FREQUENCY_METRIC = - PENDING_SINGLE_PUBLISHER_REACH_FREQUENCY_METRIC.copy { - state = Metric.State.SUCCEEDED - result = metricResult { - cmmsMeasurements += PENDING_SINGLE_PUBLISHER_REACH_FREQUENCY_MEASUREMENT.name - reachAndFrequency = - MetricResultKt.reachAndFrequencyResult { - reach = - MetricResultKt.reachResult { - value = REACH_FREQUENCY_REACH_VALUE - univariateStatistics = univariateStatistics { - standardDeviation = sqrt(VARIANCE_VALUE) - } - } - frequencyHistogram = - MetricResultKt.histogramResult { - bins += - (1..REACH_FREQUENCY_MAXIMUM_FREQUENCY).map { frequency -> - MetricResultKt.HistogramResultKt.bin { - label = frequency.toString() - binResult = - MetricResultKt.HistogramResultKt.binResult { - value = - REACH_FREQUENCY_REACH_VALUE * - REACH_FREQUENCY_FREQUENCY_VALUE.getOrDefault(frequency.toLong(), 0.0) - } - resultUnivariateStatistics = univariateStatistics { - standardDeviation = - sqrt(FREQUENCY_VARIANCES.countVariances.getValue(frequency)) - } - relativeUnivariateStatistics = univariateStatistics { - standardDeviation = - sqrt(FREQUENCY_VARIANCES.relativeVariances.getValue(frequency)) - } - kPlusUnivariateStatistics = univariateStatistics { - standardDeviation = - sqrt(FREQUENCY_VARIANCES.kPlusCountVariances.getValue(frequency)) - } - relativeKPlusUnivariateStatistics = univariateStatistics { - standardDeviation = - sqrt(FREQUENCY_VARIANCES.kPlusRelativeVariances.getValue(frequency)) - } - } - } - } + // Verify proto argument of internal MeasurementsCoroutineImplBase::batchSetCmmsMeasurementIds + verifyProtoArgument( + internalMeasurementsMock, + InternalMeasurementsGrpcKt.MeasurementsCoroutineImplBase::batchSetCmmsMeasurementIds, + ) + .ignoringRepeatedFieldOrder() + .isEqualTo( + batchSetCmmsMeasurementIdsRequest { + cmmsMeasurementConsumerId = MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId + this.measurementIds += measurementIds { + cmmsCreateMeasurementRequestId = + internalMeasurement.cmmsCreateMeasurementRequestId + cmmsMeasurementId = internalMeasurement.cmmsMeasurementId + } } - } + ) } -// Single publisher impression metrics -private val REQUESTING_SINGLE_PUBLISHER_IMPRESSION_METRIC = metric { - reportingSet = INTERNAL_SINGLE_PUBLISHER_REPORTING_SET.resourceName - timeInterval = TIME_INTERVAL - metricSpec = IMPRESSION_COUNT_METRIC_SPEC - filters += INTERNAL_REQUESTING_SINGLE_PUBLISHER_IMPRESSION_METRIC.details.filtersList -} + @Test + fun `createMetric creates measurements for single pub rf when single edp params set`() { + val cmmsMeasurementSpec = + measurementSpec { + measurementPublicKey = MEASUREMENT_CONSUMER_PUBLIC_KEY.pack() -private val PENDING_SINGLE_PUBLISHER_IMPRESSION_METRIC = - REQUESTING_SINGLE_PUBLISHER_IMPRESSION_METRIC.copy { - name = - MetricKey( - MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId, - INTERNAL_PENDING_SINGLE_PUBLISHER_IMPRESSION_METRIC.externalMetricId, - ) - .toName() - metricSpec = metricSpec { - impressionCount = impressionCountParams { - privacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = IMPRESSION_EPSILON - delta = DIFFERENTIAL_PRIVACY_DELTA + nonceHashes.add(Hashing.hashSha256(RANDOM_OUTPUT_LONG)) + + reachAndFrequency = + MeasurementSpecKt.reachAndFrequency { + reachPrivacyParams = differentialPrivacyParams { + epsilon = SINGLE_DATA_PROVIDER_REACH_FREQUENCY_REACH_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + frequencyPrivacyParams = differentialPrivacyParams { + epsilon = SINGLE_DATA_PROVIDER_REACH_FREQUENCY_FREQUENCY_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + maximumFrequency = REACH_FREQUENCY_MAXIMUM_FREQUENCY + } + vidSamplingInterval = + MeasurementSpecKt.vidSamplingInterval { + start = SINGLE_DATA_PROVIDER_REACH_FREQUENCY_VID_SAMPLING_START + width = SINGLE_DATA_PROVIDER_REACH_FREQUENCY_VID_SAMPLING_WIDTH } - maximumFrequencyPerUser = IMPRESSION_MAXIMUM_FREQUENCY_PER_USER } - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = IMPRESSION_VID_SAMPLING_START - width = IMPRESSION_VID_SAMPLING_WIDTH - } + + val internalMeasurement = internalMeasurement { + cmmsMeasurementConsumerId = MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId + cmmsMeasurementId = "cmms_id" + cmmsCreateMeasurementRequestId = "SINGLE_PUBLISHER_REACH_MEASUREMENT" + timeInterval = TIME_INTERVAL + primitiveReportingSetBases += primitiveReportingSetBasis { + externalReportingSetId = INTERNAL_SINGLE_PUBLISHER_REPORTING_SET.externalReportingSetId + filters += METRIC_FILTER + filters += PRIMITIVE_REPORTING_SET_FILTER + } + state = InternalMeasurement.State.PENDING + isSingleDataProvider = true } - state = Metric.State.RUNNING - createTime = INTERNAL_PENDING_SINGLE_PUBLISHER_IMPRESSION_METRIC.createTime - } -private val FAILED_SINGLE_PUBLISHER_IMPRESSION_METRIC = - PENDING_SINGLE_PUBLISHER_IMPRESSION_METRIC.copy { state = Metric.State.FAILED } + val pendingSingleDataProviderReachAndFrequencyMeasurementWithSingleDataProviderParams = + BASE_MEASUREMENT.copy { + dataProviders += DATA_PROVIDER_ENTRIES.getValue(DATA_PROVIDERS.keys.first()) -private val SUCCEEDED_SINGLE_PUBLISHER_IMPRESSION_METRIC = - PENDING_SINGLE_PUBLISHER_IMPRESSION_METRIC.copy { - state = Metric.State.SUCCEEDED - result = metricResult { - impressionCount = - MetricResultKt.impressionCountResult { - value = IMPRESSION_VALUE - univariateStatistics = univariateStatistics { standardDeviation = sqrt(VARIANCE_VALUE) } - } - cmmsMeasurements += PENDING_SINGLE_PUBLISHER_IMPRESSION_MEASUREMENT.name - } - } + measurementSpec = + signMeasurementSpec( + cmmsMeasurementSpec, + MEASUREMENT_CONSUMER_SIGNING_KEY_HANDLE, + ) + measurementReferenceId = + internalMeasurement.cmmsCreateMeasurementRequestId -// Cross publisher watch duration metrics -private val REQUESTING_CROSS_PUBLISHER_WATCH_DURATION_METRIC = metric { - reportingSet = INTERNAL_UNION_ALL_REPORTING_SET.resourceName - timeInterval = TIME_INTERVAL - metricSpec = WATCH_DURATION_METRIC_SPEC - filters += INTERNAL_PENDING_CROSS_PUBLISHER_WATCH_DURATION_METRIC.details.filtersList -} + name = + MeasurementKey( + MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId, + internalMeasurement.cmmsMeasurementId, + ) + .toName() + protocolConfig = REACH_FREQUENCY_PROTOCOL_CONFIG + state = Measurement.State.COMPUTING + } -private val PENDING_CROSS_PUBLISHER_WATCH_DURATION_METRIC = - REQUESTING_CROSS_PUBLISHER_WATCH_DURATION_METRIC.copy { - name = - MetricKey( - MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId, - INTERNAL_PENDING_CROSS_PUBLISHER_WATCH_DURATION_METRIC.externalMetricId, - ) - .toName() - metricSpec = metricSpec { - watchDuration = watchDurationParams { - privacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = WATCH_DURATION_EPSILON - delta = DIFFERENTIAL_PRIVACY_DELTA + val internalPendingReachAndFrequencyMetricWithSingleDataProviderParams = internalMetric { + cmmsMeasurementConsumerId = MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId + externalMetricId = "metric-id" + externalReportingSetId = INTERNAL_SINGLE_PUBLISHER_REPORTING_SET.externalReportingSetId + timeInterval = TIME_INTERVAL + metricSpec = internalMetricSpec { + reachAndFrequency = InternalMetricSpecKt.reachAndFrequencyParams { + multipleDataProviderParams = InternalMetricSpecKt.params { + privacyParams = InternalMetricSpecKt.differentialPrivacyParams { + epsilon = REACH_FREQUENCY_REACH_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + frequencyPrivacyParams = InternalMetricSpecKt.differentialPrivacyParams { + epsilon = REACH_FREQUENCY_FREQUENCY_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + vidSamplingInterval = InternalMetricSpecKt.vidSamplingInterval { + start = REACH_FREQUENCY_VID_SAMPLING_START + width = REACH_FREQUENCY_VID_SAMPLING_WIDTH + } + } + singleDataProviderParams = InternalMetricSpecKt.params { + privacyParams = InternalMetricSpecKt.differentialPrivacyParams { + epsilon = SINGLE_DATA_PROVIDER_REACH_FREQUENCY_REACH_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + frequencyPrivacyParams = InternalMetricSpecKt.differentialPrivacyParams { + epsilon = SINGLE_DATA_PROVIDER_REACH_FREQUENCY_FREQUENCY_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + vidSamplingInterval = InternalMetricSpecKt.vidSamplingInterval { + start = SINGLE_DATA_PROVIDER_REACH_FREQUENCY_VID_SAMPLING_START + width = SINGLE_DATA_PROVIDER_REACH_FREQUENCY_VID_SAMPLING_WIDTH + } } - maximumWatchDurationPerUser = MAXIMUM_WATCH_DURATION_PER_USER + maximumFrequency = REACH_FREQUENCY_MAXIMUM_FREQUENCY + } } - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = WATCH_DURATION_VID_SAMPLING_START - width = WATCH_DURATION_VID_SAMPLING_WIDTH + weightedMeasurements += weightedMeasurement { + weight = 1 + binaryRepresentation = 1 + measurement = internalMeasurement.copy { + clearCmmsMeasurementId() } + } + details = InternalMetricKt.details { + filters += METRIC_FILTER + } + createTime = INTERNAL_PENDING_SINGLE_PUBLISHER_REACH_FREQUENCY_METRIC.createTime + state = InternalMetric.State.RUNNING + } + + runBlocking { + whenever(internalMetricsMock.createMetric(any())) + .thenReturn(internalPendingReachAndFrequencyMetricWithSingleDataProviderParams) + whenever(measurementsMock.batchCreateMeasurements(any())) + .thenReturn( + batchCreateMeasurementsResponse { + measurements += pendingSingleDataProviderReachAndFrequencyMeasurementWithSingleDataProviderParams + } + ) } - state = Metric.State.RUNNING - createTime = INTERNAL_PENDING_CROSS_PUBLISHER_WATCH_DURATION_METRIC.createTime - } -private val SUCCEEDED_CROSS_PUBLISHER_WATCH_DURATION_METRIC = - PENDING_CROSS_PUBLISHER_WATCH_DURATION_METRIC.copy { - state = Metric.State.SUCCEEDED - result = metricResult { - watchDuration = - MetricResultKt.watchDurationResult { - value = TOTAL_WATCH_DURATION.seconds.toDouble() - univariateStatistics = univariateStatistics { - standardDeviation = sqrt(WATCH_DURATION_LIST.sumOf { VARIANCE_VALUE }) + val requestingReachAndFrequencyMetricWithSingleDataProviderParams = metric { + reportingSet = INTERNAL_SINGLE_PUBLISHER_REPORTING_SET.resourceName + timeInterval = TIME_INTERVAL + metricSpec = metricSpec { + reachAndFrequency = reachAndFrequencyParams { + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = MetricSpecKt.differentialPrivacyParams { + epsilon = REACH_FREQUENCY_REACH_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + frequencyPrivacyParams = MetricSpecKt.differentialPrivacyParams { + epsilon = REACH_FREQUENCY_FREQUENCY_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + vidSamplingInterval = MetricSpecKt.vidSamplingInterval { + start = REACH_FREQUENCY_VID_SAMPLING_START + width = REACH_FREQUENCY_VID_SAMPLING_WIDTH + } + } + singleDataProviderParams = MetricSpecKt.params { + privacyParams = MetricSpecKt.differentialPrivacyParams { + epsilon = SINGLE_DATA_PROVIDER_REACH_FREQUENCY_REACH_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + frequencyPrivacyParams = MetricSpecKt.differentialPrivacyParams { + epsilon = SINGLE_DATA_PROVIDER_REACH_FREQUENCY_FREQUENCY_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + vidSamplingInterval = MetricSpecKt.vidSamplingInterval { + start = SINGLE_DATA_PROVIDER_REACH_FREQUENCY_VID_SAMPLING_START + width = SINGLE_DATA_PROVIDER_REACH_FREQUENCY_VID_SAMPLING_WIDTH + } } + maximumFrequency = REACH_FREQUENCY_MAXIMUM_FREQUENCY } - cmmsMeasurements += PENDING_UNION_ALL_WATCH_DURATION_MEASUREMENT.name + } + filters += METRIC_FILTER } - } -// Population metric -val REQUESTING_POPULATION_METRIC = metric { - reportingSet = INTERNAL_POPULATION_REPORTING_SET.resourceName - timeInterval = TIME_INTERVAL - metricSpec = POPULATION_METRIC_SPEC - filters += INTERNAL_PENDING_POPULATION_METRIC.details.filtersList -} + val request = createMetricRequest { + parent = MEASUREMENT_CONSUMERS.values.first().name + metric = requestingReachAndFrequencyMetricWithSingleDataProviderParams + metricId = "metric-id" + } -val PENDING_POPULATION_METRIC = - REQUESTING_POPULATION_METRIC.copy { - name = - MetricKey( + val result = + withMeasurementConsumerPrincipal(request.parent, CONFIG) { + runBlocking { service.createMetric(request) } + } + + val pendingReachAndFrequencyMetricWithSingleDataProviderParams = requestingReachAndFrequencyMetricWithSingleDataProviderParams.copy { + name = + MetricKey( MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId, - INTERNAL_PENDING_POPULATION_METRIC.externalMetricId, + request.metricId, ) - .toName() - state = Metric.State.RUNNING - metricSpec = metricSpec { - populationCount = MetricSpec.PopulationCountParams.getDefaultInstance() - } - createTime = INTERNAL_PENDING_POPULATION_METRIC.createTime - } - -val SUCCEEDED_POPULATION_METRIC = - PENDING_POPULATION_METRIC.copy { - state = Metric.State.SUCCEEDED - result = metricResult { - populationCount = MetricResultKt.populationCountResult { value = TOTAL_POPULATION_VALUE } - cmmsMeasurements += PENDING_POPULATION_MEASUREMENT.name + .toName() + state = Metric.State.RUNNING + createTime = INTERNAL_PENDING_SINGLE_PUBLISHER_REACH_FREQUENCY_METRIC.createTime } - } -@RunWith(JUnit4::class) -class MetricsServiceTest { + assertThat(result).isEqualTo(pendingReachAndFrequencyMetricWithSingleDataProviderParams) - private val internalMetricsMock: MetricsCoroutineImplBase = mockService { - onBlocking { createMetric(any()) }.thenReturn(INTERNAL_PENDING_INITIAL_INCREMENTAL_REACH_METRIC) - onBlocking { batchCreateMetrics(any()) } - .thenReturn( - internalBatchCreateMetricsResponse { - metrics += INTERNAL_PENDING_INITIAL_INCREMENTAL_REACH_METRIC - metrics += INTERNAL_PENDING_INITIAL_SINGLE_PUBLISHER_IMPRESSION_METRIC + // Verify proto argument of the internal MetricsCoroutineImplBase::createMetric + verifyProtoArgument(internalMetricsMock, MetricsCoroutineImplBase::createMetric) + .ignoringRepeatedFieldOrder() + .isEqualTo( + internalCreateMetricRequest { + metric = internalPendingReachAndFrequencyMetricWithSingleDataProviderParams.copy { + clearState() + clearCreateTime() + clearExternalMetricId() + weightedMeasurements.clear() + weightedMeasurements += internalPendingReachAndFrequencyMetricWithSingleDataProviderParams.weightedMeasurementsList.first().copy { + measurement = measurement.copy { + clearCmmsMeasurementId() + clearState() + clearCmmsCreateMeasurementRequestId() + } + } + } + externalMetricId = request.metricId } ) - onBlocking { streamMetrics(any()) } - .thenReturn( - flowOf( - INTERNAL_PENDING_INCREMENTAL_REACH_METRIC, - INTERNAL_PENDING_SINGLE_PUBLISHER_IMPRESSION_METRIC, - ) - ) - onBlocking { batchGetMetrics(any()) } - .thenReturn( - internalBatchGetMetricsResponse { - metrics += INTERNAL_PENDING_INCREMENTAL_REACH_METRIC - metrics += INTERNAL_PENDING_SINGLE_PUBLISHER_IMPRESSION_METRIC - } - ) - } - - private val internalReportingSetsMock: - InternalReportingSetsGrpcKt.ReportingSetsCoroutineImplBase = - mockService { - onBlocking { batchGetReportingSets(any()) } - .thenAnswer { - val request = it.arguments[0] as BatchGetReportingSetsRequest - val internalReportingSetsMap = - mapOf( - INTERNAL_INCREMENTAL_REPORTING_SET.externalReportingSetId to - INTERNAL_INCREMENTAL_REPORTING_SET, - INTERNAL_UNION_ALL_REPORTING_SET.externalReportingSetId to - INTERNAL_UNION_ALL_REPORTING_SET, - INTERNAL_UNION_ALL_BUT_LAST_PUBLISHER_REPORTING_SET.externalReportingSetId to - INTERNAL_UNION_ALL_BUT_LAST_PUBLISHER_REPORTING_SET, - INTERNAL_SINGLE_PUBLISHER_REPORTING_SET.externalReportingSetId to - INTERNAL_SINGLE_PUBLISHER_REPORTING_SET, - INTERNAL_POPULATION_REPORTING_SET.externalReportingSetId to - INTERNAL_POPULATION_REPORTING_SET, - ) - batchGetReportingSetsResponse { - reportingSets += - request.externalReportingSetIdsList.map { externalReportingSetId -> - internalReportingSetsMap.getValue(externalReportingSetId) - } - } - } - } - - private val internalMeasurementsMock: InternalMeasurementsCoroutineImplBase = mockService { - onBlocking { batchSetCmmsMeasurementIds(any()) }.thenReturn(Empty.getDefaultInstance()) - onBlocking { batchSetMeasurementResults(any()) }.thenReturn(Empty.getDefaultInstance()) - onBlocking { batchSetMeasurementFailures(any()) }.thenReturn(Empty.getDefaultInstance()) - } - - private val measurementsMock: MeasurementsCoroutineImplBase = mockService { - onBlocking { batchGetMeasurements(any()) } - .thenAnswer { - val batchGetMeasurementsRequest = it.arguments[0] as BatchGetMeasurementsRequest - val measurementsMap = - mapOf( - PENDING_UNION_ALL_REACH_MEASUREMENT.name to PENDING_UNION_ALL_REACH_MEASUREMENT, - PENDING_UNION_ALL_BUT_LAST_PUBLISHER_REACH_MEASUREMENT.name to - PENDING_UNION_ALL_BUT_LAST_PUBLISHER_REACH_MEASUREMENT, - PENDING_SINGLE_PUBLISHER_IMPRESSION_MEASUREMENT.name to - PENDING_SINGLE_PUBLISHER_IMPRESSION_MEASUREMENT, - PENDING_UNION_ALL_WATCH_DURATION_MEASUREMENT.name to - PENDING_UNION_ALL_WATCH_DURATION_MEASUREMENT, - PENDING_POPULATION_MEASUREMENT.name to PENDING_POPULATION_MEASUREMENT, - ) - batchGetMeasurementsResponse { - measurements += - batchGetMeasurementsRequest.namesList.map { name -> measurementsMap.getValue(name) } - } - } - - onBlocking { batchCreateMeasurements(any()) } - .thenAnswer { - val batchCreateMeasurementsRequest = it.arguments[0] as BatchCreateMeasurementsRequest - val measurementsMap = - mapOf( - INTERNAL_PENDING_UNION_ALL_REACH_MEASUREMENT.cmmsCreateMeasurementRequestId to - PENDING_UNION_ALL_REACH_MEASUREMENT, - INTERNAL_PENDING_UNION_ALL_BUT_LAST_PUBLISHER_REACH_MEASUREMENT - .cmmsCreateMeasurementRequestId to - PENDING_UNION_ALL_BUT_LAST_PUBLISHER_REACH_MEASUREMENT, - INTERNAL_PENDING_SINGLE_PUBLISHER_IMPRESSION_MEASUREMENT - .cmmsCreateMeasurementRequestId to PENDING_SINGLE_PUBLISHER_IMPRESSION_MEASUREMENT, - INTERNAL_PENDING_POPULATION_MEASUREMENT.cmmsCreateMeasurementRequestId to - PENDING_POPULATION_MEASUREMENT, - ) - batchCreateMeasurementsResponse { - measurements += - batchCreateMeasurementsRequest.requestsList.map { createMeasurementRequest -> - measurementsMap.getValue(createMeasurementRequest.requestId) - } - } - } - } - - private val measurementConsumersMock: - MeasurementConsumersGrpcKt.MeasurementConsumersCoroutineImplBase = - mockService { - onBlocking { - getMeasurementConsumer( - eq(getMeasurementConsumerRequest { name = MEASUREMENT_CONSUMERS.values.first().name }) - ) - } - .thenReturn(MEASUREMENT_CONSUMERS.values.first()) - } - - private val dataProvidersMock: DataProvidersGrpcKt.DataProvidersCoroutineImplBase = mockService { - for (dataProvider in DATA_PROVIDERS.values) { - onBlocking { getDataProvider(eq(getDataProviderRequest { name = dataProvider.name })) } - .thenReturn(dataProvider) - } - } - - private val certificatesMock: CertificatesGrpcKt.CertificatesCoroutineImplBase = mockService { - onBlocking { getCertificate(eq(getCertificateRequest { name = AGGREGATOR_CERTIFICATE.name })) } - .thenReturn(AGGREGATOR_CERTIFICATE) - for (dataProvider in DATA_PROVIDERS.values) { - onBlocking { getCertificate(eq(getCertificateRequest { name = dataProvider.certificate })) } - .thenReturn( - certificate { - name = dataProvider.certificate - x509Der = DATA_PROVIDER_SIGNING_KEY.certificate.encoded.toByteString() - } - ) - } - for (measurementConsumer in MEASUREMENT_CONSUMERS.values) { - onBlocking { - getCertificate(eq(getCertificateRequest { name = measurementConsumer.certificate })) - } - .thenReturn( - certificate { - name = measurementConsumer.certificate - x509Der = measurementConsumer.certificateDer - } - ) - } - } - - private val randomMock: Random = mock() - - private object VariancesMock : Variances { - override fun computeMetricVariance(params: ReachMetricVarianceParams): Double = VARIANCE_VALUE - - override fun computeMeasurementVariance( - methodology: Methodology, - measurementVarianceParams: ReachMeasurementVarianceParams, - ): Double = VARIANCE_VALUE - - override fun computeMetricVariance(params: FrequencyMetricVarianceParams): FrequencyVariances = - FREQUENCY_VARIANCES - - override fun computeMeasurementVariance( - methodology: Methodology, - measurementVarianceParams: FrequencyMeasurementVarianceParams, - ): FrequencyVariances = FrequencyVariances(mapOf(), mapOf(), mapOf(), mapOf()) - - override fun computeMetricVariance(params: ImpressionMetricVarianceParams): Double = - VARIANCE_VALUE - - override fun computeMeasurementVariance( - methodology: Methodology, - measurementVarianceParams: ImpressionMeasurementVarianceParams, - ): Double = VARIANCE_VALUE - - override fun computeMetricVariance(params: WatchDurationMetricVarianceParams): Double = - VARIANCE_VALUE - - override fun computeMeasurementVariance( - methodology: Methodology, - measurementVarianceParams: WatchDurationMeasurementVarianceParams, - ): Double = VARIANCE_VALUE - } - - @get:Rule - val grpcTestServerRule = GrpcTestServerRule { - addService(internalMetricsMock) - addService(internalReportingSetsMock) - addService(internalMeasurementsMock) - addService(measurementsMock) - addService(measurementConsumersMock) - addService(dataProvidersMock) - addService(certificatesMock) - } - - private lateinit var service: MetricsService - - @Before - fun initService() { - randomMock.stub { - on { nextInt(any()) } doReturn RANDOM_OUTPUT_INT - on { nextLong() } doReturn RANDOM_OUTPUT_LONG - } - - service = - MetricsService( - METRIC_SPEC_CONFIG, - InternalReportingSetsGrpcKt.ReportingSetsCoroutineStub(grpcTestServerRule.channel), - InternalMetricsGrpcKt.MetricsCoroutineStub(grpcTestServerRule.channel), - VariancesMock, - InternalMeasurementsGrpcKt.MeasurementsCoroutineStub(grpcTestServerRule.channel), - DataProvidersGrpcKt.DataProvidersCoroutineStub(grpcTestServerRule.channel), - MeasurementsGrpcKt.MeasurementsCoroutineStub(grpcTestServerRule.channel), - CertificatesGrpcKt.CertificatesCoroutineStub(grpcTestServerRule.channel), - MeasurementConsumersGrpcKt.MeasurementConsumersCoroutineStub(grpcTestServerRule.channel), - ENCRYPTION_KEY_PAIR_STORE, - randomMock, - SECRETS_DIR, - listOf(AGGREGATOR_ROOT_CERTIFICATE, DATA_PROVIDER_ROOT_CERTIFICATE).associateBy { - it.subjectKeyIdentifier!! - }, - ) - } - - @Test - fun `createMetric creates CMMS measurements for incremental reach`() { - val request = createMetricRequest { - parent = MEASUREMENT_CONSUMERS.values.first().name - metric = REQUESTING_INCREMENTAL_REACH_METRIC - metricId = "metric-id" - } - - val result = - withMeasurementConsumerPrincipal(request.parent, CONFIG) { - runBlocking { service.createMetric(request) } - } - - val expected = PENDING_INCREMENTAL_REACH_METRIC - - // Verify proto argument of the internal MetricsCoroutineImplBase::createMetric - verifyProtoArgument(internalMetricsMock, MetricsCoroutineImplBase::createMetric) - .ignoringRepeatedFieldOrder() - .isEqualTo( - internalCreateMetricRequest { - metric = INTERNAL_REQUESTING_INCREMENTAL_REACH_METRIC - externalMetricId = "metric-id" - } - ) - - // Verify proto argument of MeasurementsCoroutineImplBase::batchCreateMeasurements - val measurementsCaptor: KArgumentCaptor = argumentCaptor() - verifyBlocking(measurementsMock, times(1)) { - batchCreateMeasurements(measurementsCaptor.capture()) - } - val capturedMeasurementRequests = measurementsCaptor.allValues - assertThat(capturedMeasurementRequests) - .ignoringRepeatedFieldOrder() - .ignoringFieldDescriptors(MEASUREMENT_SPEC_FIELD, ENCRYPTED_REQUISITION_SPEC_FIELD) - .containsExactly( - batchCreateMeasurementsRequest { - parent = request.parent - requests += createMeasurementRequest { - parent = request.parent - measurement = REQUESTING_UNION_ALL_REACH_MEASUREMENT - requestId = INTERNAL_PENDING_UNION_ALL_REACH_MEASUREMENT.cmmsCreateMeasurementRequestId - } - requests += createMeasurementRequest { - parent = request.parent - measurement = REQUESTING_UNION_ALL_BUT_LAST_PUBLISHER_REACH_MEASUREMENT - requestId = - INTERNAL_PENDING_UNION_ALL_BUT_LAST_PUBLISHER_REACH_MEASUREMENT - .cmmsCreateMeasurementRequestId - } + + // Verify proto argument of MeasurementsCoroutineImplBase::batchCreateMeasurements + val measurementsCaptor: KArgumentCaptor = argumentCaptor() + verifyBlocking(measurementsMock, times(1)) { + batchCreateMeasurements(measurementsCaptor.capture()) + } + val capturedMeasurementRequests = measurementsCaptor.allValues + assertThat(capturedMeasurementRequests) + .ignoringRepeatedFieldOrder() + .ignoringFieldDescriptors(MEASUREMENT_SPEC_FIELD, ENCRYPTED_REQUISITION_SPEC_FIELD) + .containsExactly( + batchCreateMeasurementsRequest { + parent = request.parent + requests += createMeasurementRequest { + parent = request.parent + measurement = pendingSingleDataProviderReachAndFrequencyMeasurementWithSingleDataProviderParams.copy { + clearState() + clearProtocolConfig() + clearName() + } + requestId = internalMeasurement.cmmsCreateMeasurementRequestId + } } ) @@ -2353,12 +1349,7 @@ class MetricsServiceTest { val measurementSpec: MeasurementSpec = createMeasurementRequest.measurement.measurementSpec.unpack() assertThat(measurementSpec) - .isEqualTo( - UNION_ALL_BUT_LAST_PUBLISHER_REACH_MEASUREMENT_SPEC.copy { - nonceHashes.clear() - nonceHashes += List(dataProvidersList.size) { Hashing.hashSha256(RANDOM_OUTPUT_LONG) } - } - ) + .isEqualTo(cmmsMeasurementSpec) dataProvidersList.map { dataProviderEntry -> val signedRequisitionSpec = @@ -2377,35 +1368,26 @@ class MetricsServiceTest { } } - // Verify proto argument of internal MeasurementsCoroutineImplBase::batchSetCmmsMeasurementId + // Verify proto argument of internal MeasurementsCoroutineImplBase::batchSetCmmsMeasurementIds verifyProtoArgument( - internalMeasurementsMock, - InternalMeasurementsGrpcKt.MeasurementsCoroutineImplBase::batchSetCmmsMeasurementIds, - ) + internalMeasurementsMock, + InternalMeasurementsGrpcKt.MeasurementsCoroutineImplBase::batchSetCmmsMeasurementIds, + ) .ignoringRepeatedFieldOrder() .isEqualTo( batchSetCmmsMeasurementIdsRequest { cmmsMeasurementConsumerId = MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId this.measurementIds += measurementIds { cmmsCreateMeasurementRequestId = - INTERNAL_PENDING_UNION_ALL_REACH_MEASUREMENT.cmmsCreateMeasurementRequestId - cmmsMeasurementId = INTERNAL_PENDING_UNION_ALL_REACH_MEASUREMENT.cmmsMeasurementId - } - this.measurementIds += measurementIds { - cmmsCreateMeasurementRequestId = - INTERNAL_PENDING_UNION_ALL_BUT_LAST_PUBLISHER_REACH_MEASUREMENT - .cmmsCreateMeasurementRequestId - cmmsMeasurementId = - INTERNAL_PENDING_UNION_ALL_BUT_LAST_PUBLISHER_REACH_MEASUREMENT.cmmsMeasurementId + internalMeasurement.cmmsCreateMeasurementRequestId + cmmsMeasurementId = internalMeasurement.cmmsMeasurementId } } ) - - assertThat(result).isEqualTo(expected) } @Test - fun `createMetric creates CMMS measurements for single pub reach frequency metric`() = + fun `createMetric creates CMMS measurements for single pub rf when single edp params not set`() = runBlocking { whenever(internalMetricsMock.createMetric(any())) .thenReturn(INTERNAL_PENDING_INITIAL_SINGLE_PUBLISHER_REACH_FREQUENCY_METRIC) @@ -2634,19 +1616,21 @@ class MetricsServiceTest { val internalMetricSpec = internalMetricSpec { impressionCount = InternalMetricSpecKt.impressionCountParams { - privacyParams = - InternalMetricSpecKt.differentialPrivacyParams { - this.epsilon = epsilon - this.delta = delta - } - this.maximumFrequencyPerUser = maximumFrequencyPerUser - } - vidSamplingInterval = - InternalMetricSpecKt.vidSamplingInterval { - start = vidSamplingIntervalStart - width = vidSamplingIntervalWidth - } - } + params = InternalMetricSpecKt.params { + privacyParams = + InternalMetricSpecKt.differentialPrivacyParams { + this.epsilon = epsilon + this.delta = delta + } + vidSamplingInterval = + InternalMetricSpecKt.vidSamplingInterval { + start = vidSamplingIntervalStart + width = vidSamplingIntervalWidth + } + } + this.maximumFrequencyPerUser = maximumFrequencyPerUser + } + } val internalRequestingSinglePublisherImpressionMetric = INTERNAL_REQUESTING_SINGLE_PUBLISHER_IMPRESSION_METRIC.copy { this.metricSpec = internalMetricSpec @@ -2726,6 +1710,18 @@ class MetricsServiceTest { PENDING_SINGLE_PUBLISHER_IMPRESSION_METRIC.copy { metricSpec = metricSpec { impressionCount = impressionCountParams { + params = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + this.epsilon = epsilon + this.delta = delta + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = vidSamplingIntervalStart + width = vidSamplingIntervalWidth + } + } privacyParams = MetricSpecKt.differentialPrivacyParams { this.epsilon = epsilon @@ -2971,6 +1967,7 @@ class MetricsServiceTest { primitiveReportingSetBases += primitiveReportingSetBasis { externalReportingSetId = internalSinglePublisherReportingSet.externalReportingSetId } + isSingleDataProvider = true } } details = InternalMetricKt.details {} @@ -3062,6 +2059,36 @@ class MetricsServiceTest { val numberBatchReportingSets = expectedNumberBatchGetReportingSetsRequests - 1 val numberInternalReportingSets = BATCH_GET_REPORTING_SETS_LIMIT * numberBatchReportingSets + whenever(internalReportingSetsMock.batchGetReportingSets(any())) + .thenAnswer { + val batchGetReportingSetsRequest = it.arguments[0] as BatchGetReportingSetsRequest + batchGetReportingSetsResponse { + reportingSets += + batchGetReportingSetsRequest.externalReportingSetIdsList.map { externalReportingSetId -> + INTERNAL_INCREMENTAL_REPORTING_SET.copy { + this.externalReportingSetId = externalReportingSetId + weightedSubsetUnions.clear() + weightedSubsetUnions += weightedSubsetUnion { + (0 until numberInternalReportingSets).forEach { + primitiveReportingSetBases += primitiveReportingSetBasis { + this.externalReportingSetId = it.toString() + } + } + } + } + } + } + } + .thenReturn( + batchGetReportingSetsResponse { + (0 until numberInternalReportingSets).forEach { + reportingSets += INTERNAL_UNION_ALL_REPORTING_SET.copy { + this.externalReportingSetId = it.toString() + } + } + } + ) + whenever(internalMetricsMock.createMetric(any())) .thenReturn( INTERNAL_PENDING_INITIAL_INCREMENTAL_REACH_METRIC.copy { @@ -3083,18 +2110,6 @@ class MetricsServiceTest { } ) - whenever(internalReportingSetsMock.batchGetReportingSets(any())).thenAnswer { - val batchGetReportingSetsRequest = it.arguments[0] as BatchGetReportingSetsRequest - batchGetReportingSetsResponse { - reportingSets += - batchGetReportingSetsRequest.externalReportingSetIdsList.map { externalReportingSetId -> - INTERNAL_INCREMENTAL_REPORTING_SET.copy { - this.externalReportingSetId = externalReportingSetId - } - } - } - } - val request = createMetricRequest { parent = MEASUREMENT_CONSUMERS.values.first().name metric = REQUESTING_INCREMENTAL_REACH_METRIC @@ -3290,7 +2305,7 @@ class MetricsServiceTest { } @Test - fun `createMetric with request ID when the metric exists and in terminate state`() = runBlocking { + fun `createMetric with request ID when the metric exists and in terminalstate`() = runBlocking { whenever(internalMetricsMock.createMetric(any())) .thenReturn(INTERNAL_SUCCEEDED_INCREMENTAL_REACH_METRIC) @@ -3311,7 +2326,7 @@ class MetricsServiceTest { // Verify proto argument of the internal ReportingSetsCoroutineImplBase::batchGetReportingSets val batchGetReportingSetsCaptor: KArgumentCaptor = argumentCaptor() - verifyBlocking(internalReportingSetsMock, times(1)) { + verifyBlocking(internalReportingSetsMock, times(2)) { batchGetReportingSets(batchGetReportingSetsCaptor.capture()) } @@ -5326,6 +4341,318 @@ class MetricsServiceTest { assertThat(result).isEqualTo(SUCCEEDED_INCREMENTAL_REACH_METRIC) } + @Test + fun `getMetric returns SUCCEEDED metric when metric already succeeded and single params set`() = + runBlocking { + val internalMeasurement = internalMeasurement { + cmmsMeasurementConsumerId = MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId + cmmsMeasurementId = "cmms_id" + cmmsCreateMeasurementRequestId = "SINGLE_PUBLISHER_REACH_MEASUREMENT" + timeInterval = TIME_INTERVAL + primitiveReportingSetBases += primitiveReportingSetBasis { + externalReportingSetId = INTERNAL_SINGLE_PUBLISHER_REPORTING_SET.externalReportingSetId + filters += METRIC_FILTER + filters += PRIMITIVE_REPORTING_SET_FILTER + } + state = InternalMeasurement.State.SUCCEEDED + isSingleDataProvider = true + details = + InternalMeasurementKt.details { + results += + InternalMeasurementKt.result { + reach = + InternalMeasurementKt.ResultKt.reach { + value = INCREMENTAL_REACH_VALUE + noiseMechanism = NoiseMechanism.CONTINUOUS_LAPLACE + deterministicCountDistinct = InternalDeterministicCountDistinct.getDefaultInstance() + } + } + } + } + + val internalSucceededReachMetricWithSingleDataProviderParams = internalMetric { + cmmsMeasurementConsumerId = MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId + externalMetricId = "metric-id" + externalReportingSetId = INTERNAL_SINGLE_PUBLISHER_REPORTING_SET.externalReportingSetId + timeInterval = TIME_INTERVAL + metricSpec = internalMetricSpec { + reach = InternalMetricSpecKt.reachParams { + multipleDataProviderParams = InternalMetricSpecKt.params { + privacyParams = InternalMetricSpecKt.differentialPrivacyParams { + epsilon = REACH_ONLY_REACH_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + vidSamplingInterval = InternalMetricSpecKt.vidSamplingInterval { + start = REACH_ONLY_VID_SAMPLING_START + width = REACH_ONLY_VID_SAMPLING_WIDTH + } + } + singleDataProviderParams = InternalMetricSpecKt.params { + privacyParams = InternalMetricSpecKt.differentialPrivacyParams { + epsilon = SINGLE_DATA_PROVIDER_REACH_ONLY_REACH_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + vidSamplingInterval = InternalMetricSpecKt.vidSamplingInterval { + start = SINGLE_DATA_PROVIDER_REACH_ONLY_VID_SAMPLING_START + width = SINGLE_DATA_PROVIDER_REACH_ONLY_VID_SAMPLING_WIDTH + } + } + } + } + weightedMeasurements += weightedMeasurement { + weight = 1 + binaryRepresentation = 1 + measurement = internalMeasurement + } + details = InternalMetricKt.details { + filters += METRIC_FILTER + } + createTime = INTERNAL_SUCCEEDED_INCREMENTAL_REACH_METRIC.createTime + state = InternalMetric.State.SUCCEEDED + } + + whenever(internalMetricsMock.batchGetMetrics(any())) + .thenReturn( + internalBatchGetMetricsResponse { + metrics += internalSucceededReachMetricWithSingleDataProviderParams + } + ) + + val metricName = + MetricKey( + internalSucceededReachMetricWithSingleDataProviderParams.cmmsMeasurementConsumerId, + internalSucceededReachMetricWithSingleDataProviderParams.externalMetricId + ).toName() + + val request = getMetricRequest { + name = metricName + } + + val result = + withMeasurementConsumerPrincipal(MEASUREMENT_CONSUMERS.values.first().name, CONFIG) { + runBlocking { service.getMetric(request) } + } + + // Verify proto argument of internal MetricsCoroutineImplBase::batchGetMetrics + val batchGetInternalMetricsCaptor: KArgumentCaptor = + argumentCaptor() + verifyBlocking(internalMetricsMock, times(1)) { + batchGetMetrics(batchGetInternalMetricsCaptor.capture()) + } + val capturedInternalGetMetricRequests = batchGetInternalMetricsCaptor.allValues + assertThat(capturedInternalGetMetricRequests) + .containsExactly( + internalBatchGetMetricsRequest { + cmmsMeasurementConsumerId = + internalSucceededReachMetricWithSingleDataProviderParams.cmmsMeasurementConsumerId + externalMetricIds += internalSucceededReachMetricWithSingleDataProviderParams.externalMetricId + } + ) + + // Verify proto argument of internal MeasurementsCoroutineImplBase::batchSetMeasurementResults + val batchSetMeasurementResultsCaptor: KArgumentCaptor = + argumentCaptor() + verifyBlocking(internalMeasurementsMock, never()) { + batchSetMeasurementResults(batchSetMeasurementResultsCaptor.capture()) + } + + // Verify proto argument of internal + // MeasurementsCoroutineImplBase::batchSetMeasurementFailures + val batchSetMeasurementFailuresCaptor: KArgumentCaptor = + argumentCaptor() + verifyBlocking(internalMeasurementsMock, never()) { + batchSetMeasurementFailures(batchSetMeasurementFailuresCaptor.capture()) + } + + val succeededReachMetricWithSingleDataProviderParams = metric { + name = metricName + reportingSet = INTERNAL_SINGLE_PUBLISHER_REPORTING_SET.resourceName + timeInterval = TIME_INTERVAL + metricSpec = metricSpec { + reach = reachParams { + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = MetricSpecKt.differentialPrivacyParams { + epsilon = REACH_ONLY_REACH_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + vidSamplingInterval = MetricSpecKt.vidSamplingInterval { + start = REACH_ONLY_VID_SAMPLING_START + width = REACH_ONLY_VID_SAMPLING_WIDTH + } + } + singleDataProviderParams = MetricSpecKt.params { + privacyParams = MetricSpecKt.differentialPrivacyParams { + epsilon = SINGLE_DATA_PROVIDER_REACH_ONLY_REACH_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + vidSamplingInterval = MetricSpecKt.vidSamplingInterval { + start = SINGLE_DATA_PROVIDER_REACH_ONLY_VID_SAMPLING_START + width = SINGLE_DATA_PROVIDER_REACH_ONLY_VID_SAMPLING_WIDTH + } + } + } + } + filters += METRIC_FILTER + state = Metric.State.SUCCEEDED + createTime = INTERNAL_SUCCEEDED_INCREMENTAL_REACH_METRIC.createTime + this.result = metricResult { + reach = + MetricResultKt.reachResult { + value = INCREMENTAL_REACH_VALUE + univariateStatistics = univariateStatistics { standardDeviation = sqrt(SINGLE_DATA_PROVIDER_VARIANCE_VALUE) } + } + cmmsMeasurements += MeasurementKey(internalMeasurement.cmmsMeasurementConsumerId, internalMeasurement.cmmsMeasurementId).toName() + } + } + + assertThat(result).isEqualTo(succeededReachMetricWithSingleDataProviderParams) + } + + @Test + fun `getMetric returns SUCCEEDED metric when metric already succeeded and no single params`() = + runBlocking { + val internalMeasurement = internalMeasurement { + cmmsMeasurementConsumerId = MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId + cmmsMeasurementId = "cmms_id" + cmmsCreateMeasurementRequestId = "SINGLE_PUBLISHER_REACH_MEASUREMENT" + timeInterval = TIME_INTERVAL + primitiveReportingSetBases += primitiveReportingSetBasis { + externalReportingSetId = INTERNAL_SINGLE_PUBLISHER_REPORTING_SET.externalReportingSetId + filters += METRIC_FILTER + filters += PRIMITIVE_REPORTING_SET_FILTER + } + state = InternalMeasurement.State.SUCCEEDED + isSingleDataProvider = true + details = + InternalMeasurementKt.details { + results += + InternalMeasurementKt.result { + reach = + InternalMeasurementKt.ResultKt.reach { + value = INCREMENTAL_REACH_VALUE + noiseMechanism = NoiseMechanism.CONTINUOUS_LAPLACE + deterministicCountDistinct = InternalDeterministicCountDistinct.getDefaultInstance() + } + } + } + } + + val internalSucceededReachMetric = internalMetric { + cmmsMeasurementConsumerId = MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId + externalMetricId = "metric-id" + externalReportingSetId = INTERNAL_SINGLE_PUBLISHER_REPORTING_SET.externalReportingSetId + timeInterval = TIME_INTERVAL + metricSpec = internalMetricSpec { + reach = InternalMetricSpecKt.reachParams { + multipleDataProviderParams = InternalMetricSpecKt.params { + privacyParams = InternalMetricSpecKt.differentialPrivacyParams { + epsilon = REACH_ONLY_REACH_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + vidSamplingInterval = InternalMetricSpecKt.vidSamplingInterval { + start = REACH_ONLY_VID_SAMPLING_START + width = REACH_ONLY_VID_SAMPLING_WIDTH + } + } + } + } + weightedMeasurements += weightedMeasurement { + weight = 1 + binaryRepresentation = 1 + measurement = internalMeasurement + } + details = InternalMetricKt.details { + filters += METRIC_FILTER + } + createTime = INTERNAL_SUCCEEDED_INCREMENTAL_REACH_METRIC.createTime + state = InternalMetric.State.SUCCEEDED + } + + whenever(internalMetricsMock.batchGetMetrics(any())) + .thenReturn( + internalBatchGetMetricsResponse { + metrics += internalSucceededReachMetric + } + ) + + val metricName = + MetricKey( + internalSucceededReachMetric.cmmsMeasurementConsumerId, + internalSucceededReachMetric.externalMetricId + ).toName() + + val request = getMetricRequest { + name = metricName + } + + val result = + withMeasurementConsumerPrincipal(MEASUREMENT_CONSUMERS.values.first().name, CONFIG) { + runBlocking { service.getMetric(request) } + } + + // Verify proto argument of internal MetricsCoroutineImplBase::batchGetMetrics + val batchGetInternalMetricsCaptor: KArgumentCaptor = + argumentCaptor() + verifyBlocking(internalMetricsMock, times(1)) { + batchGetMetrics(batchGetInternalMetricsCaptor.capture()) + } + val capturedInternalGetMetricRequests = batchGetInternalMetricsCaptor.allValues + assertThat(capturedInternalGetMetricRequests) + .containsExactly( + internalBatchGetMetricsRequest { + cmmsMeasurementConsumerId = + internalSucceededReachMetric.cmmsMeasurementConsumerId + externalMetricIds += internalSucceededReachMetric.externalMetricId + } + ) + + // Verify proto argument of internal MeasurementsCoroutineImplBase::batchSetMeasurementResults + val batchSetMeasurementResultsCaptor: KArgumentCaptor = + argumentCaptor() + verifyBlocking(internalMeasurementsMock, never()) { + batchSetMeasurementResults(batchSetMeasurementResultsCaptor.capture()) + } + + // Verify proto argument of internal + // MeasurementsCoroutineImplBase::batchSetMeasurementFailures + val batchSetMeasurementFailuresCaptor: KArgumentCaptor = + argumentCaptor() + verifyBlocking(internalMeasurementsMock, never()) { + batchSetMeasurementFailures(batchSetMeasurementFailuresCaptor.capture()) + } + + val succeededReachMetric = metric { + name = metricName + reportingSet = INTERNAL_SINGLE_PUBLISHER_REPORTING_SET.resourceName + timeInterval = TIME_INTERVAL + metricSpec = metricSpec { + reach = reachParams { + privacyParams = MetricSpecKt.differentialPrivacyParams { + epsilon = REACH_ONLY_REACH_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + } + vidSamplingInterval = MetricSpecKt.vidSamplingInterval { + start = REACH_ONLY_VID_SAMPLING_START + width = REACH_ONLY_VID_SAMPLING_WIDTH + } + } + filters += METRIC_FILTER + state = Metric.State.SUCCEEDED + createTime = INTERNAL_SUCCEEDED_INCREMENTAL_REACH_METRIC.createTime + this.result = metricResult { + reach = + MetricResultKt.reachResult { + value = INCREMENTAL_REACH_VALUE + univariateStatistics = univariateStatistics { standardDeviation = sqrt(VARIANCE_VALUE) } + } + cmmsMeasurements += MeasurementKey(internalMeasurement.cmmsMeasurementConsumerId, internalMeasurement.cmmsMeasurementId).toName() + } + } + + assertThat(result).isEqualTo(succeededReachMetric) + } + @Test fun `getMetric returns the metric with SUCCEEDED when the metric is already succeeded`() = runBlocking { @@ -6279,48 +5606,73 @@ class MetricsServiceTest { } @Test - fun `getMetric returns reach frequency metric with statistics not set when reach lacks info for variance`() = + fun `getMetric returns SUCCEEDED rf when metric already succeeded and single params set`() = runBlocking { + val internalSucceededReachAndFrequencyMetricWithSingleDataProviderParams = internalMetric { + cmmsMeasurementConsumerId = MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId + externalMetricId = "metric-id" + externalReportingSetId = INTERNAL_SINGLE_PUBLISHER_REPORTING_SET.externalReportingSetId + timeInterval = TIME_INTERVAL + metricSpec = internalMetricSpec { + reachAndFrequency = InternalMetricSpecKt.reachAndFrequencyParams { + multipleDataProviderParams = InternalMetricSpecKt.params { + privacyParams = InternalMetricSpecKt.differentialPrivacyParams { + epsilon = REACH_FREQUENCY_REACH_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + frequencyPrivacyParams = InternalMetricSpecKt.differentialPrivacyParams { + epsilon = REACH_FREQUENCY_FREQUENCY_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + vidSamplingInterval = InternalMetricSpecKt.vidSamplingInterval { + start = REACH_FREQUENCY_VID_SAMPLING_START + width = REACH_FREQUENCY_VID_SAMPLING_WIDTH + } + } + singleDataProviderParams = InternalMetricSpecKt.params { + privacyParams = InternalMetricSpecKt.differentialPrivacyParams { + epsilon = SINGLE_DATA_PROVIDER_REACH_FREQUENCY_REACH_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + frequencyPrivacyParams = InternalMetricSpecKt.differentialPrivacyParams { + epsilon = SINGLE_DATA_PROVIDER_REACH_FREQUENCY_FREQUENCY_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + vidSamplingInterval = InternalMetricSpecKt.vidSamplingInterval { + start = SINGLE_DATA_PROVIDER_REACH_FREQUENCY_VID_SAMPLING_START + width = SINGLE_DATA_PROVIDER_REACH_FREQUENCY_VID_SAMPLING_WIDTH + } + } + maximumFrequency = REACH_FREQUENCY_MAXIMUM_FREQUENCY + } + } + weightedMeasurements += weightedMeasurement { + weight = 1 + binaryRepresentation = 1 + measurement = INTERNAL_SUCCEEDED_SINGLE_PUBLISHER_REACH_FREQUENCY_MEASUREMENT + } + details = InternalMetricKt.details { + filters += METRIC_FILTER + } + createTime = INTERNAL_SUCCEEDED_SINGLE_PUBLISHER_REACH_FREQUENCY_METRIC.createTime + state = InternalMetric.State.SUCCEEDED + } + whenever(internalMetricsMock.batchGetMetrics(any())) .thenReturn( internalBatchGetMetricsResponse { - metrics += - INTERNAL_SUCCEEDED_SINGLE_PUBLISHER_REACH_FREQUENCY_METRIC.copy { - weightedMeasurements.clear() - weightedMeasurements += weightedMeasurement { - weight = 1 - binaryRepresentation = 1 - measurement = - INTERNAL_SUCCEEDED_SINGLE_PUBLISHER_REACH_FREQUENCY_MEASUREMENT.copy { - details = - InternalMeasurementKt.details { - results += - InternalMeasurementKt.result { - reach = - InternalMeasurementKt.ResultKt.reach { - value = REACH_FREQUENCY_REACH_VALUE - } - frequency = - InternalMeasurementKt.ResultKt.frequency { - relativeFrequencyDistribution.putAll( - REACH_FREQUENCY_FREQUENCY_VALUE - ) - noiseMechanism = NoiseMechanism.CONTINUOUS_LAPLACE - liquidLegionsDistribution = internalLiquidLegionsDistribution { - decayRate = LL_DISTRIBUTION_DECAY_RATE - maxSize = LL_DISTRIBUTION_SKETCH_SIZE - } - } - } - } - } - } - } + metrics += internalSucceededReachAndFrequencyMetricWithSingleDataProviderParams } ) + val metricName = + MetricKey( + internalSucceededReachAndFrequencyMetricWithSingleDataProviderParams.cmmsMeasurementConsumerId, + internalSucceededReachAndFrequencyMetricWithSingleDataProviderParams.externalMetricId + ).toName() + val request = getMetricRequest { - name = SUCCEEDED_SINGLE_PUBLISHER_REACH_FREQUENCY_METRIC.name + name = metricName } val result = @@ -6328,10 +5680,180 @@ class MetricsServiceTest { runBlocking { service.getMetric(request) } } - assertThat(result) - .isEqualTo( - SUCCEEDED_SINGLE_PUBLISHER_REACH_FREQUENCY_METRIC.copy { - this.result = + // Verify proto argument of internal MetricsCoroutineImplBase::batchGetMetrics + val batchGetInternalMetricsCaptor: KArgumentCaptor = + argumentCaptor() + verifyBlocking(internalMetricsMock, times(1)) { + batchGetMetrics(batchGetInternalMetricsCaptor.capture()) + } + val capturedInternalGetMetricRequests = batchGetInternalMetricsCaptor.allValues + assertThat(capturedInternalGetMetricRequests) + .containsExactly( + internalBatchGetMetricsRequest { + cmmsMeasurementConsumerId = + internalSucceededReachAndFrequencyMetricWithSingleDataProviderParams.cmmsMeasurementConsumerId + externalMetricIds += internalSucceededReachAndFrequencyMetricWithSingleDataProviderParams.externalMetricId + } + ) + + // Verify proto argument of internal MeasurementsCoroutineImplBase::batchSetMeasurementResults + val batchSetMeasurementResultsCaptor: KArgumentCaptor = + argumentCaptor() + verifyBlocking(internalMeasurementsMock, never()) { + batchSetMeasurementResults(batchSetMeasurementResultsCaptor.capture()) + } + + // Verify proto argument of internal + // MeasurementsCoroutineImplBase::batchSetMeasurementFailures + val batchSetMeasurementFailuresCaptor: KArgumentCaptor = + argumentCaptor() + verifyBlocking(internalMeasurementsMock, never()) { + batchSetMeasurementFailures(batchSetMeasurementFailuresCaptor.capture()) + } + + val succeededReachMetricWithSingleDataProviderParams = metric { + name = metricName + reportingSet = INTERNAL_SINGLE_PUBLISHER_REPORTING_SET.resourceName + timeInterval = TIME_INTERVAL + metricSpec = metricSpec { + reachAndFrequency = reachAndFrequencyParams { + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = MetricSpecKt.differentialPrivacyParams { + epsilon = REACH_FREQUENCY_REACH_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + frequencyPrivacyParams = MetricSpecKt.differentialPrivacyParams { + epsilon = REACH_FREQUENCY_FREQUENCY_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + vidSamplingInterval = MetricSpecKt.vidSamplingInterval { + start = REACH_FREQUENCY_VID_SAMPLING_START + width = REACH_FREQUENCY_VID_SAMPLING_WIDTH + } + } + singleDataProviderParams = MetricSpecKt.params { + privacyParams = MetricSpecKt.differentialPrivacyParams { + epsilon = SINGLE_DATA_PROVIDER_REACH_FREQUENCY_REACH_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + frequencyPrivacyParams = MetricSpecKt.differentialPrivacyParams { + epsilon = SINGLE_DATA_PROVIDER_REACH_FREQUENCY_FREQUENCY_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + vidSamplingInterval = MetricSpecKt.vidSamplingInterval { + start = SINGLE_DATA_PROVIDER_REACH_FREQUENCY_VID_SAMPLING_START + width = SINGLE_DATA_PROVIDER_REACH_FREQUENCY_VID_SAMPLING_WIDTH + } + } + maximumFrequency = REACH_FREQUENCY_MAXIMUM_FREQUENCY + } + } + filters += METRIC_FILTER + state = Metric.State.SUCCEEDED + createTime = INTERNAL_SUCCEEDED_SINGLE_PUBLISHER_REACH_FREQUENCY_METRIC.createTime + this.result = metricResult { + cmmsMeasurements += PENDING_SINGLE_PUBLISHER_REACH_FREQUENCY_MEASUREMENT.name + reachAndFrequency = + MetricResultKt.reachAndFrequencyResult { + reach = + MetricResultKt.reachResult { + value = REACH_FREQUENCY_REACH_VALUE + univariateStatistics = univariateStatistics { + standardDeviation = sqrt(SINGLE_DATA_PROVIDER_VARIANCE_VALUE) + } + } + frequencyHistogram = + MetricResultKt.histogramResult { + bins += + (1..REACH_FREQUENCY_MAXIMUM_FREQUENCY).map { frequency -> + MetricResultKt.HistogramResultKt.bin { + label = frequency.toString() + binResult = + MetricResultKt.HistogramResultKt.binResult { + value = + REACH_FREQUENCY_REACH_VALUE * + REACH_FREQUENCY_FREQUENCY_VALUE.getOrDefault(frequency.toLong(), 0.0) + } + resultUnivariateStatistics = univariateStatistics { + standardDeviation = + sqrt(SINGLE_DATA_PROVIDER_FREQUENCY_VARIANCES.countVariances.getValue(frequency)) + } + relativeUnivariateStatistics = univariateStatistics { + standardDeviation = + sqrt(SINGLE_DATA_PROVIDER_FREQUENCY_VARIANCES.relativeVariances.getValue(frequency)) + } + kPlusUnivariateStatistics = univariateStatistics { + standardDeviation = + sqrt(SINGLE_DATA_PROVIDER_FREQUENCY_VARIANCES.kPlusCountVariances.getValue(frequency)) + } + relativeKPlusUnivariateStatistics = univariateStatistics { + standardDeviation = + sqrt(SINGLE_DATA_PROVIDER_FREQUENCY_VARIANCES.kPlusRelativeVariances.getValue(frequency)) + } + } + } + } + } + } + } + + assertThat(result).isEqualTo(succeededReachMetricWithSingleDataProviderParams) + } + + @Test + fun `getMetric returns reach frequency metric with statistics not set when reach lacks info for variance`() = + runBlocking { + whenever(internalMetricsMock.batchGetMetrics(any())) + .thenReturn( + internalBatchGetMetricsResponse { + metrics += + INTERNAL_SUCCEEDED_SINGLE_PUBLISHER_REACH_FREQUENCY_METRIC.copy { + weightedMeasurements.clear() + weightedMeasurements += weightedMeasurement { + weight = 1 + binaryRepresentation = 1 + measurement = + INTERNAL_SUCCEEDED_SINGLE_PUBLISHER_REACH_FREQUENCY_MEASUREMENT.copy { + details = + InternalMeasurementKt.details { + results += + InternalMeasurementKt.result { + reach = + InternalMeasurementKt.ResultKt.reach { + value = REACH_FREQUENCY_REACH_VALUE + } + frequency = + InternalMeasurementKt.ResultKt.frequency { + relativeFrequencyDistribution.putAll( + REACH_FREQUENCY_FREQUENCY_VALUE + ) + noiseMechanism = NoiseMechanism.CONTINUOUS_LAPLACE + liquidLegionsDistribution = internalLiquidLegionsDistribution { + decayRate = LL_DISTRIBUTION_DECAY_RATE + maxSize = LL_DISTRIBUTION_SKETCH_SIZE + } + } + } + } + } + } + } + } + ) + + val request = getMetricRequest { + name = SUCCEEDED_SINGLE_PUBLISHER_REACH_FREQUENCY_METRIC.name + } + + val result = + withMeasurementConsumerPrincipal(MEASUREMENT_CONSUMERS.values.first().name, CONFIG) { + runBlocking { service.getMetric(request) } + } + + assertThat(result) + .isEqualTo( + SUCCEEDED_SINGLE_PUBLISHER_REACH_FREQUENCY_METRIC.copy { + this.result = this.result.copy { reachAndFrequency = MetricResultKt.reachAndFrequencyResult { @@ -8004,191 +7526,2113 @@ class MetricsServiceTest { metrics += PENDING_INCREMENTAL_REACH_METRIC metrics += PENDING_SINGLE_PUBLISHER_IMPRESSION_METRIC } - ) - } + ) + } + + @Test + fun `batchGetMetrics throws INVALID_ARGUMENT when number of requests exceeds limit`() = + runBlocking { + val request = batchGetMetricsRequest { + parent = MEASUREMENT_CONSUMERS.values.first().name + names += List(MAX_BATCH_SIZE + 1) { "metric_name" } + } + + val exception = + assertFailsWith { + withMeasurementConsumerPrincipal(MEASUREMENT_CONSUMERS.values.first().name, CONFIG) { + runBlocking { service.batchGetMetrics(request) } + } + } + + assertThat(exception.status.code).isEqualTo(Status.Code.INVALID_ARGUMENT) + assertThat(exception.status.description) + .isEqualTo("At most $MAX_BATCH_SIZE metrics can be supported in a batch.") + } + + @Test + fun `createMetric creates CMMS measurements for population`() = runBlocking { + whenever(internalMetricsMock.createMetric(any())) + .thenReturn(INTERNAL_PENDING_INITIAL_POPULATION_METRIC) + whenever(measurementsMock.batchCreateMeasurements(any())) + .thenReturn( + batchCreateMeasurementsResponse { measurements += PENDING_POPULATION_MEASUREMENT } + ) + + val request = createMetricRequest { + parent = MEASUREMENT_CONSUMERS.values.first().name + metric = REQUESTING_POPULATION_METRIC + metricId = "metric-id" + } + val result = + withMeasurementConsumerPrincipal(request.parent, CONFIG) { + runBlocking { service.createMetric(request) } + } + + val expected = PENDING_POPULATION_METRIC + + // Verify proto argument of the internal MetricsCoroutineImplBase::createMetric + verifyProtoArgument(internalMetricsMock, MetricsCoroutineImplBase::createMetric) + .ignoringRepeatedFieldOrder() + .isEqualTo( + internalCreateMetricRequest { + metric = INTERNAL_REQUESTING_POPULATION_METRIC + externalMetricId = "metric-id" + } + ) + + // Verify proto argument of MeasurementsCoroutineImplBase::createMeasurement + val measurementsCaptor: KArgumentCaptor = argumentCaptor() + verifyBlocking(measurementsMock, times(1)) { + batchCreateMeasurements(measurementsCaptor.capture()) + } + val capturedMeasurementRequests = measurementsCaptor.allValues + assertThat(capturedMeasurementRequests) + .ignoringRepeatedFieldOrder() + .ignoringFieldDescriptors(MEASUREMENT_SPEC_FIELD, ENCRYPTED_REQUISITION_SPEC_FIELD) + .containsExactly( + batchCreateMeasurementsRequest { + parent = request.parent + requests += createMeasurementRequest { + parent = request.parent + measurement = REQUESTING_POPULATION_MEASUREMENT + requestId = INTERNAL_PENDING_POPULATION_MEASUREMENT.cmmsCreateMeasurementRequestId + } + } + ) + + val capturedMeasurementRequest = capturedMeasurementRequests.single().requestsList.first() + verifyMeasurementSpec( + capturedMeasurementRequest.measurement.measurementSpec, + MEASUREMENT_CONSUMER_CERTIFICATE, + TRUSTED_MEASUREMENT_CONSUMER_ISSUER, + ) + + val dataProvidersList = + capturedMeasurementRequest.measurement.dataProvidersList.sortedBy { it.key } + + val measurementSpec: MeasurementSpec = + capturedMeasurementRequest.measurement.measurementSpec.unpack() + + assertThat(measurementSpec) + .isEqualTo( + POPULATION_MEASUREMENT_SPEC.copy { + nonceHashes.clear() + nonceHashes += List(dataProvidersList.size) { Hashing.hashSha256(RANDOM_OUTPUT_LONG) } + } + ) + + dataProvidersList.map { dataProviderEntry -> + val signedRequisitionSpec = + decryptRequisitionSpec( + dataProviderEntry.value.encryptedRequisitionSpec, + DATA_PROVIDER_PRIVATE_KEY_HANDLE, + ) + val requisitionSpec: RequisitionSpec = signedRequisitionSpec.unpack() + verifyRequisitionSpec( + signedRequisitionSpec, + requisitionSpec, + measurementSpec, + MEASUREMENT_CONSUMER_CERTIFICATE, + TRUSTED_MEASUREMENT_CONSUMER_ISSUER, + ) + } + + // Verify proto argument of internal MeasurementsCoroutineImplBase::batchSetCmmsMeasurementId + verifyProtoArgument( + internalMeasurementsMock, + InternalMeasurementsGrpcKt.MeasurementsCoroutineImplBase::batchSetCmmsMeasurementIds, + ) + .ignoringRepeatedFieldOrder() + .isEqualTo( + batchSetCmmsMeasurementIdsRequest { + cmmsMeasurementConsumerId = MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId + this.measurementIds += measurementIds { + cmmsCreateMeasurementRequestId = + INTERNAL_PENDING_POPULATION_MEASUREMENT.cmmsCreateMeasurementRequestId + cmmsMeasurementId = INTERNAL_PENDING_POPULATION_MEASUREMENT.cmmsMeasurementId + } + } + ) + assertThat(result).isEqualTo(expected) + } + + @Test + fun `getMetric returns population metric`() = runBlocking { + whenever(internalMetricsMock.batchGetMetrics(any())) + .thenReturn( + internalBatchGetMetricsResponse { metrics += INTERNAL_SUCCEEDED_POPULATION_METRIC } + ) + + val request = getMetricRequest { name = SUCCEEDED_POPULATION_METRIC.name } + + val result = + withMeasurementConsumerPrincipal(MEASUREMENT_CONSUMERS.values.first().name, CONFIG) { + runBlocking { service.getMetric(request) } + } + + // Verify proto argument of internal MetricsCoroutineImplBase::batchGetMetrics + val batchGetInternalMetricsCaptor: KArgumentCaptor = + argumentCaptor() + verifyBlocking(internalMetricsMock, times(1)) { + batchGetMetrics(batchGetInternalMetricsCaptor.capture()) + } + val capturedInternalGetMetricRequests = batchGetInternalMetricsCaptor.allValues + assertThat(capturedInternalGetMetricRequests) + .containsExactly( + internalBatchGetMetricsRequest { + cmmsMeasurementConsumerId = INTERNAL_SUCCEEDED_POPULATION_METRIC.cmmsMeasurementConsumerId + externalMetricIds += INTERNAL_SUCCEEDED_POPULATION_METRIC.externalMetricId + } + ) + + // Verify proto argument of internal MeasurementsCoroutineImplBase::batchSetMeasurementResults + val batchSetMeasurementResultsCaptor: KArgumentCaptor = + argumentCaptor() + verifyBlocking(internalMeasurementsMock, never()) { + batchSetMeasurementResults(batchSetMeasurementResultsCaptor.capture()) + } + + // Verify proto argument of internal + // MeasurementsCoroutineImplBase::batchSetMeasurementFailures + val batchSetMeasurementFailuresCaptor: KArgumentCaptor = + argumentCaptor() + verifyBlocking(internalMeasurementsMock, never()) { + batchSetMeasurementFailures(batchSetMeasurementFailuresCaptor.capture()) + } + + assertThat(result).isEqualTo(SUCCEEDED_POPULATION_METRIC) + } + + companion object { + private val MEASUREMENT_SPEC_FIELD = + Measurement.getDescriptor().findFieldByNumber(Measurement.MEASUREMENT_SPEC_FIELD_NUMBER) + private val ENCRYPTED_REQUISITION_SPEC_FIELD = + Measurement.DataProviderEntry.Value.getDescriptor() + .findFieldByNumber( + Measurement.DataProviderEntry.Value.ENCRYPTED_REQUISITION_SPEC_FIELD_NUMBER + ) + + private const val NUMBER_VID_BUCKETS = 300 + private const val REACH_ONLY_VID_SAMPLING_WIDTH = 3.0f / NUMBER_VID_BUCKETS + private const val REACH_ONLY_VID_SAMPLING_START = 0.0f + private const val REACH_ONLY_REACH_EPSILON = 0.0041 + private const val SINGLE_DATA_PROVIDER_REACH_ONLY_VID_SAMPLING_WIDTH = 4.0f / NUMBER_VID_BUCKETS + private const val SINGLE_DATA_PROVIDER_REACH_ONLY_VID_SAMPLING_START = 0.01f + private const val SINGLE_DATA_PROVIDER_REACH_ONLY_REACH_EPSILON = 0.0042 + + private const val REACH_FREQUENCY_VID_SAMPLING_WIDTH = 5.0f / NUMBER_VID_BUCKETS + private const val REACH_FREQUENCY_VID_SAMPLING_START = 48.0f / NUMBER_VID_BUCKETS + private const val REACH_FREQUENCY_REACH_EPSILON = 0.0033 + private const val REACH_FREQUENCY_FREQUENCY_EPSILON = 0.115 + private const val REACH_FREQUENCY_MAXIMUM_FREQUENCY = 5 + private const val SINGLE_DATA_PROVIDER_REACH_FREQUENCY_VID_SAMPLING_WIDTH = 5.1f / NUMBER_VID_BUCKETS + private const val SINGLE_DATA_PROVIDER_REACH_FREQUENCY_VID_SAMPLING_START = 48.1f / NUMBER_VID_BUCKETS + private const val SINGLE_DATA_PROVIDER_REACH_FREQUENCY_REACH_EPSILON = 0.0034 + private const val SINGLE_DATA_PROVIDER_REACH_FREQUENCY_FREQUENCY_EPSILON = 0.116 + + private const val IMPRESSION_VID_SAMPLING_WIDTH = 62.0f / NUMBER_VID_BUCKETS + private const val IMPRESSION_VID_SAMPLING_START = 143.0f / NUMBER_VID_BUCKETS + private const val IMPRESSION_EPSILON = 0.0011 + private const val IMPRESSION_MAXIMUM_FREQUENCY_PER_USER = 60 + private const val IMPRESSION_CUSTOM_MAXIMUM_FREQUENCY_PER_USER = 100 + + private const val WATCH_DURATION_VID_SAMPLING_WIDTH = 95.0f / NUMBER_VID_BUCKETS + private const val WATCH_DURATION_VID_SAMPLING_START = 205.0f / NUMBER_VID_BUCKETS + private const val WATCH_DURATION_EPSILON = 0.001 + private val MAXIMUM_WATCH_DURATION_PER_USER = Durations.fromSeconds(4000) + + private const val DIFFERENTIAL_PRIVACY_DELTA = 1e-12 + + private const val RANDOM_OUTPUT_INT = 0 + private const val RANDOM_OUTPUT_LONG = 0L + + private val SECRETS_DIR: File = run { + val runfiles = Runfiles.preload(buildMap { + put("RUNFILES_DIR", "src/main/k8s/testing/") + put("metric_spec_config.textproto", "metric_spec_config.textproto") + put("aggregator_cs_cert.der", "aggregator_cs_cert.der") + put("aggregator_cs_private.der", "aggregator_cs_private.der") + put("aggregator_root.pem", "aggregator_root.pem") + put("mc_root.pem", "mc_root.pem") + put("mc_cs_cert.der", "mc_cs_cert.der") + put("mc_cs_private.der", "mc_cs_private.der") + put("mc_enc_private.tink", "mc_enc_private.tink") + put("mc_enc_public.tink", "mc_enc_public.tink") + put("edp1_enc_public.tink", "edp1_enc_public.tink") + put("edp1_enc_private.tink", "edp1_enc_private.tink") + put("edp1_cs_cert.der", "edp1_cs_cert.der") + put("edp1_cs_private.der", "edp1_cs_private.der") + put("edp1_root.pem", "edp1_root.pem") + }).unmapped() + checkNotNull(runfiles.getRuntimePath(Paths.get("secretfiles"))).toFile() + } + + private val METRIC_SPEC_CONFIG = metricSpecConfig { + reachParams = + MetricSpecConfigKt.reachParams { + multipleDataProviderParams = MetricSpecConfigKt.params { + privacyParams = + MetricSpecConfigKt.differentialPrivacyParams { + epsilon = REACH_ONLY_REACH_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + vidSamplingInterval = MetricSpecConfigKt.vidSamplingInterval { + fixedStart = MetricSpecConfigKt.VidSamplingIntervalKt.fixedStart { + start = REACH_ONLY_VID_SAMPLING_START + width = REACH_ONLY_VID_SAMPLING_WIDTH + } + } + } + + singleDataProviderParams = MetricSpecConfigKt.params { + privacyParams = + MetricSpecConfigKt.differentialPrivacyParams { + epsilon = SINGLE_DATA_PROVIDER_REACH_ONLY_REACH_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + vidSamplingInterval = MetricSpecConfigKt.vidSamplingInterval { + fixedStart = MetricSpecConfigKt.VidSamplingIntervalKt.fixedStart { + start = SINGLE_DATA_PROVIDER_REACH_ONLY_VID_SAMPLING_START + width = SINGLE_DATA_PROVIDER_REACH_ONLY_VID_SAMPLING_WIDTH + } + } + } + } + + reachAndFrequencyParams = + MetricSpecConfigKt.reachAndFrequencyParams { + multipleDataProviderParams = MetricSpecConfigKt.params { + privacyParams = + MetricSpecConfigKt.differentialPrivacyParams { + epsilon = REACH_FREQUENCY_REACH_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + frequencyPrivacyParams = + MetricSpecConfigKt.differentialPrivacyParams { + epsilon = REACH_FREQUENCY_FREQUENCY_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + vidSamplingInterval = MetricSpecConfigKt.vidSamplingInterval { + fixedStart = MetricSpecConfigKt.VidSamplingIntervalKt.fixedStart { + start = REACH_FREQUENCY_VID_SAMPLING_START + width = REACH_FREQUENCY_VID_SAMPLING_WIDTH + } + } + } + + singleDataProviderParams = MetricSpecConfigKt.params { + privacyParams = + MetricSpecConfigKt.differentialPrivacyParams { + epsilon = SINGLE_DATA_PROVIDER_REACH_FREQUENCY_REACH_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + frequencyPrivacyParams = + MetricSpecConfigKt.differentialPrivacyParams { + epsilon = SINGLE_DATA_PROVIDER_REACH_FREQUENCY_FREQUENCY_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + vidSamplingInterval = MetricSpecConfigKt.vidSamplingInterval { + fixedStart = MetricSpecConfigKt.VidSamplingIntervalKt.fixedStart { + start = SINGLE_DATA_PROVIDER_REACH_FREQUENCY_VID_SAMPLING_START + width = SINGLE_DATA_PROVIDER_REACH_FREQUENCY_VID_SAMPLING_WIDTH + } + } + } + maximumFrequency = REACH_FREQUENCY_MAXIMUM_FREQUENCY + } + + impressionCountParams = + MetricSpecConfigKt.impressionCountParams { + params = MetricSpecConfigKt.params { + privacyParams = + MetricSpecConfigKt.differentialPrivacyParams { + epsilon = IMPRESSION_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + vidSamplingInterval = MetricSpecConfigKt.vidSamplingInterval { + fixedStart = MetricSpecConfigKt.VidSamplingIntervalKt.fixedStart { + start = IMPRESSION_VID_SAMPLING_START + width = IMPRESSION_VID_SAMPLING_WIDTH + } + } + } + maximumFrequencyPerUser = IMPRESSION_MAXIMUM_FREQUENCY_PER_USER + } + + watchDurationParams = + MetricSpecConfigKt.watchDurationParams { + params = MetricSpecConfigKt.params { + privacyParams = + MetricSpecConfigKt.differentialPrivacyParams { + epsilon = WATCH_DURATION_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + vidSamplingInterval = MetricSpecConfigKt.vidSamplingInterval { + fixedStart = MetricSpecConfigKt.VidSamplingIntervalKt.fixedStart { + start = WATCH_DURATION_VID_SAMPLING_START + width = WATCH_DURATION_VID_SAMPLING_WIDTH + } + } + } + maximumWatchDurationPerUser = MAXIMUM_WATCH_DURATION_PER_USER + } + + populationCountParams = MetricSpecConfig.PopulationCountParams.getDefaultInstance() + } + + // Authentication key + private const val API_AUTHENTICATION_KEY = "nR5QPN7ptx" + + // Aggregator certificate + + private val AGGREGATOR_SIGNING_KEY: SigningKeyHandle by lazy { + loadSigningKey( + SECRETS_DIR.resolve("aggregator_cs_cert.der"), + SECRETS_DIR.resolve("aggregator_cs_private.der"), + ) + } + private val AGGREGATOR_CERTIFICATE = certificate { + name = "duchies/aggregator/certificates/abc123" + x509Der = AGGREGATOR_SIGNING_KEY.certificate.encoded.toByteString() + } + private val AGGREGATOR_ROOT_CERTIFICATE: X509Certificate = + readCertificate(SECRETS_DIR.resolve("aggregator_root.pem")) + + // Measurement consumer crypto + + private val TRUSTED_MEASUREMENT_CONSUMER_ISSUER: X509Certificate = + readCertificate(SECRETS_DIR.resolve("mc_root.pem")) + private val MEASUREMENT_CONSUMER_SIGNING_KEY_HANDLE = + loadSigningKey(SECRETS_DIR.resolve("mc_cs_cert.der"), SECRETS_DIR.resolve("mc_cs_private.der")) + private val MEASUREMENT_CONSUMER_CERTIFICATE = MEASUREMENT_CONSUMER_SIGNING_KEY_HANDLE.certificate + private val MEASUREMENT_CONSUMER_PRIVATE_KEY_HANDLE: PrivateKeyHandle = + loadPrivateKey(SECRETS_DIR.resolve("mc_enc_private.tink")) + private val MEASUREMENT_CONSUMER_PUBLIC_KEY = encryptionPublicKey { + format = EncryptionPublicKey.Format.TINK_KEYSET + data = SECRETS_DIR.resolve("mc_enc_public.tink").readByteString() + } + + private val MEASUREMENT_CONSUMERS: Map = + (1L..2L).associate { + val measurementConsumerKey = MeasurementConsumerKey(ExternalId(it + 110L).apiId.value) + val certificateKey = + MeasurementConsumerCertificateKey( + measurementConsumerKey.measurementConsumerId, + ExternalId(it + 120L).apiId.value, + ) + measurementConsumerKey to + measurementConsumer { + name = measurementConsumerKey.toName() + certificate = certificateKey.toName() + certificateDer = MEASUREMENT_CONSUMER_SIGNING_KEY_HANDLE.certificate.encoded.toByteString() + publicKey = + signEncryptionPublicKey( + MEASUREMENT_CONSUMER_PUBLIC_KEY, + MEASUREMENT_CONSUMER_SIGNING_KEY_HANDLE, + ) + } + } + + private val CONFIG = measurementConsumerConfig { + apiKey = API_AUTHENTICATION_KEY + signingCertificateName = MEASUREMENT_CONSUMERS.values.first().certificate + signingPrivateKeyPath = "mc_cs_private.der" + } + + // InMemoryEncryptionKeyPairStore + private val ENCRYPTION_KEY_PAIR_STORE = + InMemoryEncryptionKeyPairStore( + MEASUREMENT_CONSUMERS.values.associateBy( + { it.name }, + { + listOf( + it.publicKey.unpack().data to MEASUREMENT_CONSUMER_PRIVATE_KEY_HANDLE + ) + }, + ) + ) + + private val DATA_PROVIDER_PUBLIC_KEY = encryptionPublicKey { + format = EncryptionPublicKey.Format.TINK_KEYSET + data = SECRETS_DIR.resolve("edp1_enc_public.tink").readByteString() + } + private val DATA_PROVIDER_PRIVATE_KEY_HANDLE = + loadPrivateKey(SECRETS_DIR.resolve("edp1_enc_private.tink")) + private val DATA_PROVIDER_SIGNING_KEY = + loadSigningKey( + SECRETS_DIR.resolve("edp1_cs_cert.der"), + SECRETS_DIR.resolve("edp1_cs_private.der"), + ) + private val DATA_PROVIDER_ROOT_CERTIFICATE = readCertificate(SECRETS_DIR.resolve("edp1_root.pem")) + + // Data providers + + private val DATA_PROVIDERS = + (1L..3L).associate { + val dataProviderKey = DataProviderKey(ExternalId(it + 550L).apiId.value) + val certificateKey = + DataProviderCertificateKey(dataProviderKey.dataProviderId, ExternalId(it + 560L).apiId.value) + dataProviderKey to + dataProvider { + name = dataProviderKey.toName() + certificate = certificateKey.toName() + publicKey = signEncryptionPublicKey(DATA_PROVIDER_PUBLIC_KEY, DATA_PROVIDER_SIGNING_KEY) + } + } + private val DATA_PROVIDERS_LIST = DATA_PROVIDERS.values.toList() + + // Event group keys + private val CMMS_EVENT_GROUP_KEYS = + DATA_PROVIDERS.keys.mapIndexed { index, dataProviderKey -> + CmmsEventGroupKey(dataProviderKey.dataProviderId, ExternalId(index + 660L).apiId.value) + } + + // Event filters + private const val INCREMENTAL_REPORTING_SET_FILTER = "AGE>18" + private const val METRIC_FILTER = "media_type==video" + private const val PRIMITIVE_REPORTING_SET_FILTER = "gender==male" + private val ALL_FILTERS = + listOf(INCREMENTAL_REPORTING_SET_FILTER, METRIC_FILTER, PRIMITIVE_REPORTING_SET_FILTER) + + // Internal reporting sets + private val INTERNAL_UNION_ALL_REPORTING_SET = internalReportingSet { + cmmsMeasurementConsumerId = MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId + externalReportingSetId = "220L" + this.primitive = + InternalReportingSetKt.primitive { + eventGroupKeys += CMMS_EVENT_GROUP_KEYS.map { it.toInternal() } + } + filter = PRIMITIVE_REPORTING_SET_FILTER + displayName = "$cmmsMeasurementConsumerId-$externalReportingSetId-$filter" + weightedSubsetUnions += weightedSubsetUnion { + primitiveReportingSetBases += primitiveReportingSetBasis { + externalReportingSetId = this@internalReportingSet.externalReportingSetId + filters += this@internalReportingSet.filter + } + weight = 1 + binaryRepresentation = 1 + } + } + private val INTERNAL_UNION_ALL_BUT_LAST_PUBLISHER_REPORTING_SET = internalReportingSet { + cmmsMeasurementConsumerId = MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId + externalReportingSetId = INTERNAL_UNION_ALL_REPORTING_SET.externalReportingSetId + "1" + this.primitive = + InternalReportingSetKt.primitive { + (0 until CMMS_EVENT_GROUP_KEYS.size - 1).map { i -> + eventGroupKeys += CMMS_EVENT_GROUP_KEYS[i].toInternal() + } + } + filter = PRIMITIVE_REPORTING_SET_FILTER + displayName = "$cmmsMeasurementConsumerId-$externalReportingSetId-$filter" + weightedSubsetUnions += weightedSubsetUnion { + primitiveReportingSetBases += primitiveReportingSetBasis { + externalReportingSetId = this@internalReportingSet.externalReportingSetId + filters += this@internalReportingSet.filter + } + weight = 1 + binaryRepresentation = 1 + } + } + private val INTERNAL_SINGLE_PUBLISHER_REPORTING_SET = internalReportingSet { + cmmsMeasurementConsumerId = MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId + externalReportingSetId = + INTERNAL_UNION_ALL_BUT_LAST_PUBLISHER_REPORTING_SET.externalReportingSetId + "1" + this.primitive = + InternalReportingSetKt.primitive { + eventGroupKeys += + (0L until 3L) + .map { index -> + CmmsEventGroupKey( + DATA_PROVIDERS.keys.first().dataProviderId, + ExternalId(index + 670L).apiId.value, + ) + } + .map { it.toInternal() } + } + filter = PRIMITIVE_REPORTING_SET_FILTER + displayName = "$cmmsMeasurementConsumerId-$externalReportingSetId-$filter" + weightedSubsetUnions += weightedSubsetUnion { + primitiveReportingSetBases += primitiveReportingSetBasis { + externalReportingSetId = this@internalReportingSet.externalReportingSetId + filters += this@internalReportingSet.filter + } + weight = 1 + binaryRepresentation = 1 + } + } + + private val INTERNAL_INCREMENTAL_REPORTING_SET = internalReportingSet { + cmmsMeasurementConsumerId = MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId + externalReportingSetId = INTERNAL_SINGLE_PUBLISHER_REPORTING_SET.externalReportingSetId + "1" + this.composite = + InternalReportingSetKt.setExpression { + operation = InternalSetExpression.Operation.DIFFERENCE + lhs = + InternalReportingSetKt.SetExpressionKt.operand { + externalReportingSetId = INTERNAL_UNION_ALL_REPORTING_SET.externalReportingSetId + } + rhs = + InternalReportingSetKt.SetExpressionKt.operand { + externalReportingSetId = + INTERNAL_UNION_ALL_BUT_LAST_PUBLISHER_REPORTING_SET.externalReportingSetId + } + } + filter = INCREMENTAL_REPORTING_SET_FILTER + displayName = "$cmmsMeasurementConsumerId-$externalReportingSetId-$filter" + weightedSubsetUnions += weightedSubsetUnion { + primitiveReportingSetBases += primitiveReportingSetBasis { + externalReportingSetId = INTERNAL_UNION_ALL_REPORTING_SET.externalReportingSetId + filters += INCREMENTAL_REPORTING_SET_FILTER + filters += INTERNAL_UNION_ALL_REPORTING_SET.filter + } + primitiveReportingSetBases += primitiveReportingSetBasis { + externalReportingSetId = + INTERNAL_UNION_ALL_BUT_LAST_PUBLISHER_REPORTING_SET.externalReportingSetId + filters += INCREMENTAL_REPORTING_SET_FILTER + filters += INTERNAL_UNION_ALL_BUT_LAST_PUBLISHER_REPORTING_SET.filter + } + weight = 1 + binaryRepresentation = 3 + } + weightedSubsetUnions += weightedSubsetUnion { + primitiveReportingSetBases += primitiveReportingSetBasis { + externalReportingSetId = + INTERNAL_UNION_ALL_BUT_LAST_PUBLISHER_REPORTING_SET.externalReportingSetId + filters += INCREMENTAL_REPORTING_SET_FILTER + filters += INTERNAL_UNION_ALL_BUT_LAST_PUBLISHER_REPORTING_SET.filter + } + weight = -1 + binaryRepresentation = 2 + } + } + private val INTERNAL_POPULATION_REPORTING_SET = internalReportingSet { + cmmsMeasurementConsumerId = MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId + externalReportingSetId = INTERNAL_INCREMENTAL_REPORTING_SET.externalReportingSetId + "1" + this.primitive = + InternalReportingSetKt.primitive { + eventGroupKeys += CMMS_EVENT_GROUP_KEYS.first().toInternal() + } + filter = INCREMENTAL_REPORTING_SET_FILTER + displayName = "$cmmsMeasurementConsumerId-$externalReportingSetId-$filter" + weightedSubsetUnions += weightedSubsetUnion { + primitiveReportingSetBases += primitiveReportingSetBasis { + externalReportingSetId = this@internalReportingSet.externalReportingSetId + } + weight = 1 + binaryRepresentation = 1 + } + } + + // Time intervals + private val START_INSTANT = Instant.now() + private val TIME_RANGE = OpenEndTimeRange(START_INSTANT, START_INSTANT.plus(Duration.ofDays(1))) + private val TIME_INTERVAL: Interval = TIME_RANGE.toInterval() + + // Requisition specs + private val REQUISITION_SPECS: Map = + CMMS_EVENT_GROUP_KEYS.groupBy( + { it.parentKey }, + { + RequisitionSpecKt.eventGroupEntry { + key = it.toName() + value = + RequisitionSpecKt.EventGroupEntryKt.value { + collectionInterval = TIME_INTERVAL + filter = + RequisitionSpecKt.eventFilter { + expression = + "($INCREMENTAL_REPORTING_SET_FILTER) AND ($METRIC_FILTER) AND ($PRIMITIVE_REPORTING_SET_FILTER)" + } + } + } + }, + ) + .mapValues { + requisitionSpec { + events = RequisitionSpecKt.events { eventGroups += it.value } + measurementPublicKey = MEASUREMENT_CONSUMERS.values.first().publicKey.message + nonce = RANDOM_OUTPUT_LONG + } + } + + // Data provider entries + private val DATA_PROVIDER_ENTRIES = + REQUISITION_SPECS.mapValues { (dataProviderKey, requisitionSpec) -> + val dataProvider = DATA_PROVIDERS.getValue(dataProviderKey) + MeasurementKt.dataProviderEntry { + key = dataProvider.name + value = + MeasurementKt.DataProviderEntryKt.value { + dataProviderCertificate = dataProvider.certificate + dataProviderPublicKey = dataProvider.publicKey.message + encryptedRequisitionSpec = + encryptRequisitionSpec( + signRequisitionSpec(requisitionSpec, MEASUREMENT_CONSUMER_SIGNING_KEY_HANDLE), + dataProvider.publicKey.unpack(), + ) + nonceHash = Hashing.hashSha256(requisitionSpec.nonce) + } + } + } + + // Measurements + private val BASE_MEASUREMENT = measurement { + measurementConsumerCertificate = MEASUREMENT_CONSUMERS.values.first().certificate + } + + private const val LL_DISTRIBUTION_DECAY_RATE = 2e-2 + private const val LL_DISTRIBUTION_SKETCH_SIZE = 20000L + private const val REACH_ONLY_LLV2_DECAY_RATE = 1e-2 + private const val REACH_ONLY_LLV2_SKETCH_SIZE = 10000L + + // Measurement values + private const val UNION_ALL_REACH_VALUE = 100_000L + private const val UNION_ALL_BUT_LAST_PUBLISHER_REACH_VALUE = 70_000L + private const val INCREMENTAL_REACH_VALUE = + UNION_ALL_REACH_VALUE - UNION_ALL_BUT_LAST_PUBLISHER_REACH_VALUE + private const val REACH_FREQUENCY_REACH_VALUE = 100_000L + private val REACH_FREQUENCY_FREQUENCY_VALUE = mapOf(1L to 0.1, 2L to 0.2, 3L to 0.3, 4L to 0.4) + private const val IMPRESSION_VALUE = 1_000_000L + private val WATCH_DURATION_SECOND_LIST = listOf(100L, 200L, 300L) + private val WATCH_DURATION_LIST = WATCH_DURATION_SECOND_LIST.map { duration { seconds = it } } + private val TOTAL_WATCH_DURATION = duration { seconds = WATCH_DURATION_SECOND_LIST.sum() } + private val TOTAL_POPULATION_VALUE = 1000L + + // Internal incremental reach measurements + private val INTERNAL_PENDING_UNION_ALL_REACH_MEASUREMENT = internalMeasurement { + cmmsMeasurementConsumerId = MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId + cmmsCreateMeasurementRequestId = "UNION_ALL_REACH_MEASUREMENT" + cmmsMeasurementId = externalIdToApiId(401L) + timeInterval = TIME_INTERVAL + primitiveReportingSetBases += primitiveReportingSetBasis { + externalReportingSetId = INTERNAL_UNION_ALL_REPORTING_SET.externalReportingSetId + filters += ALL_FILTERS + } + primitiveReportingSetBases += primitiveReportingSetBasis { + externalReportingSetId = + INTERNAL_UNION_ALL_BUT_LAST_PUBLISHER_REPORTING_SET.externalReportingSetId + filters += ALL_FILTERS + } + state = InternalMeasurement.State.PENDING + } + + private val INTERNAL_PENDING_UNION_ALL_BUT_LAST_PUBLISHER_REACH_MEASUREMENT = internalMeasurement { + cmmsMeasurementConsumerId = MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId + cmmsCreateMeasurementRequestId = "UNION_ALL_BUT_LAST_PUBLISHER_REACH_MEASUREMENT" + cmmsMeasurementId = externalIdToApiId(402L) + timeInterval = TIME_INTERVAL + primitiveReportingSetBases += primitiveReportingSetBasis { + externalReportingSetId = + INTERNAL_UNION_ALL_BUT_LAST_PUBLISHER_REPORTING_SET.externalReportingSetId + filters += ALL_FILTERS + } + state = InternalMeasurement.State.PENDING + } + + private val INTERNAL_SUCCEEDED_UNION_ALL_REACH_MEASUREMENT = + INTERNAL_PENDING_UNION_ALL_REACH_MEASUREMENT.copy { + state = InternalMeasurement.State.SUCCEEDED + details = + InternalMeasurementKt.details { + results += + InternalMeasurementKt.result { + reach = + InternalMeasurementKt.ResultKt.reach { + value = UNION_ALL_REACH_VALUE + noiseMechanism = NoiseMechanism.DISCRETE_GAUSSIAN + reachOnlyLiquidLegionsV2 = reachOnlyLiquidLegionsV2 { + sketchParams = internalReachOnlyLiquidLegionsSketchParams { + decayRate = REACH_ONLY_LLV2_DECAY_RATE + maxSize = REACH_ONLY_LLV2_SKETCH_SIZE + } + } + } + } + } + } + + private val INTERNAL_SUCCEEDED_UNION_ALL_BUT_LAST_PUBLISHER_REACH_MEASUREMENT = + INTERNAL_PENDING_UNION_ALL_BUT_LAST_PUBLISHER_REACH_MEASUREMENT.copy { + state = InternalMeasurement.State.SUCCEEDED + details = + InternalMeasurementKt.details { + results += + InternalMeasurementKt.result { + reach = + InternalMeasurementKt.ResultKt.reach { + value = UNION_ALL_BUT_LAST_PUBLISHER_REACH_VALUE + noiseMechanism = NoiseMechanism.DISCRETE_GAUSSIAN + reachOnlyLiquidLegionsV2 = reachOnlyLiquidLegionsV2 { + sketchParams = internalReachOnlyLiquidLegionsSketchParams { + decayRate = REACH_ONLY_LLV2_DECAY_RATE + maxSize = REACH_ONLY_LLV2_SKETCH_SIZE + } + } + } + } + } + } + + // Internal single publisher reach-frequency measurements + private val INTERNAL_PENDING_SINGLE_PUBLISHER_REACH_FREQUENCY_MEASUREMENT = internalMeasurement { + cmmsMeasurementConsumerId = MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId + cmmsCreateMeasurementRequestId = "SINGLE_PUBLISHER_REACH_FREQUENCY_MEASUREMENT" + cmmsMeasurementId = externalIdToApiId(443L) + timeInterval = TIME_INTERVAL + primitiveReportingSetBases += primitiveReportingSetBasis { + externalReportingSetId = INTERNAL_SINGLE_PUBLISHER_REPORTING_SET.externalReportingSetId + filters += METRIC_FILTER + filters += PRIMITIVE_REPORTING_SET_FILTER + } + state = InternalMeasurement.State.PENDING + isSingleDataProvider = true + } + + private val INTERNAL_SUCCEEDED_SINGLE_PUBLISHER_REACH_FREQUENCY_MEASUREMENT = + INTERNAL_PENDING_SINGLE_PUBLISHER_REACH_FREQUENCY_MEASUREMENT.copy { + state = InternalMeasurement.State.SUCCEEDED + details = + InternalMeasurementKt.details { + results += + InternalMeasurementKt.result { + reach = + InternalMeasurementKt.ResultKt.reach { + value = REACH_FREQUENCY_REACH_VALUE + noiseMechanism = NoiseMechanism.CONTINUOUS_LAPLACE + deterministicCountDistinct = InternalDeterministicCountDistinct.getDefaultInstance() + } + frequency = + InternalMeasurementKt.ResultKt.frequency { + relativeFrequencyDistribution.putAll(REACH_FREQUENCY_FREQUENCY_VALUE) + noiseMechanism = NoiseMechanism.CONTINUOUS_LAPLACE + liquidLegionsDistribution = internalLiquidLegionsDistribution { + decayRate = LL_DISTRIBUTION_DECAY_RATE + maxSize = LL_DISTRIBUTION_SKETCH_SIZE + } + } + } + } + } + + // Internal single publisher impression measurements + private val INTERNAL_PENDING_SINGLE_PUBLISHER_IMPRESSION_MEASUREMENT = internalMeasurement { + cmmsMeasurementConsumerId = MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId + cmmsCreateMeasurementRequestId = "SINGLE_PUBLISHER_IMPRESSION_MEASUREMENT" + cmmsMeasurementId = externalIdToApiId(403L) + timeInterval = TIME_INTERVAL + primitiveReportingSetBases += primitiveReportingSetBasis { + externalReportingSetId = INTERNAL_SINGLE_PUBLISHER_REPORTING_SET.externalReportingSetId + filters += METRIC_FILTER + filters += PRIMITIVE_REPORTING_SET_FILTER + } + state = InternalMeasurement.State.PENDING + isSingleDataProvider = true + } + + private val INTERNAL_FAILED_SINGLE_PUBLISHER_IMPRESSION_MEASUREMENT = + INTERNAL_PENDING_SINGLE_PUBLISHER_IMPRESSION_MEASUREMENT.copy { + state = InternalMeasurement.State.FAILED + details = + InternalMeasurementKt.details { + failure = + InternalMeasurementKt.failure { + reason = InternalMeasurement.Failure.Reason.REQUISITION_REFUSED + message = "Privacy budget exceeded." + } + } + } + + private val INTERNAL_SUCCEEDED_SINGLE_PUBLISHER_IMPRESSION_MEASUREMENT = + INTERNAL_PENDING_SINGLE_PUBLISHER_IMPRESSION_MEASUREMENT.copy { + state = InternalMeasurement.State.SUCCEEDED + details = + InternalMeasurementKt.details { + results += + InternalMeasurementKt.result { + impression = + InternalMeasurementKt.ResultKt.impression { + value = IMPRESSION_VALUE + noiseMechanism = NoiseMechanism.CONTINUOUS_LAPLACE + deterministicCount = InternalDeterministicCount.getDefaultInstance() + } + } + } + } + + private val INTERNAL_SUCCEEDED_SINGLE_PUBLISHER_IMPRESSION_MEASUREMENT_CUSTOM_CAP = + INTERNAL_PENDING_SINGLE_PUBLISHER_IMPRESSION_MEASUREMENT.copy { + state = InternalMeasurement.State.SUCCEEDED + details = + InternalMeasurementKt.details { + results += + InternalMeasurementKt.result { + impression = + InternalMeasurementKt.ResultKt.impression { + value = IMPRESSION_VALUE + noiseMechanism = NoiseMechanism.CONTINUOUS_LAPLACE + deterministicCount = internalDeterministicCount { + customMaximumFrequencyPerUser = IMPRESSION_CUSTOM_MAXIMUM_FREQUENCY_PER_USER + } + } + } + } + } + + // Internal cross-publisher watch duration measurements + private val INTERNAL_REQUESTING_UNION_ALL_WATCH_DURATION_MEASUREMENT = internalMeasurement { + cmmsMeasurementConsumerId = MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId + timeInterval = TIME_INTERVAL + primitiveReportingSetBases += primitiveReportingSetBasis { + externalReportingSetId = INTERNAL_UNION_ALL_REPORTING_SET.externalReportingSetId + filters += listOf(METRIC_FILTER, PRIMITIVE_REPORTING_SET_FILTER) + } + } + + private val INTERNAL_PENDING_NOT_CREATED_UNION_ALL_WATCH_DURATION_MEASUREMENT = + INTERNAL_REQUESTING_UNION_ALL_WATCH_DURATION_MEASUREMENT.copy { + cmmsMeasurementId = externalIdToApiId(414L) + cmmsCreateMeasurementRequestId = "UNION_ALL_WATCH_DURATION_MEASUREMENT" + state = InternalMeasurement.State.PENDING + } + + private val INTERNAL_PENDING_UNION_ALL_WATCH_DURATION_MEASUREMENT = + INTERNAL_PENDING_NOT_CREATED_UNION_ALL_WATCH_DURATION_MEASUREMENT.copy { + cmmsMeasurementId = externalIdToApiId(404L) + } + + private val INTERNAL_SUCCEEDED_UNION_ALL_WATCH_DURATION_MEASUREMENT = + INTERNAL_PENDING_UNION_ALL_WATCH_DURATION_MEASUREMENT.copy { + state = InternalMeasurement.State.SUCCEEDED + details = + InternalMeasurementKt.details { + results += + WATCH_DURATION_LIST.map { duration -> + InternalMeasurementKt.result { + watchDuration = + InternalMeasurementKt.ResultKt.watchDuration { + value = duration + noiseMechanism = NoiseMechanism.CONTINUOUS_LAPLACE + deterministicSum = InternalDeterministicSum.getDefaultInstance() + } + } + } + } + } + + // Internal population measurements + val INTERNAL_PENDING_POPULATION_MEASUREMENT = internalMeasurement { + cmmsMeasurementConsumerId = MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId + cmmsCreateMeasurementRequestId = "POPULATION_MEASUREMENT" + cmmsMeasurementId = externalIdToApiId(443L) + timeInterval = TIME_INTERVAL + primitiveReportingSetBases += primitiveReportingSetBasis { + externalReportingSetId = INTERNAL_POPULATION_REPORTING_SET.externalReportingSetId + filters += INCREMENTAL_REPORTING_SET_FILTER + } + state = InternalMeasurement.State.PENDING + isSingleDataProvider = true + } + + val INTERNAL_SUCCEEDED_POPULATION_MEASUREMENT = + INTERNAL_PENDING_POPULATION_MEASUREMENT.copy { + state = InternalMeasurement.State.SUCCEEDED + details = + InternalMeasurementKt.details { + results += + InternalMeasurementKt.result { + population = + InternalMeasurementKt.ResultKt.population { value = TOTAL_POPULATION_VALUE } + } + } + } + + // CMMS measurements + + private val BASE_MEASUREMENT_SPEC = measurementSpec { + measurementPublicKey = MEASUREMENT_CONSUMER_PUBLIC_KEY.pack() + // TODO(world-federation-of-advertisers/cross-media-measurement#1301): Stop setting this field. + serializedMeasurementPublicKey = measurementPublicKey.value + } + + // CMMS incremental reach measurements + private val UNION_ALL_BUT_LAST_PUBLISHER_REACH_MEASUREMENT_SPEC = BASE_MEASURMENT_SPEC.copy { + measurementPublicKey = MEASUREMENT_CONSUMER_PUBLIC_KEY.pack() + + nonceHashes += + listOf(Hashing.hashSha256(RANDOM_OUTPUT_LONG), Hashing.hashSha256(RANDOM_OUTPUT_LONG)) + + reach = + MeasurementSpecKt.reach { + privacyParams = differentialPrivacyParams { + epsilon = REACH_ONLY_REACH_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + } + vidSamplingInterval = + MeasurementSpecKt.vidSamplingInterval { + start = REACH_ONLY_VID_SAMPLING_START + width = REACH_ONLY_VID_SAMPLING_WIDTH + } + } + + private val REACH_PROTOCOL_CONFIG: ProtocolConfig = protocolConfig { + measurementType = ProtocolConfig.MeasurementType.REACH + protocols += + ProtocolConfigKt.protocol { + reachOnlyLiquidLegionsV2 = + ProtocolConfigKt.reachOnlyLiquidLegionsV2 { + sketchParams = reachOnlyLiquidLegionsSketchParams { + decayRate = REACH_ONLY_LLV2_DECAY_RATE + maxSize = REACH_ONLY_LLV2_SKETCH_SIZE + } + noiseMechanism = ProtocolConfig.NoiseMechanism.DISCRETE_GAUSSIAN + } + } + } + + private val REQUESTING_UNION_ALL_REACH_MEASUREMENT = + BASE_MEASUREMENT.copy { + dataProviders += DATA_PROVIDERS.keys.map { DATA_PROVIDER_ENTRIES.getValue(it) } + + measurementSpec = + signMeasurementSpec( + UNION_ALL_BUT_LAST_PUBLISHER_REACH_MEASUREMENT_SPEC.copy { + nonceHashes += Hashing.hashSha256(RANDOM_OUTPUT_LONG) + }, + MEASUREMENT_CONSUMER_SIGNING_KEY_HANDLE, + ) + measurementReferenceId = + INTERNAL_PENDING_UNION_ALL_REACH_MEASUREMENT.cmmsCreateMeasurementRequestId + } + private val REQUESTING_UNION_ALL_BUT_LAST_PUBLISHER_REACH_MEASUREMENT = + BASE_MEASUREMENT.copy { + dataProviders += DATA_PROVIDERS.keys.take(2).map { DATA_PROVIDER_ENTRIES.getValue(it) } + + measurementSpec = + signMeasurementSpec( + UNION_ALL_BUT_LAST_PUBLISHER_REACH_MEASUREMENT_SPEC, + MEASUREMENT_CONSUMER_SIGNING_KEY_HANDLE, + ) + measurementReferenceId = + INTERNAL_PENDING_UNION_ALL_BUT_LAST_PUBLISHER_REACH_MEASUREMENT.cmmsCreateMeasurementRequestId + } + + private val PENDING_UNION_ALL_REACH_MEASUREMENT = + REQUESTING_UNION_ALL_REACH_MEASUREMENT.copy { + name = + MeasurementKey( + MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId, + INTERNAL_PENDING_UNION_ALL_REACH_MEASUREMENT.cmmsMeasurementId, + ) + .toName() + protocolConfig = REACH_PROTOCOL_CONFIG + state = Measurement.State.COMPUTING + } + private val PENDING_UNION_ALL_BUT_LAST_PUBLISHER_REACH_MEASUREMENT = + REQUESTING_UNION_ALL_BUT_LAST_PUBLISHER_REACH_MEASUREMENT.copy { + name = + MeasurementKey( + MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId, + INTERNAL_PENDING_UNION_ALL_BUT_LAST_PUBLISHER_REACH_MEASUREMENT.cmmsMeasurementId, + ) + .toName() + protocolConfig = REACH_PROTOCOL_CONFIG + state = Measurement.State.COMPUTING + } + + private val SUCCEEDED_UNION_ALL_REACH_MEASUREMENT = + PENDING_UNION_ALL_REACH_MEASUREMENT.copy { + state = Measurement.State.SUCCEEDED + + results += resultOutput { + val result = + MeasurementKt.result { + reach = MeasurementKt.ResultKt.reach { value = UNION_ALL_REACH_VALUE } + } + encryptedResult = + encryptResult(signResult(result, AGGREGATOR_SIGNING_KEY), MEASUREMENT_CONSUMER_PUBLIC_KEY) + certificate = AGGREGATOR_CERTIFICATE.name + } + } + private val SUCCEEDED_UNION_ALL_BUT_LAST_PUBLISHER_REACH_MEASUREMENT = + PENDING_UNION_ALL_BUT_LAST_PUBLISHER_REACH_MEASUREMENT.copy { + state = Measurement.State.SUCCEEDED + + results += resultOutput { + val result = + MeasurementKt.result { + reach = MeasurementKt.ResultKt.reach { value = UNION_ALL_BUT_LAST_PUBLISHER_REACH_VALUE } + } + encryptedResult = + encryptResult(signResult(result, AGGREGATOR_SIGNING_KEY), MEASUREMENT_CONSUMER_PUBLIC_KEY) + certificate = AGGREGATOR_CERTIFICATE.name + } + } + + // CMMS single publisher reach-frequency measurements + private val SINGLE_PUBLISHER_REACH_FREQUENCY_MEASUREMENT_SPEC = BASE_MEASURMENT_SPEC.copy { + measurementPublicKey = MEASUREMENT_CONSUMER_PUBLIC_KEY.pack() + + nonceHashes.add(Hashing.hashSha256(RANDOM_OUTPUT_LONG)) + + reachAndFrequency = + MeasurementSpecKt.reachAndFrequency { + reachPrivacyParams = differentialPrivacyParams { + epsilon = REACH_FREQUENCY_REACH_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + frequencyPrivacyParams = differentialPrivacyParams { + epsilon = REACH_FREQUENCY_FREQUENCY_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + maximumFrequency = REACH_FREQUENCY_MAXIMUM_FREQUENCY + } + vidSamplingInterval = + MeasurementSpecKt.vidSamplingInterval { + start = REACH_FREQUENCY_VID_SAMPLING_START + width = REACH_FREQUENCY_VID_SAMPLING_WIDTH + } + } + + private val REACH_FREQUENCY_PROTOCOL_CONFIG: ProtocolConfig = protocolConfig { + measurementType = ProtocolConfig.MeasurementType.REACH_AND_FREQUENCY + protocols += + ProtocolConfigKt.protocol { + direct = + ProtocolConfigKt.direct { + noiseMechanisms += + listOf( + ProtocolConfig.NoiseMechanism.NONE, + ProtocolConfig.NoiseMechanism.CONTINUOUS_LAPLACE, + ProtocolConfig.NoiseMechanism.CONTINUOUS_GAUSSIAN, + ) + deterministicCount = ProtocolConfig.Direct.DeterministicCount.getDefaultInstance() + liquidLegionsDistribution = + ProtocolConfig.Direct.LiquidLegionsDistribution.getDefaultInstance() + } + } + } + + private val REQUESTING_SINGLE_PUBLISHER_REACH_FREQUENCY_MEASUREMENT = + BASE_MEASUREMENT.copy { + dataProviders += DATA_PROVIDER_ENTRIES.getValue(DATA_PROVIDERS.keys.first()) + + measurementSpec = + signMeasurementSpec( + SINGLE_PUBLISHER_REACH_FREQUENCY_MEASUREMENT_SPEC, + MEASUREMENT_CONSUMER_SIGNING_KEY_HANDLE, + ) + measurementReferenceId = + INTERNAL_PENDING_SINGLE_PUBLISHER_REACH_FREQUENCY_MEASUREMENT.cmmsCreateMeasurementRequestId + } + + private val PENDING_SINGLE_PUBLISHER_REACH_FREQUENCY_MEASUREMENT = + REQUESTING_SINGLE_PUBLISHER_REACH_FREQUENCY_MEASUREMENT.copy { + name = + MeasurementKey( + MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId, + INTERNAL_PENDING_SINGLE_PUBLISHER_REACH_FREQUENCY_MEASUREMENT.cmmsMeasurementId, + ) + .toName() + protocolConfig = REACH_FREQUENCY_PROTOCOL_CONFIG + state = Measurement.State.COMPUTING + } + + private val SUCCEEDED_SINGLE_PUBLISHER_REACH_FREQUENCY_MEASUREMENT = + PENDING_SINGLE_PUBLISHER_REACH_FREQUENCY_MEASUREMENT.copy { + state = Measurement.State.SUCCEEDED + + results += resultOutput { + val result = + MeasurementKt.result { + reach = + MeasurementKt.ResultKt.reach { + value = REACH_FREQUENCY_REACH_VALUE + noiseMechanism = ProtocolConfig.NoiseMechanism.CONTINUOUS_LAPLACE + deterministicCountDistinct = DeterministicCountDistinct.getDefaultInstance() + } + frequency = + MeasurementKt.ResultKt.frequency { + relativeFrequencyDistribution.putAll(REACH_FREQUENCY_FREQUENCY_VALUE) + noiseMechanism = ProtocolConfig.NoiseMechanism.CONTINUOUS_LAPLACE + liquidLegionsDistribution = liquidLegionsDistribution { + decayRate = LL_DISTRIBUTION_DECAY_RATE + maxSize = LL_DISTRIBUTION_SKETCH_SIZE + } + } + } + encryptedResult = + encryptResult(signResult(result, AGGREGATOR_SIGNING_KEY), MEASUREMENT_CONSUMER_PUBLIC_KEY) + certificate = AGGREGATOR_CERTIFICATE.name + } + } + + // CMMS single publisher impression measurements + private val SINGLE_PUBLISHER_IMPRESSION_MEASUREMENT_SPEC = BASE_MEASURMENT_SPEC.copy { + measurementPublicKey = MEASUREMENT_CONSUMER_PUBLIC_KEY.pack() + + nonceHashes.add(Hashing.hashSha256(RANDOM_OUTPUT_LONG)) + + impression = + MeasurementSpecKt.impression { + privacyParams = differentialPrivacyParams { + epsilon = IMPRESSION_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + maximumFrequencyPerUser = IMPRESSION_MAXIMUM_FREQUENCY_PER_USER + } + vidSamplingInterval = + MeasurementSpecKt.vidSamplingInterval { + start = IMPRESSION_VID_SAMPLING_START + width = IMPRESSION_VID_SAMPLING_WIDTH + } + } + + private val IMPRESSION_PROTOCOL_CONFIG: ProtocolConfig = protocolConfig { + measurementType = ProtocolConfig.MeasurementType.IMPRESSION + protocols += + ProtocolConfigKt.protocol { + direct = + ProtocolConfigKt.direct { + noiseMechanisms += + listOf( + ProtocolConfig.NoiseMechanism.NONE, + ProtocolConfig.NoiseMechanism.CONTINUOUS_LAPLACE, + ProtocolConfig.NoiseMechanism.CONTINUOUS_GAUSSIAN, + ) + deterministicCount = ProtocolConfig.Direct.DeterministicCount.getDefaultInstance() + } + } + } + + private val REQUESTING_SINGLE_PUBLISHER_IMPRESSION_MEASUREMENT = + BASE_MEASUREMENT.copy { + dataProviders += DATA_PROVIDER_ENTRIES.getValue(DATA_PROVIDERS.keys.first()) + + measurementSpec = + signMeasurementSpec( + SINGLE_PUBLISHER_IMPRESSION_MEASUREMENT_SPEC, + MEASUREMENT_CONSUMER_SIGNING_KEY_HANDLE, + ) + measurementReferenceId = + INTERNAL_PENDING_SINGLE_PUBLISHER_IMPRESSION_MEASUREMENT.cmmsCreateMeasurementRequestId + } + + private val PENDING_SINGLE_PUBLISHER_IMPRESSION_MEASUREMENT = + REQUESTING_SINGLE_PUBLISHER_IMPRESSION_MEASUREMENT.copy { + name = + MeasurementKey( + MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId, + INTERNAL_PENDING_SINGLE_PUBLISHER_IMPRESSION_MEASUREMENT.cmmsMeasurementId, + ) + .toName() + protocolConfig = IMPRESSION_PROTOCOL_CONFIG + state = Measurement.State.COMPUTING + } + + private val SUCCEEDED_SINGLE_PUBLISHER_IMPRESSION_MEASUREMENT = + PENDING_SINGLE_PUBLISHER_IMPRESSION_MEASUREMENT.copy { + state = Measurement.State.SUCCEEDED + results += resultOutput { + val result = + MeasurementKt.result { + impression = + MeasurementKt.ResultKt.impression { + value = IMPRESSION_VALUE + noiseMechanism = ProtocolConfig.NoiseMechanism.CONTINUOUS_LAPLACE + deterministicCount = DeterministicCount.getDefaultInstance() + } + } + encryptedResult = + encryptResult(signResult(result, AGGREGATOR_SIGNING_KEY), MEASUREMENT_CONSUMER_PUBLIC_KEY) + certificate = AGGREGATOR_CERTIFICATE.name + } + } + + private val SUCCEEDED_SINGLE_PUBLISHER_IMPRESSION_MEASUREMENT_CUSTOM_CAP = + PENDING_SINGLE_PUBLISHER_IMPRESSION_MEASUREMENT.copy { + state = Measurement.State.SUCCEEDED + results += resultOutput { + val result = + MeasurementKt.result { + impression = + MeasurementKt.ResultKt.impression { + value = IMPRESSION_VALUE + noiseMechanism = ProtocolConfig.NoiseMechanism.CONTINUOUS_LAPLACE + deterministicCount = deterministicCount { + customMaximumFrequencyPerUser = IMPRESSION_CUSTOM_MAXIMUM_FREQUENCY_PER_USER + } + } + } + encryptedResult = + encryptResult(signResult(result, AGGREGATOR_SIGNING_KEY), MEASUREMENT_CONSUMER_PUBLIC_KEY) + certificate = AGGREGATOR_CERTIFICATE.name + } + } + + // CMMS cross publisher watch duration measurements + private val UNION_ALL_WATCH_DURATION_MEASUREMENT_SPEC = BASE_MEASURMENT_SPEC.copy { + measurementPublicKey = MEASUREMENT_CONSUMER_PUBLIC_KEY.pack() + + nonceHashes += + listOf( + Hashing.hashSha256(RANDOM_OUTPUT_LONG), + Hashing.hashSha256(RANDOM_OUTPUT_LONG), + Hashing.hashSha256(RANDOM_OUTPUT_LONG), + ) + + duration = + MeasurementSpecKt.duration { + privacyParams = differentialPrivacyParams { + epsilon = WATCH_DURATION_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + privacyParams = differentialPrivacyParams { + epsilon = WATCH_DURATION_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + maximumWatchDurationPerUser = MAXIMUM_WATCH_DURATION_PER_USER + } + vidSamplingInterval = + MeasurementSpecKt.vidSamplingInterval { + start = WATCH_DURATION_VID_SAMPLING_START + width = WATCH_DURATION_VID_SAMPLING_WIDTH + } + } + + private val WATCH_DURATION_PROTOCOL_CONFIG: ProtocolConfig = protocolConfig { + measurementType = ProtocolConfig.MeasurementType.DURATION + protocols += + ProtocolConfigKt.protocol { + direct = + ProtocolConfigKt.direct { + noiseMechanisms += + listOf( + ProtocolConfig.NoiseMechanism.NONE, + ProtocolConfig.NoiseMechanism.CONTINUOUS_LAPLACE, + ProtocolConfig.NoiseMechanism.CONTINUOUS_GAUSSIAN, + ) + deterministicSum = ProtocolConfig.Direct.DeterministicSum.getDefaultInstance() + } + } + } + + private val REQUESTING_UNION_ALL_WATCH_DURATION_MEASUREMENT = + BASE_MEASUREMENT.copy { + dataProviders += DATA_PROVIDERS.keys.map { DATA_PROVIDER_ENTRIES.getValue(it) } + + measurementSpec = + signMeasurementSpec( + UNION_ALL_WATCH_DURATION_MEASUREMENT_SPEC.copy { + nonceHashes += Hashing.hashSha256(RANDOM_OUTPUT_LONG) + }, + MEASUREMENT_CONSUMER_SIGNING_KEY_HANDLE, + ) + } + + private val PENDING_UNION_ALL_WATCH_DURATION_MEASUREMENT = + REQUESTING_UNION_ALL_WATCH_DURATION_MEASUREMENT.copy { + name = + MeasurementKey( + MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId, + INTERNAL_PENDING_UNION_ALL_WATCH_DURATION_MEASUREMENT.cmmsMeasurementId, + ) + .toName() + protocolConfig = WATCH_DURATION_PROTOCOL_CONFIG + state = Measurement.State.COMPUTING + } + + private val SUCCEEDED_UNION_ALL_WATCH_DURATION_MEASUREMENT = + PENDING_UNION_ALL_WATCH_DURATION_MEASUREMENT.copy { + state = Measurement.State.SUCCEEDED + + results += + DATA_PROVIDERS.keys.zip(WATCH_DURATION_LIST).map { (dataProviderKey, watchDuration) -> + val dataProvider = DATA_PROVIDERS.getValue(dataProviderKey) + resultOutput { + val result = + MeasurementKt.result { + this.watchDuration = + MeasurementKt.ResultKt.watchDuration { + value = watchDuration + noiseMechanism = ProtocolConfig.NoiseMechanism.CONTINUOUS_LAPLACE + deterministicSum = DeterministicSum.getDefaultInstance() + } + } + encryptedResult = + encryptResult( + signResult(result, DATA_PROVIDER_SIGNING_KEY), + MEASUREMENT_CONSUMER_PUBLIC_KEY, + ) + certificate = dataProvider.certificate + } + } + } + + // CMMS population measurements + private val POPULATION_MEASUREMENT_SPEC = BASE_MEASURMENT_SPEC.copy { + measurementPublicKey = MEASUREMENT_CONSUMER_PUBLIC_KEY.pack() + + nonceHashes += + listOf( + Hashing.hashSha256(RANDOM_OUTPUT_LONG), + Hashing.hashSha256(RANDOM_OUTPUT_LONG), + Hashing.hashSha256(RANDOM_OUTPUT_LONG), + ) + + population = MeasurementSpec.Population.getDefaultInstance() + } + + private val REQUESTING_POPULATION_MEASUREMENT = + BASE_MEASUREMENT.copy { + dataProviders += DATA_PROVIDER_ENTRIES.getValue(DATA_PROVIDERS.keys.first()) + + measurementSpec = + signMeasurementSpec(POPULATION_MEASUREMENT_SPEC, MEASUREMENT_CONSUMER_SIGNING_KEY_HANDLE) + + measurementReferenceId = INTERNAL_PENDING_POPULATION_MEASUREMENT.cmmsCreateMeasurementRequestId + } + + private val PENDING_POPULATION_MEASUREMENT = + REQUESTING_POPULATION_MEASUREMENT.copy { + name = + MeasurementKey( + MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId, + INTERNAL_PENDING_POPULATION_MEASUREMENT.cmmsMeasurementId, + ) + .toName() + state = Measurement.State.COMPUTING + } + + // Metric Specs + + private val REACH_METRIC_SPEC: MetricSpec = metricSpec { + reach = reachParams { privacyParams = MetricSpec.DifferentialPrivacyParams.getDefaultInstance() } + } + private val REACH_FREQUENCY_METRIC_SPEC: MetricSpec = metricSpec { + reachAndFrequency = reachAndFrequencyParams { + reachPrivacyParams = MetricSpec.DifferentialPrivacyParams.getDefaultInstance() + frequencyPrivacyParams = MetricSpec.DifferentialPrivacyParams.getDefaultInstance() + } + } + private val IMPRESSION_COUNT_METRIC_SPEC: MetricSpec = metricSpec { + impressionCount = impressionCountParams { + privacyParams = MetricSpec.DifferentialPrivacyParams.getDefaultInstance() + } + } + private val WATCH_DURATION_METRIC_SPEC: MetricSpec = metricSpec { + watchDuration = watchDurationParams { + privacyParams = MetricSpec.DifferentialPrivacyParams.getDefaultInstance() + } + } + + private val POPULATION_METRIC_SPEC: MetricSpec = metricSpec { + populationCount = MetricSpec.PopulationCountParams.getDefaultInstance() + } + + // Metrics + + // Metric idempotency keys + private const val INCREMENTAL_REACH_METRIC_IDEMPOTENCY_KEY = "TEST_INCREMENTAL_REACH_METRIC" + + // Internal Incremental Metrics + private val INTERNAL_REQUESTING_INCREMENTAL_REACH_METRIC = internalMetric { + cmmsMeasurementConsumerId = MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId + externalReportingSetId = INTERNAL_INCREMENTAL_REPORTING_SET.externalReportingSetId + timeInterval = TIME_INTERVAL + metricSpec = internalMetricSpec { + reach = + InternalMetricSpecKt.reachParams { + multipleDataProviderParams = InternalMetricSpecKt.params { + privacyParams = + InternalMetricSpecKt.differentialPrivacyParams { + epsilon = REACH_ONLY_REACH_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + vidSamplingInterval = + InternalMetricSpecKt.vidSamplingInterval { + start = REACH_ONLY_VID_SAMPLING_START + width = REACH_ONLY_VID_SAMPLING_WIDTH + } + } + } + } + weightedMeasurements += weightedMeasurement { + weight = 1 + binaryRepresentation = 3 + measurement = + INTERNAL_PENDING_UNION_ALL_REACH_MEASUREMENT.copy { + clearCmmsCreateMeasurementRequestId() + clearCmmsMeasurementId() + clearState() + } + } + weightedMeasurements += weightedMeasurement { + weight = -1 + binaryRepresentation = 2 + measurement = + INTERNAL_PENDING_UNION_ALL_BUT_LAST_PUBLISHER_REACH_MEASUREMENT.copy { + clearCmmsCreateMeasurementRequestId() + clearCmmsMeasurementId() + clearState() + } + } + details = InternalMetricKt.details { filters += listOf(METRIC_FILTER) } + } + + private val INTERNAL_PENDING_INITIAL_INCREMENTAL_REACH_METRIC = + INTERNAL_REQUESTING_INCREMENTAL_REACH_METRIC.copy { + externalMetricId = "331L" + createTime = Instant.now().toProtoTime() + state = InternalMetric.State.RUNNING + weightedMeasurements.clear() + weightedMeasurements += weightedMeasurement { + weight = 1 + binaryRepresentation = 3 + measurement = INTERNAL_PENDING_UNION_ALL_REACH_MEASUREMENT.copy { clearCmmsMeasurementId() } + } + weightedMeasurements += weightedMeasurement { + weight = -1 + binaryRepresentation = 2 + measurement = + INTERNAL_PENDING_UNION_ALL_BUT_LAST_PUBLISHER_REACH_MEASUREMENT.copy { + clearCmmsMeasurementId() + } + } + } + + private val INTERNAL_PENDING_INCREMENTAL_REACH_METRIC = + INTERNAL_PENDING_INITIAL_INCREMENTAL_REACH_METRIC.copy { + weightedMeasurements.clear() + weightedMeasurements += weightedMeasurement { + weight = 1 + binaryRepresentation = 3 + measurement = INTERNAL_PENDING_UNION_ALL_REACH_MEASUREMENT + } + weightedMeasurements += weightedMeasurement { + weight = -1 + binaryRepresentation = 2 + measurement = INTERNAL_PENDING_UNION_ALL_BUT_LAST_PUBLISHER_REACH_MEASUREMENT + } + } + + private val INTERNAL_SUCCEEDED_INCREMENTAL_REACH_METRIC = + INTERNAL_PENDING_INCREMENTAL_REACH_METRIC.copy { + state = InternalMetric.State.SUCCEEDED + weightedMeasurements.clear() + weightedMeasurements += weightedMeasurement { + weight = 1 + binaryRepresentation = 3 + measurement = INTERNAL_SUCCEEDED_UNION_ALL_REACH_MEASUREMENT + } + weightedMeasurements += weightedMeasurement { + weight = -1 + binaryRepresentation = 2 + measurement = INTERNAL_SUCCEEDED_UNION_ALL_BUT_LAST_PUBLISHER_REACH_MEASUREMENT + } + } + + // Internal Single publisher reach-frequency metrics + private val + INTERNAL_REQUESTING_SINGLE_PUBLISHER_REACH_FREQUENCY_METRIC = internalMetric { + cmmsMeasurementConsumerId = MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId + externalReportingSetId = INTERNAL_SINGLE_PUBLISHER_REPORTING_SET.externalReportingSetId + timeInterval = TIME_INTERVAL + metricSpec = internalMetricSpec { + reachAndFrequency = + InternalMetricSpecKt.reachAndFrequencyParams { + multipleDataProviderParams = InternalMetricSpecKt.params { + privacyParams = + InternalMetricSpecKt.differentialPrivacyParams { + epsilon = REACH_FREQUENCY_REACH_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + frequencyPrivacyParams = + InternalMetricSpecKt.differentialPrivacyParams { + epsilon = REACH_FREQUENCY_FREQUENCY_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + vidSamplingInterval = + InternalMetricSpecKt.vidSamplingInterval { + start = REACH_FREQUENCY_VID_SAMPLING_START + width = REACH_FREQUENCY_VID_SAMPLING_WIDTH + } + } + maximumFrequency = REACH_FREQUENCY_MAXIMUM_FREQUENCY + } + } + weightedMeasurements += weightedMeasurement { + weight = 1 + binaryRepresentation = 1 + measurement = + INTERNAL_PENDING_SINGLE_PUBLISHER_REACH_FREQUENCY_MEASUREMENT.copy { + clearCmmsCreateMeasurementRequestId() + clearCmmsMeasurementId() + clearState() + } + } + details = InternalMetricKt.details { filters += listOf(METRIC_FILTER) } + } + + private val INTERNAL_PENDING_INITIAL_SINGLE_PUBLISHER_REACH_FREQUENCY_METRIC = + INTERNAL_REQUESTING_SINGLE_PUBLISHER_REACH_FREQUENCY_METRIC.copy { + externalMetricId = "332L" + createTime = Instant.now().toProtoTime() + state = InternalMetric.State.RUNNING + weightedMeasurements.clear() + weightedMeasurements += weightedMeasurement { + weight = 1 + binaryRepresentation = 1 + measurement = + INTERNAL_PENDING_SINGLE_PUBLISHER_REACH_FREQUENCY_MEASUREMENT.copy { + clearCmmsMeasurementId() + } + } + } + + private val INTERNAL_PENDING_SINGLE_PUBLISHER_REACH_FREQUENCY_METRIC = + INTERNAL_PENDING_INITIAL_SINGLE_PUBLISHER_REACH_FREQUENCY_METRIC.copy { + weightedMeasurements.clear() + weightedMeasurements += weightedMeasurement { + weight = 1 + binaryRepresentation = 1 + measurement = INTERNAL_PENDING_SINGLE_PUBLISHER_REACH_FREQUENCY_MEASUREMENT + } + } + + private val INTERNAL_SUCCEEDED_SINGLE_PUBLISHER_REACH_FREQUENCY_METRIC = + INTERNAL_PENDING_SINGLE_PUBLISHER_REACH_FREQUENCY_METRIC.copy { + state = InternalMetric.State.SUCCEEDED + weightedMeasurements.clear() + weightedMeasurements += weightedMeasurement { + weight = 1 + binaryRepresentation = 1 + measurement = INTERNAL_SUCCEEDED_SINGLE_PUBLISHER_REACH_FREQUENCY_MEASUREMENT + } + } + + // Internal Single publisher impression metrics + private val INTERNAL_REQUESTING_SINGLE_PUBLISHER_IMPRESSION_METRIC = internalMetric { + cmmsMeasurementConsumerId = MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId + externalReportingSetId = INTERNAL_SINGLE_PUBLISHER_REPORTING_SET.externalReportingSetId + timeInterval = TIME_INTERVAL + metricSpec = internalMetricSpec { + impressionCount = + InternalMetricSpecKt.impressionCountParams { + params = InternalMetricSpecKt.params { + privacyParams = + InternalMetricSpecKt.differentialPrivacyParams { + epsilon = IMPRESSION_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + vidSamplingInterval = + InternalMetricSpecKt.vidSamplingInterval { + start = IMPRESSION_VID_SAMPLING_START + width = IMPRESSION_VID_SAMPLING_WIDTH + } + } + maximumFrequencyPerUser = IMPRESSION_MAXIMUM_FREQUENCY_PER_USER + } + } + weightedMeasurements += weightedMeasurement { + weight = 1 + binaryRepresentation = 1 + measurement = + INTERNAL_PENDING_SINGLE_PUBLISHER_IMPRESSION_MEASUREMENT.copy { + clearCmmsCreateMeasurementRequestId() + clearCmmsMeasurementId() + clearState() + } + } + details = InternalMetricKt.details { filters += listOf(METRIC_FILTER) } + } + + private val INTERNAL_PENDING_INITIAL_SINGLE_PUBLISHER_IMPRESSION_METRIC = + INTERNAL_REQUESTING_SINGLE_PUBLISHER_IMPRESSION_METRIC.copy { + externalMetricId = "333L" + createTime = Instant.now().toProtoTime() + state = InternalMetric.State.RUNNING + weightedMeasurements.clear() + weightedMeasurements += weightedMeasurement { + weight = 1 + binaryRepresentation = 1 + measurement = + INTERNAL_PENDING_SINGLE_PUBLISHER_IMPRESSION_MEASUREMENT.copy { clearCmmsMeasurementId() } + } + } + + private val INTERNAL_PENDING_SINGLE_PUBLISHER_IMPRESSION_METRIC = + INTERNAL_PENDING_INITIAL_SINGLE_PUBLISHER_IMPRESSION_METRIC.copy { + weightedMeasurements.clear() + weightedMeasurements += weightedMeasurement { + weight = 1 + binaryRepresentation = 1 + measurement = INTERNAL_PENDING_SINGLE_PUBLISHER_IMPRESSION_MEASUREMENT + } + } + + private val INTERNAL_FAILED_SINGLE_PUBLISHER_IMPRESSION_METRIC = + INTERNAL_PENDING_SINGLE_PUBLISHER_IMPRESSION_METRIC.copy { + state = InternalMetric.State.FAILED + weightedMeasurements.clear() + weightedMeasurements += weightedMeasurement { + weight = 1 + binaryRepresentation = 1 + measurement = INTERNAL_FAILED_SINGLE_PUBLISHER_IMPRESSION_MEASUREMENT + } + } + + private val INTERNAL_SUCCEEDED_SINGLE_PUBLISHER_IMPRESSION_METRIC = + INTERNAL_PENDING_SINGLE_PUBLISHER_IMPRESSION_METRIC.copy { + state = InternalMetric.State.SUCCEEDED + weightedMeasurements.clear() + weightedMeasurements += weightedMeasurement { + weight = 1 + binaryRepresentation = 1 + measurement = INTERNAL_SUCCEEDED_SINGLE_PUBLISHER_IMPRESSION_MEASUREMENT + } + } + + private val INTERNAL_SUCCEEDED_SINGLE_PUBLISHER_IMPRESSION_METRIC_CUSTOM_CAP = + INTERNAL_PENDING_SINGLE_PUBLISHER_IMPRESSION_METRIC.copy { + state = InternalMetric.State.SUCCEEDED + weightedMeasurements.clear() + weightedMeasurements += weightedMeasurement { + weight = 1 + binaryRepresentation = 1 + measurement = INTERNAL_SUCCEEDED_SINGLE_PUBLISHER_IMPRESSION_MEASUREMENT_CUSTOM_CAP + } + } + + // Internal Cross Publisher Watch Duration Metrics + private val INTERNAL_REQUESTING_CROSS_PUBLISHER_WATCH_DURATION_METRIC = internalMetric { + cmmsMeasurementConsumerId = MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId + externalReportingSetId = INTERNAL_UNION_ALL_REPORTING_SET.externalReportingSetId + timeInterval = TIME_INTERVAL + metricSpec = internalMetricSpec { + watchDuration = + InternalMetricSpecKt.watchDurationParams { + params = InternalMetricSpecKt.params { + privacyParams = + InternalMetricSpecKt.differentialPrivacyParams { + epsilon = WATCH_DURATION_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + vidSamplingInterval = + InternalMetricSpecKt.vidSamplingInterval { + start = WATCH_DURATION_VID_SAMPLING_START + width = WATCH_DURATION_VID_SAMPLING_WIDTH + } + } + + maximumWatchDurationPerUser = MAXIMUM_WATCH_DURATION_PER_USER + } + } + weightedMeasurements += weightedMeasurement { + weight = 1 + binaryRepresentation = 1 + measurement = INTERNAL_REQUESTING_UNION_ALL_WATCH_DURATION_MEASUREMENT + } + details = InternalMetricKt.details { filters += listOf(METRIC_FILTER) } + } + + private val INTERNAL_PENDING_INITIAL_CROSS_PUBLISHER_WATCH_DURATION_METRIC = + INTERNAL_REQUESTING_CROSS_PUBLISHER_WATCH_DURATION_METRIC.copy { + externalMetricId = "334L" + createTime = Instant.now().toProtoTime() + state = InternalMetric.State.RUNNING + weightedMeasurements.clear() + weightedMeasurements += weightedMeasurement { + weight = 1 + binaryRepresentation = 1 + measurement = INTERNAL_PENDING_NOT_CREATED_UNION_ALL_WATCH_DURATION_MEASUREMENT + } + } + + private val INTERNAL_PENDING_CROSS_PUBLISHER_WATCH_DURATION_METRIC = + INTERNAL_PENDING_INITIAL_CROSS_PUBLISHER_WATCH_DURATION_METRIC.copy { + weightedMeasurements.clear() + weightedMeasurements += weightedMeasurement { + weight = 1 + binaryRepresentation = 1 + measurement = INTERNAL_PENDING_UNION_ALL_WATCH_DURATION_MEASUREMENT + } + } + + private val INTERNAL_SUCCEEDED_CROSS_PUBLISHER_WATCH_DURATION_METRIC = + INTERNAL_PENDING_CROSS_PUBLISHER_WATCH_DURATION_METRIC.copy { + state = InternalMetric.State.SUCCEEDED + weightedMeasurements.clear() + weightedMeasurements += weightedMeasurement { + weight = 1 + binaryRepresentation = 1 + measurement = INTERNAL_SUCCEEDED_UNION_ALL_WATCH_DURATION_MEASUREMENT + } + } - @Test - fun `batchGetMetrics throws INVALID_ARGUMENT when number of requests exceeds limit`() = - runBlocking { - val request = batchGetMetricsRequest { - parent = MEASUREMENT_CONSUMERS.values.first().name - names += List(MAX_BATCH_SIZE + 1) { "metric_name" } + // Internal population metric + val INTERNAL_REQUESTING_POPULATION_METRIC = internalMetric { + cmmsMeasurementConsumerId = MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId + externalReportingSetId = INTERNAL_POPULATION_REPORTING_SET.externalReportingSetId + timeInterval = TIME_INTERVAL + metricSpec = internalMetricSpec { + populationCount = InternalMetricSpec.PopulationCountParams.getDefaultInstance() } - val exception = - assertFailsWith { - withMeasurementConsumerPrincipal(MEASUREMENT_CONSUMERS.values.first().name, CONFIG) { - runBlocking { service.batchGetMetrics(request) } + weightedMeasurements += weightedMeasurement { + weight = 1 + binaryRepresentation = 1 + measurement = + INTERNAL_PENDING_POPULATION_MEASUREMENT.copy { + clearCmmsCreateMeasurementRequestId() + clearCmmsMeasurementId() + clearState() } - } - - assertThat(exception.status.code).isEqualTo(Status.Code.INVALID_ARGUMENT) - assertThat(exception.status.description) - .isEqualTo("At most $MAX_BATCH_SIZE metrics can be supported in a batch.") + } + details = InternalMetricKt.details { filters += listOf(INCREMENTAL_REPORTING_SET_FILTER) } } - @Test - fun `createMetric creates CMMS measurements for population`() = runBlocking { - whenever(internalMetricsMock.createMetric(any())) - .thenReturn(INTERNAL_PENDING_INITIAL_POPULATION_METRIC) - whenever(measurementsMock.batchCreateMeasurements(any())) - .thenReturn( - batchCreateMeasurementsResponse { measurements += PENDING_POPULATION_MEASUREMENT } - ) + private val INTERNAL_PENDING_INITIAL_POPULATION_METRIC = + INTERNAL_REQUESTING_POPULATION_METRIC.copy { + externalMetricId = "331L" + createTime = Instant.now().toProtoTime() + state = InternalMetric.State.RUNNING + weightedMeasurements.clear() - val request = createMetricRequest { - parent = MEASUREMENT_CONSUMERS.values.first().name - metric = REQUESTING_POPULATION_METRIC - metricId = "metric-id" - } - val result = - withMeasurementConsumerPrincipal(request.parent, CONFIG) { - runBlocking { service.createMetric(request) } + weightedMeasurements += weightedMeasurement { + weight = 1 + binaryRepresentation = 1 + measurement = INTERNAL_PENDING_POPULATION_MEASUREMENT.copy { clearCmmsMeasurementId() } + } } - val expected = PENDING_POPULATION_METRIC + val INTERNAL_PENDING_POPULATION_METRIC = + INTERNAL_PENDING_INITIAL_POPULATION_METRIC.copy { + weightedMeasurements.clear() + weightedMeasurements += weightedMeasurement { + weight = 1 + binaryRepresentation = 1 + measurement = INTERNAL_PENDING_POPULATION_MEASUREMENT + } + } - // Verify proto argument of the internal MetricsCoroutineImplBase::createMetric - verifyProtoArgument(internalMetricsMock, MetricsCoroutineImplBase::createMetric) - .ignoringRepeatedFieldOrder() - .isEqualTo( - internalCreateMetricRequest { - metric = INTERNAL_REQUESTING_POPULATION_METRIC - externalMetricId = "metric-id" + val INTERNAL_SUCCEEDED_POPULATION_METRIC = + INTERNAL_PENDING_POPULATION_METRIC.copy { + state = InternalMetric.State.SUCCEEDED + weightedMeasurements.clear() + weightedMeasurements += weightedMeasurement { + weight = 1 + binaryRepresentation = 1 + measurement = INTERNAL_SUCCEEDED_POPULATION_MEASUREMENT } - ) + } - // Verify proto argument of MeasurementsCoroutineImplBase::createMeasurement - val measurementsCaptor: KArgumentCaptor = argumentCaptor() - verifyBlocking(measurementsMock, times(1)) { - batchCreateMeasurements(measurementsCaptor.capture()) + // Public Metrics + + // Incremental reach metrics + private val REQUESTING_INCREMENTAL_REACH_METRIC = metric { + reportingSet = INTERNAL_INCREMENTAL_REPORTING_SET.resourceName + timeInterval = TIME_INTERVAL + metricSpec = REACH_METRIC_SPEC + filters += INTERNAL_PENDING_INCREMENTAL_REACH_METRIC.details.filtersList } - val capturedMeasurementRequests = measurementsCaptor.allValues - assertThat(capturedMeasurementRequests) - .ignoringRepeatedFieldOrder() - .ignoringFieldDescriptors(MEASUREMENT_SPEC_FIELD, ENCRYPTED_REQUISITION_SPEC_FIELD) - .containsExactly( - batchCreateMeasurementsRequest { - parent = request.parent - requests += createMeasurementRequest { - parent = request.parent - measurement = REQUESTING_POPULATION_MEASUREMENT - requestId = INTERNAL_PENDING_POPULATION_MEASUREMENT.cmmsCreateMeasurementRequestId + + private val PENDING_INCREMENTAL_REACH_METRIC = + REQUESTING_INCREMENTAL_REACH_METRIC.copy { + name = + MetricKey( + MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId, + INTERNAL_PENDING_INCREMENTAL_REACH_METRIC.externalMetricId, + ) + .toName() + state = Metric.State.RUNNING + metricSpec = metricSpec { + reach = reachParams { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = REACH_ONLY_REACH_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = REACH_ONLY_VID_SAMPLING_START + width = REACH_ONLY_VID_SAMPLING_WIDTH + } } - ) + createTime = INTERNAL_PENDING_INCREMENTAL_REACH_METRIC.createTime + } - val capturedMeasurementRequest = capturedMeasurementRequests.single().requestsList.first() - verifyMeasurementSpec( - capturedMeasurementRequest.measurement.measurementSpec, - MEASUREMENT_CONSUMER_CERTIFICATE, - TRUSTED_MEASUREMENT_CONSUMER_ISSUER, - ) + private const val VARIANCE_VALUE = 4.0 + private const val SINGLE_DATA_PROVIDER_VARIANCE_VALUE = 5.0 - val dataProvidersList = - capturedMeasurementRequest.measurement.dataProvidersList.sortedBy { it.key } + private val FREQUENCY_VARIANCE: Map = + (1..REACH_FREQUENCY_MAXIMUM_FREQUENCY).associateWith { it.toDouble().pow(2.0) } + private val FREQUENCY_VARIANCES = + FrequencyVariances(FREQUENCY_VARIANCE, FREQUENCY_VARIANCE, FREQUENCY_VARIANCE, FREQUENCY_VARIANCE) + private val SINGLE_DATA_PROVIDER_FREQUENCY_VARIANCE: Map = + (1..REACH_FREQUENCY_MAXIMUM_FREQUENCY).associateWith { it.toDouble().pow(3.0) } + private val SINGLE_DATA_PROVIDER_FREQUENCY_VARIANCES = + FrequencyVariances(SINGLE_DATA_PROVIDER_FREQUENCY_VARIANCE, SINGLE_DATA_PROVIDER_FREQUENCY_VARIANCE, SINGLE_DATA_PROVIDER_FREQUENCY_VARIANCE, SINGLE_DATA_PROVIDER_FREQUENCY_VARIANCE) - val measurementSpec: MeasurementSpec = - capturedMeasurementRequest.measurement.measurementSpec.unpack() + private val SUCCEEDED_INCREMENTAL_REACH_METRIC = + PENDING_INCREMENTAL_REACH_METRIC.copy { + state = Metric.State.SUCCEEDED - assertThat(measurementSpec) - .isEqualTo( - POPULATION_MEASUREMENT_SPEC.copy { - nonceHashes.clear() - nonceHashes += List(dataProvidersList.size) { Hashing.hashSha256(RANDOM_OUTPUT_LONG) } + result = metricResult { + reach = + MetricResultKt.reachResult { + value = INCREMENTAL_REACH_VALUE + univariateStatistics = univariateStatistics { standardDeviation = sqrt(VARIANCE_VALUE) } + } + cmmsMeasurements += PENDING_UNION_ALL_REACH_MEASUREMENT.name + cmmsMeasurements += PENDING_UNION_ALL_BUT_LAST_PUBLISHER_REACH_MEASUREMENT.name } - ) + } - dataProvidersList.map { dataProviderEntry -> - val signedRequisitionSpec = - decryptRequisitionSpec( - dataProviderEntry.value.encryptedRequisitionSpec, - DATA_PROVIDER_PRIVATE_KEY_HANDLE, - ) - val requisitionSpec: RequisitionSpec = signedRequisitionSpec.unpack() - verifyRequisitionSpec( - signedRequisitionSpec, - requisitionSpec, - measurementSpec, - MEASUREMENT_CONSUMER_CERTIFICATE, - TRUSTED_MEASUREMENT_CONSUMER_ISSUER, - ) + // Single publisher reach-frequency metrics + private val REQUESTING_SINGLE_PUBLISHER_REACH_FREQUENCY_METRIC = metric { + reportingSet = INTERNAL_SINGLE_PUBLISHER_REPORTING_SET.resourceName + timeInterval = TIME_INTERVAL + metricSpec = REACH_FREQUENCY_METRIC_SPEC + filters += INTERNAL_REQUESTING_SINGLE_PUBLISHER_REACH_FREQUENCY_METRIC.details.filtersList } - // Verify proto argument of internal MeasurementsCoroutineImplBase::batchSetCmmsMeasurementId - verifyProtoArgument( - internalMeasurementsMock, - InternalMeasurementsGrpcKt.MeasurementsCoroutineImplBase::batchSetCmmsMeasurementIds, - ) - .ignoringRepeatedFieldOrder() - .isEqualTo( - batchSetCmmsMeasurementIdsRequest { - cmmsMeasurementConsumerId = MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId - this.measurementIds += measurementIds { - cmmsCreateMeasurementRequestId = - INTERNAL_PENDING_POPULATION_MEASUREMENT.cmmsCreateMeasurementRequestId - cmmsMeasurementId = INTERNAL_PENDING_POPULATION_MEASUREMENT.cmmsMeasurementId + private val PENDING_SINGLE_PUBLISHER_REACH_FREQUENCY_METRIC = + REQUESTING_SINGLE_PUBLISHER_REACH_FREQUENCY_METRIC.copy { + name = + MetricKey( + MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId, + INTERNAL_PENDING_SINGLE_PUBLISHER_REACH_FREQUENCY_METRIC.externalMetricId, + ) + .toName() + metricSpec = metricSpec { + reachAndFrequency = reachAndFrequencyParams { + reachPrivacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = REACH_FREQUENCY_REACH_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + frequencyPrivacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = REACH_FREQUENCY_FREQUENCY_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + maximumFrequency = REACH_FREQUENCY_MAXIMUM_FREQUENCY } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = REACH_FREQUENCY_VID_SAMPLING_START + width = REACH_FREQUENCY_VID_SAMPLING_WIDTH + } } - ) - assertThat(result).isEqualTo(expected) - } + state = Metric.State.RUNNING + createTime = INTERNAL_PENDING_SINGLE_PUBLISHER_REACH_FREQUENCY_METRIC.createTime + } + + private val SUCCEEDED_SINGLE_PUBLISHER_REACH_FREQUENCY_METRIC = + PENDING_SINGLE_PUBLISHER_REACH_FREQUENCY_METRIC.copy { + state = Metric.State.SUCCEEDED + result = metricResult { + cmmsMeasurements += PENDING_SINGLE_PUBLISHER_REACH_FREQUENCY_MEASUREMENT.name + reachAndFrequency = + MetricResultKt.reachAndFrequencyResult { + reach = + MetricResultKt.reachResult { + value = REACH_FREQUENCY_REACH_VALUE + univariateStatistics = univariateStatistics { + standardDeviation = sqrt(VARIANCE_VALUE) + } + } + frequencyHistogram = + MetricResultKt.histogramResult { + bins += + (1..REACH_FREQUENCY_MAXIMUM_FREQUENCY).map { frequency -> + MetricResultKt.HistogramResultKt.bin { + label = frequency.toString() + binResult = + MetricResultKt.HistogramResultKt.binResult { + value = + REACH_FREQUENCY_REACH_VALUE * + REACH_FREQUENCY_FREQUENCY_VALUE.getOrDefault(frequency.toLong(), 0.0) + } + resultUnivariateStatistics = univariateStatistics { + standardDeviation = + sqrt(FREQUENCY_VARIANCES.countVariances.getValue(frequency)) + } + relativeUnivariateStatistics = univariateStatistics { + standardDeviation = + sqrt(FREQUENCY_VARIANCES.relativeVariances.getValue(frequency)) + } + kPlusUnivariateStatistics = univariateStatistics { + standardDeviation = + sqrt(FREQUENCY_VARIANCES.kPlusCountVariances.getValue(frequency)) + } + relativeKPlusUnivariateStatistics = univariateStatistics { + standardDeviation = + sqrt(FREQUENCY_VARIANCES.kPlusRelativeVariances.getValue(frequency)) + } + } + } + } + } + } + } - @Test - fun `getMetric returns population metric`() = runBlocking { - whenever(internalMetricsMock.batchGetMetrics(any())) - .thenReturn( - internalBatchGetMetricsResponse { metrics += INTERNAL_SUCCEEDED_POPULATION_METRIC } - ) + // Single publisher impression metrics + private val REQUESTING_SINGLE_PUBLISHER_IMPRESSION_METRIC = metric { + reportingSet = INTERNAL_SINGLE_PUBLISHER_REPORTING_SET.resourceName + timeInterval = TIME_INTERVAL + metricSpec = IMPRESSION_COUNT_METRIC_SPEC + filters += INTERNAL_REQUESTING_SINGLE_PUBLISHER_IMPRESSION_METRIC.details.filtersList + } - val request = getMetricRequest { name = SUCCEEDED_POPULATION_METRIC.name } + private val PENDING_SINGLE_PUBLISHER_IMPRESSION_METRIC = + REQUESTING_SINGLE_PUBLISHER_IMPRESSION_METRIC.copy { + name = + MetricKey( + MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId, + INTERNAL_PENDING_SINGLE_PUBLISHER_IMPRESSION_METRIC.externalMetricId, + ) + .toName() + metricSpec = metricSpec { + impressionCount = impressionCountParams { + params = params.copy { + privacyParams = MetricSpecKt.differentialPrivacyParams { + epsilon = IMPRESSION_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = IMPRESSION_VID_SAMPLING_START + width = IMPRESSION_VID_SAMPLING_WIDTH + } + } + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = IMPRESSION_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + maximumFrequencyPerUser = IMPRESSION_MAXIMUM_FREQUENCY_PER_USER + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = IMPRESSION_VID_SAMPLING_START + width = IMPRESSION_VID_SAMPLING_WIDTH + } + } + state = Metric.State.RUNNING + createTime = INTERNAL_PENDING_SINGLE_PUBLISHER_IMPRESSION_METRIC.createTime + } - val result = - withMeasurementConsumerPrincipal(MEASUREMENT_CONSUMERS.values.first().name, CONFIG) { - runBlocking { service.getMetric(request) } + private val FAILED_SINGLE_PUBLISHER_IMPRESSION_METRIC = + PENDING_SINGLE_PUBLISHER_IMPRESSION_METRIC.copy { state = Metric.State.FAILED } + + private val SUCCEEDED_SINGLE_PUBLISHER_IMPRESSION_METRIC = + PENDING_SINGLE_PUBLISHER_IMPRESSION_METRIC.copy { + state = Metric.State.SUCCEEDED + result = metricResult { + impressionCount = + MetricResultKt.impressionCountResult { + value = IMPRESSION_VALUE + univariateStatistics = univariateStatistics { standardDeviation = sqrt(VARIANCE_VALUE) } + } + cmmsMeasurements += PENDING_SINGLE_PUBLISHER_IMPRESSION_MEASUREMENT.name + } } - // Verify proto argument of internal MetricsCoroutineImplBase::batchGetMetrics - val batchGetInternalMetricsCaptor: KArgumentCaptor = - argumentCaptor() - verifyBlocking(internalMetricsMock, times(1)) { - batchGetMetrics(batchGetInternalMetricsCaptor.capture()) + // Cross publisher watch duration metrics + private val REQUESTING_CROSS_PUBLISHER_WATCH_DURATION_METRIC = metric { + reportingSet = INTERNAL_UNION_ALL_REPORTING_SET.resourceName + timeInterval = TIME_INTERVAL + metricSpec = WATCH_DURATION_METRIC_SPEC + filters += INTERNAL_PENDING_CROSS_PUBLISHER_WATCH_DURATION_METRIC.details.filtersList } - val capturedInternalGetMetricRequests = batchGetInternalMetricsCaptor.allValues - assertThat(capturedInternalGetMetricRequests) - .containsExactly( - internalBatchGetMetricsRequest { - cmmsMeasurementConsumerId = INTERNAL_SUCCEEDED_POPULATION_METRIC.cmmsMeasurementConsumerId - externalMetricIds += INTERNAL_SUCCEEDED_POPULATION_METRIC.externalMetricId + + private val PENDING_CROSS_PUBLISHER_WATCH_DURATION_METRIC = + REQUESTING_CROSS_PUBLISHER_WATCH_DURATION_METRIC.copy { + name = + MetricKey( + MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId, + INTERNAL_PENDING_CROSS_PUBLISHER_WATCH_DURATION_METRIC.externalMetricId, + ) + .toName() + metricSpec = metricSpec { + watchDuration = watchDurationParams { + params = params.copy { + privacyParams = MetricSpecKt.differentialPrivacyParams { + epsilon = WATCH_DURATION_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = WATCH_DURATION_VID_SAMPLING_START + width = WATCH_DURATION_VID_SAMPLING_WIDTH + } + } + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = WATCH_DURATION_EPSILON + delta = DIFFERENTIAL_PRIVACY_DELTA + } + maximumWatchDurationPerUser = MAXIMUM_WATCH_DURATION_PER_USER + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = WATCH_DURATION_VID_SAMPLING_START + width = WATCH_DURATION_VID_SAMPLING_WIDTH + } } - ) + state = Metric.State.RUNNING + createTime = INTERNAL_PENDING_CROSS_PUBLISHER_WATCH_DURATION_METRIC.createTime + } - // Verify proto argument of internal MeasurementsCoroutineImplBase::batchSetMeasurementResults - val batchSetMeasurementResultsCaptor: KArgumentCaptor = - argumentCaptor() - verifyBlocking(internalMeasurementsMock, never()) { - batchSetMeasurementResults(batchSetMeasurementResultsCaptor.capture()) - } + private val SUCCEEDED_CROSS_PUBLISHER_WATCH_DURATION_METRIC = + PENDING_CROSS_PUBLISHER_WATCH_DURATION_METRIC.copy { + state = Metric.State.SUCCEEDED + result = metricResult { + watchDuration = + MetricResultKt.watchDurationResult { + value = TOTAL_WATCH_DURATION.seconds.toDouble() + univariateStatistics = univariateStatistics { + standardDeviation = sqrt(WATCH_DURATION_LIST.sumOf { VARIANCE_VALUE }) + } + } + cmmsMeasurements += PENDING_UNION_ALL_WATCH_DURATION_MEASUREMENT.name + } + } - // Verify proto argument of internal - // MeasurementsCoroutineImplBase::batchSetMeasurementFailures - val batchSetMeasurementFailuresCaptor: KArgumentCaptor = - argumentCaptor() - verifyBlocking(internalMeasurementsMock, never()) { - batchSetMeasurementFailures(batchSetMeasurementFailuresCaptor.capture()) + // Population metric + val REQUESTING_POPULATION_METRIC = metric { + reportingSet = INTERNAL_POPULATION_REPORTING_SET.resourceName + timeInterval = TIME_INTERVAL + metricSpec = POPULATION_METRIC_SPEC + filters += INTERNAL_PENDING_POPULATION_METRIC.details.filtersList } - assertThat(result).isEqualTo(SUCCEEDED_POPULATION_METRIC) - } + val PENDING_POPULATION_METRIC = + REQUESTING_POPULATION_METRIC.copy { + name = + MetricKey( + MEASUREMENT_CONSUMERS.keys.first().measurementConsumerId, + INTERNAL_PENDING_POPULATION_METRIC.externalMetricId, + ) + .toName() + state = Metric.State.RUNNING + metricSpec = metricSpec { + populationCount = MetricSpec.PopulationCountParams.getDefaultInstance() + } + createTime = INTERNAL_PENDING_POPULATION_METRIC.createTime + } - companion object { - private val MEASUREMENT_SPEC_FIELD = - Measurement.getDescriptor().findFieldByNumber(Measurement.MEASUREMENT_SPEC_FIELD_NUMBER) - private val ENCRYPTED_REQUISITION_SPEC_FIELD = - Measurement.DataProviderEntry.Value.getDescriptor() - .findFieldByNumber( - Measurement.DataProviderEntry.Value.ENCRYPTED_REQUISITION_SPEC_FIELD_NUMBER - ) + val SUCCEEDED_POPULATION_METRIC = + PENDING_POPULATION_METRIC.copy { + state = Metric.State.SUCCEEDED + result = metricResult { + populationCount = MetricResultKt.populationCountResult { value = TOTAL_POPULATION_VALUE } + cmmsMeasurements += PENDING_POPULATION_MEASUREMENT.name + } + } } } diff --git a/src/test/kotlin/org/wfanet/measurement/reporting/service/api/v2alpha/ReportSchedulesServiceTest.kt b/src/test/kotlin/org/wfanet/measurement/reporting/service/api/v2alpha/ReportSchedulesServiceTest.kt index b6efd0b6796..fdf0d70b2b4 100644 --- a/src/test/kotlin/org/wfanet/measurement/reporting/service/api/v2alpha/ReportSchedulesServiceTest.kt +++ b/src/test/kotlin/org/wfanet/measurement/reporting/service/api/v2alpha/ReportSchedulesServiceTest.kt @@ -2779,16 +2779,18 @@ class ReportSchedulesServiceTest { private val INTERNAL_METRIC_SPEC: InternalMetricSpec = internalMetricSpec { reach = InternalMetricSpecKt.reachParams { - privacyParams = - InternalMetricSpecKt.differentialPrivacyParams { - epsilon = EPSILON - delta = DELTA - } - } - vidSamplingInterval = - InternalMetricSpecKt.vidSamplingInterval { - start = VID_SAMPLING_START - width = VID_SAMPLING_WIDTH + multipleDataProviderParams = InternalMetricSpecKt.params { + privacyParams = + InternalMetricSpecKt.differentialPrivacyParams { + epsilon = EPSILON + delta = DELTA + } + vidSamplingInterval = + InternalMetricSpecKt.vidSamplingInterval { + start = VID_SAMPLING_START + width = VID_SAMPLING_WIDTH + } + } } } diff --git a/src/test/kotlin/org/wfanet/measurement/reporting/service/api/v2alpha/ReportsServiceTest.kt b/src/test/kotlin/org/wfanet/measurement/reporting/service/api/v2alpha/ReportsServiceTest.kt index 7fdf2ceb18d..bff842514e1 100644 --- a/src/test/kotlin/org/wfanet/measurement/reporting/service/api/v2alpha/ReportsServiceTest.kt +++ b/src/test/kotlin/org/wfanet/measurement/reporting/service/api/v2alpha/ReportsServiceTest.kt @@ -20,7 +20,6 @@ import com.google.common.truth.Truth.assertThat import com.google.common.truth.extensions.proto.ProtoTruth.assertThat import com.google.protobuf.duration import com.google.protobuf.timestamp -import com.google.protobuf.util.Durations import com.google.type.DayOfWeek import com.google.type.Interval import com.google.type.date @@ -55,9 +54,7 @@ import org.wfanet.measurement.common.grpc.testing.mockService import org.wfanet.measurement.common.identity.ExternalId import org.wfanet.measurement.common.testing.verifyProtoArgument import org.wfanet.measurement.common.toProtoTime -import org.wfanet.measurement.config.reporting.MetricSpecConfigKt import org.wfanet.measurement.config.reporting.measurementConsumerConfig -import org.wfanet.measurement.config.reporting.metricSpecConfig import org.wfanet.measurement.internal.reporting.v2.BatchGetMetricCalculationSpecsRequest import org.wfanet.measurement.internal.reporting.v2.CreateReportRequestKt import org.wfanet.measurement.internal.reporting.v2.MetricCalculationSpec @@ -81,6 +78,16 @@ import org.wfanet.measurement.internal.reporting.v2.metricSpec as internalMetric import org.wfanet.measurement.internal.reporting.v2.report as internalReport import org.wfanet.measurement.internal.reporting.v2.streamReportsRequest import org.wfanet.measurement.internal.reporting.v2.timeIntervals as internalTimeIntervals +import com.google.devtools.build.runfiles.Runfiles +import java.nio.file.Path +import java.nio.file.Paths +import kotlin.random.Random +import org.mockito.kotlin.doReturn +import org.mockito.kotlin.mock +import org.mockito.kotlin.stub +import org.wfanet.measurement.common.getRuntimePath +import org.wfanet.measurement.common.parseTextProto +import org.wfanet.measurement.config.reporting.MetricSpecConfig import org.wfanet.measurement.reporting.service.api.v2alpha.ReportScheduleInfoServerInterceptor.Companion.withReportScheduleInfoAndMeasurementConsumerPrincipal import org.wfanet.measurement.reporting.v2alpha.BatchCreateMetricsRequest import org.wfanet.measurement.reporting.v2alpha.BatchGetMetricsRequest @@ -124,33 +131,12 @@ private const val METRIC_ID_PREFIX = "a" // Authentication key private const val API_AUTHENTICATION_KEY = "nR5QPN7ptx" -// Metric Specs -private const val NUMBER_VID_BUCKETS = 300 -private const val REACH_ONLY_VID_SAMPLING_WIDTH = 3.0f / NUMBER_VID_BUCKETS -private const val REACH_ONLY_VID_SAMPLING_START = 0.0f -private const val REACH_ONLY_REACH_EPSILON = 0.0041 - -private const val REACH_FREQUENCY_VID_SAMPLING_WIDTH = 5.0f / NUMBER_VID_BUCKETS -private const val REACH_FREQUENCY_VID_SAMPLING_START = 48.0f / NUMBER_VID_BUCKETS -private const val REACH_FREQUENCY_REACH_EPSILON = 0.0033 -private const val REACH_FREQUENCY_FREQUENCY_EPSILON = 0.115 -private const val REACH_FREQUENCY_MAXIMUM_FREQUENCY = 10 - -private const val IMPRESSION_VID_SAMPLING_WIDTH = 62.0f / NUMBER_VID_BUCKETS -private const val IMPRESSION_VID_SAMPLING_START = 143.0f / NUMBER_VID_BUCKETS -private const val IMPRESSION_EPSILON = 0.0011 -private const val IMPRESSION_MAXIMUM_FREQUENCY_PER_USER = 60 - -private const val WATCH_DURATION_VID_SAMPLING_WIDTH = 95.0f / NUMBER_VID_BUCKETS -private const val WATCH_DURATION_VID_SAMPLING_START = 205.0f / NUMBER_VID_BUCKETS -private const val WATCH_DURATION_EPSILON = 0.001 -private val MAXIMUM_WATCH_DURATION_PER_USER = Durations.fromSeconds(4000) - -private const val DIFFERENTIAL_PRIVACY_DELTA = 1e-12 - private const val BATCH_CREATE_METRICS_LIMIT = 1000 private const val BATCH_GET_METRICS_LIMIT = 100 +private const val RANDOM_OUTPUT_INT = 0 +private const val RANDOM_OUTPUT_LONG = 0L + @RunWith(JUnit4::class) class ReportsServiceTest { @@ -229,6 +215,8 @@ class ReportsServiceTest { } } + private val randomMock: Random = mock() + @get:Rule val grpcTestServerRule = GrpcTestServerRule { addService(internalReportsMock) @@ -240,12 +228,18 @@ class ReportsServiceTest { @Before fun initService() { + randomMock.stub { + on { nextInt(any()) } doReturn RANDOM_OUTPUT_INT + on { nextLong() } doReturn RANDOM_OUTPUT_LONG + } + service = ReportsService( InternalReportsCoroutineStub(grpcTestServerRule.channel), InternalMetricCalculationSpecsCoroutineStub(grpcTestServerRule.channel), MetricsCoroutineStub(grpcTestServerRule.channel), METRIC_SPEC_CONFIG, + randomMock, ) } @@ -4373,177 +4367,192 @@ class ReportsServiceTest { private val START_TIME = START_INSTANT.toProtoTime() private val END_TIME = END_INSTANT.toProtoTime() - // Metric Specs - private val METRIC_SPEC_CONFIG = metricSpecConfig { - reachParams = - MetricSpecConfigKt.reachParams { - privacyParams = - MetricSpecConfigKt.differentialPrivacyParams { - epsilon = REACH_ONLY_REACH_EPSILON - delta = DIFFERENTIAL_PRIVACY_DELTA - } - } - reachVidSamplingInterval = - MetricSpecConfigKt.vidSamplingInterval { - start = REACH_ONLY_VID_SAMPLING_START - width = REACH_ONLY_VID_SAMPLING_WIDTH - } - - reachAndFrequencyParams = - MetricSpecConfigKt.reachAndFrequencyParams { - reachPrivacyParams = - MetricSpecConfigKt.differentialPrivacyParams { - epsilon = REACH_FREQUENCY_REACH_EPSILON - delta = DIFFERENTIAL_PRIVACY_DELTA - } - frequencyPrivacyParams = - MetricSpecConfigKt.differentialPrivacyParams { - epsilon = REACH_FREQUENCY_FREQUENCY_EPSILON - delta = DIFFERENTIAL_PRIVACY_DELTA - } - maximumFrequency = REACH_FREQUENCY_MAXIMUM_FREQUENCY - } - reachAndFrequencyVidSamplingInterval = - MetricSpecConfigKt.vidSamplingInterval { - start = REACH_FREQUENCY_VID_SAMPLING_START - width = REACH_FREQUENCY_VID_SAMPLING_WIDTH - } - - impressionCountParams = - MetricSpecConfigKt.impressionCountParams { - privacyParams = - MetricSpecConfigKt.differentialPrivacyParams { - epsilon = IMPRESSION_EPSILON - delta = DIFFERENTIAL_PRIVACY_DELTA - } - maximumFrequencyPerUser = IMPRESSION_MAXIMUM_FREQUENCY_PER_USER - } - impressionCountVidSamplingInterval = - MetricSpecConfigKt.vidSamplingInterval { - start = IMPRESSION_VID_SAMPLING_START - width = IMPRESSION_VID_SAMPLING_WIDTH - } - - watchDurationParams = - MetricSpecConfigKt.watchDurationParams { - privacyParams = - MetricSpecConfigKt.differentialPrivacyParams { - epsilon = WATCH_DURATION_EPSILON - delta = DIFFERENTIAL_PRIVACY_DELTA - } - maximumWatchDurationPerUser = MAXIMUM_WATCH_DURATION_PER_USER - } - watchDurationVidSamplingInterval = - MetricSpecConfigKt.vidSamplingInterval { - start = WATCH_DURATION_VID_SAMPLING_START - width = WATCH_DURATION_VID_SAMPLING_WIDTH - } + private val SECRET_FILES_PATH: Path = run { + val runfiles = Runfiles.preload(buildMap { + put("RUNFILES_DIR", "src/main/k8s/testing/") + put("metric_spec_config.textproto", "metric_spec_config.textproto") + }).unmapped() + checkNotNull(runfiles.getRuntimePath(Paths.get("secretfiles"))) } + private val METRIC_SPEC_CONFIG: MetricSpecConfig = + parseTextProto(SECRET_FILES_PATH.resolve("metric_spec_config.textproto").toFile(), MetricSpecConfig.getDefaultInstance()) + private val REACH_METRIC_SPEC: MetricSpec = metricSpec { reach = MetricSpecKt.reachParams { - privacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = REACH_ONLY_REACH_EPSILON - delta = DIFFERENTIAL_PRIVACY_DELTA - } - } - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = REACH_ONLY_VID_SAMPLING_START - width = REACH_ONLY_VID_SAMPLING_WIDTH + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = METRIC_SPEC_CONFIG.reachParams.multipleDataProviderParams.privacyParams.epsilon + delta = METRIC_SPEC_CONFIG.reachParams.multipleDataProviderParams.privacyParams.delta + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = METRIC_SPEC_CONFIG.reachParams.multipleDataProviderParams.vidSamplingInterval.fixedStart.start + width = METRIC_SPEC_CONFIG.reachParams.multipleDataProviderParams.vidSamplingInterval.fixedStart.width + } + } + singleDataProviderParams = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = METRIC_SPEC_CONFIG.reachParams.singleDataProviderParams.privacyParams.epsilon + delta = METRIC_SPEC_CONFIG.reachParams.singleDataProviderParams.privacyParams.delta + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = METRIC_SPEC_CONFIG.reachParams.singleDataProviderParams.vidSamplingInterval.fixedStart.start + width = METRIC_SPEC_CONFIG.reachParams.singleDataProviderParams.vidSamplingInterval.fixedStart.width + } + } } } private val INTERNAL_REACH_METRIC_SPEC: InternalMetricSpec = internalMetricSpec { reach = InternalMetricSpecKt.reachParams { - privacyParams = - InternalMetricSpecKt.differentialPrivacyParams { - epsilon = REACH_ONLY_REACH_EPSILON - delta = DIFFERENTIAL_PRIVACY_DELTA - } - } - vidSamplingInterval = - InternalMetricSpecKt.vidSamplingInterval { - start = REACH_ONLY_VID_SAMPLING_START - width = REACH_ONLY_VID_SAMPLING_WIDTH + multipleDataProviderParams = InternalMetricSpecKt.params { + privacyParams = + InternalMetricSpecKt.differentialPrivacyParams { + epsilon = METRIC_SPEC_CONFIG.reachParams.multipleDataProviderParams.privacyParams.epsilon + delta = METRIC_SPEC_CONFIG.reachParams.multipleDataProviderParams.privacyParams.delta + } + vidSamplingInterval = + InternalMetricSpecKt.vidSamplingInterval { + start = METRIC_SPEC_CONFIG.reachParams.multipleDataProviderParams.vidSamplingInterval.fixedStart.start + width = METRIC_SPEC_CONFIG.reachParams.multipleDataProviderParams.vidSamplingInterval.fixedStart.width + } + } + singleDataProviderParams = InternalMetricSpecKt.params { + privacyParams = + InternalMetricSpecKt.differentialPrivacyParams { + epsilon = METRIC_SPEC_CONFIG.reachParams.singleDataProviderParams.privacyParams.epsilon + delta = METRIC_SPEC_CONFIG.reachParams.singleDataProviderParams.privacyParams.delta + } + vidSamplingInterval = + InternalMetricSpecKt.vidSamplingInterval { + start = METRIC_SPEC_CONFIG.reachParams.singleDataProviderParams.vidSamplingInterval.fixedStart.start + width = METRIC_SPEC_CONFIG.reachParams.singleDataProviderParams.vidSamplingInterval.fixedStart.width + } + } } } private val REACH_AND_FREQUENCY_METRIC_SPEC: MetricSpec = metricSpec { reachAndFrequency = MetricSpecKt.reachAndFrequencyParams { - reachPrivacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = REACH_FREQUENCY_REACH_EPSILON - delta = DIFFERENTIAL_PRIVACY_DELTA - } - frequencyPrivacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = REACH_FREQUENCY_FREQUENCY_EPSILON - delta = DIFFERENTIAL_PRIVACY_DELTA - } - maximumFrequency = REACH_FREQUENCY_MAXIMUM_FREQUENCY - } - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = REACH_FREQUENCY_VID_SAMPLING_START - width = REACH_FREQUENCY_VID_SAMPLING_WIDTH + multipleDataProviderParams = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = METRIC_SPEC_CONFIG.reachAndFrequencyParams.multipleDataProviderParams.privacyParams.epsilon + delta = METRIC_SPEC_CONFIG.reachAndFrequencyParams.multipleDataProviderParams.privacyParams.delta + } + frequencyPrivacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = METRIC_SPEC_CONFIG.reachAndFrequencyParams.multipleDataProviderParams.frequencyPrivacyParams.epsilon + delta = METRIC_SPEC_CONFIG.reachAndFrequencyParams.multipleDataProviderParams.frequencyPrivacyParams.delta + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = METRIC_SPEC_CONFIG.reachAndFrequencyParams.multipleDataProviderParams.vidSamplingInterval.fixedStart.start + width = METRIC_SPEC_CONFIG.reachAndFrequencyParams.multipleDataProviderParams.vidSamplingInterval.fixedStart.width + } + } + singleDataProviderParams = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = METRIC_SPEC_CONFIG.reachAndFrequencyParams.singleDataProviderParams.privacyParams.epsilon + delta = METRIC_SPEC_CONFIG.reachAndFrequencyParams.singleDataProviderParams.privacyParams.delta + } + frequencyPrivacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = METRIC_SPEC_CONFIG.reachAndFrequencyParams.singleDataProviderParams.frequencyPrivacyParams.epsilon + delta = METRIC_SPEC_CONFIG.reachAndFrequencyParams.singleDataProviderParams.frequencyPrivacyParams.delta + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = METRIC_SPEC_CONFIG.reachAndFrequencyParams.singleDataProviderParams.vidSamplingInterval.fixedStart.start + width = METRIC_SPEC_CONFIG.reachAndFrequencyParams.singleDataProviderParams.vidSamplingInterval.fixedStart.width + } + } + maximumFrequency = METRIC_SPEC_CONFIG.reachAndFrequencyParams.maximumFrequency } } private val INTERNAL_REACH_AND_FREQUENCY_METRIC_SPEC: InternalMetricSpec = internalMetricSpec { reachAndFrequency = InternalMetricSpecKt.reachAndFrequencyParams { - reachPrivacyParams = - InternalMetricSpecKt.differentialPrivacyParams { - epsilon = REACH_FREQUENCY_REACH_EPSILON - delta = DIFFERENTIAL_PRIVACY_DELTA - } - frequencyPrivacyParams = - InternalMetricSpecKt.differentialPrivacyParams { - epsilon = REACH_FREQUENCY_FREQUENCY_EPSILON - delta = DIFFERENTIAL_PRIVACY_DELTA - } - maximumFrequency = REACH_FREQUENCY_MAXIMUM_FREQUENCY - } - vidSamplingInterval = - InternalMetricSpecKt.vidSamplingInterval { - start = REACH_FREQUENCY_VID_SAMPLING_START - width = REACH_FREQUENCY_VID_SAMPLING_WIDTH + multipleDataProviderParams = InternalMetricSpecKt.params { + privacyParams = + InternalMetricSpecKt.differentialPrivacyParams { + epsilon = METRIC_SPEC_CONFIG.reachAndFrequencyParams.multipleDataProviderParams.privacyParams.epsilon + delta = METRIC_SPEC_CONFIG.reachAndFrequencyParams.multipleDataProviderParams.privacyParams.delta + } + frequencyPrivacyParams = + InternalMetricSpecKt.differentialPrivacyParams { + epsilon = METRIC_SPEC_CONFIG.reachAndFrequencyParams.multipleDataProviderParams.frequencyPrivacyParams.epsilon + delta = METRIC_SPEC_CONFIG.reachAndFrequencyParams.multipleDataProviderParams.frequencyPrivacyParams.delta + } + vidSamplingInterval = + InternalMetricSpecKt.vidSamplingInterval { + start = METRIC_SPEC_CONFIG.reachAndFrequencyParams.multipleDataProviderParams.vidSamplingInterval.fixedStart.start + width = METRIC_SPEC_CONFIG.reachAndFrequencyParams.multipleDataProviderParams.vidSamplingInterval.fixedStart.width + } + } + singleDataProviderParams = InternalMetricSpecKt.params { + privacyParams = + InternalMetricSpecKt.differentialPrivacyParams { + epsilon = METRIC_SPEC_CONFIG.reachAndFrequencyParams.singleDataProviderParams.privacyParams.epsilon + delta = METRIC_SPEC_CONFIG.reachAndFrequencyParams.singleDataProviderParams.privacyParams.delta + } + frequencyPrivacyParams = + InternalMetricSpecKt.differentialPrivacyParams { + epsilon = METRIC_SPEC_CONFIG.reachAndFrequencyParams.singleDataProviderParams.frequencyPrivacyParams.epsilon + delta = METRIC_SPEC_CONFIG.reachAndFrequencyParams.singleDataProviderParams.frequencyPrivacyParams.delta + } + vidSamplingInterval = + InternalMetricSpecKt.vidSamplingInterval { + start = METRIC_SPEC_CONFIG.reachAndFrequencyParams.singleDataProviderParams.vidSamplingInterval.fixedStart.start + width = METRIC_SPEC_CONFIG.reachAndFrequencyParams.singleDataProviderParams.vidSamplingInterval.fixedStart.width + } + } + maximumFrequency = METRIC_SPEC_CONFIG.reachAndFrequencyParams.maximumFrequency } } private val WATCH_DURATION_METRIC_SPEC: MetricSpec = metricSpec { watchDuration = MetricSpecKt.watchDurationParams { - privacyParams = - MetricSpecKt.differentialPrivacyParams { - epsilon = WATCH_DURATION_EPSILON - delta = DIFFERENTIAL_PRIVACY_DELTA - } - maximumWatchDurationPerUser = MAXIMUM_WATCH_DURATION_PER_USER - } - vidSamplingInterval = - MetricSpecKt.vidSamplingInterval { - start = WATCH_DURATION_VID_SAMPLING_START - width = WATCH_DURATION_VID_SAMPLING_WIDTH + params = MetricSpecKt.params { + privacyParams = + MetricSpecKt.differentialPrivacyParams { + epsilon = METRIC_SPEC_CONFIG.watchDurationParams.params.privacyParams.epsilon + delta = METRIC_SPEC_CONFIG.watchDurationParams.params.privacyParams.delta + } + vidSamplingInterval = + MetricSpecKt.vidSamplingInterval { + start = METRIC_SPEC_CONFIG.watchDurationParams.params.vidSamplingInterval.fixedStart.start + width = METRIC_SPEC_CONFIG.watchDurationParams.params.vidSamplingInterval.fixedStart.width + } + } + maximumWatchDurationPerUser = METRIC_SPEC_CONFIG.watchDurationParams.maximumWatchDurationPerUser } } private val INTERNAL_WATCH_DURATION_METRIC_SPEC: InternalMetricSpec = internalMetricSpec { watchDuration = InternalMetricSpecKt.watchDurationParams { - privacyParams = - InternalMetricSpecKt.differentialPrivacyParams { - epsilon = WATCH_DURATION_EPSILON - delta = DIFFERENTIAL_PRIVACY_DELTA - } - maximumWatchDurationPerUser = MAXIMUM_WATCH_DURATION_PER_USER - } - vidSamplingInterval = - InternalMetricSpecKt.vidSamplingInterval { - start = WATCH_DURATION_VID_SAMPLING_START - width = WATCH_DURATION_VID_SAMPLING_WIDTH + params = InternalMetricSpecKt.params { + privacyParams = + InternalMetricSpecKt.differentialPrivacyParams { + epsilon = METRIC_SPEC_CONFIG.watchDurationParams.params.privacyParams.epsilon + delta = METRIC_SPEC_CONFIG.watchDurationParams.params.privacyParams.delta + } + vidSamplingInterval = + InternalMetricSpecKt.vidSamplingInterval { + start = METRIC_SPEC_CONFIG.watchDurationParams.params.vidSamplingInterval.fixedStart.start + width = METRIC_SPEC_CONFIG.watchDurationParams.params.vidSamplingInterval.fixedStart.width + } + } + maximumWatchDurationPerUser = METRIC_SPEC_CONFIG.watchDurationParams.maximumWatchDurationPerUser } + } // Metrics