-
Notifications
You must be signed in to change notification settings - Fork 6.7k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[Autofill] Add Autofill.ShadowPredictions.* metrics
The metric is enum which represents a three way comparison between the autofill prediction, an experimental prediction, and the value the was in the field on submission. The enum also represents the original autofill prediction. The metric is uploaded on form submission, once for every alternative prediction. Bug: 1310255 Change-Id: I778673dceaa768a0272abd6612e3f6fdb572afb7 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3574693 Reviewed-by: Christos Froussios <cfroussios@chromium.org> Reviewed-by: Christoph Schwering <schwering@google.com> Reviewed-by: Caitlin Fischer <caitlinfischer@google.com> Commit-Queue: Christos Froussios <cfroussios@chromium.org> Cr-Commit-Position: refs/heads/main@{#1001060}
- Loading branch information
Showing
8 changed files
with
2,029 additions
and
3 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
87 changes: 87 additions & 0 deletions
87
components/autofill/core/browser/metrics/shadow_prediction_metrics.cc
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,87 @@ | ||
| // Copyright 2022 The Chromium Authors. All rights reserved. | ||
| // Use of this source code is governed by a BSD-style license that can be | ||
| // found in the LICENSE file. | ||
|
|
||
| #include "components/autofill/core/browser/metrics/shadow_prediction_metrics.h" | ||
|
|
||
| #include "base/metrics/histogram_functions.h" | ||
| #include "base/metrics/sparse_histogram.h" | ||
| #include "components/autofill/core/browser/form_parsing/buildflags.h" | ||
| #include "components/autofill/core/browser/form_parsing/field_candidates.h" | ||
|
|
||
| namespace autofill::metrics { | ||
|
|
||
| namespace { | ||
|
|
||
| // The number of basic comparison results (i.e. without offsetting to encode the | ||
| // field type). | ||
| constexpr int kBaseComparisonRange = 6; | ||
|
|
||
| // Encode `prediction` into `comparison_base`. | ||
| int GetTypeSpecificComparison(ServerFieldType prediction, int comparison_base) { | ||
| DCHECK_LE(comparison_base, kDifferentPredictionsValueAgreesWithBoth); | ||
| DCHECK_NE(comparison_base, kNoPrediction); | ||
|
|
||
| return static_cast<int>(prediction) * kBaseComparisonRange + comparison_base; | ||
| } | ||
|
|
||
| // Get the comparison between the predictions, without the prediction type being | ||
| // encoded in the returned value. The returned value is in the range [0,6] | ||
| // inclusive. | ||
| int GetBaseComparison(ServerFieldType current, | ||
| ServerFieldType next, | ||
| const ServerFieldTypeSet& submitted_types) { | ||
| if (current == NO_SERVER_DATA || next == NO_SERVER_DATA) { | ||
| return kNoPrediction; | ||
| } else if (current == next) { | ||
| return submitted_types.contains(current) ? kSamePredictionValueAgrees | ||
| : kSamePredictionValueDisagrees; | ||
| } else if (submitted_types.contains_all({current, next})) { | ||
| return kDifferentPredictionsValueAgreesWithBoth; | ||
| } else if (submitted_types.contains(current)) { | ||
| return kDifferentPredictionsValueAgreesWithOld; | ||
| } else if (submitted_types.contains(next)) { | ||
| return kDifferentPredictionsValueAgreesWithNew; | ||
| } else { | ||
| return kDifferentPredictionsValueAgreesWithNeither; | ||
| } | ||
| } | ||
|
|
||
| } // namespace | ||
|
|
||
| int GetShadowPrediction(ServerFieldType current, | ||
| ServerFieldType next, | ||
| const ServerFieldTypeSet& submitted_types) { | ||
| // `NO_SERVER_DATA` means that we didn't actually run any heuristics. | ||
| if (current == NO_SERVER_DATA || next == NO_SERVER_DATA) | ||
| return kNoPrediction; | ||
|
|
||
| int comparison = GetBaseComparison(current, next, submitted_types); | ||
|
|
||
| // If we compared the predictions, offset them by the field type, so that the | ||
| // type is included in the enum. | ||
| if (comparison != kNoPrediction) | ||
| comparison = GetTypeSpecificComparison(current, comparison); | ||
|
|
||
| return comparison; | ||
| } | ||
|
|
||
| void LogShadowPredictionComparison(const AutofillField& field) { | ||
| #if BUILDFLAG(USE_INTERNAL_AUTOFILL_HEADERS) | ||
| const auto& submitted_types = field.possible_types(); | ||
|
|
||
| base::UmaHistogramSparse( | ||
| "Autofill.ShadowPredictions.ExperimentalToDefault", | ||
| GetShadowPrediction(field.heuristic_type(PatternSource::kDefault), | ||
| field.heuristic_type(PatternSource::kExperimental), | ||
| submitted_types)); | ||
|
|
||
| base::UmaHistogramSparse( | ||
| "Autofill.ShadowPredictions.NextGenToDefault", | ||
| GetShadowPrediction(field.heuristic_type(PatternSource::kDefault), | ||
| field.heuristic_type(PatternSource::kNextGen), | ||
| submitted_types)); | ||
| #endif | ||
| } | ||
|
|
||
| } // namespace autofill::metrics |
38 changes: 38 additions & 0 deletions
38
components/autofill/core/browser/metrics/shadow_prediction_metrics.h
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,38 @@ | ||
| // Copyright 2022 The Chromium Authors. All rights reserved. | ||
| // Use of this source code is governed by a BSD-style license that can be | ||
| // found in the LICENSE file. | ||
|
|
||
| #ifndef COMPONENTS_AUTOFILL_CORE_BROWSER_METRICS_SHADOW_PREDICTION_METRICS_H_ | ||
| #define COMPONENTS_AUTOFILL_CORE_BROWSER_METRICS_SHADOW_PREDICTION_METRICS_H_ | ||
|
|
||
| #include "components/autofill/core/browser/form_structure.h" | ||
|
|
||
| namespace autofill::metrics { | ||
|
|
||
| // These values are persisted to logs. Entries should not be renumbered and | ||
| // numeric values should never be reused. These mirror the first entries of | ||
| // `AutofillPredictionsComparisonResult` in | ||
| // tools/metrics/histograms/metadata/autofill/histograms.xml | ||
| constexpr int kNoPrediction = 0; | ||
| constexpr int kSamePredictionValueAgrees = 1; | ||
| constexpr int kSamePredictionValueDisagrees = 2; | ||
| constexpr int kDifferentPredictionsValueAgreesWithOld = 3; | ||
| constexpr int kDifferentPredictionsValueAgreesWithNew = 4; | ||
| constexpr int kDifferentPredictionsValueAgreesWithNeither = 5; | ||
| constexpr int kDifferentPredictionsValueAgreesWithBoth = 6; | ||
|
|
||
| // Gets a 3-way comparison between | ||
| // * the `current` prediction | ||
| // * the `next` (shadow) prediction | ||
| // * the types detected in the field `submitted_types` during submission | ||
| int GetShadowPrediction(ServerFieldType current, | ||
| ServerFieldType next, | ||
| const ServerFieldTypeSet& submitted_types); | ||
|
|
||
| // Logs Autofill.ShadowPredictions.* metrics by comparing the submitted | ||
| // values to the actual and hypothetical predictions. | ||
| void LogShadowPredictionComparison(const AutofillField& field); | ||
|
|
||
| } // namespace autofill::metrics | ||
|
|
||
| #endif // COMPONENTS_AUTOFILL_CORE_BROWSER_METRICS_SHADOW_PREDICTION_METRICS_H_ |
199 changes: 199 additions & 0 deletions
199
components/autofill/core/browser/metrics/shadow_prediction_metrics_unittest.cc
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,199 @@ | ||
| // Copyright 2022 The Chromium Authors. All rights reserved. | ||
| // Use of this source code is governed by a BSD-style license that can be | ||
| // found in the LICENSE file. | ||
|
|
||
| #include "components/autofill/core/browser/metrics/shadow_prediction_metrics.h" | ||
|
|
||
| #include "base/test/metrics/histogram_tester.h" | ||
| #include "components/autofill/core/browser/autofill_form_test_utils.h" | ||
| #include "components/autofill/core/browser/autofill_test_utils.h" | ||
| #include "components/autofill/core/browser/field_types.h" | ||
| #include "components/autofill/core/browser/form_parsing/buildflags.h" | ||
| #include "components/autofill/core/browser/metrics/autofill_metrics_test_base.h" | ||
| #include "testing/gmock/include/gmock/gmock.h" | ||
| #include "testing/gtest/include/gtest/gtest.h" | ||
|
|
||
| using ::autofill::mojom::SubmissionSource; | ||
| using ::base::Bucket; | ||
| using ::testing::IsEmpty; | ||
| using ::testing::UnorderedElementsAre; | ||
|
|
||
| namespace autofill::metrics { | ||
|
|
||
| // These constants mirror the similarly named values in | ||
| // `AutofillPredictionsComparisonResult` in | ||
| // tools/metrics/histograms/metadata/autofill/histograms.xml. | ||
| constexpr int kNameFirstSamePredictionValueAgrees = 19; | ||
| constexpr int kNameFirstSamePredictionValueDisagrees = 20; | ||
| constexpr int kNameFirstDifferentPredictionsValueAgreesWithOld = 21; | ||
| constexpr int kNameFirstDifferentPredictionsValueAgreesWithBoth = 24; | ||
| constexpr int kNameFirstDifferentPredictionsValueAgreesWithNeither = 23; | ||
| constexpr int kEmailAddressDifferentPredictionsValueAgreesWithNew = 58; | ||
| #if BUILDFLAG(USE_INTERNAL_AUTOFILL_HEADERS) | ||
| constexpr int kNameFullSamePredictionValueAgrees = 43; | ||
| constexpr int kNameFullDifferentPredictionsValueAgreesWithOld = 45; | ||
| constexpr int kSearchTermSamePredictionValueDisagrees = 584; | ||
| constexpr int kSearchTermDifferentPredictionsValueAgreesWithNew = 586; | ||
| #endif | ||
|
|
||
| namespace { | ||
|
|
||
| // Get a form with 2 fields. | ||
| FormData GetFormWith2Fields(const GURL& form_origin) { | ||
| return test::GetFormData( | ||
| {.description_for_logging = "ShadowPredictions", | ||
| .fields = | ||
| { | ||
| { | ||
| .label = u"Name", | ||
| .name = u"name", | ||
| }, | ||
| { | ||
| .label = u"Email", | ||
| .name = u"email", | ||
| }, | ||
| }, | ||
| .unique_renderer_id = test::MakeFormRendererId(), | ||
| .main_frame_origin = url::Origin::Create(form_origin)}); | ||
| } | ||
|
|
||
| // Test that various combinations of predictions and values are mapped to the | ||
| // correct value in the metric enum. | ||
| TEST(AutofillShadowPredictionComparisonTest, | ||
| PredictionsMapToPredictionComparison) { | ||
| using ::autofill::metrics::GetShadowPrediction; | ||
|
|
||
| EXPECT_EQ(kNoPrediction, GetShadowPrediction(NO_SERVER_DATA, NO_SERVER_DATA, | ||
| {NO_SERVER_DATA})); | ||
|
|
||
| EXPECT_EQ(kNoPrediction, | ||
| GetShadowPrediction(NAME_FIRST, NO_SERVER_DATA, {NAME_FIRST})); | ||
|
|
||
| EXPECT_EQ(kNameFirstSamePredictionValueAgrees, | ||
| GetShadowPrediction(NAME_FIRST, NAME_FIRST, {NAME_FIRST})); | ||
|
|
||
| EXPECT_EQ(kNameFirstSamePredictionValueDisagrees, | ||
| GetShadowPrediction(NAME_FIRST, NAME_FIRST, {EMAIL_ADDRESS})); | ||
|
|
||
| EXPECT_EQ(kEmailAddressDifferentPredictionsValueAgreesWithNew, | ||
| GetShadowPrediction(EMAIL_ADDRESS, NAME_FIRST, {NAME_FIRST})); | ||
|
|
||
| EXPECT_EQ(kNameFirstDifferentPredictionsValueAgreesWithOld, | ||
| GetShadowPrediction(NAME_FIRST, EMAIL_ADDRESS, {NAME_FIRST})); | ||
|
|
||
| EXPECT_EQ(kNameFirstDifferentPredictionsValueAgreesWithNeither, | ||
| GetShadowPrediction(NAME_FIRST, EMAIL_ADDRESS, {NAME_LAST})); | ||
|
|
||
| EXPECT_EQ(kNameFirstDifferentPredictionsValueAgreesWithBoth, | ||
| GetShadowPrediction(NAME_FIRST, EMAIL_ADDRESS, | ||
| {NAME_FIRST, EMAIL_ADDRESS})); | ||
| } | ||
|
|
||
| // Test that all `ServerFieldType`s have corresponding values in the enum. | ||
| TEST(AutofillShadowPredictionComparisonTest, ComparisonContainsAllTypes) { | ||
| // If this test fails after adding a type, update | ||
| // `AutofillPredictionsComparisonResult` in | ||
| // tools/metrics/histograms/metadata/autofill/histograms.xml and set | ||
| // `last_known_type` to the last entry in the enum. | ||
| constexpr ServerFieldType last_known_type = | ||
| PHONE_HOME_CITY_AND_NUMBER_WITHOUT_TRUNK_PREFIX; | ||
| int max_comparison = | ||
| GetShadowPrediction(last_known_type, NAME_FIRST, {NAME_LAST}); | ||
|
|
||
| for (int type_int = NO_SERVER_DATA; type_int <= MAX_VALID_FIELD_TYPE; | ||
| type_int++) { | ||
| auto type = ToSafeServerFieldType(type_int, NO_SERVER_DATA); | ||
| EXPECT_LE(GetShadowPrediction(type, NAME_FIRST, {NAME_LAST}), | ||
| max_comparison) | ||
| << FieldTypeToStringPiece(type) << " has no mapping."; | ||
| } | ||
| } | ||
|
|
||
| class AutofillShadowPredictionMetricsTest | ||
| : public autofill::metrics::AutofillMetricsBaseTest { | ||
| public: | ||
| AutofillShadowPredictionMetricsTest() = default; | ||
| ~AutofillShadowPredictionMetricsTest() override = default; | ||
| }; | ||
|
|
||
| // When shadow predictions are not calculated, the shadow prediction metrics | ||
| // should report `0`. | ||
| TEST_F(AutofillShadowPredictionMetricsTest, | ||
| SubmissionWithoutShadowPredictions) { | ||
| FormData form = GetFormWith2Fields(autofill_client_->form_origin()); | ||
| form.fields[0].value = u"Elvis Aaron Presley"; // A known `NAME_FULL`. | ||
| form.fields[1].value = u"buddy@gmail.com"; // A known `EMAIL_ADDRESS`. | ||
|
|
||
| std::vector<ServerFieldType> heuristic_types = {NAME_FULL, EMAIL_ADDRESS}; | ||
| std::vector<ServerFieldType> server_types = {NAME_FULL, EMAIL_ADDRESS}; | ||
|
|
||
| // Simulate having seen this form on page load. | ||
| autofill_manager().AddSeenForm(form, heuristic_types, server_types); | ||
|
|
||
| // Simulate form submission. | ||
| base::HistogramTester histogram_tester; | ||
| autofill_manager().OnFormSubmitted(form, /*known_success=*/false, | ||
| SubmissionSource::FORM_SUBMISSION); | ||
|
|
||
| #if BUILDFLAG(USE_INTERNAL_AUTOFILL_HEADERS) | ||
| histogram_tester.ExpectBucketCount( | ||
| "Autofill.ShadowPredictions.ExperimentalToDefault", kNoPrediction, 2); | ||
| histogram_tester.ExpectBucketCount( | ||
| "Autofill.ShadowPredictions.NextGenToDefault", kNoPrediction, 2); | ||
| #else | ||
| EXPECT_THAT(histogram_tester.GetAllSamples( | ||
| "Autofill.ShadowPredictions.ExperimentalToDefault"), | ||
| IsEmpty()); | ||
| EXPECT_THAT(histogram_tester.GetAllSamples( | ||
| "Autofill.ShadowPredictions.NextGenToDefault"), | ||
| IsEmpty()); | ||
| #endif | ||
| } | ||
|
|
||
| #if BUILDFLAG(USE_INTERNAL_AUTOFILL_HEADERS) | ||
| // Test that Autofill.ShadowPredictions.* describes the differences between the | ||
| // predictions and the submitted values. | ||
| TEST_F(AutofillShadowPredictionMetricsTest, | ||
| SubmissionWithAgreeingShadowPredictions) { | ||
| FormData form = GetFormWith2Fields(autofill_client_->form_origin()); | ||
| form.fields[0].value = u"Elvis Aaron Presley"; // A known `NAME_FULL`. | ||
| form.fields[1].value = u"buddy@gmail.com"; // A known `EMAIL_ADDRESS`. | ||
|
|
||
| std::vector<ServerFieldType> server_types = {NAME_FULL, EMAIL_ADDRESS}; | ||
|
|
||
| // Simulate having seen this form on page load. | ||
| autofill_manager().AddSeenForm( | ||
| form, | ||
| {// Field 0 | ||
| {{PatternSource::kDefault, NAME_FULL}, | ||
| {PatternSource::kExperimental, NAME_FULL}, | ||
| {PatternSource::kNextGen, NAME_FIRST}}, | ||
| // Field 1 | ||
| {{PatternSource::kDefault, SEARCH_TERM}, | ||
| {PatternSource::kExperimental, EMAIL_ADDRESS}, | ||
| {PatternSource::kNextGen, SEARCH_TERM}}}, | ||
| server_types); | ||
|
|
||
| // Simulate form submission. | ||
| base::HistogramTester histogram_tester; | ||
| autofill_manager().OnFormSubmitted(form, /*known_success=*/false, | ||
| SubmissionSource::FORM_SUBMISSION); | ||
|
|
||
| EXPECT_THAT( | ||
| histogram_tester.GetAllSamples( | ||
| "Autofill.ShadowPredictions.ExperimentalToDefault"), | ||
| UnorderedElementsAre( | ||
| Bucket(kNameFullSamePredictionValueAgrees, 1), | ||
| Bucket(kSearchTermDifferentPredictionsValueAgreesWithNew, 1))); | ||
|
|
||
| EXPECT_THAT(histogram_tester.GetAllSamples( | ||
| "Autofill.ShadowPredictions.NextGenToDefault"), | ||
| UnorderedElementsAre( | ||
| Bucket(kNameFullDifferentPredictionsValueAgreesWithOld, 1), | ||
| Bucket(kSearchTermSamePredictionValueDisagrees, 1))); | ||
| } | ||
| #endif | ||
|
|
||
| } // namespace | ||
|
|
||
| } // namespace autofill::metrics |
Oops, something went wrong.