Skip to content

Commit

Permalink
Add autofill server prediction log events when predicting field types
Browse files Browse the repository at this point in the history 
We are working on improving the UKM recording of Autofill and better
understand the Autofill performance from the UKM events.
We are creating four types of log events from four field type
prediction, local heuristic, autofill server, autocomplete and
rationalizer. We log various of interesting field types,
heuristic_type, heuristic_default_type,..., server_type, html_type,
overall_type ... into UKM.

This CL is working on adding the autofill crowdsourced server prediction
log event for the fields on the form.

Autofill UKM Data Structure doc:
https://docs.google.com/document/d/1Tgp5Op4tGhf03lYT4CfWr5f_FjvcpqWhqY0qPJcR-cM/edit#heading=h.k5jx6iluw4yt
https://docs.google.com/document/d/1ZH0JbL6bES3cD4KqZWsGR6n8I-rhnkx6no6nQOgYq5w/.

Bug: 1325851
Change-Id: I4a80f1883c44cc6a1099a33a1ab2b087fb53f662
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/4167412
Reviewed-by: Dominic Battré <battre@chromium.org>
Reviewed-by: Christoph Schwering <schwering@google.com>
Commit-Queue: Lan Wei <lanwei@chromium.org>
Reviewed-by: Robert Kaplow <rkaplow@chromium.org>
Cr-Commit-Position: refs/heads/main@{#1096507}
  • Loading branch information
@LanWei22
LanWei22 authored and Chromium LUCI CQ committed Jan 25, 2023
1 parent e8ebd45 commit dcaf205
Show file tree
Hide file tree
Showing 10 changed files with 313 additions and 22 deletions.
7 changes: 6 additions & 1 deletion components/autofill/core/browser/autofill_field.cc
View file
Expand Up @@ -48,7 +48,7 @@ bool AreCollapsibleLogEvents(const AutofillField::FieldLogEventType& event1,
}

static_assert(
absl::variant_size<AutofillField::FieldLogEventType>() == 7,
absl::variant_size<AutofillField::FieldLogEventType>() == 8,
"If you add a new field event type, you need to update this function");

if (absl::holds_alternative<absl::monostate>(event1)) {
Expand Down Expand Up @@ -86,6 +86,11 @@ bool AreCollapsibleLogEvents(const AutofillField::FieldLogEventType& event1,
return AreCollapsible(absl::get<E>(event1), absl::get<E>(event2));
}

if (absl::holds_alternative<ServerPredictionFieldLogEvent>(event1)) {
using E = ServerPredictionFieldLogEvent;
return AreCollapsible(absl::get<E>(event1), absl::get<E>(event2));
}

NOTREACHED();
return false;
}
Expand Down
3 changes: 2 additions & 1 deletion components/autofill/core/browser/autofill_field.h
View file
Expand Up @@ -38,7 +38,8 @@ class AutofillField : public FormFieldData {
FillFieldLogEvent,
TypingFieldLogEvent,
HeuristicPredictionFieldLogEvent,
AutocompleteAttributeFieldLogEvent>;
AutocompleteAttributeFieldLogEvent,
ServerPredictionFieldLogEvent>;

AutofillField();
explicit AutofillField(const FormFieldData& field);
Expand Down
142 changes: 141 additions & 1 deletion components/autofill/core/browser/browser_autofill_manager_unittest.cc
View file
Expand Up @@ -863,12 +863,34 @@ class BrowserAutofillManagerTest : public testing::Test {
expected.rank_in_field_signature_group)));
}

// Matches a ServerPredictionFieldLogEvent by equality of fields.
auto Equal(const ServerPredictionFieldLogEvent& expected) {
return VariantWith<ServerPredictionFieldLogEvent>(AllOf(
Field("server_type1", &ServerPredictionFieldLogEvent::server_type1,
expected.server_type1),
Field("prediction_source1",
&ServerPredictionFieldLogEvent::prediction_source1,
expected.prediction_source1),
Field("server_type2", &ServerPredictionFieldLogEvent::server_type2,
expected.server_type2),
Field("prediction_source2",
&ServerPredictionFieldLogEvent::prediction_source2,
expected.prediction_source2),
Field(
"server_type_prediction_is_override",
&ServerPredictionFieldLogEvent::server_type_prediction_is_override,
expected.server_type_prediction_is_override),
Field("rank_in_field_signature_group",
&ServerPredictionFieldLogEvent::rank_in_field_signature_group,
expected.rank_in_field_signature_group)));
}

// Matches a vector of FieldLogEventType objects by equality of fields of each
// log event type.
auto ArrayEquals(
const std::vector<AutofillField::FieldLogEventType>& expected) {
static_assert(
absl::variant_size<AutofillField::FieldLogEventType>() == 7,
absl::variant_size<AutofillField::FieldLogEventType>() == 8,
"If you add a new field event type, you need to update this function");
std::vector<Matcher<AutofillField::FieldLogEventType>> matchers;
for (const auto& event : expected) {
Expand All @@ -889,6 +911,10 @@ class BrowserAutofillManagerTest : public testing::Test {
event)) {
matchers.push_back(
Equal(absl::get<AutocompleteAttributeFieldLogEvent>(event)));
} else if (absl::holds_alternative<ServerPredictionFieldLogEvent>(
event)) {
matchers.push_back(
Equal(absl::get<ServerPredictionFieldLogEvent>(event)));
} else {
NOTREACHED();
}
Expand Down Expand Up @@ -5945,6 +5971,120 @@ TEST_F(BrowserAutofillManagerWithLogEventsTest,
}
}

// Test that we record field log events correctly for autofill crowdsourced
// server prediction.
TEST_F(BrowserAutofillManagerWithLogEventsTest,
LogEventsParseQueryResponseServerPrediction) {
// Set up our form data.
FormData form;
form.host_frame = test::MakeLocalFrameToken();
form.unique_renderer_id = test::MakeFormRendererId();
form.name = u"MyForm";
form.url = GURL("https://myform.com/form.html");
form.action = GURL("https://myform.com/submit.html");
FormFieldData field;
test::CreateTestFormField(/*label=*/"Name", /*name=*/"name",
/*value=*/"", /*type=*/"text", /*field=*/&field);
form.fields.push_back(field);
test::CreateTestFormField(/*label=*/"Street", /*name=*/"Street",
/*value=*/"", /*type=*/"text", /*field=*/&field);
form.fields.push_back(field);
test::CreateTestFormField(/*label=*/"City", /*name=*/"city",
/*value=*/"", /*type=*/"text", /*field=*/&field);
form.fields.push_back(field);
test::CreateTestFormField(/*label=*/"State", /*name=*/"state",
/*value=*/"", /*type=*/"text", /*field=*/&field);
form.fields.push_back(field);
test::CreateTestFormField(/*label=*/"Postal Code", /*name=*/"zipcode",
/*value=*/"", /*type=*/"text", /*field=*/&field);
form.fields.push_back(field);
// Simulate having seen this form on page load.
// |form_structure_instance| will be owned by |browser_autofill_manager_|.
auto form_structure_instance = std::make_unique<FormStructure>(form);
// This pointer is valid as long as autofill manager lives.
FormStructure* form_structure = form_structure_instance.get();
form_structure->DetermineHeuristicTypes(nullptr, nullptr);
browser_autofill_manager_->AddSeenFormStructure(
std::move(form_structure_instance));

// Make API response with suggestions.
AutofillQueryResponse response;
AutofillQueryResponse::FormSuggestion* form_suggestion;
// Set suggestions for form.
form_suggestion = response.add_form_suggestions();
autofill::test::AddFieldPredictionsToForm(
form.fields[0],
{test::CreateFieldPrediction(NAME_FIRST,
FieldPrediction::SOURCE_AUTOFILL_DEFAULT),
test::CreateFieldPrediction(USERNAME,
FieldPrediction::SOURCE_PASSWORDS_DEFAULT)},
form_suggestion);
autofill::test::AddFieldPredictionsToForm(
form.fields[1],
{test::CreateFieldPrediction(ADDRESS_HOME_LINE1,
FieldPrediction::SOURCE_OVERRIDE)},
form_suggestion);
autofill::test::AddFieldPredictionToForm(form.fields[2], ADDRESS_HOME_CITY,
form_suggestion);
autofill::test::AddFieldPredictionToForm(form.fields[3], ADDRESS_HOME_STATE,
form_suggestion);
autofill::test::AddFieldPredictionToForm(form.fields[4], ADDRESS_HOME_ZIP,
form_suggestion);

std::string response_string;
ASSERT_TRUE(response.SerializeToString(&response_string));
std::string encoded_response_string;
base::Base64Encode(response_string, &encoded_response_string);

// Query autofill server for the field type prediction.
browser_autofill_manager_->OnLoadedServerPredictionsForTest(
encoded_response_string, test::GetEncodedSignatures(*form_structure));
EXPECT_EQ(NAME_FIRST, form_structure->field(0)->Type().GetStorableType());
EXPECT_EQ(ADDRESS_HOME_LINE1,
form_structure->field(1)->Type().GetStorableType());
EXPECT_EQ(ADDRESS_HOME_CITY,
form_structure->field(2)->Type().GetStorableType());
EXPECT_EQ(ADDRESS_HOME_STATE,
form_structure->field(3)->Type().GetStorableType());
EXPECT_EQ(ADDRESS_HOME_ZIP,
form_structure->field(4)->Type().GetStorableType());

// Simulate form submission.
FormSubmitted(form);

for (const auto& autofill_field_ptr : *form_structure) {
SCOPED_TRACE(autofill_field_ptr->parseable_label());
// All parsed fields share the same expected
// HeuristicPredictionFieldLogEvent.
std::vector<AutofillField::FieldLogEventType> expected_events =
ToHeuristicFieldTypeEvents(autofill_field_ptr->heuristic_type());
// The autofill server applies two predictions on the "Name" field.
ServerFieldType server_type2 =
autofill_field_ptr->parseable_label() == u"Name" ? USERNAME
: NO_SERVER_DATA;
FieldPrediction::Source prediction_source2 =
autofill_field_ptr->parseable_label() == u"Name"
? FieldPrediction::SOURCE_PASSWORDS_DEFAULT
: FieldPrediction::SOURCE_UNSPECIFIED;
// The server prediction overrides the type predicted by local heuristic on
// the field of label "Street".
bool server_type_prediction_is_override =
autofill_field_ptr->parseable_label() == u"Street" ? true : false;
expected_events.push_back(ServerPredictionFieldLogEvent{
.server_type1 = autofill_field_ptr->server_type(),
.prediction_source1 =
autofill_field_ptr->server_predictions()[0].source(),
.server_type2 = server_type2,
.prediction_source2 = prediction_source2,
.server_type_prediction_is_override =
server_type_prediction_is_override,
.rank_in_field_signature_group = 1,
});
EXPECT_THAT(autofill_field_ptr->field_log_events(),
ArrayEquals(expected_events));
}
}

// Test that when Autocomplete is enabled and Autofill is disabled, form
// submissions are still received by the SingleFieldFormFillRouter.
TEST_F(BrowserAutofillManagerTest, FormSubmittedAutocompleteEnabled) {
Expand Down
36 changes: 30 additions & 6 deletions components/autofill/core/browser/form_structure.cc
View file
Expand Up @@ -581,6 +581,9 @@ void FormStructure::ProcessQueryResponse(

// Copy the field types into the actual form.
for (FormStructure* form : forms) {
// Fields can share the same field signature. This map records for each
// signature how many fields with the same signature have been observed.
std::map<FieldSignature, size_t> field_rank_map;
for (auto& field : form->fields_) {
// Get the field prediction for |form|'s signature and the |field|'s
// host_form_signature. The former takes precedence over the latter.
Expand Down Expand Up @@ -619,6 +622,27 @@ void FormStructure::ProcessQueryResponse(

if (current_field->has_password_requirements())
field->SetPasswordRequirements(current_field->password_requirements());

++field_rank_map[field->GetFieldSignature()];
// Log the field type predicted from Autofill crowdsourced server.
field->AppendLogEventIfNotRepeated(ServerPredictionFieldLogEvent{
.server_type1 = field->server_type(),
.prediction_source1 = field->server_predictions().empty()
? FieldPrediction::SOURCE_UNSPECIFIED
: field->server_predictions()[0].source(),
.server_type2 =
field->server_predictions().size() >= 2
? ToSafeServerFieldType(field->server_predictions()[1].type(),
NO_SERVER_DATA)
: NO_SERVER_DATA,
.prediction_source2 = field->server_predictions().size() >= 2
? field->server_predictions()[1].source()
: FieldPrediction::SOURCE_UNSPECIFIED,
.server_type_prediction_is_override =
field->server_type_prediction_is_override(),
.rank_in_field_signature_group =
field_rank_map[field->GetFieldSignature()],
});
}

AutofillMetrics::LogServerResponseHasDataForForm(base::ranges::any_of(
Expand Down Expand Up @@ -1330,7 +1354,7 @@ void FormStructure::LogDetermineHeuristicTypesMetrics() {
void FormStructure::SetFieldTypesFromAutocompleteAttribute() {
has_author_specified_types_ = false;
has_author_specified_upi_vpa_hint_ = false;
std::map<FieldSignature, size_t> field_rank_id_map;
std::map<FieldSignature, size_t> field_rank_map;
for (const std::unique_ptr<AutofillField>& field : fields_) {
if (!field->parsed_autocomplete)
continue;
Expand All @@ -1353,12 +1377,12 @@ void FormStructure::SetFieldTypesFromAutocompleteAttribute() {
field->parsed_autocomplete->mode);

// Log the field type predicted from autocomplete attribute.
++field_rank_id_map[field->GetFieldSignature()];
++field_rank_map[field->GetFieldSignature()];
field->AppendLogEventIfNotRepeated(AutocompleteAttributeFieldLogEvent{
.html_type = field->parsed_autocomplete->field_type,
.html_mode = field->parsed_autocomplete->mode,
.rank_in_field_signature_group =
field_rank_id_map[field->GetFieldSignature()],
field_rank_map[field->GetFieldSignature()],
});
}
}
Expand Down Expand Up @@ -1396,22 +1420,22 @@ void FormStructure::ParseFieldTypesWithPatterns(PatternSource pattern_source,

// Fields can share the same field signature. This map records for each
// signature how many fields with the same signature have been observed.
std::map<FieldSignature, size_t> field_rank_id_map;
std::map<FieldSignature, size_t> field_rank_map;
for (const auto& field : fields_) {
auto iter = field_type_map.find(field->global_id());
if (iter == field_type_map.end())
continue;
const FieldCandidates& candidates = iter->second;
field->set_heuristic_type(pattern_source, candidates.BestHeuristicType());

++field_rank_id_map[field->GetFieldSignature()];
++field_rank_map[field->GetFieldSignature()];
// Log the field type predicted from local heuristics.
field->AppendLogEventIfNotRepeated(HeuristicPredictionFieldLogEvent{
.field_type = field->heuristic_type(pattern_source),
.pattern_source = pattern_source,
.is_active_pattern_source = GetActivePatternSource() == pattern_source,
.rank_in_field_signature_group =
field_rank_id_map[field->GetFieldSignature()],
field_rank_map[field->GetFieldSignature()],
});
}
}
Expand Down
41 changes: 36 additions & 5 deletions components/autofill/core/browser/metrics/autofill_metrics.cc
View file
Expand Up @@ -2508,6 +2508,11 @@ void AutofillMetrics::FormInteractionsUkmLogger::
// TODO(crbug.com/1325851): Add a metric in |FieldInfo| UKM event to indicate
// whether the user had any data available for the respective field type.

// If multiple fields have the same signature, this indicates the position
// within this set of fields. This allows us to understand problems related
// to duplicated field signatures.
size_t rank_in_field_signature_group = 0;

// Field types from local heuristics prediction.
// The field type from the active local heuristic pattern.
ServerFieldType heuristic_type = UNKNOWN_TYPE;
Expand All @@ -2530,16 +2535,24 @@ void AutofillMetrics::FormInteractionsUkmLogger::
HtmlFieldMode html_mode = HtmlFieldMode::kNone;
HtmlFieldType html_type = HtmlFieldType::kUnrecognized;

// If multiple fields have the same signature, this indicates the position
// within this set of fields. This allows us to understand problems related
// to duplicated field signatures.
size_t rank_in_field_signature_group = 0;
// The field type predicted by the Autofill crowdsourced server from
// majority voting.
ServerFieldType server_type1 = NO_SERVER_DATA;
FieldPrediction::Source prediction_source1 =
FieldPrediction::SOURCE_UNSPECIFIED;
ServerFieldType server_type2 = NO_SERVER_DATA;
FieldPrediction::Source prediction_source2 =
FieldPrediction::SOURCE_UNSPECIFIED;
// This is an annotation for server predicted field types which indicates
// that a manual override defines the server type.
bool server_type_is_override = false;

bool had_heuristic_type = false;
bool had_html_type = false;
bool had_server_type = false;

for (const auto& log_event : field_log_events) {
static_assert(absl::variant_size<AutofillField::FieldLogEventType>() == 7,
static_assert(absl::variant_size<AutofillField::FieldLogEventType>() == 8,
"When adding new variants check that this function does not "
"need to be updated.");
if (auto* event =
Expand Down Expand Up @@ -2611,6 +2624,16 @@ void AutofillMetrics::FormInteractionsUkmLogger::
rank_in_field_signature_group = event->rank_in_field_signature_group;
had_html_type = true;
}

if (auto* event = absl::get_if<ServerPredictionFieldLogEvent>(&log_event)) {
server_type1 = event->server_type1;
prediction_source1 = event->prediction_source1;
server_type2 = event->server_type2;
prediction_source2 = event->prediction_source2;
server_type_is_override = event->server_type_prediction_is_override;
rank_in_field_signature_group = event->rank_in_field_signature_group;
had_server_type = true;
}
}

if (had_value_after_filling != OptionalBoolean::kUndefined ||
Expand Down Expand Up @@ -2674,6 +2697,14 @@ void AutofillMetrics::FormInteractionsUkmLogger::
.SetHtmlFieldMode(static_cast<int>(html_mode));
}

if (had_server_type) {
builder.SetServerType1(server_type1)
.SetServerPredictionSource1(prediction_source1)
.SetServerType2(server_type2)
.SetServerPredictionSource2(prediction_source2)
.SetServerTypeIsOverride(server_type_is_override);
}

if (rank_in_field_signature_group) {
builder.SetRankInFieldSignatureGroup(rank_in_field_signature_group);
}
Expand Down

0 comments on commit dcaf205

Please sign in to comment.