Stop filtering the results if the key is the same.

* When the key is the same, common prefix results are not redundant
  + e.g. さかい → 堺 and 堺井
* Will fix some over filtered conversions.
* Added more comments.

#typingquality
#codehealth

PiperOrigin-RevId: 780327139

Author: hiroyuki-komatsu

src/prediction/dictionary_prediction_aggregator.cc

@@ -85,15 +85,30 @@ constexpr size_t kSuggestionMaxResultsSize = 256;
 constexpr size_t kPredictionMaxResultsSize = 100000;
 
 // Returns true if the |target| may be redundant result.
-bool MaybeRedundant(const absl::string_view reference,
-                    const absl::string_view target) {
+bool MaybeRedundant(const Result &reference_result,
+                    const Result &target_result) {
+  const absl::string_view reference = reference_result.value;
+  const absl::string_view target = target_result.value;
+
+  // Same value means the result is redundant.
+  if (reference == target) {
+    return true;
+  }
+
+  // If the key is the same, the target is not redundant as value is different.
+  if (reference_result.key == target_result.key) {
+    return false;
+  }
+
+  // target is not an appended value of the reference.
   if (!target.starts_with(reference)) {
     return false;
   }
+
+  // If the suffix is Emoji or unknown script, the result is not redundant.
+  // For example, if the reference is "東京", "東京🗼" is not redundant, but
+  // "東京タワー" is redundant.
   const absl::string_view suffix = target.substr(reference.size());
-  if (suffix.empty()) {
-    return true;
-  }
   const Util::ScriptType script_type = Util::GetScriptType(suffix);
   return (script_type != Util::EMOJI && script_type != Util::UNKNOWN_SCRIPT);
 }
@@ -949,17 +964,18 @@ void DictionaryPredictionAggregator::AggregateUnigramForMixedConversion(
 
     // Traverse all remaining elements and check if each result is redundant.
     for (Iter iter = min_iter; iter != max_iter;) {
-      // - We do not filter user dictionary word.
-      const bool should_check_redundant =
-          !(iter->candidate_attributes & converter::Candidate::USER_DICTIONARY);
-      if (should_check_redundant &&
-          MaybeRedundant(reference_result.value, iter->value)) {
-        // Swap out the redundant result.
+      // We do not filter user dictionary word.
+      if (iter->candidate_attributes & converter::Candidate::USER_DICTIONARY) {
+        ++iter;
+        continue;
+      }
+      // If the result is redundant, swap it out.
+      if (MaybeRedundant(reference_result, *iter)) {
         --max_iter;
         std::iter_swap(iter, max_iter);
-      } else {
-        ++iter;
+        continue;
       }
+      ++iter;
     }
   }
 

src/prediction/dictionary_prediction_aggregator_test.cc

@@ -1159,6 +1159,9 @@ TEST_F(DictionaryPredictionAggregatorTest, MobileUnigram) {
             {"とうきょう!", "東京!", 1100, kPosId, kPosId, Token::NONE},
             {"とうきょう!?", "東京!?", 1200, kPosId, kPosId, Token::NONE},
             {"とうきょう", "東京❤", 1300, kPosId, kPosId, Token::NONE},
+            // "とうきょう → 東京宇" is not an actual word, but an emulation of
+            // "さかい → (堺, 堺井)" and "いずみ → (泉, 泉水)".
+            {"とうきょう", "東京宇", 1400, kPosId, kPosId, Token::NONE},
         }});
   }
 
@@ -1167,6 +1170,7 @@ TEST_F(DictionaryPredictionAggregatorTest, MobileUnigram) {
   aggregator.AggregateUnigramForMixedConversion(convreq, &results);
 
   EXPECT_TRUE(FindResultByValue(results, "東京"));
+  EXPECT_TRUE(FindResultByValue(results, "東京宇"));
 
   int prefix_count = 0;
   for (const auto &result : results) {