LUCENE-8933: Validate JapaneseTokenizer user dictionary entry

lucene/MIGRATE.txt

@@ -1,5 +1,17 @@
 # Apache Lucene Migration Guide
 
+## Kuromoji user dictionary now forbids illegal segmentation (LUCENE-8933) ##
+
+User dictionary now strictly validates if the (concatenated) segment is the same as the surface form. This change avoids
+unexpected runtime exceptions or behaviours.
+For example, these entries are not allowed at all and an exception is thrown when loading the dictionary file.
+
+# concatenated "日本経済新聞" does not match the surface form "日経新聞"
+日経新聞,日本 経済 新聞,ニホン ケイザイ シンブン,カスタム名詞
+
+# concatenated "日経新聞" does not match the surface form "日本経済新聞"
+日本経済新聞,日経 新聞,ニッケイ シンブン,カスタム名詞
+
 ## TermsEnum is now fully abstract (LUCENE-8292) ##
 
 TermsEnum has been changed to be fully abstract, so non-abstract subclass must implement all it's methods.

lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UserDictionary.java

@@ -104,6 +104,8 @@ public int compare(String[] left, String[] right) {
     long ord = 0;
 
     for (String[] values : featureEntries) {
+      String surface = values[0].replaceAll("\\s", "");
+      String concatenatedSegment = values[1].replaceAll("\\s", "");
       String[] segmentation = values[1].replaceAll("  *", " ").split(" ");
       String[] readings = values[2].replaceAll("  *", " ").split(" ");
       String pos = values[3];
@@ -113,6 +115,12 @@ public int compare(String[] left, String[] right) {
                                    " - the number of segmentations (" + segmentation.length + ")" +
                                    " does not the match number of readings (" + readings.length + ")");
       }
+
+      if (!surface.equals(concatenatedSegment)) {
+        throw new RuntimeException("Illegal user dictionary entry " + values[0] +
+                                   " - the concatenated segmentation (" + concatenatedSegment + ")" +
+                                   " does not match the surface form (" + surface + ")");
+      }
 
       int[] wordIdAndLength = new int[segmentation.length + 1]; // wordId offset, length, length....
       wordIdAndLength[0] = wordId;

lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/dict/UserDictionaryTest.java

@@ -18,6 +18,7 @@
 
 
 import java.io.IOException;
+import java.io.StringReader;
 
 import org.apache.lucene.analysis.ja.TestJapaneseTokenizer;
 import org.apache.lucene.util.LuceneTestCase;
@@ -77,4 +78,19 @@ public void testRead() throws IOException {
     UserDictionary dictionary = TestJapaneseTokenizer.readDict();
     assertNotNull(dictionary);
   }
+
+  @Test(expected = RuntimeException.class)
+  public void testReadInvalid1() throws IOException {
+    // the concatenated segment must be the same as the surface form
+    String invalidEntry = "日経新聞,日本 経済 新聞,ニホン ケイザイ シンブン,カスタム名詞";
+    UserDictionary dictionary = UserDictionary.open(new StringReader(invalidEntry));
+  }
+
+  @Test(expected = RuntimeException.class)
+  public void testReadInvalid2() throws IOException {
+    // the concatenated segment must be the same as the surface form
+    String invalidEntry = "日本経済新聞,日経 新聞,ニッケイ シンブン,カスタム名詞";
+    UserDictionary dictionary = UserDictionary.open(new StringReader(invalidEntry));
+  }
+
 }