add split filter test with input normalization

Author: mh-northlander

src/test/java/com/worksap/nlp/lucene/sudachi/ja/TestSudachiSplitFilter.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022-2023 Works Applications Co., Ltd.
+ * Copyright (c) 2022-2024 Works Applications Co., Ltd.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -196,6 +196,69 @@ class TestSudachiSplitFilter : BaseTokenStreamTestCase() {
     )
   }
 
+  @Test
+  fun testWithCharNormalizationBySearchMode() {
+    val tokenStream = setUpTokenStream("search", "六三四㍿に行くカ゛カ゛カ゛")
+    assertTokenStreamContents(
+        tokenStream,
+        arrayOf("六三四", "㍿", "㍿", "", "に", "行く", "カ゛カ゛カ゛", "カ゛カ゛", "カ゛"),
+        intArrayOf(0, 3, 3, 4, 4, 5, 7, 7, 11),
+        intArrayOf(3, 4, 4, 4, 5, 7, 13, 11, 13),
+        intArrayOf(1, 1, 0, 1, 1, 1, 1, 0, 1),
+        intArrayOf(1, 2, 1, 1, 1, 1, 2, 1, 1),
+        13,
+    )
+  }
+
+  @Test
+  fun testWithCharNormalizationInNormalizedFormBySearchMode() {
+    var tokenStream = setUpTokenStream("search", "六三四㍿に行くカ゛カ゛カ゛")
+    val normFactory = SudachiNormalizedFormFilterFactory(mutableMapOf())
+    tokenStream = normFactory.create(tokenStream)
+
+    assertTokenStreamContents(
+        tokenStream,
+        arrayOf("六三四", "株式会社", "株式", "会社", "に", "行く", "ガガガ", "ガガ", "ガ"),
+        intArrayOf(0, 3, 3, 4, 4, 5, 7, 7, 11),
+        intArrayOf(3, 4, 4, 4, 5, 7, 13, 11, 13),
+        intArrayOf(1, 1, 0, 1, 1, 1, 1, 0, 1),
+        intArrayOf(1, 2, 1, 1, 1, 1, 2, 1, 1),
+        13,
+    )
+  }
+
+  @Test
+  fun testWithCharNormalizationByExtendedMode() {
+    // extending normalized form seems more natural, but we cannot calculate their offsets.
+    val tokenStream = setUpTokenStream("extended", "10㌢㍍いったソ゛")
+    assertTokenStreamContents(
+        tokenStream,
+        arrayOf("1", "0", "㌢㍍", "㌢", "㍍", "いっ", "た", "ソ゛", "ソ", "゛"),
+        intArrayOf(0, 1, 2, 2, 3, 4, 6, 7, 7, 8),
+        intArrayOf(1, 2, 4, 3, 4, 6, 7, 9, 8, 9),
+        intArrayOf(1, 1, 1, 0, 1, 1, 1, 1, 0, 1),
+        intArrayOf(1, 1, 2, 1, 1, 1, 1, 2, 1, 1),
+        9,
+    )
+  }
+
+  @Test
+  fun testWithCharNormalizationInNormalizedFormByExtendedMode() {
+    // extending normalized form seems more natural, but we cannot calculate their offsets.
+    var tokenStream = setUpTokenStream("extended", "10㌢㍍いったソ゛")
+    val normFactory = SudachiNormalizedFormFilterFactory(mutableMapOf())
+    tokenStream = normFactory.create(tokenStream)
+
+    assertTokenStreamContents(
+        tokenStream,
+        arrayOf("1", "0", "センチメートル", "㌢", "㍍", "行く", "た", "ゾ", "ソ", "゛"),
+        intArrayOf(0, 1, 2, 2, 3, 4, 6, 7, 7, 8),
+        intArrayOf(1, 2, 4, 3, 4, 6, 7, 9, 8, 9),
+        intArrayOf(1, 1, 1, 0, 1, 1, 1, 1, 0, 1),
+        intArrayOf(1, 1, 2, 1, 1, 1, 1, 2, 1, 1),
+        9)
+  }
+
   fun setUpTokenStream(mode: String, input: String): TokenStream {
     val factory =
         SudachiSplitFilterFactory(

src/test/java/com/worksap/nlp/lucene/sudachi/ja/attributes/MorphemeAttributeImplTest.kt

@@ -16,7 +16,6 @@
 
 package com.worksap.nlp.lucene.sudachi.ja.attributes
 
-import com.worksap.nlp.lucene.aliases.ToXContent
 import com.worksap.nlp.lucene.aliases.XContentBuilder
 import com.worksap.nlp.search.aliases.XContentType
 import com.worksap.nlp.sudachi.Config
@@ -65,6 +64,19 @@ class MorphemeAttributeImplTest {
     assertNull(morphemeAtt.getMorpheme())
   }
 
+  @Test
+  fun setOffsets() {
+    var morphemeAtt = MorphemeAttributeImpl()
+    assertTrue(morphemeAtt.getOffsets().isEmpty())
+
+    val intlist = listOf(1, 2, 3)
+    morphemeAtt.setOffsets(intlist)
+    assertEquals(intlist, morphemeAtt.getOffsets())
+
+    morphemeAtt.setOffsets(listOf())
+    assertTrue(morphemeAtt.getOffsets().isEmpty())
+  }
+
   @Test
   fun copyTo() {
     var morphemeAtt1 = MorphemeAttributeImpl()
@@ -85,15 +97,14 @@ class MorphemeAttributeImplTest {
     var morphemeAtt = MorphemeAttributeImpl()
     val morpheme = getFirstMorpheme("東京都")!!
     morphemeAtt.setMorpheme(morpheme)
+    val offsets = listOf(0, 3)
+    morphemeAtt.setOffsets(offsets)
 
     val builder = XContentBuilder.builder(XContentType.JSON.xContent())
     builder.startObject()
     morphemeAtt.reflectWith(
         fun(attClass, key, value) {
           assertEquals(MorphemeAttribute::class.java, attClass)
-          assertEquals("morpheme", key)
-          assertTrue(value is ToXContent)
-
           builder.field(key, value)
         })
     builder.endObject()
@@ -103,15 +114,21 @@ class MorphemeAttributeImplTest {
     val deserialized = Json.decodeFromString<MorphemeHolder>(serialized)
 
     assertNotNull(deserialized.morpheme)
+    assertNotNull(deserialized.offsetMap)
     assertEquals(morpheme.surface(), deserialized.morpheme.surface)
     assertEquals(morpheme.dictionaryForm(), deserialized.morpheme.dictionaryForm)
     assertEquals(morpheme.normalizedForm(), deserialized.morpheme.normalizedForm)
     assertEquals(morpheme.readingForm(), deserialized.morpheme.readingForm)
     assertEquals(morpheme.partOfSpeech(), deserialized.morpheme.partOfSpeech)
+    assertEquals(offsets, deserialized.offsetMap)
   }
 }
 
-@Serializable data class MorphemeHolder(val morpheme: MorphemeAttributeHolder)
+@Serializable
+data class MorphemeHolder(
+    val morpheme: MorphemeAttributeHolder,
+    val offsetMap: List<Int>,
+)
 
 @Serializable
 data class MorphemeAttributeHolder(

src/test/resources/com/worksap/nlp/lucene/sudachi/ja/sudachi.json

@@ -1,5 +1,8 @@
 {
     "systemDict" : "system_core.dic",
+    "inputTextPlugin" : [
+        { "class" : "com.worksap.nlp.sudachi.DefaultInputTextPlugin" }
+    ],
     "oovProviderPlugin" : [
         { "class" : "com.worksap.nlp.sudachi.SimpleOovProviderPlugin",
           "oovPOS" : [ "名詞", "普通名詞", "一般", "*", "*", "*" ],

src/test/resources/dict/lex.csv

@@ -37,4 +37,10 @@
 012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789,9,9,-30000,012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789,名詞,数詞,*,*,*,*,ゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウ,012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789,*,A,*,*,*,*
 特a,8,8,2914,特a,名詞,普通名詞,一般,*,*,*,トクエー,特a,*,A,*,*,*,*
 な。な,8,8,2914,な。な,名詞,普通名詞,一般,*,*,*,ナナ,な。な,*,A,*,*,*,*
-ふく,4,4,5105,ふく,動詞,非自立可能,*,*,五段-カ行,終止形-一般,フク,吹く,*,A,*,*,*,*
+ふく,4,4,5105,ふく,動詞,非自立可能,*,*,五段-カ行,終止形-一般,フク,吹く,*,A,*,*,*,*
+株式,8,8,5611,株式,名詞,普通名詞,一般,*,*,*,カブシキ,株式,*,A,*,*,*,*
+会社,8,8,2914,会社,名詞,普通名詞,一般,*,*,*,カイシャ,会社,*,A,*,*,*,*
+株式会社,8,8,6000,株式会社,名詞,普通名詞,一般,*,*,*,カブシキガイシャ,株式会社,*,C,40/41,40/41,40/41,*
+ガ,5,5,3500,ガ,副詞,*,*,*,*,*,ガ,ガ,*,A,*,*,*,*
+ガガ,5,5,5500,ガガ,副詞,*,*,*,*,*,ガガ,ガガ,*,A,*,*,*,*
+ガガガ,5,5,8494,ガガガ,副詞,*,*,*,*,*,ガガガ,ガガガ,*,B,44/43,*,44/43,*