experimental kuromoji support

mstdn-plusminus-io · Dec 26, 2020 · e77a3a3 · e77a3a3
1 parent 41e996d
commit e77a3a3
Showing 1 changed file with 17 additions and 1 deletion.
diff --git a/app/chewy/statuses_index.rb b/app/chewy/statuses_index.rb
@@ -16,13 +16,29 @@ class StatusesIndex < Chewy::Index
         language: 'possessive_english',
       },
     },
+    tokenizer: {
+      kuromoji: {
+        type: 'kuromoji_tokenizer',
+        mode: 'search',
+      },
+    },
     analyzer: {
       content: {
-        tokenizer: 'uax_url_email',
+        tokenizer: 'kuromoji',
+        type: 'custom',
+        char_filter: %w(
+          icu_normalizer
+          html_strip
+          kuromoji_iteration_mark
+        ),
         filter: %w(
           english_possessive_stemmer
           lowercase
           asciifolding
+          kuromoji_stemmer
+          kuromoji_number
+          kuromoji_baseform
+          icu_normalizer
           cjk_width
           english_stop
           english_stemmer