fix: avoid Greek stemmer UTF-8 panic

rust/lance-tokenizer/Cargo.toml

@@ -15,7 +15,26 @@ rust-version.workspace = true
 icu_segmenter = { workspace = true }
 jieba-rs = { workspace = true, optional = true }
 lindera = { workspace = true, optional = true }
-rust-stemmers = "1.2.0"
+frostem = { version = "1.20260804.0", default-features = false, features = [
+    "arabic",
+    "danish",
+    "dutch",
+    "english",
+    "finnish",
+    "french",
+    "german",
+    "greek",
+    "hungarian",
+    "italian",
+    "norwegian",
+    "portuguese",
+    "romanian",
+    "russian",
+    "spanish",
+    "swedish",
+    "tamil",
+    "turkish",
+] }
 serde = { workspace = true, features = ["derive"] }
 stop-words = { version = "0.10.0", default-features = false, features = ["iso", "nltk"] }
 unicode-normalization = "0.1.25"

rust/lance-tokenizer/src/stemmer.rs

@@ -7,7 +7,7 @@
 use std::borrow::Cow;
 use std::mem;
 
-use rust_stemmers::Algorithm;
+use frostem::Algorithm;
 use serde::{Deserialize, Serialize};
 
 use crate::{Token, TokenFilter, TokenStream, Tokenizer};
@@ -101,15 +101,15 @@ impl<T: Tokenizer> Tokenizer for StemmerFilter<T> {
     fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {
         StemmerTokenStream {
             tail: self.inner.token_stream(text),
-            stemmer: rust_stemmers::Stemmer::create(self.stemmer_algorithm),
+            stemmer: frostem::Stemmer::new(self.stemmer_algorithm),
             buffer: String::new(),
         }
     }
 }
 
 pub struct StemmerTokenStream<T> {
     tail: T,
-    stemmer: rust_stemmers::Stemmer,
+    stemmer: frostem::Stemmer,
     buffer: String,
 }
 
@@ -139,3 +139,19 @@ impl<T: TokenStream> TokenStream for StemmerTokenStream<T> {
         self.tail.token_mut()
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use crate::{Language, RawTokenizer, Stemmer, TextAnalyzer, TokenStream};
+
+    #[test]
+    fn test_greek_stemmer_handles_multibyte_suffixes() {
+        let mut analyzer = TextAnalyzer::builder(RawTokenizer::default())
+            .filter(Stemmer::new(Language::Greek))
+            .build();
+        let mut stream = analyzer.token_stream("αντιθετε");
+
+        assert!(stream.advance());
+        assert_eq!(stream.token().text, "ανετ");
+    }
+}