diff --git a/Cargo.lock b/Cargo.lock index 6d393f3ace0..36b2d605845 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3077,6 +3077,12 @@ dependencies = [ "futures-core", ] +[[package]] +name = "frostem" +version = "1.20260804.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d0ae10cfccbae085dd8612669ccc116fe88bd5dfe39d202a8fa68c24c1e546f" + [[package]] name = "fs_extra" version = "1.3.0" @@ -5144,10 +5150,10 @@ dependencies = [ name = "lance-tokenizer" version = "10.1.0-beta.2" dependencies = [ + "frostem", "icu_segmenter", "jieba-rs", "lindera", - "rust-stemmers", "serde", "stop-words", "unicode-normalization", @@ -7833,16 +7839,6 @@ dependencies = [ "ordered-multimap", ] -[[package]] -name = "rust-stemmers" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54" -dependencies = [ - "serde", - "serde_derive", -] - [[package]] name = "rustc-demangle" version = "0.1.27" diff --git a/java/lance-jni/Cargo.lock b/java/lance-jni/Cargo.lock index 1a6f06d20d2..5a605ad5711 100644 --- a/java/lance-jni/Cargo.lock +++ b/java/lance-jni/Cargo.lock @@ -2475,6 +2475,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "frostem" +version = "1.20260804.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d0ae10cfccbae085dd8612669ccc116fe88bd5dfe39d202a8fa68c24c1e546f" + [[package]] name = "fs_extra" version = "1.3.0" @@ -4277,8 +4283,8 @@ dependencies = [ name = "lance-tokenizer" version = "10.1.0-beta.2" dependencies = [ + "frostem", "icu_segmenter", - "rust-stemmers", "serde", "stop-words", "unicode-normalization", @@ -6235,16 +6241,6 @@ dependencies = [ "ordered-multimap", ] -[[package]] -name = "rust-stemmers" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54" -dependencies = [ - "serde", - "serde_derive", -] - [[package]] name = "rustc-hash" version = "2.1.3" diff --git a/python/Cargo.lock b/python/Cargo.lock index 0e40f02f788..43995772025 100644 --- a/python/Cargo.lock +++ b/python/Cargo.lock @@ -2786,6 +2786,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "frostem" +version = "1.20260804.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d0ae10cfccbae085dd8612669ccc116fe88bd5dfe39d202a8fa68c24c1e546f" + [[package]] name = "fs_extra" version = "1.3.0" @@ -4569,10 +4575,10 @@ dependencies = [ name = "lance-tokenizer" version = "10.1.0-beta.2" dependencies = [ + "frostem", "icu_segmenter", "jieba-rs", "lindera", - "rust-stemmers", "serde", "stop-words", "unicode-normalization", @@ -6933,16 +6939,6 @@ dependencies = [ "ordered-multimap", ] -[[package]] -name = "rust-stemmers" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54" -dependencies = [ - "serde", - "serde_derive", -] - [[package]] name = "rustc-hash" version = "2.1.3" diff --git a/rust/lance-tokenizer/Cargo.toml b/rust/lance-tokenizer/Cargo.toml index e1006cd93c7..ebb7b0060f8 100644 --- a/rust/lance-tokenizer/Cargo.toml +++ b/rust/lance-tokenizer/Cargo.toml @@ -15,7 +15,26 @@ rust-version.workspace = true icu_segmenter = { workspace = true } jieba-rs = { workspace = true, optional = true } lindera = { workspace = true, optional = true } -rust-stemmers = "1.2.0" +frostem = { version = "1.20260804.0", default-features = false, features = [ + "arabic", + "danish", + "dutch", + "english", + "finnish", + "french", + "german", + "greek", + "hungarian", + "italian", + "norwegian", + "portuguese", + "romanian", + "russian", + "spanish", + "swedish", + "tamil", + "turkish", +] } serde = { workspace = true, features = ["derive"] } stop-words = { version = "0.10.0", default-features = false, features = ["iso", "nltk"] } unicode-normalization = "0.1.25" diff --git a/rust/lance-tokenizer/src/stemmer.rs b/rust/lance-tokenizer/src/stemmer.rs index 03fcf118019..a79dd20d5f5 100644 --- a/rust/lance-tokenizer/src/stemmer.rs +++ b/rust/lance-tokenizer/src/stemmer.rs @@ -7,7 +7,7 @@ use std::borrow::Cow; use std::mem; -use rust_stemmers::Algorithm; +use frostem::Algorithm; use serde::{Deserialize, Serialize}; use crate::{Token, TokenFilter, TokenStream, Tokenizer}; @@ -101,7 +101,7 @@ impl Tokenizer for StemmerFilter { fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> { StemmerTokenStream { tail: self.inner.token_stream(text), - stemmer: rust_stemmers::Stemmer::create(self.stemmer_algorithm), + stemmer: frostem::Stemmer::new(self.stemmer_algorithm), buffer: String::new(), } } @@ -109,7 +109,7 @@ impl Tokenizer for StemmerFilter { pub struct StemmerTokenStream { tail: T, - stemmer: rust_stemmers::Stemmer, + stemmer: frostem::Stemmer, buffer: String, } @@ -139,3 +139,19 @@ impl TokenStream for StemmerTokenStream { self.tail.token_mut() } } + +#[cfg(test)] +mod tests { + use crate::{Language, RawTokenizer, Stemmer, TextAnalyzer, TokenStream}; + + #[test] + fn test_greek_stemmer_handles_multibyte_suffixes() { + let mut analyzer = TextAnalyzer::builder(RawTokenizer::default()) + .filter(Stemmer::new(Language::Greek)) + .build(); + let mut stream = analyzer.token_stream("αντιθετε"); + + assert!(stream.advance()); + assert_eq!(stream.token().text, "ανετ"); + } +}