From c28de69519a80de7308328eb2b5195b625f3823e Mon Sep 17 00:00:00 2001 From: Lance Gatekeeper Date: Mon, 3 Aug 2026 05:37:29 +0000 Subject: [PATCH 1/8] fix: avoid Greek stemmer UTF-8 panic --- Cargo.lock | 7 +++ java/lance-jni/Cargo.lock | 7 +++ python/Cargo.lock | 7 +++ rust/lance-tokenizer/Cargo.toml | 1 + rust/lance-tokenizer/src/stemmer.rs | 98 ++++++++++++++++++++++------- 5 files changed, 96 insertions(+), 24 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6d393f3ace0..f349d64869e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5147,6 +5147,7 @@ dependencies = [ "icu_segmenter", "jieba-rs", "lindera", + "pagefind_stem", "rust-stemmers", "serde", "stop-words", @@ -6439,6 +6440,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "pagefind_stem" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8dfa810b158f3ac364e5acd43ca4a6020a6e729d40c15ce1bed1d911237a52e5" + [[package]] name = "parking" version = "2.2.1" diff --git a/java/lance-jni/Cargo.lock b/java/lance-jni/Cargo.lock index 1a6f06d20d2..08e647fc1d3 100644 --- a/java/lance-jni/Cargo.lock +++ b/java/lance-jni/Cargo.lock @@ -4278,6 +4278,7 @@ name = "lance-tokenizer" version = "10.1.0-beta.2" dependencies = [ "icu_segmenter", + "pagefind_stem", "rust-stemmers", "serde", "stop-words", @@ -5217,6 +5218,12 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" +[[package]] +name = "pagefind_stem" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8dfa810b158f3ac364e5acd43ca4a6020a6e729d40c15ce1bed1d911237a52e5" + [[package]] name = "parking" version = "2.2.1" diff --git a/python/Cargo.lock b/python/Cargo.lock index 0e40f02f788..1f653c20584 100644 --- a/python/Cargo.lock +++ b/python/Cargo.lock @@ -4572,6 +4572,7 @@ dependencies = [ "icu_segmenter", "jieba-rs", "lindera", + "pagefind_stem", "rust-stemmers", "serde", "stop-words", @@ -5627,6 +5628,12 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" +[[package]] +name = "pagefind_stem" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8dfa810b158f3ac364e5acd43ca4a6020a6e729d40c15ce1bed1d911237a52e5" + [[package]] name = "parking" version = "2.2.1" diff --git a/rust/lance-tokenizer/Cargo.toml b/rust/lance-tokenizer/Cargo.toml index e1006cd93c7..482d2dfab0a 100644 --- a/rust/lance-tokenizer/Cargo.toml +++ b/rust/lance-tokenizer/Cargo.toml @@ -15,6 +15,7 @@ rust-version.workspace = true icu_segmenter = { workspace = true } jieba-rs = { workspace = true, optional = true } lindera = { workspace = true, optional = true } +pagefind_stem = { version = "1.0.0", default-features = false, features = ["greek"] } rust-stemmers = "1.2.0" serde = { workspace = true, features = ["derive"] } stop-words = { version = "0.10.0", default-features = false, features = ["iso", "nltk"] } diff --git a/rust/lance-tokenizer/src/stemmer.rs b/rust/lance-tokenizer/src/stemmer.rs index 03fcf118019..97b3a8bccd8 100644 --- a/rust/lance-tokenizer/src/stemmer.rs +++ b/rust/lance-tokenizer/src/stemmer.rs @@ -7,7 +7,6 @@ use std::borrow::Cow; use std::mem; -use rust_stemmers::Algorithm; use serde::{Deserialize, Serialize}; use crate::{Token, TokenFilter, TokenStream, Tokenizer}; @@ -35,33 +34,68 @@ pub enum Language { } impl Language { - fn algorithm(self) -> Algorithm { + fn algorithm(self) -> StemmerAlgorithm { match self { - Self::Arabic => Algorithm::Arabic, - Self::Danish => Algorithm::Danish, - Self::Dutch => Algorithm::Dutch, - Self::English => Algorithm::English, - Self::Finnish => Algorithm::Finnish, - Self::French => Algorithm::French, - Self::German => Algorithm::German, - Self::Greek => Algorithm::Greek, - Self::Hungarian => Algorithm::Hungarian, - Self::Italian => Algorithm::Italian, - Self::Norwegian => Algorithm::Norwegian, - Self::Portuguese => Algorithm::Portuguese, - Self::Romanian => Algorithm::Romanian, - Self::Russian => Algorithm::Russian, - Self::Spanish => Algorithm::Spanish, - Self::Swedish => Algorithm::Swedish, - Self::Tamil => Algorithm::Tamil, - Self::Turkish => Algorithm::Turkish, + Self::Arabic => StemmerAlgorithm::Legacy(rust_stemmers::Algorithm::Arabic), + Self::Danish => StemmerAlgorithm::Legacy(rust_stemmers::Algorithm::Danish), + Self::Dutch => StemmerAlgorithm::Legacy(rust_stemmers::Algorithm::Dutch), + Self::English => StemmerAlgorithm::Legacy(rust_stemmers::Algorithm::English), + Self::Finnish => StemmerAlgorithm::Legacy(rust_stemmers::Algorithm::Finnish), + Self::French => StemmerAlgorithm::Legacy(rust_stemmers::Algorithm::French), + Self::German => StemmerAlgorithm::Legacy(rust_stemmers::Algorithm::German), + Self::Greek => StemmerAlgorithm::Greek, + Self::Hungarian => StemmerAlgorithm::Legacy(rust_stemmers::Algorithm::Hungarian), + Self::Italian => StemmerAlgorithm::Legacy(rust_stemmers::Algorithm::Italian), + Self::Norwegian => StemmerAlgorithm::Legacy(rust_stemmers::Algorithm::Norwegian), + Self::Portuguese => StemmerAlgorithm::Legacy(rust_stemmers::Algorithm::Portuguese), + Self::Romanian => StemmerAlgorithm::Legacy(rust_stemmers::Algorithm::Romanian), + Self::Russian => StemmerAlgorithm::Legacy(rust_stemmers::Algorithm::Russian), + Self::Spanish => StemmerAlgorithm::Legacy(rust_stemmers::Algorithm::Spanish), + Self::Swedish => StemmerAlgorithm::Legacy(rust_stemmers::Algorithm::Swedish), + Self::Tamil => StemmerAlgorithm::Legacy(rust_stemmers::Algorithm::Tamil), + Self::Turkish => StemmerAlgorithm::Legacy(rust_stemmers::Algorithm::Turkish), + } + } +} + +#[derive(Copy, Clone)] +enum StemmerAlgorithm { + Legacy(rust_stemmers::Algorithm), + // The legacy generated Greek algorithm can retain stale UTF-8 byte offsets + // after shortening a word and panic when it slices the resulting stem. + Greek, +} + +impl StemmerAlgorithm { + fn create(self) -> StemmerBackend { + match self { + Self::Legacy(algorithm) => { + StemmerBackend::Legacy(rust_stemmers::Stemmer::create(algorithm)) + } + Self::Greek => StemmerBackend::Greek(pagefind_stem::Stemmer::create( + pagefind_stem::Algorithm::Greek, + )), + } + } +} + +enum StemmerBackend { + Legacy(rust_stemmers::Stemmer), + Greek(pagefind_stem::Stemmer), +} + +impl StemmerBackend { + fn stem<'a>(&self, input: &'a str) -> Cow<'a, str> { + match self { + Self::Legacy(stemmer) => stemmer.stem(input), + Self::Greek(stemmer) => stemmer.stem(input), } } } #[derive(Clone)] pub struct Stemmer { - stemmer_algorithm: Algorithm, + stemmer_algorithm: StemmerAlgorithm, } impl Stemmer { @@ -91,7 +125,7 @@ impl TokenFilter for Stemmer { #[derive(Clone)] pub struct StemmerFilter { - stemmer_algorithm: Algorithm, + stemmer_algorithm: StemmerAlgorithm, inner: T, } @@ -101,7 +135,7 @@ impl Tokenizer for StemmerFilter { fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> { StemmerTokenStream { tail: self.inner.token_stream(text), - stemmer: rust_stemmers::Stemmer::create(self.stemmer_algorithm), + stemmer: self.stemmer_algorithm.create(), buffer: String::new(), } } @@ -109,7 +143,7 @@ impl Tokenizer for StemmerFilter { pub struct StemmerTokenStream { tail: T, - stemmer: rust_stemmers::Stemmer, + stemmer: StemmerBackend, buffer: String, } @@ -139,3 +173,19 @@ impl TokenStream for StemmerTokenStream { self.tail.token_mut() } } + +#[cfg(test)] +mod tests { + use crate::{Language, RawTokenizer, Stemmer, TextAnalyzer, TokenStream}; + + #[test] + fn test_greek_stemmer_handles_multibyte_suffixes() { + let mut analyzer = TextAnalyzer::builder(RawTokenizer::default()) + .filter(Stemmer::new(Language::Greek)) + .build(); + let mut stream = analyzer.token_stream("αντιθετε"); + + assert!(stream.advance()); + assert_eq!(stream.token().text, "ανετ"); + } +} From 889db02aa671d5e896c2f82d6a38904c58f83a94 Mon Sep 17 00:00:00 2001 From: Lance Gatekeeper Date: Mon, 3 Aug 2026 11:18:14 +0000 Subject: [PATCH 2/8] fix: preserve legacy Greek stemming semantics --- protos/index_old.proto | 12 +++ rust/lance-index/src/scalar/inverted/index.rs | 65 +++++++++++++ .../src/scalar/inverted/tokenizer.rs | 92 ++++++++++++++++++- rust/lance-tokenizer/src/stemmer.rs | 71 ++++++++++---- 4 files changed, 221 insertions(+), 19 deletions(-) diff --git a/protos/index_old.proto b/protos/index_old.proto index bdba4ad0252..4a155b23b9a 100644 --- a/protos/index_old.proto +++ b/protos/index_old.proto @@ -41,6 +41,14 @@ message ZoneMapIndexDetails { optional bool use_seeds = 2; } message InvertedIndexDetails { + enum GreekStemmer { + // Legacy rust-stemmers 1.2.0 semantics. An absent field also selects this + // value so indexes written before stemmer versioning remain compatible. + GREEK_STEMMER_LEGACY = 0; + // Snowball 3 Greek semantics, including corrected UTF-8 offset handling. + GREEK_STEMMER_SNOWBALL_3 = 1; + } + message CodeTokenizerConfig { // Split one lexical identifier into subwords, e.g. getUserName -> // get/user/name. @@ -84,4 +92,8 @@ message InvertedIndexDetails { // configuration used to build the index; absence means there is no // code-specific configuration to apply. CodeTokenizerConfig code_config = 13; + // Greek stemming semantics used to build this index. An absent value means + // the legacy rust-stemmers 1.2.0 implementation; new indexes record the + // corrected Snowball 3 implementation. + optional GreekStemmer greek_stemmer = 14; } diff --git a/rust/lance-index/src/scalar/inverted/index.rs b/rust/lance-index/src/scalar/inverted/index.rs index 2aa76bf1513..d0701e98306 100644 --- a/rust/lance-index/src/scalar/inverted/index.rs +++ b/rust/lance-index/src/scalar/inverted/index.rs @@ -8334,6 +8334,7 @@ mod tests { use std::sync::Arc; use std::sync::atomic::{AtomicU32, Ordering}; + use crate::scalar::inverted::tokenizer::GreekStemmerVersion; use crate::scalar::inverted::tokenizer::document_tokenizer::TextTokenizer; use lance_tokenizer::{Language, SimpleTokenizer, StopWordFilter, TextAnalyzer}; @@ -8552,6 +8553,70 @@ mod tests { InvertedIndex::load(store, None, &LanceCache::no_cache()).await } + #[tokio::test] + async fn test_legacy_greek_stemmer_load_query_and_update() -> Result<()> { + let src_dir = TempObjDir::default(); + let dest_dir = TempObjDir::default(); + let src_store = Arc::new(LanceIndexStore::new( + ObjectStore::local().into(), + src_dir.clone(), + Arc::new(LanceCache::no_cache()), + )); + let dest_store = Arc::new(LanceIndexStore::new( + ObjectStore::local().into(), + dest_dir.clone(), + Arc::new(LanceCache::no_cache()), + )); + + let current_params = InvertedIndexParams::new("raw".to_string(), Language::Greek) + .max_token_length(None) + .remove_stop_words(false) + .ascii_folding(false); + let mut legacy_json = serde_json::to_value(current_params)?; + legacy_json + .as_object_mut() + .expect("inverted index params should serialize to an object") + .remove("greek_stemmer"); + let legacy_params: InvertedIndexParams = serde_json::from_value(legacy_json)?; + assert_eq!(legacy_params.greek_stemmer, GreekStemmerVersion::Legacy); + + // rust-stemmers 1.2.0 produced an empty term for this word. The + // manually written term and missing version field model a persisted + // index created before Greek stemmer versioning. + let index = + write_single_partition_index(src_store, legacy_params, TokenSetFormat::Fst, "", 100) + .await?; + assert_eq!(index.params.greek_stemmer, GreekStemmerVersion::Legacy); + + let matches = index.do_search("ίσα").await?; + let row_ids = matches[ROW_ID].as_primitive::(); + assert_eq!(row_ids.values(), &[100]); + + let schema = Arc::new(Schema::new(vec![ + Field::new("doc", DataType::Utf8, true), + Field::new(ROW_ID, DataType::UInt64, false), + ])); + let docs = Arc::new(StringArray::from(vec![Some("ίσα")])); + let row_ids = Arc::new(UInt64Array::from(vec![101_u64])); + let batch = RecordBatch::try_new(schema.clone(), vec![docs, row_ids])?; + let stream = RecordBatchStreamAdapter::new(schema, stream::iter(vec![Ok(batch)])); + index + .update(Box::pin(stream), dest_store.as_ref(), None) + .await?; + + let updated = InvertedIndex::load(dest_store, None, &LanceCache::no_cache()).await?; + assert_eq!(updated.params.greek_stemmer, GreekStemmerVersion::Legacy); + let matches = updated.do_search("ίσα").await?; + let mut row_ids = matches[ROW_ID] + .as_primitive::() + .values() + .to_vec(); + row_ids.sort_unstable(); + assert_eq!(row_ids, vec![100, 101]); + + Ok(()) + } + fn empty_doc_stream() -> SendableRecordBatchStream { let schema = Arc::new(Schema::new(vec![ Field::new("doc", DataType::Utf8, true), diff --git a/rust/lance-index/src/scalar/inverted/tokenizer.rs b/rust/lance-index/src/scalar/inverted/tokenizer.rs index 5978621c462..972254781a6 100644 --- a/rust/lance-index/src/scalar/inverted/tokenizer.rs +++ b/rust/lance-index/src/scalar/inverted/tokenizer.rs @@ -44,6 +44,43 @@ pub const DEFAULT_BLOCK_SIZE: usize = 128; pub const VALID_BLOCK_SIZES: [usize; 2] = [128, 256]; const LANCE_FTS_FORMAT_VERSION_ENV_KEY: &str = "LANCE_FTS_FORMAT_VERSION"; +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub(crate) enum GreekStemmerVersion { + Legacy, + Snowball3, +} + +impl GreekStemmerVersion { + fn is_legacy(&self) -> bool { + matches!(self, Self::Legacy) + } + + fn from_proto(value: Option) -> Result { + use pbold::inverted_index_details::GreekStemmer; + + match value { + None => Ok(Self::Legacy), + Some(value) => match GreekStemmer::try_from(value) { + Ok(GreekStemmer::Legacy) => Ok(Self::Legacy), + Ok(GreekStemmer::Snowball3) => Ok(Self::Snowball3), + Err(_) => Err(Error::invalid_input(format!( + "unknown Greek stemmer version {value}" + ))), + }, + } + } + + fn to_proto(self) -> Option { + use pbold::inverted_index_details::GreekStemmer; + + match self { + Self::Legacy => None, + Self::Snowball3 => Some(GreekStemmer::Snowball3 as i32), + } + } +} + /// Tokenizer configs #[derive(Debug, Clone, Serialize, PartialEq)] pub struct InvertedIndexParams { @@ -77,6 +114,13 @@ pub struct InvertedIndexParams { /// this is only used when `stem` or `remove_stop_words` is true pub(crate) language: Language, + /// Greek stemming semantics persisted with the index vocabulary. + /// + /// Missing serialized values select [`GreekStemmerVersion::Legacy`]. New + /// indexes use [`GreekStemmerVersion::Snowball3`]. + #[serde(skip_serializing_if = "GreekStemmerVersion::is_legacy")] + pub(crate) greek_stemmer: GreekStemmerVersion, + /// If true, store the position of the term in the document /// This can significantly increase the size of the index /// If false, only store the frequency of the term in the document @@ -180,6 +224,7 @@ struct RawInvertedIndexParams { lance_tokenizer: Option, base_tokenizer: Option, language: Option, + greek_stemmer: Option, with_position: Option, #[serde(default, deserialize_with = "deserialize_explicit_option")] max_token_length: Option>, @@ -280,6 +325,7 @@ impl RawInvertedIndexParams { if let Some(language) = self.language { params.language = language; } + params.greek_stemmer = self.greek_stemmer.unwrap_or(GreekStemmerVersion::Legacy); if let Some(with_position) = self.with_position { params.with_position = with_position; } @@ -358,6 +404,7 @@ impl TryFrom<&InvertedIndexParams> for pbold::InvertedIndexDetails { index_operators: params.index_operators, }, ), + greek_stemmer: params.greek_stemmer.to_proto(), }) } } @@ -377,6 +424,7 @@ impl TryFrom<&pbold::InvertedIndexDetails> for InvertedIndexParams { Some(block_size) => validate_block_size(block_size as usize)?, None => LEGACY_BLOCK_SIZE, }, + greek_stemmer: GreekStemmerVersion::Legacy, ..Self::default() }; return Ok(params); @@ -396,6 +444,7 @@ impl TryFrom<&pbold::InvertedIndexDetails> for InvertedIndexParams { } else { serde_json::from_str(details.language.as_str())? }; + params.greek_stemmer = GreekStemmerVersion::from_proto(details.greek_stemmer)?; params.with_position = details.with_position; params.max_token_length = details.max_token_length.map(|l| l as usize); params.lower_case = details.lower_case; @@ -557,6 +606,7 @@ impl InvertedIndexParams { lance_tokenizer: None, base_tokenizer, language, + greek_stemmer: GreekStemmerVersion::Snowball3, with_position: false, max_token_length: Some(40), lower_case: true, @@ -917,7 +967,11 @@ impl InvertedIndexParams { builder = builder.filter_dynamic(LowerCaser); } if self.stem { - builder = builder.filter_dynamic(Stemmer::new(self.language)); + let stemmer = match self.greek_stemmer { + GreekStemmerVersion::Legacy => Stemmer::new_legacy(self.language), + GreekStemmerVersion::Snowball3 => Stemmer::new(self.language), + }; + builder = builder.filter_dynamic(stemmer); } if self.remove_stop_words { builder = builder.filter_dynamic(self.stop_word_filter()?); @@ -1041,7 +1095,7 @@ pub fn language_model_home() -> Option { mod tests { use crate::pbold; - use super::{InvertedIndexParams, InvertedListFormatVersion, Language}; + use super::{GreekStemmerVersion, InvertedIndexParams, InvertedListFormatVersion, Language}; use lance_core::Error; use lance_tokenizer::TokenStream; use rstest::rstest; @@ -1424,6 +1478,39 @@ mod tests { assert_eq!(params.block_size, 128); } + #[test] + fn test_greek_stemmer_version_metadata() { + let current = InvertedIndexParams::new("raw".to_string(), Language::Greek); + let mut json = serde_json::to_value(¤t).unwrap(); + assert_eq!(json["greek_stemmer"], "snowball3"); + + json.as_object_mut().unwrap().remove("greek_stemmer"); + let legacy: InvertedIndexParams = serde_json::from_value(json).unwrap(); + assert_eq!(legacy.greek_stemmer, GreekStemmerVersion::Legacy); + assert!( + serde_json::to_value(&legacy).unwrap()["greek_stemmer"].is_null(), + "legacy metadata should remain absent when rewritten" + ); + + let current_details = pbold::InvertedIndexDetails::try_from(¤t).unwrap(); + assert!(current_details.greek_stemmer.is_some()); + assert_eq!( + InvertedIndexParams::try_from(¤t_details) + .unwrap() + .greek_stemmer, + GreekStemmerVersion::Snowball3 + ); + + let mut legacy_details = current_details; + legacy_details.greek_stemmer = None; + assert_eq!( + InvertedIndexParams::try_from(&legacy_details) + .unwrap() + .greek_stemmer, + GreekStemmerVersion::Legacy + ); + } + #[test] fn test_block_size_details_conversion() { let params = InvertedIndexParams::default().block_size(256).unwrap(); @@ -1444,6 +1531,7 @@ mod tests { prefix_only: false, block_size: None, code_config: None, + greek_stemmer: None, }; let params = InvertedIndexParams::try_from(&old_details).unwrap(); assert_eq!(params.block_size, 128); diff --git a/rust/lance-tokenizer/src/stemmer.rs b/rust/lance-tokenizer/src/stemmer.rs index 97b3a8bccd8..88a48fa7ec1 100644 --- a/rust/lance-tokenizer/src/stemmer.rs +++ b/rust/lance-tokenizer/src/stemmer.rs @@ -36,24 +36,31 @@ pub enum Language { impl Language { fn algorithm(self) -> StemmerAlgorithm { match self { - Self::Arabic => StemmerAlgorithm::Legacy(rust_stemmers::Algorithm::Arabic), - Self::Danish => StemmerAlgorithm::Legacy(rust_stemmers::Algorithm::Danish), - Self::Dutch => StemmerAlgorithm::Legacy(rust_stemmers::Algorithm::Dutch), - Self::English => StemmerAlgorithm::Legacy(rust_stemmers::Algorithm::English), - Self::Finnish => StemmerAlgorithm::Legacy(rust_stemmers::Algorithm::Finnish), - Self::French => StemmerAlgorithm::Legacy(rust_stemmers::Algorithm::French), - Self::German => StemmerAlgorithm::Legacy(rust_stemmers::Algorithm::German), Self::Greek => StemmerAlgorithm::Greek, - Self::Hungarian => StemmerAlgorithm::Legacy(rust_stemmers::Algorithm::Hungarian), - Self::Italian => StemmerAlgorithm::Legacy(rust_stemmers::Algorithm::Italian), - Self::Norwegian => StemmerAlgorithm::Legacy(rust_stemmers::Algorithm::Norwegian), - Self::Portuguese => StemmerAlgorithm::Legacy(rust_stemmers::Algorithm::Portuguese), - Self::Romanian => StemmerAlgorithm::Legacy(rust_stemmers::Algorithm::Romanian), - Self::Russian => StemmerAlgorithm::Legacy(rust_stemmers::Algorithm::Russian), - Self::Spanish => StemmerAlgorithm::Legacy(rust_stemmers::Algorithm::Spanish), - Self::Swedish => StemmerAlgorithm::Legacy(rust_stemmers::Algorithm::Swedish), - Self::Tamil => StemmerAlgorithm::Legacy(rust_stemmers::Algorithm::Tamil), - Self::Turkish => StemmerAlgorithm::Legacy(rust_stemmers::Algorithm::Turkish), + language => StemmerAlgorithm::Legacy(language.legacy_algorithm()), + } + } + + fn legacy_algorithm(self) -> rust_stemmers::Algorithm { + match self { + Self::Arabic => rust_stemmers::Algorithm::Arabic, + Self::Danish => rust_stemmers::Algorithm::Danish, + Self::Dutch => rust_stemmers::Algorithm::Dutch, + Self::English => rust_stemmers::Algorithm::English, + Self::Finnish => rust_stemmers::Algorithm::Finnish, + Self::French => rust_stemmers::Algorithm::French, + Self::German => rust_stemmers::Algorithm::German, + Self::Greek => rust_stemmers::Algorithm::Greek, + Self::Hungarian => rust_stemmers::Algorithm::Hungarian, + Self::Italian => rust_stemmers::Algorithm::Italian, + Self::Norwegian => rust_stemmers::Algorithm::Norwegian, + Self::Portuguese => rust_stemmers::Algorithm::Portuguese, + Self::Romanian => rust_stemmers::Algorithm::Romanian, + Self::Russian => rust_stemmers::Algorithm::Russian, + Self::Spanish => rust_stemmers::Algorithm::Spanish, + Self::Swedish => rust_stemmers::Algorithm::Swedish, + Self::Tamil => rust_stemmers::Algorithm::Tamil, + Self::Turkish => rust_stemmers::Algorithm::Turkish, } } } @@ -104,6 +111,18 @@ impl Stemmer { stemmer_algorithm: language.algorithm(), } } + + /// Create a stemmer with the semantics used by indexes written before the + /// corrected Greek stemmer was introduced. + /// + /// This is only intended for reading and incrementally updating persisted + /// index metadata that does not identify its Greek stemmer version. + #[doc(hidden)] + pub fn new_legacy(language: Language) -> Self { + Self { + stemmer_algorithm: StemmerAlgorithm::Legacy(language.legacy_algorithm()), + } + } } impl Default for Stemmer { @@ -188,4 +207,22 @@ mod tests { assert!(stream.advance()); assert_eq!(stream.token().text, "ανετ"); } + + #[test] + fn test_legacy_greek_stemmer_preserves_existing_terms() { + let mut legacy = TextAnalyzer::builder(RawTokenizer::default()) + .filter(Stemmer::new_legacy(Language::Greek)) + .build(); + let mut current = TextAnalyzer::builder(RawTokenizer::default()) + .filter(Stemmer::new(Language::Greek)) + .build(); + + let mut legacy_stream = legacy.token_stream("ίσα"); + assert!(legacy_stream.advance()); + assert_eq!(legacy_stream.token().text, ""); + + let mut current_stream = current.token_stream("ίσα"); + assert!(current_stream.advance()); + assert_eq!(current_stream.token().text, "ισ"); + } } From b21bcaec86304b6dbe8b50206ec9a7c259b9ee63 Mon Sep 17 00:00:00 2001 From: Lance Gatekeeper Date: Mon, 3 Aug 2026 12:06:30 +0000 Subject: [PATCH 3/8] fix: gate corrected Greek stemming by capability --- protos/index_old.proto | 5 +- rust/lance-index/src/scalar/inverted.rs | 34 ++++- rust/lance-index/src/scalar/inverted/index.rs | 26 +++- .../src/scalar/inverted/tokenizer.rs | 134 ++++++++++++++---- rust/lance/src/dataset/mem_wal/index.rs | 12 +- .../src/dataset/mem_wal/memtable/flush.rs | 4 +- rust/lance/src/dataset/mem_wal/write.rs | 27 +++- 7 files changed, 192 insertions(+), 50 deletions(-) diff --git a/protos/index_old.proto b/protos/index_old.proto index 4a155b23b9a..34805892a09 100644 --- a/protos/index_old.proto +++ b/protos/index_old.proto @@ -93,7 +93,8 @@ message InvertedIndexDetails { // code-specific configuration to apply. CodeTokenizerConfig code_config = 13; // Greek stemming semantics used to build this index. An absent value means - // the legacy rust-stemmers 1.2.0 implementation; new indexes record the - // corrected Snowball 3 implementation. + // the legacy rust-stemmers 1.2.0 implementation. The Snowball 3 value is + // present only when Greek stemming is active and requires inverted-index + // capability version 4 so older readers reject the segment. optional GreekStemmer greek_stemmer = 14; } diff --git a/rust/lance-index/src/scalar/inverted.rs b/rust/lance-index/src/scalar/inverted.rs index 3554b155c98..f8ef01707ff 100644 --- a/rust/lance-index/src/scalar/inverted.rs +++ b/rust/lance-index/src/scalar/inverted.rs @@ -157,6 +157,7 @@ impl InvertedIndexPlugin { params.validate_format_version()?; let format_version = params.resolved_format_version(); + let index_version = index::index_version_for_params(¶ms, format_version); let details = pbold::InvertedIndexDetails::try_from(¶ms)?; let mut inverted_index = InvertedIndexBuilder::new_with_fragment_mask(params, fragment_mask) @@ -164,7 +165,7 @@ impl InvertedIndexPlugin { let files = inverted_index.update(data, index_store, None).await?; Ok(CreatedIndex { index_details: prost_types::Any::from_msg(&details).unwrap(), - index_version: format_version.index_version(), + index_version, files, }) } @@ -276,7 +277,7 @@ impl ScalarIndexPlugin for InvertedIndexPlugin { } fn version(&self) -> u32 { - INVERTED_INDEX_VERSION_V3 + INVERTED_INDEX_VERSION_V4 } fn new_query_parser( @@ -328,9 +329,34 @@ mod tests { use crate::scalar::{BuiltinIndexType, ScalarIndexParams}; #[test] - fn test_plugin_version_tracks_v3_capability_gate() { + fn test_plugin_version_tracks_v4_capability_gate() { let plugin = InvertedIndexPlugin; - assert_eq!(plugin.version(), INVERTED_INDEX_VERSION_V3); + assert_eq!(plugin.version(), INVERTED_INDEX_VERSION_V4); + } + + #[test] + fn test_snowball_greek_uses_v4_capability_gate() { + let current = InvertedIndexParams::new("raw".to_string(), Language::Greek); + assert_eq!( + index::index_version_for_params(¤t, InvertedListFormatVersion::V2), + INVERTED_INDEX_VERSION_V4 + ); + + let mut legacy_json = serde_json::to_value(¤t).unwrap(); + legacy_json.as_object_mut().unwrap().remove("greek_stemmer"); + let legacy: InvertedIndexParams = serde_json::from_value(legacy_json).unwrap(); + assert_eq!( + index::index_version_for_params(&legacy, InvertedListFormatVersion::V2), + INVERTED_INDEX_VERSION_V2 + ); + + assert_eq!( + index::index_version_for_params( + &InvertedIndexParams::default(), + InvertedListFormatVersion::V2, + ), + INVERTED_INDEX_VERSION_V2 + ); } #[test] diff --git a/rust/lance-index/src/scalar/inverted/index.rs b/rust/lance-index/src/scalar/inverted/index.rs index d0701e98306..b2b02dd9918 100644 --- a/rust/lance-index/src/scalar/inverted/index.rs +++ b/rust/lance-index/src/scalar/inverted/index.rs @@ -98,9 +98,11 @@ use std::str::FromStr; // Version 1: Fst TokenSetFormat with per-doc compressed positions // Version 2: Fst TokenSetFormat with shared posting-list position streams. // Version 3: Version 2 layout with configurable posting blocks and analyzer metadata. +// Version 4: Version 3 capabilities plus Snowball 3 Greek vocabulary semantics. pub const INVERTED_INDEX_VERSION_V1: u32 = 1; pub const INVERTED_INDEX_VERSION_V2: u32 = 2; pub const INVERTED_INDEX_VERSION_V3: u32 = 3; +pub const INVERTED_INDEX_VERSION_V4: u32 = 4; pub const TOKENS_FILE: &str = "tokens.lance"; pub const INVERT_LIST_FILE: &str = "invert.lance"; pub const DOCS_FILE: &str = "docs.lance"; @@ -169,6 +171,20 @@ pub fn resolve_fts_format_version( } } +/// Return the manifest capability version required by an inverted-index +/// analyzer and its physical posting-list format. +#[doc(hidden)] +pub fn index_version_for_params( + params: &InvertedIndexParams, + format_version: InvertedListFormatVersion, +) -> u32 { + if params.uses_snowball3_greek() { + INVERTED_INDEX_VERSION_V4 + } else { + format_version.index_version() + } +} + pub fn default_fts_format_version() -> InvertedListFormatVersion { InvertedListFormatVersion::V2 } @@ -734,6 +750,9 @@ impl InvertedIndex { } fn index_version(&self) -> u32 { + if self.params.uses_snowball3_greek() { + return INVERTED_INDEX_VERSION_V4; + } match (self.token_set_format, self.format_version()) { ( TokenSetFormat::Arrow, @@ -8334,7 +8353,6 @@ mod tests { use std::sync::Arc; use std::sync::atomic::{AtomicU32, Ordering}; - use crate::scalar::inverted::tokenizer::GreekStemmerVersion; use crate::scalar::inverted::tokenizer::document_tokenizer::TextTokenizer; use lance_tokenizer::{Language, SimpleTokenizer, StopWordFilter, TextAnalyzer}; @@ -8578,7 +8596,7 @@ mod tests { .expect("inverted index params should serialize to an object") .remove("greek_stemmer"); let legacy_params: InvertedIndexParams = serde_json::from_value(legacy_json)?; - assert_eq!(legacy_params.greek_stemmer, GreekStemmerVersion::Legacy); + assert_eq!(legacy_params.greek_stemmer, None); // rust-stemmers 1.2.0 produced an empty term for this word. The // manually written term and missing version field model a persisted @@ -8586,7 +8604,7 @@ mod tests { let index = write_single_partition_index(src_store, legacy_params, TokenSetFormat::Fst, "", 100) .await?; - assert_eq!(index.params.greek_stemmer, GreekStemmerVersion::Legacy); + assert_eq!(index.params.greek_stemmer, None); let matches = index.do_search("ίσα").await?; let row_ids = matches[ROW_ID].as_primitive::(); @@ -8605,7 +8623,7 @@ mod tests { .await?; let updated = InvertedIndex::load(dest_store, None, &LanceCache::no_cache()).await?; - assert_eq!(updated.params.greek_stemmer, GreekStemmerVersion::Legacy); + assert_eq!(updated.params.greek_stemmer, None); let matches = updated.do_search("ίσα").await?; let mut row_ids = matches[ROW_ID] .as_primitive::() diff --git a/rust/lance-index/src/scalar/inverted/tokenizer.rs b/rust/lance-index/src/scalar/inverted/tokenizer.rs index 972254781a6..dfca4eff506 100644 --- a/rust/lance-index/src/scalar/inverted/tokenizer.rs +++ b/rust/lance-index/src/scalar/inverted/tokenizer.rs @@ -52,18 +52,14 @@ pub(crate) enum GreekStemmerVersion { } impl GreekStemmerVersion { - fn is_legacy(&self) -> bool { - matches!(self, Self::Legacy) - } - - fn from_proto(value: Option) -> Result { + fn from_proto(value: Option) -> Result> { use pbold::inverted_index_details::GreekStemmer; match value { - None => Ok(Self::Legacy), + None => Ok(None), Some(value) => match GreekStemmer::try_from(value) { - Ok(GreekStemmer::Legacy) => Ok(Self::Legacy), - Ok(GreekStemmer::Snowball3) => Ok(Self::Snowball3), + Ok(GreekStemmer::Legacy) => Ok(Some(Self::Legacy)), + Ok(GreekStemmer::Snowball3) => Ok(Some(Self::Snowball3)), Err(_) => Err(Error::invalid_input(format!( "unknown Greek stemmer version {value}" ))), @@ -71,12 +67,12 @@ impl GreekStemmerVersion { } } - fn to_proto(self) -> Option { + fn to_proto(self) -> i32 { use pbold::inverted_index_details::GreekStemmer; match self { - Self::Legacy => None, - Self::Snowball3 => Some(GreekStemmer::Snowball3 as i32), + Self::Legacy => GreekStemmer::Legacy as i32, + Self::Snowball3 => GreekStemmer::Snowball3 as i32, } } } @@ -116,10 +112,10 @@ pub struct InvertedIndexParams { /// Greek stemming semantics persisted with the index vocabulary. /// - /// Missing serialized values select [`GreekStemmerVersion::Legacy`]. New - /// indexes use [`GreekStemmerVersion::Snowball3`]. - #[serde(skip_serializing_if = "GreekStemmerVersion::is_legacy")] - pub(crate) greek_stemmer: GreekStemmerVersion, + /// Missing serialized values select legacy semantics. New indexes persist + /// [`GreekStemmerVersion::Snowball3`] only when Greek stemming is active. + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) greek_stemmer: Option, /// If true, store the position of the term in the document /// This can significantly increase the size of the index @@ -325,7 +321,7 @@ impl RawInvertedIndexParams { if let Some(language) = self.language { params.language = language; } - params.greek_stemmer = self.greek_stemmer.unwrap_or(GreekStemmerVersion::Legacy); + params.greek_stemmer = self.greek_stemmer; if let Some(with_position) = self.with_position { params.with_position = with_position; } @@ -404,7 +400,9 @@ impl TryFrom<&InvertedIndexParams> for pbold::InvertedIndexDetails { index_operators: params.index_operators, }, ), - greek_stemmer: params.greek_stemmer.to_proto(), + greek_stemmer: params + .active_greek_stemmer() + .map(GreekStemmerVersion::to_proto), }) } } @@ -424,7 +422,7 @@ impl TryFrom<&pbold::InvertedIndexDetails> for InvertedIndexParams { Some(block_size) => validate_block_size(block_size as usize)?, None => LEGACY_BLOCK_SIZE, }, - greek_stemmer: GreekStemmerVersion::Legacy, + greek_stemmer: None, ..Self::default() }; return Ok(params); @@ -606,7 +604,7 @@ impl InvertedIndexParams { lance_tokenizer: None, base_tokenizer, language, - greek_stemmer: GreekStemmerVersion::Snowball3, + greek_stemmer: (language == Language::Greek).then_some(GreekStemmerVersion::Snowball3), with_position: false, max_token_length: Some(40), lower_case: true, @@ -642,6 +640,7 @@ impl InvertedIndexParams { self.stem = true; self.remove_stop_words = true; self.index_operators = false; + self.select_current_greek_stemmer(); } fn apply_code_defaults(&mut self) { @@ -654,6 +653,7 @@ impl InvertedIndexParams { self.stem = false; self.remove_stop_words = false; self.index_operators = false; + self.select_current_greek_stemmer(); } /// Create parameters for the code analyzer profile. @@ -726,6 +726,7 @@ impl InvertedIndexParams { // need to convert to valid JSON string let language = serde_json::from_str(format!("\"{}\"", language).as_str())?; self.language = language; + self.select_current_greek_stemmer(); Ok(self) } @@ -756,6 +757,7 @@ impl InvertedIndexParams { pub fn stem(mut self, stem: bool) -> Self { self.stem = stem; + self.select_current_greek_stemmer(); self } @@ -872,8 +874,9 @@ impl InvertedIndexParams { /// If unset, new index creation falls back to /// `LANCE_FTS_FORMAT_VERSION`. Without either override, text analysis with /// 128-document blocks writes v2, while code analysis or 256-document blocks - /// write v3. Existing indexes keep their own format during update and - /// optimize operations. + /// write v3. Snowball 3 Greek stemming also requires the v3 physical + /// format and advertises manifest capability version 4. Existing indexes + /// keep their own format during update and optimize operations. pub fn format_version(mut self, format_version: InvertedListFormatVersion) -> Self { self.format_version = Some(format_version); self @@ -883,7 +886,7 @@ impl InvertedIndexParams { /// the configured analyzer and block size. pub fn resolved_format_version(&self) -> InvertedListFormatVersion { self.format_version.unwrap_or_else(|| { - if self.base_tokenizer == "code" { + if self.base_tokenizer == "code" || self.uses_snowball3_greek() { InvertedListFormatVersion::V3 } else { default_fts_format_version_for_block_size(self.block_size) @@ -897,6 +900,12 @@ impl InvertedIndexParams { pub fn validate_format_version(&self) -> Result<()> { let format_version = self.resolved_format_version(); validate_format_version_block_size(format_version, self.block_size)?; + if self.uses_snowball3_greek() && format_version != InvertedListFormatVersion::V3 { + return Err(Error::invalid_input(format!( + "Snowball 3 Greek stemming requires FTS format_version=3, got {}", + format_version.index_version() + ))); + } if self.base_tokenizer == "code" && format_version != InvertedListFormatVersion::V3 { return Err(Error::invalid_input(format!( "base_tokenizer='code' requires FTS format_version=3, got {}", @@ -912,6 +921,12 @@ impl InvertedIndexParams { let object = value .as_object_mut() .expect("inverted index params should serialize to a JSON object"); + if self.language == Language::Greek && self.stem && self.greek_stemmer.is_none() { + object.insert( + "greek_stemmer".to_string(), + serde_json::Value::from("legacy"), + ); + } if let Some(memory_limit_mb) = self.memory_limit_mb { object.insert( "memory_limit".to_string(), @@ -942,12 +957,20 @@ impl InvertedIndexParams { let supplied = supplied.as_object().ok_or_else(|| { Error::invalid_input("FTS inverted index params must be a JSON object".to_string()) })?; + let has_explicit_greek_stemmer = supplied.contains_key("greek_stemmer"); let object = value .as_object_mut() .expect("inverted index params should serialize to a JSON object"); object.extend(supplied.clone()); let mut params: Self = serde_json::from_value(value)?; + if params.language == Language::Greek && params.stem { + if !has_explicit_greek_stemmer { + params.greek_stemmer = Some(GreekStemmerVersion::Snowball3); + } + } else { + params.greek_stemmer = None; + } let default_format_version = params.resolved_format_version(); params.format_version = Some(resolve_creation_format_version( params.format_version, @@ -967,9 +990,9 @@ impl InvertedIndexParams { builder = builder.filter_dynamic(LowerCaser); } if self.stem { - let stemmer = match self.greek_stemmer { - GreekStemmerVersion::Legacy => Stemmer::new_legacy(self.language), - GreekStemmerVersion::Snowball3 => Stemmer::new(self.language), + let stemmer = match self.active_greek_stemmer() { + Some(GreekStemmerVersion::Snowball3) => Stemmer::new(self.language), + None | Some(GreekStemmerVersion::Legacy) => Stemmer::new_legacy(self.language), }; builder = builder.filter_dynamic(stemmer); } @@ -1009,6 +1032,11 @@ impl InvertedIndexParams { fn validate(&self) -> Result<()> { validate_block_size(self.block_size)?; + if self.greek_stemmer.is_some() && (self.language != Language::Greek || !self.stem) { + return Err(Error::invalid_input( + "greek_stemmer requires language='Greek' and stem=true".to_string(), + )); + } if self.base_tokenizer != "code" && (self.split_identifiers || self.split_on_numerics @@ -1022,6 +1050,21 @@ impl InvertedIndexParams { Ok(()) } + fn select_current_greek_stemmer(&mut self) { + self.greek_stemmer = (self.language == Language::Greek && self.stem) + .then_some(GreekStemmerVersion::Snowball3); + } + + fn active_greek_stemmer(&self) -> Option { + (self.language == Language::Greek && self.stem) + .then_some(self.greek_stemmer) + .flatten() + } + + pub(crate) fn uses_snowball3_greek(&self) -> bool { + self.active_greek_stemmer() == Some(GreekStemmerVersion::Snowball3) + } + fn build_base_tokenizer(&self) -> Result { match self.base_tokenizer.as_str() { "simple" => Ok(TextAnalyzer::builder(SimpleTokenizer::default()).dynamic()), @@ -1484,9 +1527,26 @@ mod tests { let mut json = serde_json::to_value(¤t).unwrap(); assert_eq!(json["greek_stemmer"], "snowball3"); + let english = InvertedIndexParams::default(); + assert!( + serde_json::to_value(&english).unwrap()["greek_stemmer"].is_null(), + "non-Greek analyzers should not persist a Greek stemmer version" + ); + let stemming_disabled = current.clone().stem(false); + assert!( + serde_json::to_value(&stemming_disabled).unwrap()["greek_stemmer"].is_null(), + "a disabled Greek stemmer should not persist a version" + ); + assert!( + pbold::InvertedIndexDetails::try_from(&stemming_disabled) + .unwrap() + .greek_stemmer + .is_none() + ); + json.as_object_mut().unwrap().remove("greek_stemmer"); let legacy: InvertedIndexParams = serde_json::from_value(json).unwrap(); - assert_eq!(legacy.greek_stemmer, GreekStemmerVersion::Legacy); + assert_eq!(legacy.greek_stemmer, None); assert!( serde_json::to_value(&legacy).unwrap()["greek_stemmer"].is_null(), "legacy metadata should remain absent when rewritten" @@ -1498,7 +1558,7 @@ mod tests { InvertedIndexParams::try_from(¤t_details) .unwrap() .greek_stemmer, - GreekStemmerVersion::Snowball3 + Some(GreekStemmerVersion::Snowball3) ); let mut legacy_details = current_details; @@ -1507,7 +1567,25 @@ mod tests { InvertedIndexParams::try_from(&legacy_details) .unwrap() .greek_stemmer, - GreekStemmerVersion::Legacy + None + ); + + let legacy_training_json = legacy.to_training_json().unwrap(); + assert_eq!(legacy_training_json["greek_stemmer"], "legacy"); + let legacy_training = InvertedIndexParams::from_training_json( + &serde_json::to_string(&legacy_training_json).unwrap(), + ) + .unwrap(); + assert_eq!( + legacy_training.greek_stemmer, + Some(GreekStemmerVersion::Legacy) + ); + + let current_training = + InvertedIndexParams::from_training_json(r#"{"language":"Greek"}"#).unwrap(); + assert_eq!( + current_training.greek_stemmer, + Some(GreekStemmerVersion::Snowball3) ); } diff --git a/rust/lance/src/dataset/mem_wal/index.rs b/rust/lance/src/dataset/mem_wal/index.rs index b3da5b5b67e..c2261618bf4 100644 --- a/rust/lance/src/dataset/mem_wal/index.rs +++ b/rust/lance/src/dataset/mem_wal/index.rs @@ -35,7 +35,7 @@ use lance_core::datatypes::Schema as LanceSchema; use lance_core::{Error, Result}; use lance_index::pbold; use lance_index::scalar::InvertedIndexParams; -use lance_index::scalar::inverted::InvertedListFormatVersion; +use lance_index::scalar::inverted::{INVERTED_INDEX_VERSION_V4, InvertedListFormatVersion}; use lance_index::vector::hnsw::builder::HnswBuildParams; use lance_linalg::distance::DistanceType; use lance_table::format::IndexMetadata; @@ -388,8 +388,11 @@ impl MemIndexConfig { 0 | 1 => Ok(InvertedListFormatVersion::V1), 2 => Ok(InvertedListFormatVersion::V2), 3 => Ok(InvertedListFormatVersion::V3), + version if version == INVERTED_INDEX_VERSION_V4 as i32 => { + Ok(InvertedListFormatVersion::V3) + } version => Err(Error::invalid_input(format!( - "FTS index '{}' has unsupported index_version {}; expected 0, 1, 2, or 3", + "FTS index '{}' has unsupported index_version {}; expected 0, 1, 2, 3, or 4", index_meta.name, version ))), } @@ -1568,6 +1571,7 @@ mod tests { (1, InvertedListFormatVersion::V1), (2, InvertedListFormatVersion::V2), (3, InvertedListFormatVersion::V3), + (4, InvertedListFormatVersion::V3), ] { let config = MemIndexConfig::fts_from_metadata(&fts_index_metadata(index_version), &schema) @@ -1590,9 +1594,9 @@ mod tests { let arrow_schema = create_test_schema(); let schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap(); - let err = MemIndexConfig::fts_from_metadata(&fts_index_metadata(4), &schema).unwrap_err(); + let err = MemIndexConfig::fts_from_metadata(&fts_index_metadata(5), &schema).unwrap_err(); assert!( - err.to_string().contains("unsupported index_version 4"), + err.to_string().contains("unsupported index_version 5"), "{err}" ); } diff --git a/rust/lance/src/dataset/mem_wal/memtable/flush.rs b/rust/lance/src/dataset/mem_wal/memtable/flush.rs index 1bb2cbae039..a76ccd19bd8 100644 --- a/rust/lance/src/dataset/mem_wal/memtable/flush.rs +++ b/rust/lance/src/dataset/mem_wal/memtable/flush.rs @@ -13,6 +13,7 @@ use lance_core::utils::deletion::DeletionVector; use lance_core::{Error, Result}; use lance_index::IndexType; use lance_index::mem_wal::{ShardManifest, SsTable}; +use lance_index::scalar::inverted::index_version_for_params; use lance_index::scalar::{IndexStore, ScalarIndexParams}; use lance_io::object_store::{ObjectStore, ObjectStoreParams}; use lance_table::format::IndexMetadata; @@ -775,6 +776,7 @@ impl MemTableFlusher { let fragment_ids: roaring::RoaringBitmap = dataset.fragment_bitmap.as_ref().clone(); let format_version = fts_cfg.params.resolved_format_version(); + let index_version = index_version_for_params(&fts_cfg.params, format_version); let index_meta = IndexMetadata { uuid: index_uuid, @@ -783,7 +785,7 @@ impl MemTableFlusher { dataset_version: dataset.version().version, fragment_bitmap: Some(fragment_ids), index_details: Some(Arc::new(index_details)), - index_version: format_version.index_version() as i32, + index_version: index_version as i32, created_at: None, base_id: None, files: None, diff --git a/rust/lance/src/dataset/mem_wal/write.rs b/rust/lance/src/dataset/mem_wal/write.rs index 7f05cbfff3e..79cb417b4cf 100644 --- a/rust/lance/src/dataset/mem_wal/write.rs +++ b/rust/lance/src/dataset/mem_wal/write.rs @@ -7527,8 +7527,19 @@ mod shard_writer_tests { writer.close().await.unwrap(); } + #[rstest::rstest] + #[case::v1( + lance_tokenizer::Language::English, + Some(InvertedListFormatVersion::V1), + 1 + )] + #[case::snowball3_greek(lance_tokenizer::Language::Greek, None, 4)] #[tokio::test] - async fn test_mem_wal_maintained_fts_v1_flush_preserves_format() { + async fn test_mem_wal_maintained_fts_flush_preserves_version( + #[case] language: lance_tokenizer::Language, + #[case] format_version: Option, + #[case] expected_index_version: i32, + ) { use tempfile::TempDir; let vector_dim = 32; @@ -7542,8 +7553,10 @@ mod shard_writer_tests { .await .expect("Failed to create dataset"); - let fts_params = - InvertedIndexParams::default().format_version(InvertedListFormatVersion::V1); + let mut fts_params = InvertedIndexParams::new("simple".to_string(), language); + if let Some(format_version) = format_version { + fts_params = fts_params.format_version(format_version); + } dataset .create_index( &["text"], @@ -7553,10 +7566,10 @@ mod shard_writer_tests { false, ) .await - .expect("Failed to create v1 FTS index"); + .expect("Failed to create FTS index"); let base_indices = dataset.load_indices().await.unwrap(); assert_eq!(base_indices.len(), 1); - assert_eq!(base_indices[0].index_version, 1); + assert_eq!(base_indices[0].index_version, expected_index_version); dataset .initialize_mem_wal() @@ -7598,8 +7611,8 @@ mod shard_writer_tests { assert_eq!(sstable_indices.len(), 1); assert_eq!(sstable_indices[0].name, "text_fts"); assert_eq!( - sstable_indices[0].index_version, 1, - "maintained v1 FTS index must flush as v1" + sstable_indices[0].index_version, expected_index_version, + "maintained FTS index capability version changed during flush" ); let results = sstable From fadb9d7e06f9c7493e90e23485b2562dab54701d Mon Sep 17 00:00:00 2001 From: Lance Gatekeeper Date: Mon, 3 Aug 2026 12:31:53 +0000 Subject: [PATCH 4/8] fix: canonicalize legacy Greek stemmer metadata --- .../src/scalar/inverted/tokenizer.rs | 13 ++- rust/lance/src/index/append.rs | 105 ++++++++++++++++++ 2 files changed, 114 insertions(+), 4 deletions(-) diff --git a/rust/lance-index/src/scalar/inverted/tokenizer.rs b/rust/lance-index/src/scalar/inverted/tokenizer.rs index dfca4eff506..6d63bff6b29 100644 --- a/rust/lance-index/src/scalar/inverted/tokenizer.rs +++ b/rust/lance-index/src/scalar/inverted/tokenizer.rs @@ -67,12 +67,12 @@ impl GreekStemmerVersion { } } - fn to_proto(self) -> i32 { + fn to_proto(self) -> Option { use pbold::inverted_index_details::GreekStemmer; match self { - Self::Legacy => GreekStemmer::Legacy as i32, - Self::Snowball3 => GreekStemmer::Snowball3 as i32, + Self::Legacy => None, + Self::Snowball3 => Some(GreekStemmer::Snowball3 as i32), } } } @@ -402,7 +402,7 @@ impl TryFrom<&InvertedIndexParams> for pbold::InvertedIndexDetails { ), greek_stemmer: params .active_greek_stemmer() - .map(GreekStemmerVersion::to_proto), + .and_then(GreekStemmerVersion::to_proto), }) } } @@ -1580,6 +1580,11 @@ mod tests { legacy_training.greek_stemmer, Some(GreekStemmerVersion::Legacy) ); + assert_eq!( + pbold::InvertedIndexDetails::try_from(&legacy_training).unwrap(), + legacy_details, + "the internal legacy training hint should retain absent canonical metadata" + ); let current_training = InvertedIndexParams::from_training_json(r#"{"language":"Greek"}"#).unwrap(); diff --git a/rust/lance/src/index/append.rs b/rust/lance/src/index/append.rs index b6fc5a786d9..3c156b1add1 100644 --- a/rust/lance/src/index/append.rs +++ b/rust/lance/src/index/append.rs @@ -1971,6 +1971,111 @@ mod tests { ); } + #[tokio::test] + async fn test_append_legacy_greek_fts_preserves_canonical_details() { + use lance_index::pbold::InvertedIndexDetails; + use lance_index::scalar::FullTextSearchQuery; + use lance_index::scalar::inverted::{InvertedIndexParams, Language}; + use prost::Message; + + const INDEX_NAME: &str = "body_idx"; + + let test_dir = TempStrDir::default(); + let schema = Arc::new(Schema::new(vec![Field::new("body", DataType::Utf8, false)])); + let initial_batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(StringArray::from(vec!["ίσα"]))], + ) + .unwrap(); + let mut dataset = Dataset::write( + RecordBatchIterator::new(vec![Ok(initial_batch)], schema.clone()), + test_dir.as_str(), + None, + ) + .await + .unwrap(); + + let current_params = InvertedIndexParams::new("raw".to_string(), Language::Greek) + .max_token_length(None) + .remove_stop_words(false) + .ascii_folding(false); + let mut legacy_json = serde_json::to_value(current_params).unwrap(); + legacy_json + .as_object_mut() + .expect("inverted index params should serialize to an object") + .remove("greek_stemmer"); + let legacy_params: InvertedIndexParams = serde_json::from_value(legacy_json).unwrap(); + + let fragment_ids = dataset + .get_fragments() + .into_iter() + .map(|fragment| fragment.id() as u32) + .collect(); + let mut legacy_segment = + CreateIndexBuilder::new(&mut dataset, &["body"], IndexType::Inverted, &legacy_params) + .name(INDEX_NAME.to_string()) + .fragments(fragment_ids) + .execute_uncommitted() + .await + .unwrap(); + + // Model the manifest details written before the Greek stemmer + // discriminator existed. The index files still select legacy stemming. + let details_any = legacy_segment + .index_details + .as_ref() + .expect("inverted segment should include index details"); + let mut details = InvertedIndexDetails::decode(details_any.value.as_slice()).unwrap(); + details.greek_stemmer = None; + legacy_segment.index_details = Some(Arc::new( + prost_types::Any::from_msg(&details).expect("details should encode"), + )); + dataset + .commit_existing_index_segments(INDEX_NAME, "body", vec![legacy_segment]) + .await + .unwrap(); + + let appended_batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(StringArray::from(vec!["ίσα"]))], + ) + .unwrap(); + dataset + .append( + RecordBatchIterator::new(vec![Ok(appended_batch)], schema), + None, + ) + .await + .unwrap(); + dataset + .optimize_indices(&OptimizeOptions::append()) + .await + .unwrap(); + + let dataset = DatasetBuilder::from_uri(test_dir.as_str()) + .load() + .await + .unwrap(); + let segments = dataset.load_indices_by_name(INDEX_NAME).await.unwrap(); + assert_eq!(segments.len(), 2); + for segment in segments { + let details_any = segment + .index_details + .expect("inverted segment should include index details"); + let details = InvertedIndexDetails::decode(details_any.value.as_slice()).unwrap(); + assert_eq!(details.greek_stemmer, None); + } + + let result = dataset + .scan() + .full_text_search(FullTextSearchQuery::new("ίσα".to_string())) + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!(result.num_rows(), 2); + } + #[tokio::test] async fn test_append_index() { const DIM: usize = 64; From 7e9366610c820dc7f4ce764953d141ddcff2ea65 Mon Sep 17 00:00:00 2001 From: Lance Gatekeeper Date: Mon, 3 Aug 2026 13:23:23 +0000 Subject: [PATCH 5/8] fix: normalize loaded legacy Greek stemmer params --- rust/lance-index/src/scalar/inverted/index.rs | 20 +++- .../src/scalar/inverted/tokenizer.rs | 9 ++ rust/lance/src/index/append.rs | 101 +++++++++++++++--- 3 files changed, 108 insertions(+), 22 deletions(-) diff --git a/rust/lance-index/src/scalar/inverted/index.rs b/rust/lance-index/src/scalar/inverted/index.rs index b2b02dd9918..6d7b21ede40 100644 --- a/rust/lance-index/src/scalar/inverted/index.rs +++ b/rust/lance-index/src/scalar/inverted/index.rs @@ -1756,7 +1756,8 @@ impl InvertedIndex { .metadata .get("params") .ok_or(Error::index("params not found in metadata".to_owned()))?; - Ok(serde_json::from_str::(params)?) + Ok(serde_json::from_str::(params)? + .normalize_loaded_greek_stemmer()) } Err(metadata_error) => { // Legacy format: params live in the tokens file (see @@ -1772,7 +1773,8 @@ impl InvertedIndex { .get("tokenizer") .map(|s| serde_json::from_str::(s)) .transpose()? - .unwrap_or_default()) + .unwrap_or_default() + .normalize_loaded_greek_stemmer()) } } } @@ -1796,7 +1798,8 @@ impl InvertedIndex { .metadata .get("params") .ok_or(Error::index("params not found in metadata".to_owned()))?; - let params = serde_json::from_str::(params)?; + let params = serde_json::from_str::(params)? + .normalize_loaded_greek_stemmer(); let partitions = reader .schema() .metadata @@ -8353,6 +8356,7 @@ mod tests { use std::sync::Arc; use std::sync::atomic::{AtomicU32, Ordering}; + use crate::scalar::inverted::tokenizer::GreekStemmerVersion; use crate::scalar::inverted::tokenizer::document_tokenizer::TextTokenizer; use lance_tokenizer::{Language, SimpleTokenizer, StopWordFilter, TextAnalyzer}; @@ -8604,7 +8608,10 @@ mod tests { let index = write_single_partition_index(src_store, legacy_params, TokenSetFormat::Fst, "", 100) .await?; - assert_eq!(index.params.greek_stemmer, None); + assert_eq!( + index.params.greek_stemmer, + Some(GreekStemmerVersion::Legacy) + ); let matches = index.do_search("ίσα").await?; let row_ids = matches[ROW_ID].as_primitive::(); @@ -8623,7 +8630,10 @@ mod tests { .await?; let updated = InvertedIndex::load(dest_store, None, &LanceCache::no_cache()).await?; - assert_eq!(updated.params.greek_stemmer, None); + assert_eq!( + updated.params.greek_stemmer, + Some(GreekStemmerVersion::Legacy) + ); let matches = updated.do_search("ίσα").await?; let mut row_ids = matches[ROW_ID] .as_primitive::() diff --git a/rust/lance-index/src/scalar/inverted/tokenizer.rs b/rust/lance-index/src/scalar/inverted/tokenizer.rs index 6d63bff6b29..daa4cb0c2f9 100644 --- a/rust/lance-index/src/scalar/inverted/tokenizer.rs +++ b/rust/lance-index/src/scalar/inverted/tokenizer.rs @@ -1065,6 +1065,15 @@ impl InvertedIndexParams { self.active_greek_stemmer() == Some(GreekStemmerVersion::Snowball3) } + /// Normalize historical field-absent physical metadata to the explicit + /// internal hint used when rebuilding or merging legacy Greek segments. + pub(crate) fn normalize_loaded_greek_stemmer(mut self) -> Self { + if self.language == Language::Greek && self.stem && self.greek_stemmer.is_none() { + self.greek_stemmer = Some(GreekStemmerVersion::Legacy); + } + self + } + fn build_base_tokenizer(&self) -> Result { match self.base_tokenizer.as_str() { "simple" => Ok(TextAnalyzer::builder(SimpleTokenizer::default()).dynamic()), diff --git a/rust/lance/src/index/append.rs b/rust/lance/src/index/append.rs index 3c156b1add1..9e02f2ade48 100644 --- a/rust/lance/src/index/append.rs +++ b/rust/lance/src/index/append.rs @@ -1974,8 +1974,8 @@ mod tests { #[tokio::test] async fn test_append_legacy_greek_fts_preserves_canonical_details() { use lance_index::pbold::InvertedIndexDetails; - use lance_index::scalar::FullTextSearchQuery; - use lance_index::scalar::inverted::{InvertedIndexParams, Language}; + use lance_index::scalar::inverted::{InvertedIndexParams, Language, METADATA_FILE}; + use lance_index::scalar::{FullTextSearchQuery, IndexStore}; use prost::Message; const INDEX_NAME: &str = "body_idx"; @@ -1996,6 +1996,7 @@ mod tests { .unwrap(); let current_params = InvertedIndexParams::new("raw".to_string(), Language::Greek) + .lance_tokenizer("text".to_string()) .max_token_length(None) .remove_stop_words(false) .ascii_folding(false); @@ -2011,30 +2012,63 @@ mod tests { .into_iter() .map(|fragment| fragment.id() as u32) .collect(); - let mut legacy_segment = + let legacy_segment = CreateIndexBuilder::new(&mut dataset, &["body"], IndexType::Inverted, &legacy_params) .name(INDEX_NAME.to_string()) .fragments(fragment_ids) .execute_uncommitted() .await .unwrap(); - - // Model the manifest details written before the Greek stemmer - // discriminator existed. The index files still select legacy stemming. - let details_any = legacy_segment - .index_details - .as_ref() - .expect("inverted segment should include index details"); - let mut details = InvertedIndexDetails::decode(details_any.value.as_slice()).unwrap(); - details.greek_stemmer = None; - legacy_segment.index_details = Some(Arc::new( - prost_types::Any::from_msg(&details).expect("details should encode"), - )); dataset .commit_existing_index_segments(INDEX_NAME, "body", vec![legacy_segment]) .await .unwrap(); + // Model a released segment whose physical metadata predates the Greek + // stemmer field. Preserve the JSON byte length so the committed file + // size remains valid after rewriting the metadata file. + let committed = dataset.load_indices_by_name(INDEX_NAME).await.unwrap(); + let legacy_store = LanceIndexStore::from_dataset_for_existing(&dataset, &committed[0]) + .await + .unwrap(); + let metadata_reader = legacy_store.open_index_file(METADATA_FILE).await.unwrap(); + let mut physical_metadata = metadata_reader.schema().metadata.clone(); + let explicit_params = physical_metadata + .get("params") + .expect("physical metadata should contain params"); + assert!(explicit_params.contains("greek_stemmer")); + let mut absent_params = serde_json::to_string(&legacy_params).unwrap(); + assert!(!absent_params.contains("greek_stemmer")); + absent_params.extend(std::iter::repeat_n( + ' ', + explicit_params.len() - absent_params.len(), + )); + physical_metadata.insert("params".to_string(), absent_params); + + let metadata_batch = metadata_reader.read_range(0..1, None).await.unwrap(); + let metadata_schema = Arc::new(Schema::new(metadata_batch.schema().fields().clone())); + let metadata_batch = + RecordBatch::try_new(metadata_schema.clone(), metadata_batch.columns().to_vec()) + .unwrap(); + drop(metadata_reader); + let mut metadata_writer = legacy_store + .new_index_file(METADATA_FILE, metadata_schema) + .await + .unwrap(); + metadata_writer + .write_record_batch(metadata_batch) + .await + .unwrap(); + metadata_writer + .finish_with_metadata(physical_metadata) + .await + .unwrap(); + + dataset = DatasetBuilder::from_uri(test_dir.as_str()) + .load() + .await + .unwrap(); + let appended_batch = RecordBatch::try_new( schema.clone(), vec![Arc::new(StringArray::from(vec!["ίσα"]))], @@ -2052,15 +2086,23 @@ mod tests { .await .unwrap(); - let dataset = DatasetBuilder::from_uri(test_dir.as_str()) + let mut dataset = DatasetBuilder::from_uri(test_dir.as_str()) .load() .await .unwrap(); let segments = dataset.load_indices_by_name(INDEX_NAME).await.unwrap(); assert_eq!(segments.len(), 2); - for segment in segments { + let first_params = crate::index::scalar::load_segment_params(&dataset, &segments[0]) + .await + .unwrap(); + let second_params = crate::index::scalar::load_segment_params(&dataset, &segments[1]) + .await + .unwrap(); + assert_eq!(first_params, second_params); + for segment in &segments { let details_any = segment .index_details + .as_ref() .expect("inverted segment should include index details"); let details = InvertedIndexDetails::decode(details_any.value.as_slice()).unwrap(); assert_eq!(details.greek_stemmer, None); @@ -2074,6 +2116,31 @@ mod tests { .await .unwrap(); assert_eq!(result.num_rows(), 2); + + dataset + .optimize_indices(&OptimizeOptions::merge(2)) + .await + .unwrap(); + let dataset = DatasetBuilder::from_uri(test_dir.as_str()) + .load() + .await + .unwrap(); + assert_eq!( + dataset + .load_indices_by_name(INDEX_NAME) + .await + .unwrap() + .len(), + 1 + ); + let result = dataset + .scan() + .full_text_search(FullTextSearchQuery::new("ίσα".to_string())) + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!(result.num_rows(), 2); } #[tokio::test] From 043b06cabaf65fd95b639764b4c90e21fe061582 Mon Sep 17 00:00:00 2001 From: Lance Gatekeeper Date: Tue, 4 Aug 2026 06:39:22 +0000 Subject: [PATCH 6/8] fix: use frostem for corrected Greek stemming --- Cargo.lock | 14 +++++++------- java/lance-jni/Cargo.lock | 18 +++++++++--------- python/Cargo.lock | 18 +++++++++--------- rust/lance-tokenizer/Cargo.toml | 3 ++- rust/lance-tokenizer/src/stemmer.rs | 6 ++---- 5 files changed, 29 insertions(+), 30 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f349d64869e..d52a40b12c9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3077,6 +3077,12 @@ dependencies = [ "futures-core", ] +[[package]] +name = "frostem" +version = "1.20260804.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d0ae10cfccbae085dd8612669ccc116fe88bd5dfe39d202a8fa68c24c1e546f" + [[package]] name = "fs_extra" version = "1.3.0" @@ -5144,10 +5150,10 @@ dependencies = [ name = "lance-tokenizer" version = "10.1.0-beta.2" dependencies = [ + "frostem", "icu_segmenter", "jieba-rs", "lindera", - "pagefind_stem", "rust-stemmers", "serde", "stop-words", @@ -6440,12 +6446,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "pagefind_stem" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8dfa810b158f3ac364e5acd43ca4a6020a6e729d40c15ce1bed1d911237a52e5" - [[package]] name = "parking" version = "2.2.1" diff --git a/java/lance-jni/Cargo.lock b/java/lance-jni/Cargo.lock index 08e647fc1d3..a79ebc62e51 100644 --- a/java/lance-jni/Cargo.lock +++ b/java/lance-jni/Cargo.lock @@ -2475,6 +2475,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "frostem" +version = "1.20260804.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d0ae10cfccbae085dd8612669ccc116fe88bd5dfe39d202a8fa68c24c1e546f" + [[package]] name = "fs_extra" version = "1.3.0" @@ -4277,8 +4283,8 @@ dependencies = [ name = "lance-tokenizer" version = "10.1.0-beta.2" dependencies = [ + "frostem", "icu_segmenter", - "pagefind_stem", "rust-stemmers", "serde", "stop-words", @@ -5218,12 +5224,6 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" -[[package]] -name = "pagefind_stem" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8dfa810b158f3ac364e5acd43ca4a6020a6e729d40c15ce1bed1d911237a52e5" - [[package]] name = "parking" version = "2.2.1" @@ -5541,7 +5541,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" dependencies = [ "heck", - "itertools 0.11.0", + "itertools 0.14.0", "log", "multimap", "petgraph", @@ -5560,7 +5560,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.11.0", + "itertools 0.14.0", "proc-macro2", "quote", "syn 2.0.118", diff --git a/python/Cargo.lock b/python/Cargo.lock index 1f653c20584..c5ec394170e 100644 --- a/python/Cargo.lock +++ b/python/Cargo.lock @@ -2786,6 +2786,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "frostem" +version = "1.20260804.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d0ae10cfccbae085dd8612669ccc116fe88bd5dfe39d202a8fa68c24c1e546f" + [[package]] name = "fs_extra" version = "1.3.0" @@ -4569,10 +4575,10 @@ dependencies = [ name = "lance-tokenizer" version = "10.1.0-beta.2" dependencies = [ + "frostem", "icu_segmenter", "jieba-rs", "lindera", - "pagefind_stem", "rust-stemmers", "serde", "stop-words", @@ -5628,12 +5634,6 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" -[[package]] -name = "pagefind_stem" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8dfa810b158f3ac364e5acd43ca4a6020a6e729d40c15ce1bed1d911237a52e5" - [[package]] name = "parking" version = "2.2.1" @@ -6026,7 +6026,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" dependencies = [ "heck", - "itertools 0.11.0", + "itertools 0.14.0", "log", "multimap", "petgraph", @@ -6045,7 +6045,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.11.0", + "itertools 0.14.0", "proc-macro2", "quote", "syn 2.0.118", diff --git a/rust/lance-tokenizer/Cargo.toml b/rust/lance-tokenizer/Cargo.toml index 482d2dfab0a..355b300b8b8 100644 --- a/rust/lance-tokenizer/Cargo.toml +++ b/rust/lance-tokenizer/Cargo.toml @@ -15,7 +15,8 @@ rust-version.workspace = true icu_segmenter = { workspace = true } jieba-rs = { workspace = true, optional = true } lindera = { workspace = true, optional = true } -pagefind_stem = { version = "1.0.0", default-features = false, features = ["greek"] } +# Stemming output is persisted in indexes, so pin the generated Snowball revision. +frostem = { version = "=1.20260804.0", default-features = false, features = ["greek"] } rust-stemmers = "1.2.0" serde = { workspace = true, features = ["derive"] } stop-words = { version = "0.10.0", default-features = false, features = ["iso", "nltk"] } diff --git a/rust/lance-tokenizer/src/stemmer.rs b/rust/lance-tokenizer/src/stemmer.rs index 88a48fa7ec1..032f500b4ba 100644 --- a/rust/lance-tokenizer/src/stemmer.rs +++ b/rust/lance-tokenizer/src/stemmer.rs @@ -79,16 +79,14 @@ impl StemmerAlgorithm { Self::Legacy(algorithm) => { StemmerBackend::Legacy(rust_stemmers::Stemmer::create(algorithm)) } - Self::Greek => StemmerBackend::Greek(pagefind_stem::Stemmer::create( - pagefind_stem::Algorithm::Greek, - )), + Self::Greek => StemmerBackend::Greek(frostem::Stemmer::new(frostem::Algorithm::Greek)), } } } enum StemmerBackend { Legacy(rust_stemmers::Stemmer), - Greek(pagefind_stem::Stemmer), + Greek(frostem::Stemmer), } impl StemmerBackend { From e83ce437e4a5f1473f028a5f9498d2336bd950cc Mon Sep 17 00:00:00 2001 From: Lance Gatekeeper Date: Tue, 4 Aug 2026 07:05:59 +0000 Subject: [PATCH 7/8] fix: replace rust-stemmers with frostem --- Cargo.lock | 11 - java/lance-jni/Cargo.lock | 15 +- protos/index_old.proto | 13 -- python/Cargo.lock | 15 +- rust/lance-index/src/scalar/inverted.rs | 34 +--- rust/lance-index/src/scalar/inverted/index.rs | 99 +-------- .../src/scalar/inverted/tokenizer.rs | 190 +----------------- rust/lance-tokenizer/Cargo.toml | 22 +- rust/lance-tokenizer/src/stemmer.rs | 117 +++-------- rust/lance/src/dataset/mem_wal/index.rs | 12 +- .../src/dataset/mem_wal/memtable/flush.rs | 4 +- rust/lance/src/dataset/mem_wal/write.rs | 27 +-- rust/lance/src/index/append.rs | 172 ---------------- 13 files changed, 72 insertions(+), 659 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d52a40b12c9..36b2d605845 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5154,7 +5154,6 @@ dependencies = [ "icu_segmenter", "jieba-rs", "lindera", - "rust-stemmers", "serde", "stop-words", "unicode-normalization", @@ -7840,16 +7839,6 @@ dependencies = [ "ordered-multimap", ] -[[package]] -name = "rust-stemmers" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54" -dependencies = [ - "serde", - "serde_derive", -] - [[package]] name = "rustc-demangle" version = "0.1.27" diff --git a/java/lance-jni/Cargo.lock b/java/lance-jni/Cargo.lock index a79ebc62e51..5a605ad5711 100644 --- a/java/lance-jni/Cargo.lock +++ b/java/lance-jni/Cargo.lock @@ -4285,7 +4285,6 @@ version = "10.1.0-beta.2" dependencies = [ "frostem", "icu_segmenter", - "rust-stemmers", "serde", "stop-words", "unicode-normalization", @@ -5541,7 +5540,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" dependencies = [ "heck", - "itertools 0.14.0", + "itertools 0.11.0", "log", "multimap", "petgraph", @@ -5560,7 +5559,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.11.0", "proc-macro2", "quote", "syn 2.0.118", @@ -6242,16 +6241,6 @@ dependencies = [ "ordered-multimap", ] -[[package]] -name = "rust-stemmers" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54" -dependencies = [ - "serde", - "serde_derive", -] - [[package]] name = "rustc-hash" version = "2.1.3" diff --git a/protos/index_old.proto b/protos/index_old.proto index 34805892a09..bdba4ad0252 100644 --- a/protos/index_old.proto +++ b/protos/index_old.proto @@ -41,14 +41,6 @@ message ZoneMapIndexDetails { optional bool use_seeds = 2; } message InvertedIndexDetails { - enum GreekStemmer { - // Legacy rust-stemmers 1.2.0 semantics. An absent field also selects this - // value so indexes written before stemmer versioning remain compatible. - GREEK_STEMMER_LEGACY = 0; - // Snowball 3 Greek semantics, including corrected UTF-8 offset handling. - GREEK_STEMMER_SNOWBALL_3 = 1; - } - message CodeTokenizerConfig { // Split one lexical identifier into subwords, e.g. getUserName -> // get/user/name. @@ -92,9 +84,4 @@ message InvertedIndexDetails { // configuration used to build the index; absence means there is no // code-specific configuration to apply. CodeTokenizerConfig code_config = 13; - // Greek stemming semantics used to build this index. An absent value means - // the legacy rust-stemmers 1.2.0 implementation. The Snowball 3 value is - // present only when Greek stemming is active and requires inverted-index - // capability version 4 so older readers reject the segment. - optional GreekStemmer greek_stemmer = 14; } diff --git a/python/Cargo.lock b/python/Cargo.lock index c5ec394170e..43995772025 100644 --- a/python/Cargo.lock +++ b/python/Cargo.lock @@ -4579,7 +4579,6 @@ dependencies = [ "icu_segmenter", "jieba-rs", "lindera", - "rust-stemmers", "serde", "stop-words", "unicode-normalization", @@ -6026,7 +6025,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" dependencies = [ "heck", - "itertools 0.14.0", + "itertools 0.11.0", "log", "multimap", "petgraph", @@ -6045,7 +6044,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.11.0", "proc-macro2", "quote", "syn 2.0.118", @@ -6940,16 +6939,6 @@ dependencies = [ "ordered-multimap", ] -[[package]] -name = "rust-stemmers" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54" -dependencies = [ - "serde", - "serde_derive", -] - [[package]] name = "rustc-hash" version = "2.1.3" diff --git a/rust/lance-index/src/scalar/inverted.rs b/rust/lance-index/src/scalar/inverted.rs index f8ef01707ff..3554b155c98 100644 --- a/rust/lance-index/src/scalar/inverted.rs +++ b/rust/lance-index/src/scalar/inverted.rs @@ -157,7 +157,6 @@ impl InvertedIndexPlugin { params.validate_format_version()?; let format_version = params.resolved_format_version(); - let index_version = index::index_version_for_params(¶ms, format_version); let details = pbold::InvertedIndexDetails::try_from(¶ms)?; let mut inverted_index = InvertedIndexBuilder::new_with_fragment_mask(params, fragment_mask) @@ -165,7 +164,7 @@ impl InvertedIndexPlugin { let files = inverted_index.update(data, index_store, None).await?; Ok(CreatedIndex { index_details: prost_types::Any::from_msg(&details).unwrap(), - index_version, + index_version: format_version.index_version(), files, }) } @@ -277,7 +276,7 @@ impl ScalarIndexPlugin for InvertedIndexPlugin { } fn version(&self) -> u32 { - INVERTED_INDEX_VERSION_V4 + INVERTED_INDEX_VERSION_V3 } fn new_query_parser( @@ -329,34 +328,9 @@ mod tests { use crate::scalar::{BuiltinIndexType, ScalarIndexParams}; #[test] - fn test_plugin_version_tracks_v4_capability_gate() { + fn test_plugin_version_tracks_v3_capability_gate() { let plugin = InvertedIndexPlugin; - assert_eq!(plugin.version(), INVERTED_INDEX_VERSION_V4); - } - - #[test] - fn test_snowball_greek_uses_v4_capability_gate() { - let current = InvertedIndexParams::new("raw".to_string(), Language::Greek); - assert_eq!( - index::index_version_for_params(¤t, InvertedListFormatVersion::V2), - INVERTED_INDEX_VERSION_V4 - ); - - let mut legacy_json = serde_json::to_value(¤t).unwrap(); - legacy_json.as_object_mut().unwrap().remove("greek_stemmer"); - let legacy: InvertedIndexParams = serde_json::from_value(legacy_json).unwrap(); - assert_eq!( - index::index_version_for_params(&legacy, InvertedListFormatVersion::V2), - INVERTED_INDEX_VERSION_V2 - ); - - assert_eq!( - index::index_version_for_params( - &InvertedIndexParams::default(), - InvertedListFormatVersion::V2, - ), - INVERTED_INDEX_VERSION_V2 - ); + assert_eq!(plugin.version(), INVERTED_INDEX_VERSION_V3); } #[test] diff --git a/rust/lance-index/src/scalar/inverted/index.rs b/rust/lance-index/src/scalar/inverted/index.rs index 6d7b21ede40..2aa76bf1513 100644 --- a/rust/lance-index/src/scalar/inverted/index.rs +++ b/rust/lance-index/src/scalar/inverted/index.rs @@ -98,11 +98,9 @@ use std::str::FromStr; // Version 1: Fst TokenSetFormat with per-doc compressed positions // Version 2: Fst TokenSetFormat with shared posting-list position streams. // Version 3: Version 2 layout with configurable posting blocks and analyzer metadata. -// Version 4: Version 3 capabilities plus Snowball 3 Greek vocabulary semantics. pub const INVERTED_INDEX_VERSION_V1: u32 = 1; pub const INVERTED_INDEX_VERSION_V2: u32 = 2; pub const INVERTED_INDEX_VERSION_V3: u32 = 3; -pub const INVERTED_INDEX_VERSION_V4: u32 = 4; pub const TOKENS_FILE: &str = "tokens.lance"; pub const INVERT_LIST_FILE: &str = "invert.lance"; pub const DOCS_FILE: &str = "docs.lance"; @@ -171,20 +169,6 @@ pub fn resolve_fts_format_version( } } -/// Return the manifest capability version required by an inverted-index -/// analyzer and its physical posting-list format. -#[doc(hidden)] -pub fn index_version_for_params( - params: &InvertedIndexParams, - format_version: InvertedListFormatVersion, -) -> u32 { - if params.uses_snowball3_greek() { - INVERTED_INDEX_VERSION_V4 - } else { - format_version.index_version() - } -} - pub fn default_fts_format_version() -> InvertedListFormatVersion { InvertedListFormatVersion::V2 } @@ -750,9 +734,6 @@ impl InvertedIndex { } fn index_version(&self) -> u32 { - if self.params.uses_snowball3_greek() { - return INVERTED_INDEX_VERSION_V4; - } match (self.token_set_format, self.format_version()) { ( TokenSetFormat::Arrow, @@ -1756,8 +1737,7 @@ impl InvertedIndex { .metadata .get("params") .ok_or(Error::index("params not found in metadata".to_owned()))?; - Ok(serde_json::from_str::(params)? - .normalize_loaded_greek_stemmer()) + Ok(serde_json::from_str::(params)?) } Err(metadata_error) => { // Legacy format: params live in the tokens file (see @@ -1773,8 +1753,7 @@ impl InvertedIndex { .get("tokenizer") .map(|s| serde_json::from_str::(s)) .transpose()? - .unwrap_or_default() - .normalize_loaded_greek_stemmer()) + .unwrap_or_default()) } } } @@ -1798,8 +1777,7 @@ impl InvertedIndex { .metadata .get("params") .ok_or(Error::index("params not found in metadata".to_owned()))?; - let params = serde_json::from_str::(params)? - .normalize_loaded_greek_stemmer(); + let params = serde_json::from_str::(params)?; let partitions = reader .schema() .metadata @@ -8356,7 +8334,6 @@ mod tests { use std::sync::Arc; use std::sync::atomic::{AtomicU32, Ordering}; - use crate::scalar::inverted::tokenizer::GreekStemmerVersion; use crate::scalar::inverted::tokenizer::document_tokenizer::TextTokenizer; use lance_tokenizer::{Language, SimpleTokenizer, StopWordFilter, TextAnalyzer}; @@ -8575,76 +8552,6 @@ mod tests { InvertedIndex::load(store, None, &LanceCache::no_cache()).await } - #[tokio::test] - async fn test_legacy_greek_stemmer_load_query_and_update() -> Result<()> { - let src_dir = TempObjDir::default(); - let dest_dir = TempObjDir::default(); - let src_store = Arc::new(LanceIndexStore::new( - ObjectStore::local().into(), - src_dir.clone(), - Arc::new(LanceCache::no_cache()), - )); - let dest_store = Arc::new(LanceIndexStore::new( - ObjectStore::local().into(), - dest_dir.clone(), - Arc::new(LanceCache::no_cache()), - )); - - let current_params = InvertedIndexParams::new("raw".to_string(), Language::Greek) - .max_token_length(None) - .remove_stop_words(false) - .ascii_folding(false); - let mut legacy_json = serde_json::to_value(current_params)?; - legacy_json - .as_object_mut() - .expect("inverted index params should serialize to an object") - .remove("greek_stemmer"); - let legacy_params: InvertedIndexParams = serde_json::from_value(legacy_json)?; - assert_eq!(legacy_params.greek_stemmer, None); - - // rust-stemmers 1.2.0 produced an empty term for this word. The - // manually written term and missing version field model a persisted - // index created before Greek stemmer versioning. - let index = - write_single_partition_index(src_store, legacy_params, TokenSetFormat::Fst, "", 100) - .await?; - assert_eq!( - index.params.greek_stemmer, - Some(GreekStemmerVersion::Legacy) - ); - - let matches = index.do_search("ίσα").await?; - let row_ids = matches[ROW_ID].as_primitive::(); - assert_eq!(row_ids.values(), &[100]); - - let schema = Arc::new(Schema::new(vec![ - Field::new("doc", DataType::Utf8, true), - Field::new(ROW_ID, DataType::UInt64, false), - ])); - let docs = Arc::new(StringArray::from(vec![Some("ίσα")])); - let row_ids = Arc::new(UInt64Array::from(vec![101_u64])); - let batch = RecordBatch::try_new(schema.clone(), vec![docs, row_ids])?; - let stream = RecordBatchStreamAdapter::new(schema, stream::iter(vec![Ok(batch)])); - index - .update(Box::pin(stream), dest_store.as_ref(), None) - .await?; - - let updated = InvertedIndex::load(dest_store, None, &LanceCache::no_cache()).await?; - assert_eq!( - updated.params.greek_stemmer, - Some(GreekStemmerVersion::Legacy) - ); - let matches = updated.do_search("ίσα").await?; - let mut row_ids = matches[ROW_ID] - .as_primitive::() - .values() - .to_vec(); - row_ids.sort_unstable(); - assert_eq!(row_ids, vec![100, 101]); - - Ok(()) - } - fn empty_doc_stream() -> SendableRecordBatchStream { let schema = Arc::new(Schema::new(vec![ Field::new("doc", DataType::Utf8, true), diff --git a/rust/lance-index/src/scalar/inverted/tokenizer.rs b/rust/lance-index/src/scalar/inverted/tokenizer.rs index daa4cb0c2f9..5978621c462 100644 --- a/rust/lance-index/src/scalar/inverted/tokenizer.rs +++ b/rust/lance-index/src/scalar/inverted/tokenizer.rs @@ -44,39 +44,6 @@ pub const DEFAULT_BLOCK_SIZE: usize = 128; pub const VALID_BLOCK_SIZES: [usize; 2] = [128, 256]; const LANCE_FTS_FORMAT_VERSION_ENV_KEY: &str = "LANCE_FTS_FORMAT_VERSION"; -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub(crate) enum GreekStemmerVersion { - Legacy, - Snowball3, -} - -impl GreekStemmerVersion { - fn from_proto(value: Option) -> Result> { - use pbold::inverted_index_details::GreekStemmer; - - match value { - None => Ok(None), - Some(value) => match GreekStemmer::try_from(value) { - Ok(GreekStemmer::Legacy) => Ok(Some(Self::Legacy)), - Ok(GreekStemmer::Snowball3) => Ok(Some(Self::Snowball3)), - Err(_) => Err(Error::invalid_input(format!( - "unknown Greek stemmer version {value}" - ))), - }, - } - } - - fn to_proto(self) -> Option { - use pbold::inverted_index_details::GreekStemmer; - - match self { - Self::Legacy => None, - Self::Snowball3 => Some(GreekStemmer::Snowball3 as i32), - } - } -} - /// Tokenizer configs #[derive(Debug, Clone, Serialize, PartialEq)] pub struct InvertedIndexParams { @@ -110,13 +77,6 @@ pub struct InvertedIndexParams { /// this is only used when `stem` or `remove_stop_words` is true pub(crate) language: Language, - /// Greek stemming semantics persisted with the index vocabulary. - /// - /// Missing serialized values select legacy semantics. New indexes persist - /// [`GreekStemmerVersion::Snowball3`] only when Greek stemming is active. - #[serde(skip_serializing_if = "Option::is_none")] - pub(crate) greek_stemmer: Option, - /// If true, store the position of the term in the document /// This can significantly increase the size of the index /// If false, only store the frequency of the term in the document @@ -220,7 +180,6 @@ struct RawInvertedIndexParams { lance_tokenizer: Option, base_tokenizer: Option, language: Option, - greek_stemmer: Option, with_position: Option, #[serde(default, deserialize_with = "deserialize_explicit_option")] max_token_length: Option>, @@ -321,7 +280,6 @@ impl RawInvertedIndexParams { if let Some(language) = self.language { params.language = language; } - params.greek_stemmer = self.greek_stemmer; if let Some(with_position) = self.with_position { params.with_position = with_position; } @@ -400,9 +358,6 @@ impl TryFrom<&InvertedIndexParams> for pbold::InvertedIndexDetails { index_operators: params.index_operators, }, ), - greek_stemmer: params - .active_greek_stemmer() - .and_then(GreekStemmerVersion::to_proto), }) } } @@ -422,7 +377,6 @@ impl TryFrom<&pbold::InvertedIndexDetails> for InvertedIndexParams { Some(block_size) => validate_block_size(block_size as usize)?, None => LEGACY_BLOCK_SIZE, }, - greek_stemmer: None, ..Self::default() }; return Ok(params); @@ -442,7 +396,6 @@ impl TryFrom<&pbold::InvertedIndexDetails> for InvertedIndexParams { } else { serde_json::from_str(details.language.as_str())? }; - params.greek_stemmer = GreekStemmerVersion::from_proto(details.greek_stemmer)?; params.with_position = details.with_position; params.max_token_length = details.max_token_length.map(|l| l as usize); params.lower_case = details.lower_case; @@ -604,7 +557,6 @@ impl InvertedIndexParams { lance_tokenizer: None, base_tokenizer, language, - greek_stemmer: (language == Language::Greek).then_some(GreekStemmerVersion::Snowball3), with_position: false, max_token_length: Some(40), lower_case: true, @@ -640,7 +592,6 @@ impl InvertedIndexParams { self.stem = true; self.remove_stop_words = true; self.index_operators = false; - self.select_current_greek_stemmer(); } fn apply_code_defaults(&mut self) { @@ -653,7 +604,6 @@ impl InvertedIndexParams { self.stem = false; self.remove_stop_words = false; self.index_operators = false; - self.select_current_greek_stemmer(); } /// Create parameters for the code analyzer profile. @@ -726,7 +676,6 @@ impl InvertedIndexParams { // need to convert to valid JSON string let language = serde_json::from_str(format!("\"{}\"", language).as_str())?; self.language = language; - self.select_current_greek_stemmer(); Ok(self) } @@ -757,7 +706,6 @@ impl InvertedIndexParams { pub fn stem(mut self, stem: bool) -> Self { self.stem = stem; - self.select_current_greek_stemmer(); self } @@ -874,9 +822,8 @@ impl InvertedIndexParams { /// If unset, new index creation falls back to /// `LANCE_FTS_FORMAT_VERSION`. Without either override, text analysis with /// 128-document blocks writes v2, while code analysis or 256-document blocks - /// write v3. Snowball 3 Greek stemming also requires the v3 physical - /// format and advertises manifest capability version 4. Existing indexes - /// keep their own format during update and optimize operations. + /// write v3. Existing indexes keep their own format during update and + /// optimize operations. pub fn format_version(mut self, format_version: InvertedListFormatVersion) -> Self { self.format_version = Some(format_version); self @@ -886,7 +833,7 @@ impl InvertedIndexParams { /// the configured analyzer and block size. pub fn resolved_format_version(&self) -> InvertedListFormatVersion { self.format_version.unwrap_or_else(|| { - if self.base_tokenizer == "code" || self.uses_snowball3_greek() { + if self.base_tokenizer == "code" { InvertedListFormatVersion::V3 } else { default_fts_format_version_for_block_size(self.block_size) @@ -900,12 +847,6 @@ impl InvertedIndexParams { pub fn validate_format_version(&self) -> Result<()> { let format_version = self.resolved_format_version(); validate_format_version_block_size(format_version, self.block_size)?; - if self.uses_snowball3_greek() && format_version != InvertedListFormatVersion::V3 { - return Err(Error::invalid_input(format!( - "Snowball 3 Greek stemming requires FTS format_version=3, got {}", - format_version.index_version() - ))); - } if self.base_tokenizer == "code" && format_version != InvertedListFormatVersion::V3 { return Err(Error::invalid_input(format!( "base_tokenizer='code' requires FTS format_version=3, got {}", @@ -921,12 +862,6 @@ impl InvertedIndexParams { let object = value .as_object_mut() .expect("inverted index params should serialize to a JSON object"); - if self.language == Language::Greek && self.stem && self.greek_stemmer.is_none() { - object.insert( - "greek_stemmer".to_string(), - serde_json::Value::from("legacy"), - ); - } if let Some(memory_limit_mb) = self.memory_limit_mb { object.insert( "memory_limit".to_string(), @@ -957,20 +892,12 @@ impl InvertedIndexParams { let supplied = supplied.as_object().ok_or_else(|| { Error::invalid_input("FTS inverted index params must be a JSON object".to_string()) })?; - let has_explicit_greek_stemmer = supplied.contains_key("greek_stemmer"); let object = value .as_object_mut() .expect("inverted index params should serialize to a JSON object"); object.extend(supplied.clone()); let mut params: Self = serde_json::from_value(value)?; - if params.language == Language::Greek && params.stem { - if !has_explicit_greek_stemmer { - params.greek_stemmer = Some(GreekStemmerVersion::Snowball3); - } - } else { - params.greek_stemmer = None; - } let default_format_version = params.resolved_format_version(); params.format_version = Some(resolve_creation_format_version( params.format_version, @@ -990,11 +917,7 @@ impl InvertedIndexParams { builder = builder.filter_dynamic(LowerCaser); } if self.stem { - let stemmer = match self.active_greek_stemmer() { - Some(GreekStemmerVersion::Snowball3) => Stemmer::new(self.language), - None | Some(GreekStemmerVersion::Legacy) => Stemmer::new_legacy(self.language), - }; - builder = builder.filter_dynamic(stemmer); + builder = builder.filter_dynamic(Stemmer::new(self.language)); } if self.remove_stop_words { builder = builder.filter_dynamic(self.stop_word_filter()?); @@ -1032,11 +955,6 @@ impl InvertedIndexParams { fn validate(&self) -> Result<()> { validate_block_size(self.block_size)?; - if self.greek_stemmer.is_some() && (self.language != Language::Greek || !self.stem) { - return Err(Error::invalid_input( - "greek_stemmer requires language='Greek' and stem=true".to_string(), - )); - } if self.base_tokenizer != "code" && (self.split_identifiers || self.split_on_numerics @@ -1050,30 +968,6 @@ impl InvertedIndexParams { Ok(()) } - fn select_current_greek_stemmer(&mut self) { - self.greek_stemmer = (self.language == Language::Greek && self.stem) - .then_some(GreekStemmerVersion::Snowball3); - } - - fn active_greek_stemmer(&self) -> Option { - (self.language == Language::Greek && self.stem) - .then_some(self.greek_stemmer) - .flatten() - } - - pub(crate) fn uses_snowball3_greek(&self) -> bool { - self.active_greek_stemmer() == Some(GreekStemmerVersion::Snowball3) - } - - /// Normalize historical field-absent physical metadata to the explicit - /// internal hint used when rebuilding or merging legacy Greek segments. - pub(crate) fn normalize_loaded_greek_stemmer(mut self) -> Self { - if self.language == Language::Greek && self.stem && self.greek_stemmer.is_none() { - self.greek_stemmer = Some(GreekStemmerVersion::Legacy); - } - self - } - fn build_base_tokenizer(&self) -> Result { match self.base_tokenizer.as_str() { "simple" => Ok(TextAnalyzer::builder(SimpleTokenizer::default()).dynamic()), @@ -1147,7 +1041,7 @@ pub fn language_model_home() -> Option { mod tests { use crate::pbold; - use super::{GreekStemmerVersion, InvertedIndexParams, InvertedListFormatVersion, Language}; + use super::{InvertedIndexParams, InvertedListFormatVersion, Language}; use lance_core::Error; use lance_tokenizer::TokenStream; use rstest::rstest; @@ -1530,79 +1424,6 @@ mod tests { assert_eq!(params.block_size, 128); } - #[test] - fn test_greek_stemmer_version_metadata() { - let current = InvertedIndexParams::new("raw".to_string(), Language::Greek); - let mut json = serde_json::to_value(¤t).unwrap(); - assert_eq!(json["greek_stemmer"], "snowball3"); - - let english = InvertedIndexParams::default(); - assert!( - serde_json::to_value(&english).unwrap()["greek_stemmer"].is_null(), - "non-Greek analyzers should not persist a Greek stemmer version" - ); - let stemming_disabled = current.clone().stem(false); - assert!( - serde_json::to_value(&stemming_disabled).unwrap()["greek_stemmer"].is_null(), - "a disabled Greek stemmer should not persist a version" - ); - assert!( - pbold::InvertedIndexDetails::try_from(&stemming_disabled) - .unwrap() - .greek_stemmer - .is_none() - ); - - json.as_object_mut().unwrap().remove("greek_stemmer"); - let legacy: InvertedIndexParams = serde_json::from_value(json).unwrap(); - assert_eq!(legacy.greek_stemmer, None); - assert!( - serde_json::to_value(&legacy).unwrap()["greek_stemmer"].is_null(), - "legacy metadata should remain absent when rewritten" - ); - - let current_details = pbold::InvertedIndexDetails::try_from(¤t).unwrap(); - assert!(current_details.greek_stemmer.is_some()); - assert_eq!( - InvertedIndexParams::try_from(¤t_details) - .unwrap() - .greek_stemmer, - Some(GreekStemmerVersion::Snowball3) - ); - - let mut legacy_details = current_details; - legacy_details.greek_stemmer = None; - assert_eq!( - InvertedIndexParams::try_from(&legacy_details) - .unwrap() - .greek_stemmer, - None - ); - - let legacy_training_json = legacy.to_training_json().unwrap(); - assert_eq!(legacy_training_json["greek_stemmer"], "legacy"); - let legacy_training = InvertedIndexParams::from_training_json( - &serde_json::to_string(&legacy_training_json).unwrap(), - ) - .unwrap(); - assert_eq!( - legacy_training.greek_stemmer, - Some(GreekStemmerVersion::Legacy) - ); - assert_eq!( - pbold::InvertedIndexDetails::try_from(&legacy_training).unwrap(), - legacy_details, - "the internal legacy training hint should retain absent canonical metadata" - ); - - let current_training = - InvertedIndexParams::from_training_json(r#"{"language":"Greek"}"#).unwrap(); - assert_eq!( - current_training.greek_stemmer, - Some(GreekStemmerVersion::Snowball3) - ); - } - #[test] fn test_block_size_details_conversion() { let params = InvertedIndexParams::default().block_size(256).unwrap(); @@ -1623,7 +1444,6 @@ mod tests { prefix_only: false, block_size: None, code_config: None, - greek_stemmer: None, }; let params = InvertedIndexParams::try_from(&old_details).unwrap(); assert_eq!(params.block_size, 128); diff --git a/rust/lance-tokenizer/Cargo.toml b/rust/lance-tokenizer/Cargo.toml index 355b300b8b8..0dfb7a4b102 100644 --- a/rust/lance-tokenizer/Cargo.toml +++ b/rust/lance-tokenizer/Cargo.toml @@ -16,8 +16,26 @@ icu_segmenter = { workspace = true } jieba-rs = { workspace = true, optional = true } lindera = { workspace = true, optional = true } # Stemming output is persisted in indexes, so pin the generated Snowball revision. -frostem = { version = "=1.20260804.0", default-features = false, features = ["greek"] } -rust-stemmers = "1.2.0" +frostem = { version = "=1.20260804.0", default-features = false, features = [ + "arabic", + "danish", + "dutch", + "english", + "finnish", + "french", + "german", + "greek", + "hungarian", + "italian", + "norwegian", + "portuguese", + "romanian", + "russian", + "spanish", + "swedish", + "tamil", + "turkish", +] } serde = { workspace = true, features = ["derive"] } stop-words = { version = "0.10.0", default-features = false, features = ["iso", "nltk"] } unicode-normalization = "0.1.25" diff --git a/rust/lance-tokenizer/src/stemmer.rs b/rust/lance-tokenizer/src/stemmer.rs index 032f500b4ba..a79dd20d5f5 100644 --- a/rust/lance-tokenizer/src/stemmer.rs +++ b/rust/lance-tokenizer/src/stemmer.rs @@ -7,6 +7,7 @@ use std::borrow::Cow; use std::mem; +use frostem::Algorithm; use serde::{Deserialize, Serialize}; use crate::{Token, TokenFilter, TokenStream, Tokenizer}; @@ -34,73 +35,33 @@ pub enum Language { } impl Language { - fn algorithm(self) -> StemmerAlgorithm { + fn algorithm(self) -> Algorithm { match self { - Self::Greek => StemmerAlgorithm::Greek, - language => StemmerAlgorithm::Legacy(language.legacy_algorithm()), - } - } - - fn legacy_algorithm(self) -> rust_stemmers::Algorithm { - match self { - Self::Arabic => rust_stemmers::Algorithm::Arabic, - Self::Danish => rust_stemmers::Algorithm::Danish, - Self::Dutch => rust_stemmers::Algorithm::Dutch, - Self::English => rust_stemmers::Algorithm::English, - Self::Finnish => rust_stemmers::Algorithm::Finnish, - Self::French => rust_stemmers::Algorithm::French, - Self::German => rust_stemmers::Algorithm::German, - Self::Greek => rust_stemmers::Algorithm::Greek, - Self::Hungarian => rust_stemmers::Algorithm::Hungarian, - Self::Italian => rust_stemmers::Algorithm::Italian, - Self::Norwegian => rust_stemmers::Algorithm::Norwegian, - Self::Portuguese => rust_stemmers::Algorithm::Portuguese, - Self::Romanian => rust_stemmers::Algorithm::Romanian, - Self::Russian => rust_stemmers::Algorithm::Russian, - Self::Spanish => rust_stemmers::Algorithm::Spanish, - Self::Swedish => rust_stemmers::Algorithm::Swedish, - Self::Tamil => rust_stemmers::Algorithm::Tamil, - Self::Turkish => rust_stemmers::Algorithm::Turkish, - } - } -} - -#[derive(Copy, Clone)] -enum StemmerAlgorithm { - Legacy(rust_stemmers::Algorithm), - // The legacy generated Greek algorithm can retain stale UTF-8 byte offsets - // after shortening a word and panic when it slices the resulting stem. - Greek, -} - -impl StemmerAlgorithm { - fn create(self) -> StemmerBackend { - match self { - Self::Legacy(algorithm) => { - StemmerBackend::Legacy(rust_stemmers::Stemmer::create(algorithm)) - } - Self::Greek => StemmerBackend::Greek(frostem::Stemmer::new(frostem::Algorithm::Greek)), - } - } -} - -enum StemmerBackend { - Legacy(rust_stemmers::Stemmer), - Greek(frostem::Stemmer), -} - -impl StemmerBackend { - fn stem<'a>(&self, input: &'a str) -> Cow<'a, str> { - match self { - Self::Legacy(stemmer) => stemmer.stem(input), - Self::Greek(stemmer) => stemmer.stem(input), + Self::Arabic => Algorithm::Arabic, + Self::Danish => Algorithm::Danish, + Self::Dutch => Algorithm::Dutch, + Self::English => Algorithm::English, + Self::Finnish => Algorithm::Finnish, + Self::French => Algorithm::French, + Self::German => Algorithm::German, + Self::Greek => Algorithm::Greek, + Self::Hungarian => Algorithm::Hungarian, + Self::Italian => Algorithm::Italian, + Self::Norwegian => Algorithm::Norwegian, + Self::Portuguese => Algorithm::Portuguese, + Self::Romanian => Algorithm::Romanian, + Self::Russian => Algorithm::Russian, + Self::Spanish => Algorithm::Spanish, + Self::Swedish => Algorithm::Swedish, + Self::Tamil => Algorithm::Tamil, + Self::Turkish => Algorithm::Turkish, } } } #[derive(Clone)] pub struct Stemmer { - stemmer_algorithm: StemmerAlgorithm, + stemmer_algorithm: Algorithm, } impl Stemmer { @@ -109,18 +70,6 @@ impl Stemmer { stemmer_algorithm: language.algorithm(), } } - - /// Create a stemmer with the semantics used by indexes written before the - /// corrected Greek stemmer was introduced. - /// - /// This is only intended for reading and incrementally updating persisted - /// index metadata that does not identify its Greek stemmer version. - #[doc(hidden)] - pub fn new_legacy(language: Language) -> Self { - Self { - stemmer_algorithm: StemmerAlgorithm::Legacy(language.legacy_algorithm()), - } - } } impl Default for Stemmer { @@ -142,7 +91,7 @@ impl TokenFilter for Stemmer { #[derive(Clone)] pub struct StemmerFilter { - stemmer_algorithm: StemmerAlgorithm, + stemmer_algorithm: Algorithm, inner: T, } @@ -152,7 +101,7 @@ impl Tokenizer for StemmerFilter { fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> { StemmerTokenStream { tail: self.inner.token_stream(text), - stemmer: self.stemmer_algorithm.create(), + stemmer: frostem::Stemmer::new(self.stemmer_algorithm), buffer: String::new(), } } @@ -160,7 +109,7 @@ impl Tokenizer for StemmerFilter { pub struct StemmerTokenStream { tail: T, - stemmer: StemmerBackend, + stemmer: frostem::Stemmer, buffer: String, } @@ -205,22 +154,4 @@ mod tests { assert!(stream.advance()); assert_eq!(stream.token().text, "ανετ"); } - - #[test] - fn test_legacy_greek_stemmer_preserves_existing_terms() { - let mut legacy = TextAnalyzer::builder(RawTokenizer::default()) - .filter(Stemmer::new_legacy(Language::Greek)) - .build(); - let mut current = TextAnalyzer::builder(RawTokenizer::default()) - .filter(Stemmer::new(Language::Greek)) - .build(); - - let mut legacy_stream = legacy.token_stream("ίσα"); - assert!(legacy_stream.advance()); - assert_eq!(legacy_stream.token().text, ""); - - let mut current_stream = current.token_stream("ίσα"); - assert!(current_stream.advance()); - assert_eq!(current_stream.token().text, "ισ"); - } } diff --git a/rust/lance/src/dataset/mem_wal/index.rs b/rust/lance/src/dataset/mem_wal/index.rs index c2261618bf4..b3da5b5b67e 100644 --- a/rust/lance/src/dataset/mem_wal/index.rs +++ b/rust/lance/src/dataset/mem_wal/index.rs @@ -35,7 +35,7 @@ use lance_core::datatypes::Schema as LanceSchema; use lance_core::{Error, Result}; use lance_index::pbold; use lance_index::scalar::InvertedIndexParams; -use lance_index::scalar::inverted::{INVERTED_INDEX_VERSION_V4, InvertedListFormatVersion}; +use lance_index::scalar::inverted::InvertedListFormatVersion; use lance_index::vector::hnsw::builder::HnswBuildParams; use lance_linalg::distance::DistanceType; use lance_table::format::IndexMetadata; @@ -388,11 +388,8 @@ impl MemIndexConfig { 0 | 1 => Ok(InvertedListFormatVersion::V1), 2 => Ok(InvertedListFormatVersion::V2), 3 => Ok(InvertedListFormatVersion::V3), - version if version == INVERTED_INDEX_VERSION_V4 as i32 => { - Ok(InvertedListFormatVersion::V3) - } version => Err(Error::invalid_input(format!( - "FTS index '{}' has unsupported index_version {}; expected 0, 1, 2, 3, or 4", + "FTS index '{}' has unsupported index_version {}; expected 0, 1, 2, or 3", index_meta.name, version ))), } @@ -1571,7 +1568,6 @@ mod tests { (1, InvertedListFormatVersion::V1), (2, InvertedListFormatVersion::V2), (3, InvertedListFormatVersion::V3), - (4, InvertedListFormatVersion::V3), ] { let config = MemIndexConfig::fts_from_metadata(&fts_index_metadata(index_version), &schema) @@ -1594,9 +1590,9 @@ mod tests { let arrow_schema = create_test_schema(); let schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap(); - let err = MemIndexConfig::fts_from_metadata(&fts_index_metadata(5), &schema).unwrap_err(); + let err = MemIndexConfig::fts_from_metadata(&fts_index_metadata(4), &schema).unwrap_err(); assert!( - err.to_string().contains("unsupported index_version 5"), + err.to_string().contains("unsupported index_version 4"), "{err}" ); } diff --git a/rust/lance/src/dataset/mem_wal/memtable/flush.rs b/rust/lance/src/dataset/mem_wal/memtable/flush.rs index a76ccd19bd8..1bb2cbae039 100644 --- a/rust/lance/src/dataset/mem_wal/memtable/flush.rs +++ b/rust/lance/src/dataset/mem_wal/memtable/flush.rs @@ -13,7 +13,6 @@ use lance_core::utils::deletion::DeletionVector; use lance_core::{Error, Result}; use lance_index::IndexType; use lance_index::mem_wal::{ShardManifest, SsTable}; -use lance_index::scalar::inverted::index_version_for_params; use lance_index::scalar::{IndexStore, ScalarIndexParams}; use lance_io::object_store::{ObjectStore, ObjectStoreParams}; use lance_table::format::IndexMetadata; @@ -776,7 +775,6 @@ impl MemTableFlusher { let fragment_ids: roaring::RoaringBitmap = dataset.fragment_bitmap.as_ref().clone(); let format_version = fts_cfg.params.resolved_format_version(); - let index_version = index_version_for_params(&fts_cfg.params, format_version); let index_meta = IndexMetadata { uuid: index_uuid, @@ -785,7 +783,7 @@ impl MemTableFlusher { dataset_version: dataset.version().version, fragment_bitmap: Some(fragment_ids), index_details: Some(Arc::new(index_details)), - index_version: index_version as i32, + index_version: format_version.index_version() as i32, created_at: None, base_id: None, files: None, diff --git a/rust/lance/src/dataset/mem_wal/write.rs b/rust/lance/src/dataset/mem_wal/write.rs index 79cb417b4cf..7f05cbfff3e 100644 --- a/rust/lance/src/dataset/mem_wal/write.rs +++ b/rust/lance/src/dataset/mem_wal/write.rs @@ -7527,19 +7527,8 @@ mod shard_writer_tests { writer.close().await.unwrap(); } - #[rstest::rstest] - #[case::v1( - lance_tokenizer::Language::English, - Some(InvertedListFormatVersion::V1), - 1 - )] - #[case::snowball3_greek(lance_tokenizer::Language::Greek, None, 4)] #[tokio::test] - async fn test_mem_wal_maintained_fts_flush_preserves_version( - #[case] language: lance_tokenizer::Language, - #[case] format_version: Option, - #[case] expected_index_version: i32, - ) { + async fn test_mem_wal_maintained_fts_v1_flush_preserves_format() { use tempfile::TempDir; let vector_dim = 32; @@ -7553,10 +7542,8 @@ mod shard_writer_tests { .await .expect("Failed to create dataset"); - let mut fts_params = InvertedIndexParams::new("simple".to_string(), language); - if let Some(format_version) = format_version { - fts_params = fts_params.format_version(format_version); - } + let fts_params = + InvertedIndexParams::default().format_version(InvertedListFormatVersion::V1); dataset .create_index( &["text"], @@ -7566,10 +7553,10 @@ mod shard_writer_tests { false, ) .await - .expect("Failed to create FTS index"); + .expect("Failed to create v1 FTS index"); let base_indices = dataset.load_indices().await.unwrap(); assert_eq!(base_indices.len(), 1); - assert_eq!(base_indices[0].index_version, expected_index_version); + assert_eq!(base_indices[0].index_version, 1); dataset .initialize_mem_wal() @@ -7611,8 +7598,8 @@ mod shard_writer_tests { assert_eq!(sstable_indices.len(), 1); assert_eq!(sstable_indices[0].name, "text_fts"); assert_eq!( - sstable_indices[0].index_version, expected_index_version, - "maintained FTS index capability version changed during flush" + sstable_indices[0].index_version, 1, + "maintained v1 FTS index must flush as v1" ); let results = sstable diff --git a/rust/lance/src/index/append.rs b/rust/lance/src/index/append.rs index 9e02f2ade48..b6fc5a786d9 100644 --- a/rust/lance/src/index/append.rs +++ b/rust/lance/src/index/append.rs @@ -1971,178 +1971,6 @@ mod tests { ); } - #[tokio::test] - async fn test_append_legacy_greek_fts_preserves_canonical_details() { - use lance_index::pbold::InvertedIndexDetails; - use lance_index::scalar::inverted::{InvertedIndexParams, Language, METADATA_FILE}; - use lance_index::scalar::{FullTextSearchQuery, IndexStore}; - use prost::Message; - - const INDEX_NAME: &str = "body_idx"; - - let test_dir = TempStrDir::default(); - let schema = Arc::new(Schema::new(vec![Field::new("body", DataType::Utf8, false)])); - let initial_batch = RecordBatch::try_new( - schema.clone(), - vec![Arc::new(StringArray::from(vec!["ίσα"]))], - ) - .unwrap(); - let mut dataset = Dataset::write( - RecordBatchIterator::new(vec![Ok(initial_batch)], schema.clone()), - test_dir.as_str(), - None, - ) - .await - .unwrap(); - - let current_params = InvertedIndexParams::new("raw".to_string(), Language::Greek) - .lance_tokenizer("text".to_string()) - .max_token_length(None) - .remove_stop_words(false) - .ascii_folding(false); - let mut legacy_json = serde_json::to_value(current_params).unwrap(); - legacy_json - .as_object_mut() - .expect("inverted index params should serialize to an object") - .remove("greek_stemmer"); - let legacy_params: InvertedIndexParams = serde_json::from_value(legacy_json).unwrap(); - - let fragment_ids = dataset - .get_fragments() - .into_iter() - .map(|fragment| fragment.id() as u32) - .collect(); - let legacy_segment = - CreateIndexBuilder::new(&mut dataset, &["body"], IndexType::Inverted, &legacy_params) - .name(INDEX_NAME.to_string()) - .fragments(fragment_ids) - .execute_uncommitted() - .await - .unwrap(); - dataset - .commit_existing_index_segments(INDEX_NAME, "body", vec![legacy_segment]) - .await - .unwrap(); - - // Model a released segment whose physical metadata predates the Greek - // stemmer field. Preserve the JSON byte length so the committed file - // size remains valid after rewriting the metadata file. - let committed = dataset.load_indices_by_name(INDEX_NAME).await.unwrap(); - let legacy_store = LanceIndexStore::from_dataset_for_existing(&dataset, &committed[0]) - .await - .unwrap(); - let metadata_reader = legacy_store.open_index_file(METADATA_FILE).await.unwrap(); - let mut physical_metadata = metadata_reader.schema().metadata.clone(); - let explicit_params = physical_metadata - .get("params") - .expect("physical metadata should contain params"); - assert!(explicit_params.contains("greek_stemmer")); - let mut absent_params = serde_json::to_string(&legacy_params).unwrap(); - assert!(!absent_params.contains("greek_stemmer")); - absent_params.extend(std::iter::repeat_n( - ' ', - explicit_params.len() - absent_params.len(), - )); - physical_metadata.insert("params".to_string(), absent_params); - - let metadata_batch = metadata_reader.read_range(0..1, None).await.unwrap(); - let metadata_schema = Arc::new(Schema::new(metadata_batch.schema().fields().clone())); - let metadata_batch = - RecordBatch::try_new(metadata_schema.clone(), metadata_batch.columns().to_vec()) - .unwrap(); - drop(metadata_reader); - let mut metadata_writer = legacy_store - .new_index_file(METADATA_FILE, metadata_schema) - .await - .unwrap(); - metadata_writer - .write_record_batch(metadata_batch) - .await - .unwrap(); - metadata_writer - .finish_with_metadata(physical_metadata) - .await - .unwrap(); - - dataset = DatasetBuilder::from_uri(test_dir.as_str()) - .load() - .await - .unwrap(); - - let appended_batch = RecordBatch::try_new( - schema.clone(), - vec![Arc::new(StringArray::from(vec!["ίσα"]))], - ) - .unwrap(); - dataset - .append( - RecordBatchIterator::new(vec![Ok(appended_batch)], schema), - None, - ) - .await - .unwrap(); - dataset - .optimize_indices(&OptimizeOptions::append()) - .await - .unwrap(); - - let mut dataset = DatasetBuilder::from_uri(test_dir.as_str()) - .load() - .await - .unwrap(); - let segments = dataset.load_indices_by_name(INDEX_NAME).await.unwrap(); - assert_eq!(segments.len(), 2); - let first_params = crate::index::scalar::load_segment_params(&dataset, &segments[0]) - .await - .unwrap(); - let second_params = crate::index::scalar::load_segment_params(&dataset, &segments[1]) - .await - .unwrap(); - assert_eq!(first_params, second_params); - for segment in &segments { - let details_any = segment - .index_details - .as_ref() - .expect("inverted segment should include index details"); - let details = InvertedIndexDetails::decode(details_any.value.as_slice()).unwrap(); - assert_eq!(details.greek_stemmer, None); - } - - let result = dataset - .scan() - .full_text_search(FullTextSearchQuery::new("ίσα".to_string())) - .unwrap() - .try_into_batch() - .await - .unwrap(); - assert_eq!(result.num_rows(), 2); - - dataset - .optimize_indices(&OptimizeOptions::merge(2)) - .await - .unwrap(); - let dataset = DatasetBuilder::from_uri(test_dir.as_str()) - .load() - .await - .unwrap(); - assert_eq!( - dataset - .load_indices_by_name(INDEX_NAME) - .await - .unwrap() - .len(), - 1 - ); - let result = dataset - .scan() - .full_text_search(FullTextSearchQuery::new("ίσα".to_string())) - .unwrap() - .try_into_batch() - .await - .unwrap(); - assert_eq!(result.num_rows(), 2); - } - #[tokio::test] async fn test_append_index() { const DIM: usize = 64; From 336734fb1a471db00aa879bd93162910b92b54bb Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Tue, 4 Aug 2026 15:23:05 +0800 Subject: [PATCH 8/8] Update Cargo.toml --- rust/lance-tokenizer/Cargo.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/rust/lance-tokenizer/Cargo.toml b/rust/lance-tokenizer/Cargo.toml index 0dfb7a4b102..ebb7b0060f8 100644 --- a/rust/lance-tokenizer/Cargo.toml +++ b/rust/lance-tokenizer/Cargo.toml @@ -15,8 +15,7 @@ rust-version.workspace = true icu_segmenter = { workspace = true } jieba-rs = { workspace = true, optional = true } lindera = { workspace = true, optional = true } -# Stemming output is persisted in indexes, so pin the generated Snowball revision. -frostem = { version = "=1.20260804.0", default-features = false, features = [ +frostem = { version = "1.20260804.0", default-features = false, features = [ "arabic", "danish", "dutch",