refactor: move mask code into new crate

.bumpversion.toml

@@ -95,6 +95,11 @@ filename = "Cargo.toml"
 search = 'lance-namespace-impls = {{ version = "={current_version}"'
 replace = 'lance-namespace-impls = {{ version = "={new_version}"'
 
+[[tool.bumpversion.files]]
+filename = "Cargo.toml"
+search = 'lance-select = {{ version = "={current_version}"'
+replace = 'lance-select = {{ version = "={new_version}"'
+
 [[tool.bumpversion.files]]
 filename = "Cargo.toml"
 search = 'lance-table = {{ version = "={current_version}"'

Cargo.toml

@@ -14,6 +14,7 @@ members = [
     "rust/lance-namespace",
     "rust/lance-namespace-impls",
     "rust/lance-namespace-datafusion",
+    "rust/lance-select",
     "rust/lance-tokenizer",
     "rust/lance-table",
     "rust/lance-test-macros",
@@ -70,6 +71,7 @@ lance-namespace = { version = "=7.1.0-beta.1", path = "./rust/lance-namespace" }
 lance-namespace-impls = { version = "=7.1.0-beta.1", path = "./rust/lance-namespace-impls" }
 lance-namespace-datafusion = { version = "=7.0.0-beta.9", path = "./rust/lance-namespace-datafusion" }
 lance-namespace-reqwest-client = "0.7.7"
+lance-select = { version = "=7.1.0-beta.1", path = "./rust/lance-select" }
 lance-tokenizer = { version = "=7.1.0-beta.1", path = "./rust/lance-tokenizer" }
 lance-table = { version = "=7.1.0-beta.1", path = "./rust/lance-table" }
 lance-test-macros = { version = "=7.1.0-beta.1", path = "./rust/lance-test-macros" }

rust/lance-core/Cargo.toml

@@ -48,16 +48,11 @@ log.workspace = true
 libc = { version = "0.2" }
 
 [dev-dependencies]
-criterion.workspace = true
 proptest.workspace = true
 rstest.workspace = true
 
 [features]
 datafusion = ["dep:datafusion-common", "dep:datafusion-sql"]
 
-[[bench]]
-name = "row_addr_mask"
-harness = false
-
 [lints]
 workspace = true

rust/lance-core/src/utils.rs

@@ -11,7 +11,6 @@ pub mod cpu;
 pub mod deletion;
 pub mod futures;
 pub mod hash;
-pub mod mask;
 pub mod parse;
 pub mod path;
 pub mod tempfile;

rust/lance-index/Cargo.toml

@@ -45,6 +45,7 @@ lance-file.workspace = true
 lance-geo = { workspace = true, optional = true }
 lance-io.workspace = true
 lance-linalg.workspace = true
+lance-select.workspace = true
 lance-tokenizer.workspace = true
 lance-table.workspace = true
 libm.workspace = true

rust/lance-index/src/frag_reuse.rs

@@ -8,8 +8,8 @@ use arrow_array::{Array, ArrayRef, PrimitiveArray, RecordBatch, UInt64Array};
 use async_trait::async_trait;
 use deepsize::{Context, DeepSizeOf};
 use itertools::Itertools;
-use lance_core::utils::mask::RowAddrTreeMap;
 use lance_core::{Error, Result};
+use lance_select::RowAddrTreeMap;
 use lance_table::format::pb::fragment_reuse_index_details::InlineContent;
 use lance_table::format::{ExternalFile, Fragment, pb};
 use roaring::{RoaringBitmap, RoaringTreemap};

rust/lance-index/src/prefilter.rs

@@ -5,7 +5,7 @@ use std::sync::Arc;
 
 use async_trait::async_trait;
 use lance_core::Result;
-use lance_core::utils::mask::RowAddrMask;
+use lance_select::RowAddrMask;
 
 /// A trait to be implemented by anything supplying a prefilter row addr mask
 ///

rust/lance-index/src/scalar.rs

@@ -21,9 +21,9 @@ use datafusion_expr::Expr;
 use datafusion_expr::expr::ScalarFunction;
 use deepsize::DeepSizeOf;
 use inverted::query::{FtsQuery, FtsQueryNode, FtsSearchParams, MatchQuery, fill_fts_query_column};
-use lance_core::utils::mask::{NullableRowAddrSet, RowAddrTreeMap, RowSetOps};
 use lance_core::{Error, Result};
 use lance_io::stream::{RecordBatchStream, RecordBatchStreamAdapter};
+use lance_select::{NullableRowAddrSet, RowAddrTreeMap, RowSetOps};
 use roaring::RoaringBitmap;
 use serde::Serialize;
 

rust/lance-index/src/scalar/bitmap.rs

@@ -23,17 +23,14 @@ use lance_arrow::ipc::{
     read_ipc_stream_single_at, read_len_prefixed_bytes_at, write_ipc_stream,
     write_len_prefixed_bytes,
 };
-use lance_core::utils::mask::RowSetOps;
 use lance_core::{
     Error, ROW_ID, Result,
     cache::{CacheCodec, CacheCodecImpl, CacheKey, LanceCache, WeakLanceCache},
     error::LanceOptionExt,
-    utils::{
-        mask::{NullableRowAddrSet, RowAddrTreeMap},
-        tokio::get_num_compute_intensive_cpus,
-    },
+    utils::tokio::get_num_compute_intensive_cpus,
 };
 use lance_io::object_store::ObjectStore;
+use lance_select::{NullableRowAddrSet, RowAddrTreeMap, RowSetOps};
 use object_store::path::Path;
 use roaring::RoaringBitmap;
 use serde::{Deserialize, Serialize};
@@ -1761,9 +1758,9 @@ mod tests {
     }
     use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
     use futures::stream;
-    use lance_core::utils::mask::RowSetOps;
     use lance_core::utils::{address::RowAddress, tempfile::TempObjDir};
     use lance_io::object_store::ObjectStore;
+    use lance_select::RowSetOps;
     use std::collections::HashMap;
 
     fn assert_state_roundtrips(state: &BitmapIndexState) {
@@ -1947,8 +1944,8 @@ mod tests {
         use arrow_schema::DataType;
         use datafusion_common::ScalarValue;
         use lance_core::cache::LanceCache;
-        use lance_core::utils::mask::RowAddrTreeMap;
         use lance_io::object_store::ObjectStore;
+        use lance_select::RowAddrTreeMap;
         use std::collections::HashMap;
         use std::sync::Arc;
 

rust/lance-index/src/scalar/bloomfilter.rs

@@ -1154,12 +1154,9 @@ mod tests {
     use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
     use datafusion_common::ScalarValue;
     use futures::{StreamExt, stream};
-    use lance_core::{
-        ROW_ADDR,
-        cache::LanceCache,
-        utils::{mask::RowAddrTreeMap, tempfile::TempObjDir},
-    };
+    use lance_core::{ROW_ADDR, cache::LanceCache, utils::tempfile::TempObjDir};
     use lance_io::object_store::ObjectStore;
+    use lance_select::RowAddrTreeMap;
 
     use crate::scalar::{
         BloomFilterQuery, ScalarIndex, SearchResult,

rust/lance-index/src/scalar/btree.rs

@@ -50,7 +50,6 @@ use lance_core::{
     cache::{CacheCodec, CacheCodecImpl, CacheKey, LanceCache, WeakLanceCache},
     error::LanceOptionExt,
     utils::{
-        mask::NullableRowAddrSet,
         tokio::get_num_compute_intensive_cpus,
         tracing::{IO_TYPE_LOAD_SCALAR_PART, TRACE_IO_EVENTS},
     },
@@ -60,6 +59,7 @@ use lance_datafusion::{
     exec::{LanceExecutionOptions, OneShotExec, execute_plan},
 };
 use lance_io::object_store::ObjectStore;
+use lance_select::NullableRowAddrSet;
 use log::{debug, warn};
 use object_store::{Error as ObjectStoreError, path::Path};
 use rangemap::RangeInclusiveMap;
@@ -2987,12 +2987,12 @@ mod tests {
     use deepsize::DeepSizeOf;
     use futures::TryStreamExt;
     use futures::stream;
-    use lance_core::utils::mask::RowSetOps;
+    use lance_core::cache::LanceCache;
     use lance_core::utils::tempfile::TempObjDir;
-    use lance_core::{cache::LanceCache, utils::mask::RowAddrTreeMap};
     use lance_datafusion::{chunker::break_stream, datagen::DatafusionDatagenExt};
     use lance_datagen::{ArrayGeneratorExt, BatchCount, RowCount, array, gen_batch};
     use lance_io::object_store::ObjectStore;
+    use lance_select::{RowAddrTreeMap, RowSetOps};
     use object_store::path::Path;
 
     use crate::metrics::LocalMetricsCollector;

rust/lance-index/src/scalar/btree/flat.rs

@@ -18,7 +18,7 @@ use lance_arrow::ipc::{read_ipc_stream_single_at, read_len_prefixed_bytes_at, wr
 use lance_core::Result;
 use lance_core::cache::CacheCodecImpl;
 use lance_core::utils::address::RowAddress;
-use lance_core::utils::mask::{NullableRowAddrSet, RowAddrTreeMap, RowSetOps};
+use lance_select::{NullableRowAddrSet, RowAddrTreeMap, RowSetOps};
 use roaring::RoaringBitmap;
 use tracing::instrument;