Create indexes that store only one of the bands (not finished)

This allow to create a distributed index between different processes/machines.
ZJaume · Jun 12, 2023 · d0c54b3 · d0c54b3
1 parent 810b976
commit d0c54b3
Show file tree

Hide file tree

Showing 2 changed files with 21 additions and 9 deletions.
diff --git a/gaoya/src/minhash/minhash_index.rs b/gaoya/src/minhash/minhash_index.rs
@@ -340,7 +340,7 @@ impl<T, Id> MinHashIndex<T, Id>
 {
     /// Create a new MinHashIndex
     pub fn new(num_bands: usize, band_width: usize, jaccard_threshold: f64) -> Self {
-        MinHashIndex::<T, Id, HashSetContainer<Id>>::new_index(num_bands, band_width, jaccard_threshold)
+        MinHashIndex::<T, Id, HashSetContainer<Id>>::new_index(num_bands, band_width, jaccard_threshold, -1)
     }
 }
 
@@ -352,14 +352,24 @@ where
     C: IdContainer<Id>
 {
     /// Create a new MinHashIndex
-    pub fn new_index(num_bands: usize, band_width: usize, jaccard_threshold: f64) -> Self {
-        let build_hasher = RandomState::new();
+    pub fn new_index(num_bands: usize,
+                     band_width: usize,
+                     jaccard_threshold: f64,
+                     band_id: isize) -> Self {
+        let build_hasher = RandomState::with_seed(42);
         let mut bands = Vec::new();
-        for i in 0..num_bands {
-            let (start, end) = (i * band_width, (i + 1) * band_width);
+        if band_id < 0 {
+            for i in 0..num_bands {
+                let (start, end) = (i * band_width, (i + 1) * band_width);
+                bands.push(MinHashBand::<T, Id, C>::new(start, end, build_hasher.clone()));
+            }
+        } else {
+            // Index with only one of all possible bands for partitioned indexing
+            let uband_id: usize = band_id.try_into().unwrap();
+            let (start, end) = (uband_id * band_width, (uband_id + 1) * band_width);
             bands.push(MinHashBand::<T, Id, C>::new(start, end, build_hasher.clone()));
         }
-        let mut hash_table = HashMap::with_hasher(ahash::RandomState::new());
+        let mut hash_table = HashMap::with_hasher(ahash::RandomState::with_seed(42));
         hash_table.reserve(1000);
         MinHashIndex {
             bands,

diff --git a/py-gaoya/src/min_hash.rs b/py-gaoya/src/min_hash.rs
@@ -36,7 +36,8 @@ macro_rules! py_minhash_index {
                 num_hashes = "126",
                 analyzer = "\"word\"",
                 lowercase = "false",
-                ngram_range = "(1,1)"
+                ngram_range = "(1,1)",
+                band_id = "-1",
             )]
             pub fn new(
                 jaccard_threshold: f64,
@@ -46,10 +47,11 @@ macro_rules! py_minhash_index {
                 analyzer: Option<&str>,
                 lowercase: Option<bool>,
                 ngram_range: Option<(usize, usize)>,
+                band_id: Option<isize>,
             ) -> PyResult<Self> {
                 if let (Some(num_bands), Some(band_width)) = (num_bands, band_width) {
                     let index = $name {
-                        inner: gaoya::minhash::MinHashIndex::<_, _, $container_type>::new_index(num_bands, band_width, jaccard_threshold),
+                        inner: gaoya::minhash::MinHashIndex::<_, _, $container_type>::new_index(num_bands, band_width, jaccard_threshold, band_id.unwrap()),
                         min_hash: $minhash::new(num_bands * band_width),
                         tokenizer: TokenizerSpecification::new(analyzer.unwrap_or("word"), ngram_range),
                         lowercase: lowercase.unwrap_or(false),
@@ -59,7 +61,7 @@ macro_rules! py_minhash_index {
                 if let Some(num_hashes) = num_hashes {
                     let (num_bands, band_width) = calculate_minhash_params(jaccard_threshold, num_hashes);
                     let index = $name {
-                        inner: gaoya::minhash::MinHashIndex::<_,_, $container_type>::new_index(num_bands, band_width, jaccard_threshold),
+                        inner: gaoya::minhash::MinHashIndex::<_,_, $container_type>::new_index(num_bands, band_width, jaccard_threshold, band_id.unwrap()),
                         min_hash: $minhash::new(num_bands * band_width),
                         tokenizer: TokenizerSpecification::new(analyzer.unwrap_or("word"), ngram_range),
                         lowercase: lowercase.unwrap_or(false),