feat: Fix the ranking algorithm to sort only the needed documents

raptor-indexer/src/main.rs

@@ -31,6 +31,12 @@ where P: AsRef<Path>
     fs::set_permissions(&path, perms)
 }
 
+fn is_readonly<P>(path: P) -> io::Result<bool>
+where P: AsRef<Path>
+{
+    fs::metadata(&path).map(|m| m.permissions().readonly())
+}
+
 fn main() {
     let data = File::open("products.json_lines").unwrap();
     let data = BufReader::new(data);
@@ -54,6 +60,18 @@ fn main() {
         }
     };
 
+    let map_file = "map.fst";
+    let values_file = "values.vecs";
+
+    for file in &[map_file, values_file] {
+        match is_readonly(file) {
+            Ok(true) => panic!("the {:?} file is readonly, please make it writeable", file),
+            Err(ref e) if e.kind() == io::ErrorKind::NotFound => (),
+            Err(e) => panic!("{:?}", e),
+            _ => (),
+        }
+    }
+
     let mut builder = DocIndexMapBuilder::new();
     for line in data.lines() {
         let line = line.unwrap();
@@ -83,13 +101,13 @@ fn main() {
         }
     }
 
-    let map = File::create("map.fst").unwrap();
-    let values = File::create("values.vecs").unwrap();
+    let map = File::create(map_file).unwrap();
+    let values = File::create(values_file).unwrap();
 
     let (map, values) = builder.build(map, values).unwrap();
 
-    set_readonly("map.fst", true).unwrap();
-    set_readonly("values.vecs", true).unwrap();
+    set_readonly(map_file, true).unwrap();
+    set_readonly(values_file, true).unwrap();
 
     println!("Checking the dump consistency...");
     unsafe { DocIndexMap::from_paths("map.fst", "values.vecs").unwrap() };

raptor-search/src/main.rs

@@ -3,8 +3,7 @@ extern crate fst;
 extern crate raptor;
 extern crate elapsed;
 
-use std::{env, fs};
-use std::process::Command;
+use std::env;
 use std::io::{self, Write};
 use elapsed::measure_time;
 use fst::Streamer;
@@ -21,7 +20,8 @@ fn search(map: &DocIndexMap, lev_builder: &LevBuilder, query: &str) {
     while let Some(document_id) = stream.next() {
         print!("{:?}", document_id);
 
-        /* only here to debug !
+        // /* only here to debug !
+        use std::{fs, process::Command};
         if let Ok(_) = fs::File::open("products.json_lines") {
             let output = Command::new("rg")
                                 .arg(document_id.to_string())

src/map.rs

@@ -86,7 +86,7 @@ impl<T> Values<T> {
         let cap = raw.iter().map(Vec::len).sum();
         let mut values = Vec::with_capacity(cap);
 
-        for v in &raw {
+        for mut v in &raw {
             let len = v.len() as u64;
             let start = ranges.last().map(|&Range { end, .. }| end).unwrap_or(0);