Merge pull request #3 from dib-lab/rust_ffi

Calling MQF from Rust

.gitignore

@@ -75,3 +75,6 @@ ThirdParty/stxxl
 
 .idea/*
 build/*
+
+Cargo.lock
+target

.travis.yml

@@ -5,22 +5,36 @@ compiler:
 - g++
 os:
 - linux
-
-
-before_install:
-  - sudo apt-get -qq update
-  - sudo apt-get install -y make automake autotools-dev cmake
-script:
-  - mkdir -p build
-  - cd build
-  - cmake ..
-  - make
-  - ctest
-  - gcov -n -o . src/gqf.cpp > /dev/null;
 branches:
   only:
   - mqfDevelopmenet
   - master
+cache:
+  directories:
+    - "$HOME/.cargo"
+    - "target"
 
-after_success:
-  - bash <(curl -s https://codecov.io/bash)
+jobs:
+  include:
+    - &test
+      stage: test
+      before_install:
+        - sudo apt-get -qq update
+        - sudo apt-get install -y make automake autotools-dev cmake
+      script:
+        - mkdir -p build
+        - cd build
+        - cmake ..
+        - make
+        - ctest
+        - gcov -n -o . src/gqf.cpp > /dev/null;
+      after_success:
+        - bash <(curl -s https://codecov.io/bash)
+    - <<: *test
+      name: Rust FFI bindings
+      language: rust
+      rust: stable
+      env:
+        - RUSTFLAGS="-Clink-arg=-fuse-ld=gold"
+      script:
+        - cargo test

Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "mqf"
+version = "0.1.0"
+authors = ["Luiz Irber <rust@luizirber.org>"]
+links = "libmqf"
+
+[build-dependencies]
+bindgen = "0.51"
+cmake = "0.1.42"
+
+[dev-dependencies]
+tempfile = "3.1.0"

build.rs

@@ -0,0 +1,65 @@
+use std::env;
+use std::path::PathBuf;
+
+extern crate cmake;
+use cmake::Config;
+
+fn main() {
+    let dst = Config::new(".")
+        .define("BUILD_STATIC_LIBS", "ON")
+        .build_target("MQF")
+        .build();
+
+    // TODO: there are probably better ways to do this...
+    let target = env::var("TARGET").unwrap();
+    if target.contains("apple") {
+        println!("cargo:rustc-link-lib=dylib=c++");
+    } else if target.contains("linux") {
+        println!("cargo:rustc-link-lib=dylib=stdc++");
+    } else {
+        unimplemented!();
+    }
+
+    println!("cargo:rustc-link-search=native={}/build/src", dst.display());
+    println!("cargo:rustc-link-lib=static=MQF");
+
+    println!(
+        "cargo:rustc-link-search=native={}/build/ThirdParty/stxxl/lib",
+        dst.display()
+    );
+
+    let mode = match env::var("PROFILE").unwrap().as_ref() {
+        "debug" => "_debug",
+        _ => "",
+    };
+    println!("cargo:rustc-link-lib=static=stxxl{}", mode);
+
+    let bindings = bindgen::Builder::default()
+        .clang_arg("-I./include")
+        .clang_arg("-x")
+        .clang_arg("c++")
+        .clang_arg("-std=c++11")
+        .header("include/gqf.h")
+        .whitelist_type("QF")
+        .whitelist_type("QFi")
+        .whitelist_function("qf_init")
+        .whitelist_function("qf_insert")
+        .whitelist_function("qf_count_key")
+        .whitelist_function("qf_destroy")
+        .whitelist_function("qf_copy")
+        .whitelist_function("qf_serialize")
+        .whitelist_function("qf_deserialize")
+        .whitelist_function("qf_migrate")
+        .whitelist_function("qf_iterator")
+        .whitelist_function("qfi_get")
+        .whitelist_function("qfi_next")
+        .whitelist_function("qfi_end")
+        .blacklist_type("std::*")
+        .generate()
+        .expect("Unable to generate bindings");
+
+    let out_path = PathBuf::from(env::var("OUT_DIR").unwrap());
+    bindings
+        .write_to_file(out_path.join("bindings.rs"))
+        .expect("couldn't write bindings!");
+}

include/gqf.h

@@ -109,7 +109,7 @@ extern "C" {
 
 	void qf_destroy(QF *qf);
 
-	void qf_copy(QF *dest, QF *src);
+	void qf_copy(QF *dest, const QF *src);
 
 	/*!
 	 	@breif Increment the counter for this item by count.

src/gqf.cpp

@@ -1982,7 +1982,7 @@ bool qf_getBlockLabel_pointer_byItem(const QF *qf, uint64_t key,char *&res){
 
 /* The caller should call qf_init on the dest QF before calling this function.
  */
-void qf_copy(QF *dest, QF *src)
+void qf_copy(QF *dest, const QF *src)
 {
 	memcpy(dest->mem, src->mem, sizeof(qfmem));
 	memcpy(dest->metadata, src->metadata, sizeof(qfmetadata));

src/lib.rs

@@ -0,0 +1,223 @@
+mod raw;
+
+use std::ffi::CString;
+use std::path::Path;
+use std::ptr;
+
+#[derive(Debug)]
+pub struct MQF {
+    inner: raw::QF,
+}
+
+impl Default for MQF {
+    fn default() -> MQF {
+        MQF {
+            inner: raw::QF {
+                mem: ptr::null_mut(),
+                metadata: ptr::null_mut(),
+                blocks: ptr::null_mut(),
+            },
+        }
+    }
+}
+
+impl Drop for MQF {
+    fn drop(&mut self) {
+        unsafe { raw::qf_destroy(&mut self.inner) };
+    }
+}
+
+impl Clone for MQF {
+    fn clone(&self) -> Self {
+        let mut new_qf = MQF::default();
+        unsafe {
+            raw::qf_copy(&mut new_qf.inner, &self.inner);
+        };
+        new_qf
+    }
+}
+
+unsafe impl Sync for MQF {}
+
+impl MQF {
+    pub fn new(counter_size: u64, qbits: u64) -> MQF {
+        let mut mqf = MQF::default();
+
+        let num_hash_bits = qbits + 8;
+
+        let s = CString::new("").unwrap();
+
+        unsafe {
+            raw::qf_init(
+                &mut mqf.inner,
+                1u64 << qbits, // nslots
+                num_hash_bits, // key_bits
+                0,             // label_bits
+                counter_size,  // fixed_counter_size
+                0,             // blocksLabelSize
+                true,          // mem
+                s.as_ptr(),    // path
+                2038074760,    // seed (doesn't matter)
+            );
+        };
+
+        mqf
+    }
+
+    pub fn insert(&mut self, key: u64, count: u64) {
+        unsafe { raw::qf_insert(&mut self.inner, key, count, true, true) };
+        //unsafe { raw::qf_insert(&mut self.inner, key, count, false, false) };
+    }
+
+    pub fn count_key(&self, key: u64) -> u64 {
+        unsafe { raw::qf_count_key(&self.inner, key) }
+    }
+
+    pub fn iter(&mut self) -> MQFIter {
+        let mut cfi = raw::QFi {
+            qf: ptr::null_mut(),
+            run: 0,
+            current: 0,
+            cur_start_index: 0,
+            cur_length: 0,
+            num_clusters: 0,
+            c_info: ptr::null_mut(),
+        };
+
+        // TODO: treat false
+        let _ = unsafe { raw::qf_iterator(&mut self.inner, &mut cfi, 0) };
+
+        MQFIter { inner: cfi }
+    }
+
+    pub fn serialize<P: AsRef<Path>>(&self, path: P) -> Result<(), Box<dyn std::error::Error>> {
+        let s = CString::new(path.as_ref().to_str().unwrap())?;
+
+        unsafe {
+            raw::qf_serialize(&self.inner, s.as_ptr());
+        }
+
+        Ok(())
+    }
+
+    pub fn deserialize<P: AsRef<Path>>(path: P) -> Result<MQF, Box<dyn std::error::Error>> {
+        let mut qf = MQF::default();
+        let s = CString::new(path.as_ref().to_str().unwrap())?;
+
+        unsafe {
+            raw::qf_deserialize(&mut qf.inner, s.as_ptr());
+        }
+
+        Ok(qf)
+    }
+
+    pub fn merge(&mut self, other: &mut MQF) -> Result<(), Box<dyn std::error::Error>> {
+        unsafe {
+            raw::qf_migrate(&mut other.inner, &mut self.inner);
+        }
+
+        Ok(())
+    }
+}
+
+pub struct MQFIter {
+    inner: raw::QFi,
+}
+
+impl Iterator for MQFIter {
+    type Item = (u64, u64, u64);
+
+    fn next(&mut self) -> Option<Self::Item> {
+        if unsafe { raw::qfi_end(&mut self.inner) } != 0 {
+            None
+        } else {
+            let mut key = 0;
+            let mut value = 0;
+            let mut count = 0;
+
+            unsafe {
+                raw::qfi_get(&mut self.inner, &mut key, &mut value, &mut count);
+                raw::qfi_next(&mut self.inner)
+            };
+            Some((key, value, count))
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn simple_counting_test_api() {
+        //except first item is inserted 5 times to full test _insert1
+        let counter_size = 2;
+        let qbits = 5;
+        let mut qf: MQF = MQF::new(counter_size, qbits);
+
+        let mut count = 0;
+        let mut fixed_counter = 0;
+
+        for i in 0..=10 {
+            qf.insert(100, 1);
+            count = qf.count_key(100);
+            dbg!((count, fixed_counter));
+            assert_eq!(count, 1 + i);
+        }
+
+        qf.insert(1500, 50);
+
+        count = qf.count_key(1500);
+        dbg!((count, fixed_counter));
+        assert_eq!(count, 50);
+
+        qf.insert(1600, 60);
+        count = qf.count_key(1600);
+        dbg!((count, fixed_counter));
+        assert_eq!(count, 60);
+
+        qf.insert(2000, 4000);
+        count = qf.count_key(2000);
+        dbg!((count, fixed_counter));
+        assert_eq!(count, 4000);
+    }
+
+    #[test]
+    fn big_count() {
+        let mut qf = MQF::new(4, 5);
+        qf.insert(100, 100000);
+        let mut count = qf.count_key(100);
+        assert_eq!(count, 100000);
+    }
+
+    #[test]
+    fn iter_next() {
+        let mut qf = MQF::new(4, 5);
+        qf.insert(100, 100000);
+        qf.insert(101, 10000);
+        qf.insert(102, 1000);
+        qf.insert(103, 100);
+
+        let vals: Vec<(u64, u64, u64)> = qf.iter().collect();
+        dbg!(&vals);
+        assert_eq!(vals.len(), 4);
+    }
+
+    #[test]
+    fn serde() {
+        let mut qf = MQF::new(4, 5);
+        qf.insert(100, 100000);
+        qf.insert(101, 10000);
+        qf.insert(102, 1000);
+        qf.insert(103, 100);
+
+        let vals: Vec<(u64, u64, u64)> = qf.iter().collect();
+
+        let mut file = tempfile::NamedTempFile::new().unwrap();
+        qf.serialize(file.path()).unwrap();
+
+        let mut new_qf = MQF::deserialize(file.path()).unwrap();
+        let new_vals: Vec<(u64, u64, u64)> = new_qf.iter().collect();
+        assert_eq!(vals, new_vals);
+    }
+}