design base trait for ANN algorithms

rust/Cargo.toml

@@ -14,4 +14,10 @@
 # limitations under the License.
 #
 [workspace]
-members = ["libs/proto", "libs/ngt", "bin/agent"]
+members = [
+    "libs/proto",
+    "bin/agent",
+    "libs/algorithm",
+    "libs/kvs",
+    "libs/ngt-rs",
+]

rust/libs/algorithm/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "algorithm"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+anyhow = "1.0.79"
+serde = { version = "1.0.196", features = ["derive"] }
+ngt-rs = { path = "../ngt-rs" }
+num-traits = "0.2.18"

rust/libs/algorithm/src/base.rs

@@ -0,0 +1,56 @@
+pub mod algorithm1 {
+    use anyhow::Result;
+
+    pub trait Base<T, U, SP, SR, IP, IR, UP, UR, RP, RR, CP, CR, P>: Send + Sync
+    where
+        T: num::Num,
+        U: num::sign::Unsigned,
+    {
+        fn search(&self, v: &[T], p: Option<SP>) -> Result<SR>;
+        fn insert(&self, v: &[T], id: &U, p: Option<IP>) -> Result<IR>;
+        fn update(&self, v: &[T], id: &U, p: Option<UP>) -> Result<UR>;
+        fn remove(&self, id: &U, p: Option<RP>) -> Result<RR>;
+        fn commit(&self, p: Option<CP>) -> Result<CR>;
+    }
+}
+
+pub mod algorithm2 {
+    use anyhow::Result;
+
+    pub trait Search<T: num::Num, P, R>: Send + Sync {
+        fn search(&self, v: &[T], p: Option<P>) -> Result<R>;
+    }
+
+    pub trait Insert<T: num::Num, U: num::sign::Unsigned, P, R>: Send + Sync {
+        fn insert(&self, v: &[T], id: &U, p: Option<P>) -> Result<R>;
+    }
+
+    pub trait Update<T: num::Num, U: num::sign::Unsigned, P, R>: Send + Sync {
+        fn update(&self, v: &[T], id: &U, p: Option<P>) -> Result<R>;
+    }
+
+    pub trait Remove<U: num::sign::Unsigned, P, R>: Send + Sync {
+        fn remove(&self, id: &U, p: Option<P>) -> Result<R>;
+    }
+
+    pub trait Commit<P, R>: Send + Sync {
+        fn commit(&self, p: Option<P>) -> Result<R>;
+    }
+}
+
+// inspired by https://github.com/jonluca/hora
+pub mod algorithm3 {
+    use anyhow::Result;
+
+    pub trait Base<T, U, SP, SR, IP, UP, RP, CP>: Send + Sync
+    where
+        T: num::Num,
+        U: num::sign::Unsigned,
+    {
+        fn search(&self, v: &[T], p: Option<SP>) -> Result<Vec<SR>>;
+        fn insert(&mut self, v: &[T], id: &U, p: Option<IP>) -> Result<()>;
+        fn update(&mut self, v: &[T], id: &U, p: Option<UP>) -> Result<()>;
+        fn remove(&mut self, id: &U, p: Option<RP>) -> Result<()>;
+        fn commit(&mut self, p: Option<CP>) -> Result<()>;
+    }
+}

rust/libs/algorithm/src/lib.rs

@@ -0,0 +1,14 @@
+pub fn add(left: usize, right: usize) -> usize {
+    left + right
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn it_works() {
+        let result = add(2, 2);
+        assert_eq!(result, 4);
+    }
+}

rust/libs/algorithm/src/ngt.rs

@@ -0,0 +1,70 @@
+pub mod ngt {
+    use crate::algorithm::Base;
+    use anyhow::{anyhow, Ok, Result};
+
+    #[derive(Debug)]
+    struct Index {}
+
+    #[derive(Debug)]
+    pub struct NGT {
+        index: Index,
+    }
+    #[derive(Debug)]
+    pub struct SearchResult {
+        id: String,
+        distance: f32,
+    }
+
+    #[derive(Debug)]
+    pub struct SearchResponse {
+        results: Vec<SearchResult>,
+    }
+
+    #[derive(Debug)]
+    pub struct SearchParam {
+        epsilon: f32,
+        radius: f32,
+        num: u32,
+    }
+
+    #[derive(Debug)]
+    pub struct InsertParam {}
+
+    #[derive(Debug)]
+    pub struct UpdateParam {}
+
+    #[derive(Debug)]
+    pub struct RemoveParam {}
+
+    #[derive(Debug)]
+    pub struct CommitParam {}
+
+    #[derive(Debug)]
+    pub struct NGTParam {}
+
+    impl
+        Base<
+            f32,
+            String,
+            SearchParam,
+            SearchResponse,
+            InsertParam,
+            (),
+            UpdateParam,
+            (),
+            RemoveParam,
+            (),
+            CommitParam,
+            (),
+            NGTParam,
+        > for NGT
+    {
+        fn search(&self, v: &Vec<T>, p: Option<SearchParam>) -> Result<SearchResponse>;
+        fn insert(&self, v: &Vec<T>, id: &U, p: Option<InsertParam>) -> Result<()>;
+        fn update(&self, v: &Vec<T>, id: &U, p: Option<UpdateParam>) -> Result<()>;
+        fn remove(&self, id: &U, p: Option<RemoveParam>) -> Result<()>;
+        fn commit(&self, p: Option<CommitParam>) -> Result<()>;
+
+        fn new(p: Option<NGTParam>) -> Result<Self>;
+    }
+}

rust/libs/kvs/Cargo.toml

@@ -0,0 +1,8 @@
+[package]
+name = "kvs"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]

rust/libs/kvs/src/kvs.rs

@@ -0,0 +1,12 @@
+use anyhow::Result;
+
+pub mod kvs {
+    pub trait KVS<K, V> {
+        fn set(k: &K, v: &V) -> Result<()>;
+        fn get_value(k: &K) -> Result<&V>;
+        fn get_key(v: &V) -> Result<&K>;
+
+        fn new(p: &str) -> Result<&Self>;
+        fn open(p: &str) -> Result<&Self>;
+    }
+}

rust/libs/kvs/src/lib.rs

@@ -0,0 +1,14 @@
+pub fn add(left: usize, right: usize) -> usize {
+    left + right
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn it_works() {
+        let result = add(2, 2);
+        assert_eq!(result, 4);
+    }
+}

rust/libs/ngt-rs/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "ngt-rs"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+cxx = { version = "1.0.117", features = ["c++20"] }
+
+[build-dependencies]
+cxx-build = "1.0.117"
+miette = { version = "7.1.0", features = ["fancy"] }

rust/libs/ngt-rs/build.rs

@@ -0,0 +1,14 @@
+fn main() -> miette::Result<()> {let current_dir = std::env::current_dir().unwrap();
+    println!("cargo:rustc-link-search=native={}", current_dir.display());
+    println!("cargo:rustc-link-search=native={}", current_dir.display());
+
+    cxx_build::bridge("src/lib.rs")
+        .file("src/input.cpp")
+        .flag_if_supported("-std=c++20")
+        .compile("ngt-rs");
+
+    println!("cargo:rustc-link-lib=static=ngt");
+    println!("cargo:rustc-link-lib=static=gomp");
+    println!("cargo:rerun-if-changed=src/*");
+    Ok(())
+}

rust/libs/ngt-rs/src/input.cpp

@@ -0,0 +1,42 @@
+#include "ngt-rs/src/input.h"
+#include "ngt-rs/src/lib.rs.h"
+
+Property::Property(): p() {}
+
+void Property::set_dimension(rust::i32 dimension) {
+    p.dimension = dimension;
+}
+
+void Property::set_distance_type(const DistanceType t) {
+    p.distanceType = static_cast<NGT::Property::DistanceType>(t);
+}
+
+void Property::set_object_type(const ObjectType t) {
+    p.objectType = static_cast<NGT::Property::ObjectType>(t);
+}
+
+Index::Index(Property& p) {
+    index = new NGT::GraphAndTreeIndex(p.p);
+}
+
+rust::u32 Index::insert(const rust::Vec<rust::f32>& object) {
+    std::vector<float> v;
+    std::copy(object.begin(), object.end(), std::back_inserter(v));
+    return index->insert(v);
+}
+
+void Index::create_index(rust::u32 pool_size) {
+    index->createIndex(pool_size);
+}
+
+void Index::remove(rust::u32 id) {
+    index->remove(id);
+}
+
+std::unique_ptr<Property> new_property() {
+    return std::make_unique<Property>();
+}
+
+std::unique_ptr<Index> new_index(Property& p) {
+    return std::make_unique<Index>(p);
+}

rust/libs/ngt-rs/src/input.h

@@ -0,0 +1,29 @@
+#pragma once
+
+#include "NGT/Index.h"
+#include <memory>
+#include "rust/cxx.h"
+
+enum class DistanceType;
+enum class ObjectType;
+
+class Property {
+public:
+    NGT::Property p;
+    Property();
+    void set_dimension(rust::i32);
+    void set_object_type(const ObjectType);
+    void set_distance_type(const DistanceType);
+};
+
+class Index {
+    NGT::Index* index;
+public:
+    Index(Property&);
+    rust::u32 insert(const rust::Vec<rust::f32>& object);
+    void create_index(rust::u32);
+    void remove(rust::u32);
+};
+
+std::unique_ptr<Property> new_property();
+std::unique_ptr<Index> new_index(Property& p);

rust/libs/ngt-rs/src/lib.rs

@@ -0,0 +1,54 @@
+#[cxx::bridge]
+pub mod ffi {
+    #[repr(i32)]
+    enum ObjectType {
+        None = 0,
+        Uint8 = 1,
+        Float = 2,
+        Float16 = 3,
+    }
+
+    #[repr(i32)]
+    enum DistanceType {
+        None = -1,
+        L1 = 0,
+        L2 = 1,
+        Hamming = 2,
+        Angle = 3,
+        Cosine = 4,
+        NormalizedAngle = 5,
+        NormalizedCosine = 6,
+        Jaccard = 7,
+        SparseJaccard = 8,
+        NormalizedL2 = 9,
+        Poincare = 100,
+        Lorentz = 101,
+    }
+
+    unsafe extern "C++" {
+        include!("ngt-rs/src/input.h");
+
+        type Property;
+        fn new_property() -> UniquePtr<Property>;
+        fn set_dimension(self: Pin<&mut Property>, dimension: i32);
+        fn set_object_type(self: Pin<&mut Property>, t: ObjectType);
+        fn set_distance_type(self: Pin<&mut Property>, t: DistanceType);
+
+        type Index;
+        fn new_index(p: Pin<&mut Property>) -> UniquePtr<Index>;
+        fn insert(self: Pin<&mut Index>, v: &Vec<f32>) -> u32;
+        fn create_index(self: Pin<&mut Index>, pool_size: u32);
+        fn remove(self: Pin<&mut Index>, id: u32);
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn it_works() {
+        let result = add(2, 2);
+        assert_eq!(result, 4);
+    }
+}