Add lmdb store

Cargo.toml

@@ -5,4 +5,6 @@ members = [
   "crates/holochain_persistence_mem",
   "crates/holochain_persistence_file",
   "crates/holochain_persistence_pickle",
+  "crates/holochain_persistence_lmdb",
+  # "benchmarks",
 ]

benchmarks/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+name = "benchmarks"
+version = "0.0.8"
+authors = ["Holochain Core Dev Team <devcore@holochain.org>"]
+
+[dependencies]
+holochain_persistence_api = { path = "../crates/holochain_persistence_api" }
+holochain_persistence_file = { path = "../crates/holochain_persistence_file" }
+holochain_persistence_mem = { path = "../crates/holochain_persistence_mem" }
+holochain_persistence_pickle = { path = "../crates/holochain_persistence_pickle" }
+holochain_persistence_lmdb = { path = "../crates/holochain_persistence_lmdb" }
+bencher = "=0.1.5"
+tempfile = "=3.0.7"
+
+
+
+[[bench]]
+name = "my_benchmark"
+harness = false

benchmarks/benches/my_benchmark.rs

@@ -0,0 +1,143 @@
+#[macro_use]
+extern crate bencher;
+extern crate holochain_persistence_api;
+extern crate holochain_persistence_file;
+extern crate holochain_persistence_mem;
+extern crate holochain_persistence_pickle;
+extern crate holochain_persistence_lmdb;
+extern crate tempfile;
+
+use self::tempfile::tempdir;
+use bencher::Bencher;
+use holochain_persistence_api::{
+    cas::{content::ExampleAddressableContent, storage::EavTestSuite},
+    eav::eavi::ExampleAttribute,
+};
+use holochain_persistence_file::eav::file::EavFileStorage;
+use holochain_persistence_mem::eav::memory::EavMemoryStorage;
+use holochain_persistence_pickle::eav::pickle::EavPickleStorage;
+use holochain_persistence_lmdb::eav::lmdb::EavLmdbStorage;
+
+/*----------  Memory Storage  ----------*/
+
+
+fn bench_memory_eav_one_to_many(b: &mut Bencher) {
+    b.iter(|| {
+        let eav_storage = EavMemoryStorage::new();
+        EavTestSuite::test_one_to_many::<
+            ExampleAddressableContent,
+            ExampleAttribute,
+            EavMemoryStorage<ExampleAttribute>,
+        >(eav_storage.clone(), &ExampleAttribute::WithoutPayload)
+    })
+}
+
+fn bench_memory_eav_many_to_one(b: &mut Bencher) {
+    b.iter(|| {
+        let eav_storage = EavMemoryStorage::new();
+        EavTestSuite::test_one_to_many::<
+            ExampleAddressableContent,
+            ExampleAttribute,
+            EavMemoryStorage<ExampleAttribute>,
+        >(eav_storage.clone(), &ExampleAttribute::WithoutPayload)
+    })
+}
+
+/*----------  File Storage  ----------*/
+
+
+fn bench_file_eav_one_to_many(b: &mut Bencher) {
+    b.iter(|| {
+        let temp = tempdir().expect("test was supposed to create temp dir");
+        let temp_path = String::from(temp.path().to_str().expect("temp dir could not be string"));
+        let eav_storage = EavFileStorage::new(temp_path).unwrap();
+        EavTestSuite::test_one_to_many::<
+            ExampleAddressableContent,
+            ExampleAttribute,
+            EavFileStorage<ExampleAttribute>,
+        >(eav_storage.clone(), &&ExampleAttribute::WithoutPayload)
+    })
+}
+
+fn bench_file_eav_many_to_one(b: &mut Bencher) {
+    b.iter(|| {
+        let temp = tempdir().expect("test was supposed to create temp dir");
+        let temp_path = String::from(temp.path().to_str().expect("temp dir could not be string"));
+        let eav_storage = EavFileStorage::new(temp_path).unwrap();
+        EavTestSuite::test_many_to_one::<
+            ExampleAddressableContent,
+            ExampleAttribute,
+            EavFileStorage<ExampleAttribute>,
+        >(eav_storage.clone(), &ExampleAttribute::WithoutPayload)
+    })
+}
+
+/*----------  Pickle Storage  ----------*/
+
+
+fn bench_pickle_eav_one_to_many(b: &mut Bencher) {
+    b.iter(|| {
+        let temp = tempdir().expect("test was supposed to create temp dir");
+        let temp_path = String::from(temp.path().to_str().expect("temp dir could not be string"));
+        let eav_storage = EavPickleStorage::new(temp_path);
+        EavTestSuite::test_one_to_many::<
+            ExampleAddressableContent,
+            ExampleAttribute,
+            EavPickleStorage<ExampleAttribute>,
+        >(eav_storage.clone(), &ExampleAttribute::WithoutPayload)
+    })
+}
+
+fn bench_pickle_eav_many_to_one(b: &mut Bencher) {
+    b.iter(|| {
+        let temp = tempdir().expect("test was supposed to create temp dir");
+        let temp_path = String::from(temp.path().to_str().expect("temp dir could not be string"));
+        let eav_storage = EavPickleStorage::new(temp_path);
+        EavTestSuite::test_many_to_one::<
+            ExampleAddressableContent,
+            ExampleAttribute,
+            EavPickleStorage<ExampleAttribute>,
+        >(eav_storage.clone(), &ExampleAttribute::WithoutPayload)
+    })
+}
+
+/*----------  LMDB Storage  ----------*/
+
+fn bench_lmdb_eav_one_to_many(b: &mut Bencher) {
+    b.iter(|| {
+        let temp = tempdir().expect("test was supposed to create temp dir");
+        let temp_path = String::from(temp.path().to_str().expect("temp dir could not be string"));
+        let eav_storage = EavLmdbStorage::new(temp_path);
+        EavTestSuite::test_one_to_many::<
+            ExampleAddressableContent,
+            ExampleAttribute,
+            EavLmdbStorage<ExampleAttribute>,
+        >(eav_storage.clone(), &ExampleAttribute::WithoutPayload)
+    })
+}
+
+fn bench_lmdb_eav_many_to_one(b: &mut Bencher) {
+    b.iter(|| {
+        let temp = tempdir().expect("test was supposed to create temp dir");
+        let temp_path = String::from(temp.path().to_str().expect("temp dir could not be string"));
+        let eav_storage = EavLmdbStorage::new(temp_path);
+        EavTestSuite::test_many_to_one::<
+            ExampleAddressableContent,
+            ExampleAttribute,
+            EavLmdbStorage<ExampleAttribute>,
+        >(eav_storage.clone(), &ExampleAttribute::WithoutPayload)
+    })
+}
+
+benchmark_group!(
+    benches,
+    bench_memory_eav_many_to_one,
+    bench_memory_eav_one_to_many,
+    bench_file_eav_one_to_many,
+    bench_file_eav_many_to_one,
+    bench_pickle_eav_many_to_one,
+    bench_pickle_eav_one_to_many,
+    bench_lmdb_eav_many_to_one,
+    bench_lmdb_eav_one_to_many,
+);
+benchmark_main!(benches);

crates/holochain_persistence_api/Cargo.toml

@@ -35,6 +35,7 @@ objekt= "=0.1.2"
 holochain_json_api = "=0.0.17"
 holochain_json_derive = "=0.0.17"
 uuid = { version = "=0.7.1", features = ["v4"] }
+rand = "0.7.2"
 
 [dev-dependencies]
 maplit = "=1.0.1"

crates/holochain_persistence_api/src/cas/storage.rs

@@ -4,7 +4,7 @@
 //! A test suite for CAS is also implemented here.
 
 use crate::{
-    cas::content::{Address, AddressableContent, Content},
+    cas::content::{Address, AddressableContent, Content, ExampleAddressableContent},
     eav::{
         Attribute, EavFilter, EaviQuery, EntityAttributeValueIndex, EntityAttributeValueStorage,
         IndexFilter,
@@ -811,6 +811,42 @@ impl EavTestSuite {
     }
 }
 
+pub struct CasBencher;
+
+impl CasBencher {
+
+    fn random_addressable_content() -> ExampleAddressableContent {
+        let s: String = (0..4).map(|_| rand::random::<char>()).collect();
+        ExampleAddressableContent::try_from_content(&RawString::from(s).into()).unwrap()
+    }
+
+    pub fn bench_add(
+        b: &mut test::Bencher,
+        mut store: impl ContentAddressableStorage,
+    ) {
+        b.iter(|| {
+            store.add(&CasBencher::random_addressable_content())     
+        })
+    }
+
+    pub fn bench_fetch(
+        b: &mut test::Bencher,
+        mut store: impl ContentAddressableStorage,       
+    ) {
+        // add some values to make it realistic
+        for _ in 0..100 {
+            store.add(&CasBencher::random_addressable_content()).unwrap();
+        }
+
+        let test_content = CasBencher::random_addressable_content();
+        store.add(&test_content).unwrap();
+
+        b.iter(|| {
+            store.fetch(&test_content.address())     
+        })        
+    }
+}
+
 #[cfg(test)]
 pub mod tests {
     use crate::cas::{

crates/holochain_persistence_api/src/eav/query.rs

@@ -154,32 +154,39 @@ impl<'a, A: Attribute> EaviQuery<'a, A> {
 }
 
 /// Represents a filter type which takes in a function to match on
-pub struct EavFilter<'a, T: 'a + Eq>(Box<dyn Fn(T) -> bool + 'a>);
+// pub struct EavFilter<'a, T: 'a + Eq>(Box<dyn Fn(T) -> bool + 'a>);
+pub enum EavFilter<'a, T: 'a + Eq> {
+    Exact(T),
+    Predicate(Box<dyn Fn(T) -> bool + 'a>),
+}
 
 impl<'a, T: 'a + Eq> EavFilter<'a, T> {
     pub fn single(val: T) -> Self {
-        Self(Box::new(move |v| v == val))
+        Self::Exact(val)
     }
 
     pub fn multiple(vals: Vec<T>) -> Self {
-        Self(Box::new(move |val| vals.iter().any(|v| *v == val)))
+        Self::Predicate(Box::new(move |val| vals.iter().any(|v| *v == val)))
     }
 
     pub fn predicate<F>(predicate: F) -> Self
     where
         F: Fn(T) -> bool + 'a,
     {
-        Self(Box::new(predicate))
+        Self::Predicate(Box::new(predicate))
     }
 
-    pub fn check(&self, val: T) -> bool {
-        self.0(val)
+    pub fn check(&self, b: T) -> bool {
+        match self {
+            Self::Exact(a) => a == &b,
+            Self::Predicate(f) => f(b) 
+        }
     }
 }
 
 impl<'a, T: Eq> Default for EavFilter<'a, T> {
     fn default() -> EavFilter<'a, T> {
-        Self(Box::new(|_| true))
+        Self::Predicate(Box::new(|_| true))
     }
 }
 

crates/holochain_persistence_api/src/eav/storage.rs

@@ -1,4 +1,17 @@
-use eav::{eavi::EntityAttributeValueIndex, query::EaviQuery, Attribute};
+use eav::{
+    eavi::{
+        EntityAttributeValueIndex,
+        ExampleAttribute,
+    },
+    query::EaviQuery, Attribute, EavFilter, IndexFilter,
+};
+use crate::holochain_json_api::{
+    json::{RawString},
+};
+use cas::content::{
+    ExampleAddressableContent,
+    AddressableContent
+};
 use error::PersistenceResult;
 use objekt;
 use reporting::ReportStorage;
@@ -99,3 +112,80 @@ pub fn increment_key_till_no_collision<A: Attribute>(
         Ok(eav)
     }
 }
+
+pub struct EavBencher;
+
+impl EavBencher {
+
+    fn random_addressable_content() -> ExampleAddressableContent {
+        let s: String = (0..4).map(|_| rand::random::<char>()).collect();
+        ExampleAddressableContent::try_from_content(&RawString::from(s).into()).unwrap()
+    }
+
+    pub fn bench_add(
+        b: &mut test::Bencher,
+        mut store: impl EntityAttributeValueStorage<ExampleAttribute>,
+    ) {
+        b.iter(|| {
+            let eav = EntityAttributeValueIndex::new(
+                &Self::random_addressable_content().address(),
+                &ExampleAttribute::WithPayload("favourite-color".to_string()),
+                &Self::random_addressable_content().address(),
+            )
+            .expect("Could create entityAttributeValue");
+            store.add_eavi(&eav)     
+        })
+    }
+
+    pub fn bench_fetch_all(
+        b: &mut test::Bencher,
+        mut store: impl EntityAttributeValueStorage<ExampleAttribute>,       
+    ) {
+        // add some values to make it realistic
+        for _ in 0..100 {
+            let eav = EntityAttributeValueIndex::new(
+                &Self::random_addressable_content().address(),
+                &ExampleAttribute::WithPayload("favourite-color".to_string()),
+                &Self::random_addressable_content().address(),
+            ).expect("Could create entityAttributeValue");
+            store.add_eavi(&eav).unwrap();
+        }
+
+        b.iter(|| {
+            store.fetch_eavi(&EaviQuery::default())     
+        })        
+    }
+
+    pub fn bench_fetch_exact(
+        b: &mut test::Bencher,
+        mut store: impl EntityAttributeValueStorage<ExampleAttribute>,       
+    ) {
+        // add some values to make it realistic
+        for _ in 0..100 {
+            let eav = EntityAttributeValueIndex::new(
+                &Self::random_addressable_content().address(),
+                &ExampleAttribute::WithPayload("favourite-color".to_string()),
+                &Self::random_addressable_content().address(),
+            ).expect("Could create entityAttributeValue");
+            store.add_eavi(&eav).unwrap();
+        }
+
+        // add the one entry we want to test retrieval of
+        let eav = EntityAttributeValueIndex::new(
+            &Self::random_addressable_content().address(),
+            &ExampleAttribute::WithPayload("favourite-color".to_string()),
+            &Self::random_addressable_content().address(),
+        ).expect("Could create entityAttributeValue");
+        store.add_eavi(&eav).unwrap();
+
+        b.iter(|| {
+            store.fetch_eavi(&EaviQuery::new(
+                EavFilter::single(eav.entity()),
+                EavFilter::default(),
+                EavFilter::default(),
+                IndexFilter::LatestByAttribute,
+                None,
+            ))  
+        })        
+    }
+}

crates/holochain_persistence_api/src/lib.rs

@@ -4,6 +4,8 @@
 #![feature(try_trait)]
 #![feature(never_type)]
 #![warn(unused_extern_crates)]
+#![feature(test)]
+extern crate test;  
 #[macro_use]
 extern crate lazy_static;