WIP: MMAP backed bucket map that is suitable as a KV store for the accounts index

Cargo.toml

@@ -9,6 +9,7 @@ members = [
     "banks-client",
     "banks-interface",
     "banks-server",
+    "bucket_map",
     "clap-utils",
     "cli-config",
     "cli-output",

bucket_map/Cargo.toml

@@ -0,0 +1,28 @@
+[package]
+name = "solana-bucket-map"
+version = "1.8.0"
+description = "solana-bucket-map"
+homepage = "https://solana.com/"
+documentation = "https://docs.rs/solana-bucket-map"
+readme = "../README.md"
+repository = "https://github.com/solana-labs/solana"
+authors = ["Solana Maintainers <maintainers@solana.foundation>"]
+license = "Apache-2.0"
+edition = "2018"
+
+[dependencies]
+rayon = "1.5.0"
+solana-logger = { path = "../logger", version = "=1.8.0" }
+solana-sdk = { path = "../sdk", version = "=1.8.0" }
+memmap2 = "0.2.3"
+log = { version = "0.4.11" }
+solana-measure = { path = "../measure", version = "=1.8.0" }
+rand = "0.7.0"
+fs_extra = "1.2.0"
+
+[lib]
+crate-type = ["lib"]
+name = "solana_bucket_map"
+
+[[bench]]
+name = "bucket_map"

bucket_map/benches/bucket_map.rs

@@ -0,0 +1,78 @@
+#![feature(test)]
+
+extern crate test;
+use rayon::prelude::*;
+use solana_bucket_map::bucket_map::BucketMap;
+use solana_sdk::pubkey::Pubkey;
+use std::collections::hash_map::HashMap;
+use std::sync::Arc;
+use std::sync::RwLock;
+use test::Bencher;
+
+type AccountInfo = u64;
+
+#[bench]
+fn bucket_map_bench_hashmap_baseline(bencher: &mut Bencher) {
+    let mut index = HashMap::new();
+    bencher.iter(|| {
+        let key = Pubkey::new_unique();
+        index.insert(key, vec![(0, AccountInfo::default())]);
+    });
+}
+
+#[bench]
+fn bucket_map_bench_insert_1(bencher: &mut Bencher) {
+    let tmpdir = std::env::temp_dir().join("bucket_map_bench_insert_1");
+    std::fs::create_dir_all(tmpdir.clone()).unwrap();
+    let drives = Arc::new(vec![tmpdir.clone()]);
+    let index = BucketMap::new(1, drives);
+    bencher.iter(|| {
+        let key = Pubkey::new_unique();
+        index.update(&key, |_| Some(vec![(0, AccountInfo::default())]));
+    });
+    std::fs::remove_dir_all(tmpdir).unwrap();
+}
+
+#[bench]
+fn bucket_map_bench_insert_16x32_baseline(bencher: &mut Bencher) {
+    let index = RwLock::new(HashMap::new());
+    (0..16).into_iter().into_par_iter().for_each(|_| {
+        let key = Pubkey::new_unique();
+        index
+            .write()
+            .unwrap()
+            .insert(key, vec![(0, AccountInfo::default())]);
+    });
+    bencher.iter(|| {
+        (0..16).into_iter().into_par_iter().for_each(|_| {
+            for _ in 0..32 {
+                let key = Pubkey::new_unique();
+                index
+                    .write()
+                    .unwrap()
+                    .insert(key, vec![(0, AccountInfo::default())]);
+            }
+        })
+    });
+}
+
+#[bench]
+fn bucket_map_bench_insert_16x32(bencher: &mut Bencher) {
+    let tmpdir = std::env::temp_dir().join("bucket_map_bench_insert_16x32");
+    std::fs::create_dir_all(tmpdir.clone()).unwrap();
+    let drives = Arc::new(vec![tmpdir.clone()]);
+    let index = BucketMap::new(4, drives);
+    (0..16).into_iter().into_par_iter().for_each(|_| {
+        let key = Pubkey::new_unique();
+        index.update(&key, |_| Some(vec![(0, AccountInfo::default())]));
+    });
+    bencher.iter(|| {
+        (0..16).into_iter().into_par_iter().for_each(|_| {
+            for _ in 0..32 {
+                let key = Pubkey::new_unique();
+                index.update(&key, |_| Some(vec![(0, AccountInfo::default())]));
+            }
+        })
+    });
+    std::fs::remove_dir_all(tmpdir).unwrap();
+}