Skip to content
This repository has been archived by the owner on Aug 14, 2023. It is now read-only.

Commit

Permalink
Introducing namespace to the kv-store API. (#108)
Browse files Browse the repository at this point in the history
  • Loading branch information
YaronWittenstein authored Mar 26, 2020
1 parent 7d731a4 commit c07557b
Show file tree
Hide file tree
Showing 30 changed files with 413 additions and 259 deletions.
8 changes: 5 additions & 3 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ libc = "0.2"
byteorder = "1.3.2"
tiny-keccak = "1.4.2"
log = "0.4"
serde = { version = "1.0.98", features = ["derive"] }
lazy_static = "1.4.0"
svm-common = { path = "crates/svm-common" }
svm-kv = { path = "crates/svm-kv" }
svm-gas = { path = "crates/svm-gas" }
Expand Down
3 changes: 3 additions & 0 deletions crates/svm-app/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ version = "0.4"
[dependencies.bit-vec]
version = "0.6.1"

[dependencies.lazy_static]
version = "1.4.0"

[features]
default = ["memory", "default-rocksdb"]
memory = ["svm-kv/memory"]
Expand Down
26 changes: 18 additions & 8 deletions crates/svm-app/src/rocksdb/template_store.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,18 @@ use crate::{

use svm_kv::{rocksdb::Rocksdb, traits::KVStore};

use lazy_static::lazy_static;
use log::info;

lazy_static! {
static ref TEMPLATE_NS: Vec<u8> = vec![b't', b'e', b'm', b'p'];
static ref CODE_NS: Vec<u8> = vec![b'c', b'o', b'd', b'e'];
}

/// `AppTemplate` store backed by `rocksdb`
pub struct RocksdbAppTemplateStore<S, D> {
db: Rocksdb,
_phantom: PhantomData<(S, D)>,
phantom: PhantomData<(S, D)>,
}

impl<S, D> RocksdbAppTemplateStore<S, D>
Expand All @@ -28,7 +34,7 @@ where
{
Self {
db: Rocksdb::new(path),
_phantom: PhantomData,
phantom: PhantomData,
}
}
}
Expand All @@ -51,21 +57,25 @@ where

let bytes = S::serialize(template, author);

let addr_hash = (addr.inner().as_slice(), &hash.0[..]);
let hash_wasm = (&hash.0[..], &bytes[..]);
self.db.store(&[addr_hash, hash_wasm]);
// template addr -> code-hash
let entry1 = (&TEMPLATE_NS[..], addr.inner().as_slice(), &hash.0[..]);

// code-hash -> code
let entry2 = (&CODE_NS[..], &hash.0[..], &bytes[..]);

self.db.store(&[entry1, entry2]);

Ok(())
}

fn load(&self, addr: &TemplateAddr) -> Option<(AppTemplate, AuthorAddr)> {
let addr = addr.inner().as_slice();

info!("loading `AppTemplate` account {:?}", addr);
info!("Loading `AppTemplate` account {:?}", addr);

self.db.get(addr).and_then(|hash| {
self.db.get(&TEMPLATE_NS, addr).and_then(|hash| {
self.db
.get(&hash)
.get(&CODE_NS[..], &hash)
.and_then(|bytes| D::deserialize(&bytes[..]))
})
}
Expand Down
53 changes: 53 additions & 0 deletions crates/svm-kv/src/key.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
/// Concatenates a namespace and a key into a fully-qualified key.
#[allow(unused)]
pub fn concat_ns_to_key<NS, K>(ns: NS, key: K) -> Vec<u8>
where
NS: AsRef<[u8]>,
K: AsRef<[u8]>,
{
let ns = ns.as_ref();
let key = key.as_ref();

let cap = if ns.len() > 0 {
ns.len() + 1 + key.len()
} else {
key.len()
};

let mut buf = Vec::with_capacity(cap);

if ns.len() > 0 {
buf.extend_from_slice(ns);
buf.extend_from_slice(&[b':']);
}

buf.extend_from_slice(key);
buf
}

#[cfg(test)]
mod tests {
use super::*;

#[test]
fn concat_ns_to_key_empty_ns() {
let ns = vec![];
let key = vec![b'a', b'b', b'c'];

let actual = concat_ns_to_key(&ns, &key);
let expected = "abc".as_bytes();

assert_eq!(&expected[..], &actual[..]);
}

#[test]
fn concat_ns_to_key_non_empty_ns() {
let ns = vec![b'n', b's'];
let key = vec![b'a', b'b', b'c'];

let actual = concat_ns_to_key(&ns, &key);
let expected = "ns:abc".as_bytes();

assert_eq!(&expected[..], &actual[..]);
}
}
6 changes: 4 additions & 2 deletions crates/svm-kv/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,14 @@
#![deny(dead_code)]
#![deny(unreachable_code)]

//! The `svm-kv` is responsible on providing different implementations for the `KVStore` trait.
//! (defined in `traits.rs`).
//! The `svm-kv` crate is responsible on providing different implementations for the `KVStore` trait.

/// Defines the `KVStore` trait.
pub mod traits;

/// Helpers for composing keys.
pub mod key;

/// An in-memory implementation for `KVStore`
#[cfg(feature = "memory")]
pub mod memory;
Expand Down
27 changes: 14 additions & 13 deletions crates/svm-kv/src/memory.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
use crate::traits::KVStore;
use crate::{key::concat_ns_to_key, traits::KVStore};

use std::collections::{hash_map, HashMap};

use log::{debug, info};
Expand Down Expand Up @@ -38,27 +39,27 @@ impl MemKVStore {
}

impl KVStore for MemKVStore {
fn get(&self, key: &[u8]) -> Option<Vec<u8>> {
let entry = self.map.get(key);
fn get(&self, ns: &[u8], key: &[u8]) -> Option<Vec<u8>> {
let key = concat_ns_to_key(ns, key);

if let Some(entry) = entry {
Some(entry.clone())
} else {
None
}
let entry = self.map.get(&key);
entry.cloned()
}

fn store(&mut self, changes: &[(&[u8], &[u8])]) {
info!("storing in-memory kv changeset");
fn store(&mut self, changes: &[(&[u8], &[u8], &[u8])]) {
info!("Storing in-memory kv changeset");

for (ns, k, v) in changes {
let k = concat_ns_to_key(ns, k);
let v = v.as_ref().to_vec();

for (k, v) in changes {
self.map.insert(k.to_vec(), v.to_vec());
self.map.insert(k, v);
}
}
}

impl Drop for MemKVStore {
fn drop(&mut self) {
debug!("dropping `MemKVStore`...")
debug!("Dropping `MemKVStore`...")
}
}
34 changes: 22 additions & 12 deletions crates/svm-kv/src/rocksdb/db.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
use crate::traits::KVStore;
use std::path::Path;

use crate::{key::concat_ns_to_key, traits::KVStore};

use log::info;

/// An implementation of `KVStore` trait against `rocksdb`.
Expand All @@ -21,8 +22,10 @@ impl Rocksdb {

impl KVStore for Rocksdb {
#[allow(clippy::match_wild_err_arm)]
fn get(&self, key: &[u8]) -> Option<Vec<u8>> {
match self.db.get(key) {
fn get(&self, ns: &[u8], key: &[u8]) -> Option<Vec<u8>> {
let key = concat_ns_to_key(ns, key);

match self.db.get(&key) {
Ok(dbvec) => match dbvec {
None => None,
Some(dbvec) => Some(dbvec.to_vec()),
Expand All @@ -31,11 +34,13 @@ impl KVStore for Rocksdb {
}
}

fn store(&mut self, changes: &[(&[u8], &[u8])]) {
fn store(&mut self, changes: &[(&[u8], &[u8], &[u8])]) {
let mut batch = rocksdb::WriteBatch::default();

for (k, v) in changes {
let res = batch.put(k, v);
for (ns, k, v) in changes {
let k = concat_ns_to_key(ns, k);

let res = batch.put(k, v.as_ref());

if res.is_err() {
panic!("failed `put`-ing bach data");
Expand All @@ -52,7 +57,7 @@ impl KVStore for Rocksdb {

impl Drop for Rocksdb {
fn drop(&mut self) {
info!("dropping `Rocksdb`");
info!("Dropping `Rocksdb`...");
}
}

Expand All @@ -64,15 +69,20 @@ mod tests {
fn rocksdb_sanity() {
let mut db = Rocksdb::new("rocksdb-tests");

db.store(&[(&[10, 20, 30], &[40, 50, 60])]);
let ns = vec![0xFF, 0xFF];
let key = vec![10, 20, 30];
let val = vec![40, 50, 60];

let change = (&ns[..], &key[..], &val[..]);
db.store(&[change]);

let v = db.get(&[10, 20, 30]).unwrap();
assert_eq!(vec![40, 50, 60], v);
let v = db.get(&ns, &key).unwrap();
assert_eq!(val, v);

drop(db);

let db = Rocksdb::new("rocksdb-tests");
let v = db.get(&[10, 20, 30]).unwrap();
assert_eq!(vec![40, 50, 60], v);
let v = db.get(&ns, &key).unwrap();
assert_eq!(val, v);
}
}
8 changes: 4 additions & 4 deletions crates/svm-kv/src/traits.rs
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
/// `KVStore` is a trait for defining an interface against key-value stores. for example `in-memory / rocksdb`
/// `KVStore` is a trait for defining an interface against key-value stores, for example `rocksdb/leveldb`.
pub trait KVStore {
/// Retrieves the value pointed by `key` (Optional).
#[must_use]
fn get(&self, key: &[u8]) -> Option<Vec<u8>>;
fn get(&self, ns: &[u8], key: &[u8]) -> Option<Vec<u8>>;

/// Stores a batch of changes. Each change is `key` -> `value` association.
fn store(&mut self, changes: &[(&[u8], &[u8])]);
/// Stores a batch of changes. Each change is `(ns, key) -> value` association.
fn store(&mut self, changes: &[(&[u8], &[u8], &[u8])]);
}
12 changes: 6 additions & 6 deletions crates/svm-kv/tests/asserts.rs
Original file line number Diff line number Diff line change
@@ -1,29 +1,29 @@
#[macro_export]
macro_rules! assert_key_value {
($kv: expr, $key: expr, $expected: expr) => {{
let actual = $kv.get(&$key).unwrap();
($kv:expr, $ns:expr, $key:expr, $expected:expr) => {{
let actual = $kv.get(&$ns, &$key).unwrap();
assert_eq!($expected, &actual[..]);
}};
}

#[macro_export]
macro_rules! assert_no_key {
($kv: expr, $key: expr) => {{
assert!($kv.get(&$key).is_none());
($kv:expr, $ns:expr, $key:expr) => {{
assert!($kv.get(&$ns, &$key).is_none());
}};
}

#[macro_export]
macro_rules! kv_keys_vec {
($kv: ident) => {{
($kv:ident) => {{
let keys: Vec<Vec<u8>> = $kv.borrow().keys().map(|key| key.clone()).collect();
keys
}};
}

#[macro_export]
macro_rules! assert_same_keys {
($expected: expr, $actual: expr) => {{
($expected:expr, $actual:expr) => {{
let mut expected = $expected
.iter()
.map(|k| k.to_vec())
Expand Down
Loading

0 comments on commit c07557b

Please sign in to comment.