diff --git a/CHANGELOG.md b/CHANGELOG.md index f96c92b7f8..2dc9fa769c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,14 @@ namespace in `PrefixedStorage::new`, `PrefixedStorage::multilevel`, `ReadonlyPrefixedStorage::new`, `ReadonlyPrefixedStorage::multilevel`, `prefixed` and `prefixed_read`. +- Remove `Bucket::multilevel` as it was a misnomer. Provide support for + composite `PrimaryKey`s in storage, using `Pk2` and `Pk3` for 2 or 3 items + keys. Add `Bucket::range_prefixed` to provide the first section(s) of the + composite key and iterate under that. +- Bucket stores all data under `(namespace, "_pk", key)`, where the first two + are length-prefixed. This is a different layout from previous form + where it was `(namespace, key)` and this is intended to support co-existence + with `IndexedBucket`. (It only changes the raw storage layout, APIs don't change). **cosmwasm-vm** diff --git a/MIGRATING.md b/MIGRATING.md index 8e105915f8..9e869b8895 100644 --- a/MIGRATING.md +++ b/MIGRATING.md @@ -162,6 +162,52 @@ major releases of `cosmwasm`. Note that you can also view the }) ``` +- Remove `Bukcet::multilevel` constructor. The bucket now has exactly one namespace + and if we need `multilevel`, it was really to provide composite primary keys. + We now do that explicitly in a way that allows better compatibility + with `IndexedBucket`. The test `bucket_with_composite_pk` in `bucket.rs` is + a good (tested) example of the new API. + + ```rust + // before (from cw20-base) + pub fn allowances<'a, S: Storage>( + storage: &'a mut S, + owner: &CanonicalAddr, + ) -> Bucket<'a, S, AllowanceResponse> { + Bucket::multilevel(&[PREFIX_ALLOWANCE, owner.as_slice()], storage) + } + + allowances(&mut deps.storage, owner_raw).update(spender_raw.as_slice(), |allow| { + // code omitted... + })?; + + let allowances: StdResult> = allowances_read(&deps.storage, &owner_raw) + .range(start.as_deref(), None, Order::Ascending) + .take(limit) + .map(|item| { + // code omitted... + }) + .collect(); + + // after + pub fn allowances(storage: &mut S) -> Bucket { + Bucket::new(PREFIX_ALLOWANCE, storage) + } + + allowances(&mut deps.storage).update(&Pk2(owner_raw.as_slice(), spender_raw.as_slice()).pk(), |allow| { + // code omitted... + })?; + + let allowances: StdResult> = allowances(&deps.storage) + .range_prefixed(owner_raw.as_slice(), start.as_deref(), None, Order::Ascending) + .take(limit) + .map(|item| { + // code omitted... + }) + .collect(); + ``` + + ## 0.9 -> 0.10 Integration tests: diff --git a/packages/storage/src/bucket.rs b/packages/storage/src/bucket.rs index b1d0c957d5..5aa16a6773 100644 --- a/packages/storage/src/bucket.rs +++ b/packages/storage/src/bucket.rs @@ -5,16 +5,19 @@ use cosmwasm_std::{to_vec, ReadonlyStorage, StdError, StdResult, Storage}; #[cfg(feature = "iterator")] use cosmwasm_std::{Order, KV}; -use crate::length_prefixed::{to_length_prefixed, to_length_prefixed_nested}; +use crate::length_prefixed::{ + decode_length, length_prefixed_with_key, namespaces_with_key, nested_namespaces_with_key, +}; #[cfg(feature = "iterator")] use crate::namespace_helpers::range_with_prefix; -use crate::namespace_helpers::{get_with_prefix, remove_with_prefix, set_with_prefix}; #[cfg(feature = "iterator")] use crate::type_helpers::deserialize_kv; use crate::type_helpers::{may_deserialize, must_deserialize}; +pub(crate) const PREFIX_PK: &[u8] = b"_pk"; + /// An alias of Bucket::new for less verbose usage -pub fn bucket<'a, S, T>(storage: &'a mut S, namespace: &[u8]) -> Bucket<'a, S, T> +pub fn bucket<'a, 'b, S, T>(storage: &'a mut S, namespace: &'b [u8]) -> Bucket<'a, 'b, S, T> where S: Storage, T: Serialize + DeserializeOwned, @@ -23,7 +26,10 @@ where } /// An alias of ReadonlyBucket::new for less verbose usage -pub fn bucket_read<'a, S, T>(storage: &'a S, namespace: &[u8]) -> ReadonlyBucket<'a, S, T> +pub fn bucket_read<'a, 'b, S, T>( + storage: &'a S, + namespace: &'b [u8], +) -> ReadonlyBucket<'a, 'b, S, T> where S: ReadonlyStorage, T: Serialize + DeserializeOwned, @@ -31,70 +37,116 @@ where ReadonlyBucket::new(storage, namespace) } -pub struct Bucket<'a, S, T> +/// Bucket stores all data under a series of length-prefixed steps: (namespace, "_pk"). +/// After this is created (each step length-prefixed), we just append the bucket key at the end. +/// +/// The reason for the "_pk" at the end is to allow easy extensibility with IndexedBuckets, which +/// can also store indexes under the same namespace +pub struct Bucket<'a, 'b, S, T> where S: Storage, T: Serialize + DeserializeOwned, { - storage: &'a mut S, - prefix: Vec, + pub(crate) storage: &'a mut S, + pub namespace: &'b [u8], // see https://doc.rust-lang.org/std/marker/struct.PhantomData.html#unused-type-parameters for why this is needed data: PhantomData, } -impl<'a, S, T> Bucket<'a, S, T> +impl<'a, 'b, S, T> Bucket<'a, 'b, S, T> where S: Storage, T: Serialize + DeserializeOwned, { - pub fn new(storage: &'a mut S, namespace: &[u8]) -> Self { + /// After some reflection, I removed multilevel, as what I really wanted was a composite primary key. + /// For eg. allowances, the multilevel design would make it ("bucket", owner, "_pk", spender) + /// and then maybe ("bucket", owner, "expires", expires_idx, pk) as a secondary index. This is NOT what we want. + /// + /// What we want is ("bucket", "_pk", owner, spender) for the first key. And + /// ("bucket", "expires", expires_idx, pk) as the secondary index. This is not done with "multi-level" + /// but by supporting CompositeKeys. Looking something like: + /// + /// `Bucket::new(storage, "bucket).save(&(owner, spender).pk(), &allowance)` + /// + /// Need to figure out the most ergonomic approach for the composite keys, but it should + /// live outside the Bucket, and just convert into a normal `&[u8]` for this array. + pub fn new(storage: &'a mut S, namespace: &'b [u8]) -> Self { Bucket { storage, - prefix: to_length_prefixed(namespace), + namespace, data: PhantomData, } } - pub fn multilevel(storage: &'a mut S, namespaces: &[&[u8]]) -> Self { - Bucket { - storage, - prefix: to_length_prefixed_nested(namespaces), - data: PhantomData, - } + /// This provides the raw storage key that we use to access a given "bucket key". + /// Calling this with `key = b""` will give us the pk prefix for range queries + pub fn build_primary_key(&self, key: &[u8]) -> Vec { + namespaces_with_key(&[&self.namespace, PREFIX_PK], key) + } + + /// This provides the raw storage key that we use to access a secondary index + /// Calling this with `key = b""` will give us the index prefix for range queries + pub fn build_secondary_key(&self, path: &[&[u8]], key: &[u8]) -> Vec { + nested_namespaces_with_key(&[self.namespace], path, key) } /// save will serialize the model and store, returns an error on serialization issues pub fn save(&mut self, key: &[u8], data: &T) -> StdResult<()> { - set_with_prefix(self.storage, &self.prefix, key, &to_vec(data)?); + let key = self.build_primary_key(key); + self.storage.set(&key, &to_vec(data)?); Ok(()) } pub fn remove(&mut self, key: &[u8]) { - remove_with_prefix(self.storage, &self.prefix, key) + let key = self.build_primary_key(key); + self.storage.remove(&key); } /// load will return an error if no data is set at the given key, or on parse error pub fn load(&self, key: &[u8]) -> StdResult { - let value = get_with_prefix(self.storage, &self.prefix, key); + let key = self.build_primary_key(key); + let value = self.storage.get(&key); must_deserialize(&value) } /// may_load will parse the data stored at the key if present, returns Ok(None) if no data there. /// returns an error on issues parsing pub fn may_load(&self, key: &[u8]) -> StdResult> { - let value = get_with_prefix(self.storage, &self.prefix, key); + let key = self.build_primary_key(key); + let value = self.storage.get(&key); may_deserialize(&value) } #[cfg(feature = "iterator")] - pub fn range<'b>( - &'b self, + pub fn range<'c>( + &'c self, + start: Option<&[u8]>, + end: Option<&[u8]>, + order: Order, + ) -> Box>> + 'c> { + let namespace = self.build_primary_key(b""); + let mapped = + range_with_prefix(self.storage, &namespace, start, end, order).map(deserialize_kv::); + Box::new(mapped) + } + + // TODO: test this, just an idea now, need more work on Pks + // Also, how much do we trim from the keys? Leave just the last part of the PK, right? + + /// This lets us grab all items under the beginning of a composite key. + /// If we store under `Pk2(owner, spender)`, then we pass `prefixes: &[owner]` here + /// To list all spenders under the owner + #[cfg(feature = "iterator")] + pub fn range_prefixed<'c>( + &'c self, + prefixes: &[&[u8]], start: Option<&[u8]>, end: Option<&[u8]>, order: Order, - ) -> Box>> + 'b> { - let mapped = range_with_prefix(self.storage, &self.prefix, start, end, order) - .map(deserialize_kv::); + ) -> Box>> + 'c> { + let namespace = nested_namespaces_with_key(&[&self.namespace, PREFIX_PK], prefixes, b""); + let mapped = + range_with_prefix(self.storage, &namespace, start, end, order).map(deserialize_kv::); Box::new(mapped) } @@ -114,60 +166,135 @@ where } } -pub struct ReadonlyBucket<'a, S, T> +pub trait PrimaryKey { + type Output; + + fn pk(&self) -> Vec; + fn parse(data: &[u8]) -> Self::Output; + + // convert a PK into an owned variant (Vec rather than &[u8]) + // this can be done brute force, but please override for a cheaper version + // FIXME: better name for this function - to_owned() sounded good, but uses Cloned. Other ideas? + fn to_output(&self) -> Self::Output { + Self::parse(&self.pk()) + } + + fn from_kv(kv: (Vec, T)) -> (Self::Output, T) + where + Self: std::marker::Sized, + { + let (k, v) = kv; + (Self::parse(&k), v) + } +} + +#[derive(Debug, Clone, PartialEq)] +pub struct Pk2<'a>(pub &'a [u8], pub &'a [u8]); + +impl<'a> PrimaryKey for Pk2<'a> { + type Output = (Vec, Vec); + + fn pk(&self) -> Vec { + length_prefixed_with_key(self.0, self.1) + } + + fn to_output(&self) -> Self::Output { + (self.0.to_vec(), self.1.to_vec()) + } + + fn parse(pk: &[u8]) -> Self::Output { + let l = decode_length(&pk[..2]); + let first = pk[2..l + 2].to_vec(); + let second = pk[l + 2..].to_vec(); + (first, second) + } +} + +#[derive(Debug, Clone, PartialEq)] +pub struct Pk3<'a>(&'a [u8], &'a [u8], &'a [u8]); + +impl<'a> PrimaryKey for Pk3<'a> { + type Output = (Vec, Vec, Vec); + + fn pk(&self) -> Vec { + namespaces_with_key(&[self.0, self.1], self.2) + } + + fn to_output(&self) -> Self::Output { + (self.0.to_vec(), self.1.to_vec(), self.2.to_vec()) + } + + fn parse(pk: &[u8]) -> Self::Output { + let l = decode_length(&pk[..2]); + let l2 = decode_length(&pk[l + 2..l + 4]); + let first = pk[2..l + 2].to_vec(); + let second = pk[l + 4..l + l2 + 4].to_vec(); + let third = pk[l + l2 + 4..].to_vec(); + (first, second, third) + } +} + +pub struct ReadonlyBucket<'a, 'b, S, T> where S: ReadonlyStorage, T: Serialize + DeserializeOwned, { - storage: &'a S, - prefix: Vec, + pub(crate) storage: &'a S, + pub(crate) namespace: &'b [u8], // see https://doc.rust-lang.org/std/marker/struct.PhantomData.html#unused-type-parameters for why this is needed data: PhantomData, } -impl<'a, S, T> ReadonlyBucket<'a, S, T> +impl<'a, 'b, S, T> ReadonlyBucket<'a, 'b, S, T> where S: ReadonlyStorage, T: Serialize + DeserializeOwned, { - pub fn new(storage: &'a S, namespace: &[u8]) -> Self { + pub fn new(storage: &'a S, namespace: &'b [u8]) -> Self { ReadonlyBucket { storage, - prefix: to_length_prefixed(namespace), + namespace, data: PhantomData, } } - pub fn multilevel(storage: &'a S, namespaces: &[&[u8]]) -> Self { - ReadonlyBucket { - storage, - prefix: to_length_prefixed_nested(namespaces), - data: PhantomData, - } + /// This provides the raw storage key that we use to access a given "bucket key". + /// Calling this with `key = b""` will give us the pk prefix for range queries + pub fn build_primary_key(&self, key: &[u8]) -> Vec { + namespaces_with_key(&[&self.namespace, PREFIX_PK], key) + } + + /// This provides the raw storage key that we use to access a secondary index + /// Calling this with `key = b""` will give us the index prefix for range queries + pub fn build_secondary_key(&self, path: &[&[u8]], key: &[u8]) -> Vec { + nested_namespaces_with_key(&[self.namespace], path, key) } /// load will return an error if no data is set at the given key, or on parse error pub fn load(&self, key: &[u8]) -> StdResult { - let value = get_with_prefix(self.storage, &self.prefix, key); + let key = self.build_primary_key(key); + let value = self.storage.get(&key); must_deserialize(&value) } /// may_load will parse the data stored at the key if present, returns Ok(None) if no data there. /// returns an error on issues parsing pub fn may_load(&self, key: &[u8]) -> StdResult> { - let value = get_with_prefix(self.storage, &self.prefix, key); + let key = self.build_primary_key(key); + let value = self.storage.get(&key); may_deserialize(&value) } #[cfg(feature = "iterator")] - pub fn range<'b>( - &'b self, + pub fn range<'c>( + &'c self, start: Option<&[u8]>, end: Option<&[u8]>, order: Order, - ) -> Box>> + 'b> { - let mapped = range_with_prefix(self.storage, &self.prefix, start, end, order) - .map(deserialize_kv::); + ) -> Box>> + 'c> { + let namespace = self.build_primary_key(b""); + let mapped = + range_with_prefix(self.storage, &namespace, start, end, order).map(deserialize_kv::); Box::new(mapped) } } @@ -463,4 +590,82 @@ mod test { assert_eq!(data[0], (b"jose".to_vec(), jose)); assert_eq!(data[1], (b"maria".to_vec(), maria)); } + + #[test] + fn composite_keys() { + let composite = Pk2(b"its", b"windy"); + let key = composite.pk(); + assert_eq!(10, key.len()); + let parsed = Pk2::parse(&key); + assert_eq!(parsed, composite.to_output()); + + let composite = Pk3(b"winters", b"really", b"windy"); + let key = composite.pk(); + assert_eq!(22, key.len()); + let parsed = Pk3::parse(&key); + assert_eq!(parsed, composite.to_output()); + } + + #[test] + fn composite_keys_parsing() { + // Try from a KV (as if we got a range) + let john = Data { + name: "John".to_string(), + age: 123, + }; + let composite = Pk3(b"lots", b"of", b"text"); + + // demo usage as if we mapped over a range iterator + let mut it = vec![(composite.pk(), john.clone())] + .into_iter() + .map(Pk3::from_kv); + let (k1, v1) = it.next().unwrap(); + assert_eq!(k1, composite.to_output()); + assert_eq!(v1, john); + assert!(it.next().is_none()); + } + + #[test] + #[cfg(features = "iterator")] + fn bucket_with_composite_pk() { + let mut store = MockStorage::new(); + let mut bucket = Bucket::<_, 64>::new(&mut store, b"allowance"); + + let owner1: &[u8] = b"john"; + let owner2: &[u8] = b"juan"; + let spender1: &[u8] = b"marco"; + let spender2: &[u8] = b"maria"; + let spender3: &[u8] = b"martian"; + + // store some data with composite key + bucket.save(&Pk2(owner1, spender1).pk(), &100).unwrap(); + bucket.save(&Pk2(owner1, spender2).pk(), &250).unwrap(); + bucket.save(&Pk2(owner2, spender1).pk(), &77).unwrap(); + bucket.save(&Pk2(owner2, spender3).pk(), &444).unwrap(); + + // query by full key + assert_eq!(100, bucket.load(&Pk2(owner1, spender1).pk()).unwrap()); + assert_eq!(444, bucket.load(&Pk2(owner2, spender3).pk()).unwrap()); + + // range over one owner. since it is prefixed, we only get the remaining part of the pk (spender) + let spenders: StdResult> = bucket + .range_prefixed(&[owner1], None, None, Order::Ascending) + .collect(); + let spenders = spenders.unwrap(); + assert_eq!(2, spenders.len()); + assert_eq!(spenders[0], (spender1.to_vec(), 100)); + assert_eq!(spenders[1], (spender2.to_vec(), 250)); + + // range over all data. use Pk2::from_kv to parse out the composite key (owner, spender) + let spenders: StdResult> = bucket + .range(None, None, Order::Ascending) + .map(Pk2::from_kv) + .collect(); + let spenders = spenders.unwrap(); + assert_eq!(4, spenders.len()); + assert_eq!(spenders[0], (Pk2(owner1, spender1).to_output(), 100)); + assert_eq!(spenders[1], (Pk2(owner1, spender2).to_output(), 250)); + assert_eq!(spenders[2], (Pk2(owner2, spender1).to_output(), 77)); + assert_eq!(spenders[3], (Pk2(owner2, spender3).to_output(), 444)); + } } diff --git a/packages/storage/src/indexed_bucket.rs b/packages/storage/src/indexed_bucket.rs new file mode 100644 index 0000000000..32c550f0a9 --- /dev/null +++ b/packages/storage/src/indexed_bucket.rs @@ -0,0 +1,402 @@ +// this module requires iterator to be useful at all +#![cfg(feature = "iterator")] + +use cosmwasm_std::{Order, StdError, StdResult, Storage, KV}; +use serde::de::DeserializeOwned; +use serde::Serialize; + +use crate::bucket::Bucket; +use crate::indexes::{Index, MultiIndex, UniqueIndex}; + +/// reserved name, no index may register +const PREFIX_PK: &[u8] = b"_pk"; + +/// IndexedBucket works like a bucket but has a secondary index +/// This is a WIP. +/// Step 1 - allow exactly 1 secondary index, no multi-prefix on primary key +/// Step 2 - allow multiple named secondary indexes, no multi-prefix on primary key +/// Step 3 - allow unique indexes - They store {pk: Vec, value: T} so we don't need to re-lookup +/// Step 4 - allow multiple named secondary indexes, clean composite key support +/// +/// Current Status: 2 +pub struct IndexedBucket<'a, 'b, 'x, S, T> +where + S: Storage + 'x, + T: Serialize + DeserializeOwned + Clone + 'x, +{ + bucket: Bucket<'a, 'b, S, T>, + indexes: Vec + 'x>>, +} + +impl<'a, 'b, 'x, S, T> IndexedBucket<'a, 'b, 'x, S, T> +where + S: Storage, + T: Serialize + DeserializeOwned + Clone, +{ + pub fn new(storage: &'a mut S, namespace: &'b [u8]) -> Self { + IndexedBucket { + bucket: Bucket::new(storage, namespace), + indexes: vec![], + } + } + + /// Usage: + /// let mut bucket = IndexedBucket::new(&mut storeage, b"foobar") + /// .with_unique_index("name", |x| x.name.clone())? + /// .with_index("age", by_age)?; + pub fn with_index(mut self, name: &'x str, indexer: fn(&T) -> Vec) -> StdResult { + self.can_add_index(name)?; + let index = MultiIndex::new(indexer, name); + self.indexes.push(Box::new(index)); + Ok(self) + } + + /// Usage: + /// let mut bucket = IndexedBucket::new(&mut storeage, b"foobar") + /// .with_unique_index("name", |x| x.name.clone())? + /// .with_index("age", by_age)?; + pub fn with_unique_index( + mut self, + name: &'x str, + indexer: fn(&T) -> Vec, + ) -> StdResult { + self.can_add_index(name)?; + let index = UniqueIndex::new(indexer, name); + self.indexes.push(Box::new(index)); + Ok(self) + } + + fn can_add_index(&self, name: &str) -> StdResult<()> { + if name.as_bytes() == PREFIX_PK { + return Err(StdError::generic_err( + "Index _pk is reserved for the primary key", + )); + } + let dup = self.get_index(name); + match dup { + Some(_) => Err(StdError::generic_err(format!( + "Attempt to write index {} 2 times", + name + ))), + None => Ok(()), + } + } + + fn get_index(&self, name: &str) -> Option<&dyn Index> { + for existing in self.indexes.iter() { + if existing.name() == name { + return Some(existing.as_ref()); + } + } + None + } + + /// save will serialize the model and store, returns an error on serialization issues. + /// this must load the old value to update the indexes properly + /// if you loaded the old value earlier in the same function, use replace to avoid needless db reads + pub fn save(&mut self, key: &[u8], data: &T) -> StdResult<()> { + let old_data = self.may_load(key)?; + self.replace(key, Some(data), old_data.as_ref()) + } + + pub fn remove(&mut self, key: &[u8]) -> StdResult<()> { + let old_data = self.may_load(key)?; + self.replace(key, None, old_data.as_ref()) + } + + /// replace writes data to key. old_data must be the current stored value (from a previous load) + /// and is used to properly update the index. This is used by save, replace, and update + /// and can be called directly if you want to optimize + pub fn replace(&mut self, key: &[u8], data: Option<&T>, old_data: Option<&T>) -> StdResult<()> { + if let Some(old) = old_data { + // Note: this didn't work as we cannot mutably borrow self (remove_from_index) inside the iterator + for index in self.indexes.iter() { + index.remove(&mut self.bucket, key, old)?; + } + } + if let Some(updated) = data { + for index in self.indexes.iter() { + index.insert(&mut self.bucket, key, updated)?; + } + self.bucket.save(key, updated)?; + } else { + self.bucket.remove(key); + } + Ok(()) + } + + /// Loads the data, perform the specified action, and store the result + /// in the database. This is shorthand for some common sequences, which may be useful. + /// + /// If the data exists, `action(Some(value))` is called. Otherwise `action(None)` is called. + pub fn update(&mut self, key: &[u8], action: A) -> Result + where + A: FnOnce(Option) -> Result, + E: From, + { + let input = self.may_load(key)?; + let old_val = input.clone(); + let output = action(input)?; + self.replace(key, Some(&output), old_val.as_ref())?; + Ok(output) + } + + // Everything else, that doesn't touch indexers, is just pass-through from self.core, + // thus can be used from while iterating over indexes + + /// load will return an error if no data is set at the given key, or on parse error + pub fn load(&self, key: &[u8]) -> StdResult { + self.bucket.load(key) + } + + /// may_load will parse the data stored at the key if present, returns Ok(None) if no data there. + /// returns an error on issues parsing + pub fn may_load(&self, key: &[u8]) -> StdResult> { + self.bucket.may_load(key) + } + + /// iterates over the items in pk order + pub fn range<'c>( + &'c self, + start: Option<&[u8]>, + end: Option<&[u8]>, + order: Order, + ) -> Box>> + 'c> { + self.bucket.range(start, end, order) + } + + /// returns all pks that where stored under this secondary index, always Ascending + /// this is mainly an internal function, but can be used direcly if you just want to list ids cheaply + pub fn pks_by_index<'c>( + &'c self, + index_name: &str, + idx: &[u8], + ) -> StdResult> + 'c>> { + let index = self + .get_index(index_name) + .ok_or_else(|| StdError::not_found(index_name))?; + Ok(index.pks_by_index(&self.bucket, idx)) + } + + /// returns all items that match this secondary index, always by pk Ascending + pub fn items_by_index<'c>( + &'c self, + index_name: &str, + idx: &[u8], + ) -> StdResult>> + 'c>> { + let index = self + .get_index(index_name) + .ok_or_else(|| StdError::not_found(index_name))?; + Ok(index.items_by_index(&self.bucket, idx)) + } + + // this will return None for 0 items, Some(x) for 1 item, + // and an error for > 1 item. Only meant to be called on unique + // indexes that can return 0 or 1 item + pub fn load_unique_index(&self, index_name: &str, idx: &[u8]) -> StdResult>> { + let mut it = self.items_by_index(index_name, idx)?; + let first = it.next().transpose()?; + match first { + None => Ok(None), + Some(one) => match it.next() { + None => Ok(Some(one)), + Some(_) => Err(StdError::generic_err("Unique Index returned 2 matches")), + }, + } + } +} + +#[cfg(test)] +mod test { + use super::*; + + use crate::indexes::{index_i32, index_string}; + use cosmwasm_std::testing::MockStorage; + use cosmwasm_std::MemoryStorage; + use serde::{Deserialize, Serialize}; + + #[derive(Serialize, Deserialize, PartialEq, Debug, Clone)] + struct Data { + pub name: String, + pub age: i32, + } + + fn build_bucket(store: &mut S) -> IndexedBucket { + IndexedBucket::::new(store, b"data") + .with_index("name", |d| index_string(&d.name)) + .unwrap() + .with_unique_index("age", |d| index_i32(d.age)) + .unwrap() + } + + #[test] + fn store_and_load_by_index() { + let mut store = MockStorage::new(); + let mut bucket = build_bucket(&mut store); + + // save data + let data = Data { + name: "Maria".to_string(), + age: 42, + }; + let pk: &[u8] = b"5627"; + bucket.save(pk, &data).unwrap(); + + // load it properly + let loaded = bucket.load(pk).unwrap(); + assert_eq!(data, loaded); + + // load it by secondary index (we must know how to compute this) + let marias: StdResult> = bucket + .items_by_index("name", &index_string("Maria")) + .unwrap() + .collect(); + let marias = marias.unwrap(); + assert_eq!(1, marias.len()); + let (k, v) = &marias[0]; + assert_eq!(pk, k.as_slice()); + assert_eq!(&data, v); + + // other index doesn't match (1 byte after) + let marias: StdResult> = bucket + .items_by_index("name", &index_string("Marib")) + .unwrap() + .collect(); + assert_eq!(0, marias.unwrap().len()); + + // other index doesn't match (1 byte before) + let marias: StdResult> = bucket + .items_by_index("name", &index_string("Mari`")) + .unwrap() + .collect(); + assert_eq!(0, marias.unwrap().len()); + + // other index doesn't match (longer) + let marias: StdResult> = bucket + .items_by_index("name", &index_string("Maria5")) + .unwrap() + .collect(); + assert_eq!(0, marias.unwrap().len()); + + // match on proper age + let proper = index_i32(42); + let marias: StdResult> = bucket.items_by_index("age", &proper).unwrap().collect(); + let marias = marias.unwrap(); + assert_eq!(1, marias.len()); + + // no match on wrong age + let too_old = index_i32(43); + let marias: StdResult> = bucket.items_by_index("age", &too_old).unwrap().collect(); + assert_eq!(0, marias.unwrap().len()); + } + + #[test] + fn unique_index_enforced() { + let mut store = MockStorage::new(); + let mut bucket = build_bucket(&mut store); + + // first data + let data1 = Data { + name: "Maria".to_string(), + age: 42, + }; + let pk1: &[u8] = b"5627"; + bucket.save(pk1, &data1).unwrap(); + + // same name (multi-index), different age => ok + let data2 = Data { + name: "Maria".to_string(), + age: 23, + }; + let pk2: &[u8] = b"7326"; + bucket.save(pk2, &data2).unwrap(); + + // different name, same age => error + let data3 = Data { + name: "Marta".to_string(), + age: 42, + }; + let pk3: &[u8] = b"8263"; + // enforce this returns some error + bucket.save(pk3, &data3).unwrap_err(); + + // query by unique key + // match on proper age + let age42 = index_i32(42); + let (k, v) = bucket.load_unique_index("age", &age42).unwrap().unwrap(); + assert_eq!(k.as_slice(), pk1); + assert_eq!(&v.name, "Maria"); + assert_eq!(v.age, 42); + + // match on other age + let age23 = index_i32(23); + let (k, v) = bucket.load_unique_index("age", &age23).unwrap().unwrap(); + assert_eq!(k.as_slice(), pk2); + assert_eq!(&v.name, "Maria"); + assert_eq!(v.age, 23); + + // if we delete the first one, we can add the blocked one + bucket.remove(pk1).unwrap(); + bucket.save(pk3, &data3).unwrap(); + // now 42 is the new owner + let (k, v) = bucket.load_unique_index("age", &age42).unwrap().unwrap(); + assert_eq!(k.as_slice(), pk3); + assert_eq!(&v.name, "Marta"); + assert_eq!(v.age, 42); + } + + #[test] + fn remove_and_update_reflected_on_indexes() { + let mut store = MockStorage::new(); + let mut bucket = build_bucket(&mut store); + + let name_count = |bucket: &IndexedBucket, name: &str| -> usize { + bucket + .items_by_index("name", &index_string(name)) + .unwrap() + .count() + }; + + // set up some data + let data1 = Data { + name: "John".to_string(), + age: 22, + }; + let pk1: &[u8] = b"john"; + bucket.save(pk1, &data1).unwrap(); + let data2 = Data { + name: "John".to_string(), + age: 25, + }; + let pk2: &[u8] = b"john2"; + bucket.save(pk2, &data2).unwrap(); + let data3 = Data { + name: "Fred".to_string(), + age: 33, + }; + let pk3: &[u8] = b"fred"; + bucket.save(pk3, &data3).unwrap(); + + // find 2 Johns, 1 Fred, and no Mary + assert_eq!(name_count(&bucket, "John"), 2); + assert_eq!(name_count(&bucket, "Fred"), 1); + assert_eq!(name_count(&bucket, "Mary"), 0); + + // remove john 2 + bucket.remove(pk2).unwrap(); + // change fred to mary + bucket + .update(pk3, |d| -> StdResult<_> { + let mut x = d.unwrap(); + assert_eq!(&x.name, "Fred"); + x.name = "Mary".to_string(); + Ok(x) + }) + .unwrap(); + + // find 1 Johns, no Fred, and 1 Mary + assert_eq!(name_count(&bucket, "John"), 1); + assert_eq!(name_count(&bucket, "Fred"), 0); + assert_eq!(name_count(&bucket, "Mary"), 1); + } +} diff --git a/packages/storage/src/indexes.rs b/packages/storage/src/indexes.rs new file mode 100644 index 0000000000..236f3c80b7 --- /dev/null +++ b/packages/storage/src/indexes.rs @@ -0,0 +1,239 @@ +// this module requires iterator to be useful at all +#![cfg(feature = "iterator")] + +use cosmwasm_std::{from_slice, to_vec, Binary, Order, StdError, StdResult, Storage, KV}; +use serde::de::DeserializeOwned; +use serde::{Deserialize, Serialize}; +use std::marker::PhantomData; + +use crate::namespace_helpers::range_with_prefix; +use crate::Bucket; + +/// MARKER is stored in the multi-index as value, but we only look at the key (which is pk) +const MARKER: &[u8] = b"1"; + +pub fn index_string(data: &str) -> Vec { + data.as_bytes().to_vec() +} + +// Look at https://docs.rs/endiannezz/0.4.1/endiannezz/trait.Primitive.html +// if you want to make this generic over all ints +pub fn index_u64(data: u64) -> Vec { + data.to_be_bytes().into() +} + +pub fn index_i32(data: i32) -> Vec { + data.to_be_bytes().into() +} + +// 2 main variants: +// * store (namespace, index_name, idx_value, key) -> b"1" - allows many and references pk +// * store (namespace, index_name, idx_value) -> {key, value} - allows one and copies pk and data +// // this would be the primary key - we abstract that too??? +// * store (namespace, index_name, pk) -> value - allows one with data +pub(crate) trait Index +where + S: Storage, + T: Serialize + DeserializeOwned + Clone, +{ + // TODO: do we make this any Vec ? + fn name(&self) -> String; + fn index(&self, data: &T) -> Vec; + + fn insert(&self, bucket: &mut Bucket, pk: &[u8], data: &T) -> StdResult<()>; + fn remove(&self, bucket: &mut Bucket, pk: &[u8], old_data: &T) -> StdResult<()>; + + // these should be implemented by all + fn pks_by_index<'c>( + &self, + bucket: &'c Bucket, + idx: &[u8], + ) -> Box> + 'c>; + + /// returns all items that match this secondary index, always by pk Ascending + fn items_by_index<'c>( + &self, + bucket: &'c Bucket, + idx: &[u8], + ) -> Box>> + 'c>; + + // TODO: range over secondary index values? (eg. all results with 30 < age < 40) +} + +pub(crate) struct MultiIndex<'a, S, T> +where + S: Storage, + T: Serialize + DeserializeOwned + Clone, +{ + idx_fn: fn(&T) -> Vec, + _name: &'a str, + _phantom: PhantomData, +} + +impl<'a, S, T> MultiIndex<'a, S, T> +where + S: Storage, + T: Serialize + DeserializeOwned + Clone, +{ + pub fn new(idx_fn: fn(&T) -> Vec, name: &'a str) -> Self { + MultiIndex { + idx_fn, + _name: name, + _phantom: Default::default(), + } + } +} + +impl<'a, S, T> Index for MultiIndex<'a, S, T> +where + S: Storage, + T: Serialize + DeserializeOwned + Clone, +{ + fn name(&self) -> String { + self._name.to_string() + } + + fn index(&self, data: &T) -> Vec { + (self.idx_fn)(data) + } + + fn insert(&self, bucket: &mut Bucket, pk: &[u8], data: &T) -> StdResult<()> { + let idx = self.index(data); + let key = bucket.build_secondary_key(&[self._name.as_bytes(), &idx], pk); + bucket.storage.set(&key, MARKER); + Ok(()) + } + + fn remove(&self, bucket: &mut Bucket, pk: &[u8], old_data: &T) -> StdResult<()> { + let idx = self.index(old_data); + let key = bucket.build_secondary_key(&[self._name.as_bytes(), &idx], pk); + bucket.storage.remove(&key); + Ok(()) + } + + fn pks_by_index<'c>( + &self, + bucket: &'c Bucket, + idx: &[u8], + ) -> Box> + 'c> { + let namespace = bucket.build_secondary_key(&[self._name.as_bytes(), &idx], b""); + let mapped = range_with_prefix(bucket.storage, &namespace, None, None, Order::Ascending) + .map(|(k, _)| k); + Box::new(mapped) + } + + /// returns all items that match this secondary index, always by pk Ascending + fn items_by_index<'c>( + &self, + bucket: &'c Bucket, + idx: &[u8], + ) -> Box>> + 'c> { + let mapped = self.pks_by_index(bucket, idx).map(move |pk| { + let v = bucket.load(&pk)?; + Ok((pk, v)) + }); + Box::new(mapped) + } +} + +#[derive(Deserialize, Serialize, Clone)] +pub(crate) struct UniqueRef { + pk: Binary, + value: T, +} + +pub(crate) struct UniqueIndex<'a, S, T> +where + S: Storage, + T: Serialize + DeserializeOwned + Clone, +{ + idx_fn: fn(&T) -> Vec, + _name: &'a str, + _phantom: PhantomData, +} + +impl<'a, S, T> UniqueIndex<'a, S, T> +where + S: Storage, + T: Serialize + DeserializeOwned + Clone, +{ + pub fn new(idx_fn: fn(&T) -> Vec, name: &'a str) -> Self { + UniqueIndex { + idx_fn, + _name: name, + _phantom: Default::default(), + } + } +} + +impl<'a, S, T> Index for UniqueIndex<'a, S, T> +where + S: Storage, + T: Serialize + DeserializeOwned + Clone, +{ + fn name(&self) -> String { + self._name.to_string() + } + + fn index(&self, data: &T) -> Vec { + (self.idx_fn)(data) + } + + // we store (namespace, index_name, idx_value) -> { pk, value } + fn insert(&self, bucket: &mut Bucket, pk: &[u8], data: &T) -> StdResult<()> { + let idx = self.index(data); + let key = bucket.build_secondary_key(&[self._name.as_bytes()], &idx); + // error if this is already set + if bucket.storage.get(&key).is_some() { + return Err(StdError::generic_err(format!( + "Violates unique constraint on index `{}`", + self._name + ))); + } + + let reference = UniqueRef:: { + pk: pk.into(), + value: data.clone(), + }; + bucket.storage.set(&key, &to_vec(&reference)?); + Ok(()) + } + + // we store (namespace, index_name, idx_value) -> { pk, value } + fn remove(&self, bucket: &mut Bucket, _pk: &[u8], old_data: &T) -> StdResult<()> { + let idx = self.index(old_data); + let key = bucket.build_secondary_key(&[self._name.as_bytes()], &idx); + bucket.storage.remove(&key); + Ok(()) + } + + // there is exactly 0 or 1 here... + fn pks_by_index<'c>( + &self, + bucket: &'c Bucket, + idx: &[u8], + ) -> Box> + 'c> { + // TODO: update types to return StdResult> ? + // should never really happen, but I dislike unwrap + let mapped = self.items_by_index(bucket, idx).map(|res| res.unwrap().0); + Box::new(mapped) + } + + /// returns all items that match this secondary index, always by pk Ascending + fn items_by_index<'c>( + &self, + bucket: &'c Bucket, + idx: &[u8], + ) -> Box>> + 'c> { + let key = bucket.build_secondary_key(&[self._name.as_bytes()], &idx); + let data = match bucket.storage.get(&key) { + Some(bin) => vec![bin], + None => vec![], + }; + let mapped = data.into_iter().map(|bin| { + let parsed: UniqueRef = from_slice(&bin)?; + Ok((parsed.pk.into_vec(), parsed.value)) + }); + Box::new(mapped) + } +} diff --git a/packages/storage/src/length_prefixed.rs b/packages/storage/src/length_prefixed.rs index 6f6cc7f2d3..bc8528ca4e 100644 --- a/packages/storage/src/length_prefixed.rs +++ b/packages/storage/src/length_prefixed.rs @@ -29,6 +29,56 @@ pub fn to_length_prefixed_nested(namespaces: &[&[u8]]) -> Vec { out } +pub fn length_prefixed_with_key(namespace: &[u8], key: &[u8]) -> Vec { + let mut out = Vec::with_capacity(namespace.len() + 2 + key.len()); + out.extend_from_slice(&encode_length(namespace)); + out.extend_from_slice(namespace); + out.extend_from_slice(key); + out +} + +/// This is equivalent concat(to_length_prefixed_nested(namespaces), key) +/// But more efficient when the intermediate namespaces often must be recalculated +#[allow(dead_code)] +pub fn namespaces_with_key(namespaces: &[&[u8]], key: &[u8]) -> Vec { + let mut size = key.len(); + for &namespace in namespaces { + size += namespace.len() + 2; + } + + let mut out = Vec::with_capacity(size); + for &namespace in namespaces { + out.extend_from_slice(&encode_length(namespace)); + out.extend_from_slice(namespace); + } + out.extend_from_slice(key); + out +} + +/// Customization of namespaces_with_key for when +/// there are multiple sets we do not want to combine just to call this +pub fn nested_namespaces_with_key(top_names: &[&[u8]], sub_names: &[&[u8]], key: &[u8]) -> Vec { + let mut size = key.len(); + for &namespace in top_names { + size += namespace.len() + 2; + } + for &namespace in sub_names { + size += namespace.len() + 2; + } + + let mut out = Vec::with_capacity(size); + for &namespace in top_names { + out.extend_from_slice(&encode_length(namespace)); + out.extend_from_slice(namespace); + } + for &namespace in sub_names { + out.extend_from_slice(&encode_length(namespace)); + out.extend_from_slice(namespace); + } + out.extend_from_slice(key); + out +} + /// Encodes the length of a given namespace as a 2 byte big endian encoded integer fn encode_length(namespace: &[u8]) -> [u8; 2] { if namespace.len() > 0xFFFF { @@ -38,6 +88,12 @@ fn encode_length(namespace: &[u8]) -> [u8; 2] { [length_bytes[2], length_bytes[3]] } +// pub(crate) fn decode_length(prefix: [u8; 2]) -> usize { +pub(crate) fn decode_length(prefix: &[u8]) -> usize { + // TODO: enforce exactly 2 bytes somehow, but usable with slices + (prefix[0] as usize) * 256 + (prefix[1] as usize) +} + #[cfg(test)] mod test { use super::*; diff --git a/packages/storage/src/lib.rs b/packages/storage/src/lib.rs index e9973c5575..7276e6eb8d 100644 --- a/packages/storage/src/lib.rs +++ b/packages/storage/src/lib.rs @@ -1,4 +1,6 @@ mod bucket; +mod indexed_bucket; +mod indexes; mod length_prefixed; mod namespace_helpers; mod prefixed_storage; @@ -8,7 +10,11 @@ mod transactions; mod type_helpers; mod typed; -pub use bucket::{bucket, bucket_read, Bucket, ReadonlyBucket}; +pub use bucket::{bucket, bucket_read, Bucket, Pk2, Pk3, PrimaryKey, ReadonlyBucket}; +#[cfg(feature = "iterator")] +pub use indexed_bucket::IndexedBucket; +#[cfg(feature = "iterator")] +pub use indexes::{index_i32, index_string, index_u64}; pub use length_prefixed::{to_length_prefixed, to_length_prefixed_nested}; pub use prefixed_storage::{prefixed, prefixed_read, PrefixedStorage, ReadonlyPrefixedStorage}; pub use sequence::{currval, nextval, sequence};