feat(db): codec encoding/decoding

crates/codecs/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "reth-codecs"
+version = "0.1.0"
+edition = "2021"
+
+[features]
+default = ["scale"]
+scale = ["codecs-derive/scale"]
+postcard = ["codecs-derive/postcard"]
+no_codec = ["codecs-derive/no_codec"]
+
+[dependencies]
+codecs-derive = { version = "0.1.0", path = "./derive", default-features = false }

crates/codecs/derive/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "codecs-derive"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+proc-macro2 = "1.0"
+quote = "1.0"
+syn = { version = "1.0", features = ["full"] }
+
+# codecs
+serde = { version = "1.0.*", default-features = false }
+parity-scale-codec = { version = "3.2.1", features = ["derive", "bytes"] }
+
+[lib]
+proc-macro = true
+
+[features]
+default = ["scale"]
+scale = []
+postcard = []
+no_codec = []

crates/codecs/derive/src/lib.rs

@@ -0,0 +1,66 @@
+use proc_macro::{self, TokenStream};
+use quote::quote;
+use syn::{parse_macro_input, DeriveInput};
+
+#[proc_macro_attribute]
+#[rustfmt::skip]
+#[allow(unreachable_code)]
+pub fn main_codec(args: TokenStream, input: TokenStream) -> TokenStream {
+    #[cfg(feature = "scale")]
+    return use_scale(args, input);
+
+    #[cfg(feature = "postcard")]
+    return use_postcard(args, input);
+
+    #[cfg(feature = "no_codec")]
+    return no_codec(args, input);
+
+    // no features
+    no_codec(args, input)
+}
+
+#[proc_macro_attribute]
+pub fn use_scale(_args: TokenStream, input: TokenStream) -> TokenStream {
+    let mut ast = parse_macro_input!(input as DeriveInput);
+    let compactable_types = ["u8", "u16", "u32", "i32", "i64", "u64", "f32", "f64"];
+
+    if let syn::Data::Struct(ref mut data) = &mut ast.data {
+        if let syn::Fields::Named(fields) = &mut data.fields {
+            for field in fields.named.iter_mut() {
+                if let syn::Type::Path(ref path) = field.ty {
+                    if !path.path.segments.is_empty() {
+                        let _type = format!("{}", path.path.segments[0].ident);
+                        if compactable_types.contains(&_type.as_str()) {
+                            field.attrs.push(syn::parse_quote! {
+                                #[codec(compact)]
+                            });
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    quote! {
+        #[derive(parity_scale_codec::Encode, parity_scale_codec::Decode)]
+        #ast
+    }
+    .into()
+}
+
+#[proc_macro_attribute]
+pub fn use_postcard(_args: TokenStream, input: TokenStream) -> TokenStream {
+    let ast = parse_macro_input!(input as DeriveInput);
+
+    quote! {
+        #[derive(serde::Serialize, serde::Deserialize)]
+        #ast
+    }
+    .into()
+}
+
+#[proc_macro_attribute]
+pub fn no_codec(_args: TokenStream, input: TokenStream) -> TokenStream {
+    let ast = parse_macro_input!(input as DeriveInput);
+    quote! { #ast }.into()
+}

crates/codecs/src/lib.rs

@@ -0,0 +1 @@
+pub use codecs_derive::*;

crates/db/Cargo.toml

@@ -11,6 +11,12 @@ description = "Staged syncing primitives used in reth."
 # reth
 reth-primitives = { path = "../primitives" }
 
+# codecs
+serde = { version = "1.0.*", default-features = false }
+postcard = { version = "1.0.2", features = ["heapless"] }
+heapless = "0.7.16"
+parity-scale-codec = { version = "3.2.1", features = ["bytes"] }
+
 # misc
 bytes = "1.2.1"
 libmdbx = "0.1.8"

crates/db/src/kv/codecs/mod.rs

@@ -0,0 +1,2 @@
+mod postcard;
+mod scale;

crates/db/src/kv/codecs/postcard.rs

@@ -0,0 +1,35 @@
+#![allow(unused)]
+
+use crate::kv::{Decode, Encode, KVError};
+use heapless::Vec;
+use postcard::{from_bytes, to_vec};
+use reth_primitives::Account;
+
+// Just add `Serialize` and `Deserialize`, and set impl_heapless_postcard!(T, MaxSize(T))
+//
+//
+// use serde::{Deserialize, Serialize};
+//
+// #[derive(Serialize, Deserialize )]
+// pub struct T {
+// }
+//
+// impl_heapless_postcard!(T, MaxSize(T))
+
+macro_rules! impl_heapless_postcard {
+    ($name:tt, $static_size:tt) => {
+        impl Encode for $name {
+            type Encoded = Vec<u8, $static_size>;
+
+            fn encode(self) -> Self::Encoded {
+                to_vec(&self).expect("Failed to encode")
+            }
+        }
+
+        impl Decode for $name {
+            fn decode<B: Into<bytes::Bytes>>(value: B) -> Result<Self, KVError> {
+                from_bytes(&value.into()).map_err(|_| KVError::InvalidValue)
+            }
+        }
+    };
+}

crates/db/src/kv/codecs/scale.rs

@@ -0,0 +1,43 @@
+use crate::kv::{Decode, Encode, KVError};
+use parity_scale_codec::decode_from_bytes;
+use reth_primitives::*;
+
+mod sealed {
+    pub trait Sealed {}
+}
+/// Marker trait type to restrict the TableEncode and TableDecode with scale to chosen types.
+pub trait ScaleOnly: sealed::Sealed {}
+
+impl<T> Encode for T
+where
+    T: ScaleOnly + parity_scale_codec::Encode + Sync + Send + std::fmt::Debug,
+{
+    type Encoded = Vec<u8>;
+
+    fn encode(self) -> Self::Encoded {
+        parity_scale_codec::Encode::encode(&self)
+    }
+}
+
+impl<T> Decode for T
+where
+    T: ScaleOnly + parity_scale_codec::Decode + Sync + Send + std::fmt::Debug,
+{
+    fn decode<B: Into<bytes::Bytes>>(value: B) -> Result<T, KVError> {
+        decode_from_bytes(value.into()).map_err(|_| KVError::InvalidValue)
+    }
+}
+
+macro_rules! impl_scale {
+    ($($name:tt),+) => {
+        $(
+            impl ScaleOnly for $name {}
+            impl sealed::Sealed for $name {}
+        )+
+    };
+}
+
+impl_scale!(u16, H256, U256, H160, u8, u64, Header, Account, Log, Receipt, TxType);
+
+impl ScaleOnly for Vec<u8> {}
+impl sealed::Sealed for Vec<u8> {}

crates/db/src/kv/error.rs

@@ -31,6 +31,6 @@ pub enum KVError {
     #[error("{0:?}")]
     InitTransaction(Error),
     /// Failed to decode or encode a key or value coming from a table..
-    #[error("{0:?}")]
-    InvalidValue(Option<String>),
+    #[error("Error decoding value.")]
+    InvalidValue,
 }

crates/db/src/kv/mod.rs

@@ -15,12 +15,17 @@ use tables::TABLES;
 
 pub mod cursor;
 
+pub mod models;
+pub use models::*;
+
 pub mod tx;
 use tx::Tx;
 
 mod error;
 pub use error::KVError;
 
+mod codecs;
+
 /// Environment used when opening a MDBX environment. RO/RW.
 #[derive(Debug)]
 pub enum EnvKind {
@@ -164,9 +169,12 @@ pub mod test_utils {
 
 #[cfg(test)]
 mod tests {
-    use super::{tables::PlainState, test_utils, Env, EnvKind};
+    use super::{
+        tables::{Headers, PlainState},
+        test_utils, Env, EnvKind,
+    };
     use libmdbx::{NoWriteMap, WriteMap};
-    use reth_primitives::Address;
+    use reth_primitives::{Account, Address, Header, H256, U256};
     use std::str::FromStr;
     use tempfile::TempDir;
 
@@ -187,18 +195,17 @@ mod tests {
     fn db_manual_put_get() {
         let env = test_utils::create_test_db::<NoWriteMap>(EnvKind::RW);
 
-        let value = vec![1, 3, 3, 7];
-        let key = Address::from_str("0xa2c122be93b0074270ebee7f6b7292c7deb45047")
-            .expect(ERROR_ETH_ADDRESS);
+        let value = Header::default();
+        let key = (1u64, H256::zero());
 
         // PUT
         let tx = env.begin_mut_tx().expect(ERROR_INIT_TX);
-        tx.put::<PlainState>(key, value.clone()).expect(ERROR_PUT);
+        tx.put::<Headers>(key.into(), value.clone()).expect(ERROR_PUT);
         tx.commit().expect(ERROR_COMMIT);
 
         // GET
         let tx = env.begin_tx().expect(ERROR_INIT_TX);
-        let result = tx.get::<PlainState>(key).expect(ERROR_GET);
+        let result = tx.get::<Headers>(key.into()).expect(ERROR_GET);
         assert!(result.expect(ERROR_RETURN_VALUE) == value);
         tx.commit().expect(ERROR_COMMIT);
     }
@@ -207,7 +214,11 @@ mod tests {
     fn db_closure_put_get() {
         let path = TempDir::new().expect(test_utils::ERROR_TEMPDIR).into_path();
 
-        let value = vec![1, 3, 3, 7];
+        let value = Account {
+            nonce: 18446744073709551615,
+            bytecode_hash: H256::random(),
+            balance: U256::max_value(),
+        };
         let key = Address::from_str("0xa2c122be93b0074270ebee7f6b7292c7deb45047")
             .expect(ERROR_ETH_ADDRESS);
 
@@ -216,7 +227,7 @@ mod tests {
 
             // PUT
             let result = env.update(|tx| {
-                tx.put::<PlainState>(key, value.clone()).expect(ERROR_PUT);
+                tx.put::<PlainState>(key, value).expect(ERROR_PUT);
                 200
             });
             assert!(result.expect(ERROR_RETURN_VALUE) == 200);

crates/db/src/kv/models/blocks.rs

@@ -0,0 +1,64 @@
+//! Block related models and types.
+
+use crate::kv::{
+    table::{Decode, Encode},
+    KVError,
+};
+use bytes::Bytes;
+use reth_primitives::{BlockHash, BlockNumber, H256};
+
+/// Total chain number of transactions. Key for [`CumulativeTxCount`].
+pub type NumTransactions = u64;
+
+/// Number of transactions in the block. Value for [`BlockBodies`].
+pub type NumTxesInBlock = u16;
+
+/// Hash of the block header. Value for [`CanonicalHeaders`]
+pub type HeaderHash = H256;
+
+/// BlockNumber concatenated with BlockHash. Used as a key for multiple tables. Having the first
+/// element as BlockNumber, helps out with querying/sorting.
+///
+/// Since it's used as a key, the `BlockNumber` is not compressed when encoding it.
+#[derive(Debug)]
+#[allow(non_camel_case_types)]
+pub struct BlockNumHash((BlockNumber, BlockHash));
+
+impl BlockNumHash {
+    /// Consumes `Self` and returns [`BlockNumber`], [`BlockHash`]
+    pub fn take(self) -> (BlockNumber, BlockHash) {
+        (self.0 .0, self.0 .1)
+    }
+}
+
+impl From<(u64, H256)> for BlockNumHash {
+    fn from(tpl: (u64, H256)) -> Self {
+        BlockNumHash(tpl)
+    }
+}
+
+impl Encode for BlockNumHash {
+    type Encoded = [u8; 40];
+
+    fn encode(self) -> Self::Encoded {
+        let number = self.0 .0;
+        let hash = self.0 .1;
+
+        let mut rnum = [0; 40];
+
+        rnum[..8].copy_from_slice(&number.to_be_bytes());
+        rnum[8..].copy_from_slice(&hash.encode());
+        rnum
+    }
+}
+
+impl Decode for BlockNumHash {
+    fn decode<B: Into<Bytes>>(value: B) -> Result<Self, KVError> {
+        let value: bytes::Bytes = value.into();
+
+        let num = u64::from_be_bytes(value.as_ref().try_into().map_err(|_| KVError::InvalidValue)?);
+        let hash = H256::decode(value.slice(8..))?;
+
+        Ok(BlockNumHash((num, hash)))
+    }
+}

crates/db/src/kv/models/mod.rs

@@ -0,0 +1,3 @@
+//! Implements data structures specific to the database
+
+pub mod blocks;