diff --git a/Cargo.lock b/Cargo.lock index 59655e44877..44e4c6d769b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -90,6 +90,15 @@ dependencies = [ "syn", ] +[[package]] +name = "atomic-polyfill" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c041a8d9751a520ee19656232a18971f18946a7900f1520ee4400002244dd89" +dependencies = [ + "critical-section", +] + [[package]] name = "atty" version = "0.2.14" @@ -119,6 +128,21 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "bare-metal" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5deb64efa5bd81e31fcd1938615a6d98c82eafcbcd787162b6f63b91d6bac5b3" +dependencies = [ + "rustc_version 0.2.3", +] + +[[package]] +name = "bare-metal" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fe8f5a8a398345e52358e18ff07cc17a568fbca5c6f73873d3a62056309603" + [[package]] name = "base16ct" version = "0.1.1" @@ -180,6 +204,18 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" +[[package]] +name = "bit_field" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcb6dd1c2376d2e096796e234a70e17e94cc2d5d54ff8ce42b28cef1d0d359a4" + +[[package]] +name = "bitfield" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46afbd2983a5d5a7bd740ccb198caf5b82f45c40c09c0eed36052d91cb92e719" + [[package]] name = "bitflags" version = "1.3.2" @@ -416,6 +452,23 @@ dependencies = [ "os_str_bytes", ] +[[package]] +name = "cobs" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67ba02a97a2bd10f4b59b25c7973101c79642302776489e030cd13cdab09ed15" + +[[package]] +name = "codecs-derive" +version = "0.1.0" +dependencies = [ + "parity-scale-codec", + "proc-macro2", + "quote", + "serde", + "syn", +] + [[package]] name = "const-oid" version = "0.9.0" @@ -444,6 +497,18 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" +[[package]] +name = "cortex-m" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70858629a458fdfd39f9675c4dc309411f2a3f83bede76988d81bf1a0ecee9e0" +dependencies = [ + "bare-metal 0.2.5", + "bitfield", + "embedded-hal", + "volatile-register", +] + [[package]] name = "cpufeatures" version = "0.2.5" @@ -498,6 +563,18 @@ dependencies = [ "itertools 0.10.5", ] +[[package]] +name = "critical-section" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95da181745b56d4bd339530ec393508910c909c784e8962d15d722bacf0bcbcd" +dependencies = [ + "bare-metal 1.0.0", + "cfg-if", + "cortex-m", + "riscv", +] + [[package]] name = "crossbeam-channel" version = "0.5.6" @@ -597,7 +674,7 @@ dependencies = [ "convert_case", "proc-macro2", "quote", - "rustc_version", + "rustc_version 0.4.0", "syn", ] @@ -671,6 +748,16 @@ dependencies = [ "zeroize", ] +[[package]] +name = "embedded-hal" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35949884794ad573cf46071e41c9b60efb0cb311e3ca01f7af807af1debc66ff" +dependencies = [ + "nb 0.1.3", + "void", +] + [[package]] name = "enr" version = "0.6.2" @@ -700,7 +787,7 @@ dependencies = [ "num-traits", "proc-macro2", "quote", - "rustc_version", + "rustc_version 0.4.0", "syn", ] @@ -1084,6 +1171,15 @@ version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" +[[package]] +name = "hash32" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c35f58762feb77d74ebe43bdbc3210f09be9fe6742234d573bacc26ed92b67" +dependencies = [ + "byteorder", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -1093,6 +1189,20 @@ dependencies = [ "ahash", ] +[[package]] +name = "heapless" +version = "0.7.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db04bc24a18b9ea980628ecf00e6c0264f3c1426dac36c00cb49b6fbad8b0743" +dependencies = [ + "atomic-polyfill", + "hash32", + "rustc_version 0.4.0", + "serde", + "spin 0.9.4", + "stable_deref_trait", +] + [[package]] name = "heck" version = "0.4.0" @@ -1516,7 +1626,7 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" dependencies = [ - "spin", + "spin 0.5.2", ] [[package]] @@ -1632,6 +1742,21 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "nb" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "801d31da0513b6ec5214e9bf433a77966320625a37860f910be265be6e18d06f" +dependencies = [ + "nb 1.0.0", +] + +[[package]] +name = "nb" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "546c37ac5d9e56f55e73b677106873d9d9f5190605e41a856503623648488cae" + [[package]] name = "nom" version = "7.1.1" @@ -1798,6 +1923,7 @@ dependencies = [ "arrayvec", "bitvec", "byte-slice-cast", + "bytes", "impl-trait-for-tuples", "parity-scale-codec-derive", "serde", @@ -1920,6 +2046,17 @@ dependencies = [ "plotters-backend", ] +[[package]] +name = "postcard" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c2b180dc0bade59f03fd005cb967d3f1e5f69b13922dad0cd6e047cb8af2363" +dependencies = [ + "cobs", + "heapless", + "serde", +] + [[package]] name = "ppv-lite86" version = "0.2.16" @@ -2144,6 +2281,13 @@ dependencies = [ "walkdir", ] +[[package]] +name = "reth-codecs" +version = "0.1.0" +dependencies = [ + "codecs-derive", +] + [[package]] name = "reth-crate-template" version = "0.1.0" @@ -2153,9 +2297,13 @@ name = "reth-db" version = "0.1.0" dependencies = [ "bytes", + "heapless", "libmdbx", "page_size", + "parity-scale-codec", + "postcard", "reth-primitives", + "serde", "tempfile", "thiserror", ] @@ -2263,6 +2411,8 @@ dependencies = [ "ethers-core", "hex-literal", "maplit", + "parity-scale-codec", + "reth-codecs", "reth-rlp", "serde", "serde_json", @@ -2418,7 +2568,7 @@ dependencies = [ "cc", "libc", "once_cell", - "spin", + "spin 0.5.2", "untrusted", "web-sys", "winapi", @@ -2433,6 +2583,27 @@ dependencies = [ "digest 0.10.5", ] +[[package]] +name = "riscv" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6907ccdd7a31012b70faf2af85cd9e5ba97657cc3987c4f13f8e4d2c2a088aba" +dependencies = [ + "bare-metal 1.0.0", + "bit_field", + "riscv-target", +] + +[[package]] +name = "riscv-target" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88aa938cda42a0cf62a20cfe8d139ff1af20c2e681212b5b34adb5a58333f222" +dependencies = [ + "lazy_static", + "regex", +] + [[package]] name = "rlp" version = "0.5.1" @@ -2477,13 +2648,22 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e75f6a532d0fd9f7f13144f392b6ad56a32696bfcd9c78f797f16bbb6f072d6" +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +dependencies = [ + "semver 0.9.0", +] + [[package]] name = "rustc_version" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" dependencies = [ - "semver", + "semver 1.0.14", ] [[package]] @@ -2634,12 +2814,27 @@ dependencies = [ "libc", ] +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +dependencies = [ + "semver-parser", +] + [[package]] name = "semver" version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e25dfac463d778e353db5be2449d1cce89bd6fd23c9f1ea21310ce6e5a1b29c4" +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" + [[package]] name = "send_wrapper" version = "0.4.0" @@ -2782,6 +2977,15 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +[[package]] +name = "spin" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f6002a767bff9e83f8eeecf883ecb8011875a21ae8da43bffb817a57e78cc09" +dependencies = [ + "lock_api", +] + [[package]] name = "spki" version = "0.6.0" @@ -2792,6 +2996,12 @@ dependencies = [ "der", ] +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "static_assertions" version = "1.1.0" @@ -3106,12 +3316,33 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" +[[package]] +name = "vcell" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77439c1b53d2303b20d9459b1ade71a83c716e3f9c34f3228c00e6f185d6c002" + [[package]] name = "version_check" version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "void" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" + +[[package]] +name = "volatile-register" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ee8f19f9d74293faf70901bc20ad067dc1ad390d2cbf1e3f75f721ffee908b6" +dependencies = [ + "vcell", +] + [[package]] name = "wait-timeout" version = "0.2.0" diff --git a/crates/codecs/Cargo.toml b/crates/codecs/Cargo.toml new file mode 100644 index 00000000000..6b41ed90efb --- /dev/null +++ b/crates/codecs/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "reth-codecs" +version = "0.1.0" +edition = "2021" + +[features] +default = ["scale"] +scale = ["codecs-derive/scale"] +postcard = ["codecs-derive/postcard"] +no_codec = ["codecs-derive/no_codec"] + +[dependencies] +codecs-derive = { version = "0.1.0", path = "./derive", default-features = false } \ No newline at end of file diff --git a/crates/codecs/README.md b/crates/codecs/README.md new file mode 100644 index 00000000000..567894bbfdb --- /dev/null +++ b/crates/codecs/README.md @@ -0,0 +1,22 @@ +# codecs + +This crate allows to easily configure different codecs for different purposes (benchmarks, user configuration) with minimal changes. Having them to be configurable through annotations allows us to contain their implementations/leakage to isolated portions of the project. + + +Example: + +[Header struct](../primitives/src/header.rs) + +[DB usage](../db/src/kv/codecs/scale.rs) + + +## Features + +Feature defines what is the main codec used by `#[main_codec]`. However it is still possible to define them directly: `#[use_scale]`, `#[use_postcat]`, `#[no_codec]`. + +```rust +default = ["scale"] +scale = ["codecs-derive/scale"] +postcard = ["codecs-derive/postcard"] +no_codec = ["codecs-derive/no_codec"] +``` \ No newline at end of file diff --git a/crates/codecs/derive/Cargo.toml b/crates/codecs/derive/Cargo.toml new file mode 100644 index 00000000000..568e5dfe4e4 --- /dev/null +++ b/crates/codecs/derive/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "codecs-derive" +version = "0.1.0" +edition = "2021" + +[lib] +proc-macro = true + +[dependencies] +proc-macro2 = "1.0" +quote = "1.0" +syn = { version = "1.0", features = ["full"] } + +# codecs +serde = { version = "1.0.*", default-features = false } +parity-scale-codec = { version = "3.2.1", features = ["derive", "bytes"] } + +[features] +default = ["scale"] +scale = [] +postcard = [] +no_codec = [] \ No newline at end of file diff --git a/crates/codecs/derive/src/lib.rs b/crates/codecs/derive/src/lib.rs new file mode 100644 index 00000000000..5dd662a3919 --- /dev/null +++ b/crates/codecs/derive/src/lib.rs @@ -0,0 +1,66 @@ +use proc_macro::{self, TokenStream}; +use quote::quote; +use syn::{parse_macro_input, DeriveInput}; + +#[proc_macro_attribute] +#[rustfmt::skip] +#[allow(unreachable_code)] +pub fn main_codec(args: TokenStream, input: TokenStream) -> TokenStream { + #[cfg(feature = "scale")] + return use_scale(args, input); + + #[cfg(feature = "postcard")] + return use_postcard(args, input); + + #[cfg(feature = "no_codec")] + return no_codec(args, input); + + // no features + no_codec(args, input) +} + +#[proc_macro_attribute] +pub fn use_scale(_args: TokenStream, input: TokenStream) -> TokenStream { + let mut ast = parse_macro_input!(input as DeriveInput); + let compactable_types = ["u8", "u16", "u32", "i32", "i64", "u64", "f32", "f64"]; + + if let syn::Data::Struct(ref mut data) = &mut ast.data { + if let syn::Fields::Named(fields) = &mut data.fields { + for field in fields.named.iter_mut() { + if let syn::Type::Path(ref path) = field.ty { + if !path.path.segments.is_empty() { + let _type = format!("{}", path.path.segments[0].ident); + if compactable_types.contains(&_type.as_str()) { + field.attrs.push(syn::parse_quote! { + #[codec(compact)] + }); + } + } + } + } + } + } + + quote! { + #[derive(parity_scale_codec::Encode, parity_scale_codec::Decode)] + #ast + } + .into() +} + +#[proc_macro_attribute] +pub fn use_postcard(_args: TokenStream, input: TokenStream) -> TokenStream { + let ast = parse_macro_input!(input as DeriveInput); + + quote! { + #[derive(serde::Serialize, serde::Deserialize)] + #ast + } + .into() +} + +#[proc_macro_attribute] +pub fn no_codec(_args: TokenStream, input: TokenStream) -> TokenStream { + let ast = parse_macro_input!(input as DeriveInput); + quote! { #ast }.into() +} diff --git a/crates/codecs/src/lib.rs b/crates/codecs/src/lib.rs new file mode 100644 index 00000000000..69174635301 --- /dev/null +++ b/crates/codecs/src/lib.rs @@ -0,0 +1 @@ +pub use codecs_derive::*; diff --git a/crates/db/Cargo.toml b/crates/db/Cargo.toml index 464051d782c..7232b36d3a7 100644 --- a/crates/db/Cargo.toml +++ b/crates/db/Cargo.toml @@ -11,6 +11,12 @@ description = "Staged syncing primitives used in reth." # reth reth-primitives = { path = "../primitives" } +# codecs +serde = { version = "1.0.*", default-features = false } +postcard = { version = "1.0.2", features = ["heapless"] } +heapless = "0.7.16" +parity-scale-codec = { version = "3.2.1", features = ["bytes"] } + # misc bytes = "1.2.1" libmdbx = "0.1.8" diff --git a/crates/db/src/kv/codecs/mod.rs b/crates/db/src/kv/codecs/mod.rs new file mode 100644 index 00000000000..8869d6ad96b --- /dev/null +++ b/crates/db/src/kv/codecs/mod.rs @@ -0,0 +1,2 @@ +mod postcard; +mod scale; diff --git a/crates/db/src/kv/codecs/postcard.rs b/crates/db/src/kv/codecs/postcard.rs new file mode 100644 index 00000000000..171ee5ea06b --- /dev/null +++ b/crates/db/src/kv/codecs/postcard.rs @@ -0,0 +1,35 @@ +#![allow(unused)] + +use crate::kv::{Decode, Encode, KVError}; +use heapless::Vec; +use postcard::{from_bytes, to_vec}; +use reth_primitives::Account; + +// Just add `Serialize` and `Deserialize`, and set impl_heapless_postcard!(T, MaxSize(T)) +// +// +// use serde::{Deserialize, Serialize}; +// +// #[derive(Serialize, Deserialize )] +// pub struct T { +// } +// +// impl_heapless_postcard!(T, MaxSize(T)) + +macro_rules! impl_heapless_postcard { + ($name:tt, $static_size:tt) => { + impl Encode for $name { + type Encoded = Vec; + + fn encode(self) -> Self::Encoded { + to_vec(&self).expect("Failed to encode") + } + } + + impl Decode for $name { + fn decode>(value: B) -> Result { + from_bytes(&value.into()).map_err(|_| KVError::InvalidValue) + } + } + }; +} diff --git a/crates/db/src/kv/codecs/scale.rs b/crates/db/src/kv/codecs/scale.rs new file mode 100644 index 00000000000..499b8c83665 --- /dev/null +++ b/crates/db/src/kv/codecs/scale.rs @@ -0,0 +1,43 @@ +use crate::kv::{Decode, Encode, KVError}; +use parity_scale_codec::decode_from_bytes; +use reth_primitives::*; + +mod sealed { + pub trait Sealed {} +} +/// Marker trait type to restrict the TableEncode and TableDecode with scale to chosen types. +pub trait ScaleOnly: sealed::Sealed {} + +impl Encode for T +where + T: ScaleOnly + parity_scale_codec::Encode + Sync + Send + std::fmt::Debug, +{ + type Encoded = Vec; + + fn encode(self) -> Self::Encoded { + parity_scale_codec::Encode::encode(&self) + } +} + +impl Decode for T +where + T: ScaleOnly + parity_scale_codec::Decode + Sync + Send + std::fmt::Debug, +{ + fn decode>(value: B) -> Result { + decode_from_bytes(value.into()).map_err(|_| KVError::InvalidValue) + } +} + +macro_rules! impl_scale { + ($($name:tt),+) => { + $( + impl ScaleOnly for $name {} + impl sealed::Sealed for $name {} + )+ + }; +} + +impl_scale!(u16, H256, U256, H160, u8, u64, Header, Account, Log, Receipt, TxType); + +impl ScaleOnly for Vec {} +impl sealed::Sealed for Vec {} diff --git a/crates/db/src/kv/error.rs b/crates/db/src/kv/error.rs index 7d46779d60f..096196c96c4 100644 --- a/crates/db/src/kv/error.rs +++ b/crates/db/src/kv/error.rs @@ -31,6 +31,6 @@ pub enum KVError { #[error("{0:?}")] InitTransaction(Error), /// Failed to decode or encode a key or value coming from a table.. - #[error("{0:?}")] - InvalidValue(Option), + #[error("Error decoding value.")] + InvalidValue, } diff --git a/crates/db/src/kv/mod.rs b/crates/db/src/kv/mod.rs index a7724450646..67740a78396 100644 --- a/crates/db/src/kv/mod.rs +++ b/crates/db/src/kv/mod.rs @@ -15,12 +15,17 @@ use tables::TABLES; pub mod cursor; +pub mod models; +pub use models::*; + pub mod tx; use tx::Tx; mod error; pub use error::KVError; +mod codecs; + /// Environment used when opening a MDBX environment. RO/RW. #[derive(Debug)] pub enum EnvKind { @@ -164,9 +169,12 @@ pub mod test_utils { #[cfg(test)] mod tests { - use super::{tables::PlainState, test_utils, Env, EnvKind}; + use super::{ + tables::{Headers, PlainState}, + test_utils, Env, EnvKind, + }; use libmdbx::{NoWriteMap, WriteMap}; - use reth_primitives::Address; + use reth_primitives::{Account, Address, Header, H256, U256}; use std::str::FromStr; use tempfile::TempDir; @@ -187,18 +195,17 @@ mod tests { fn db_manual_put_get() { let env = test_utils::create_test_db::(EnvKind::RW); - let value = vec![1, 3, 3, 7]; - let key = Address::from_str("0xa2c122be93b0074270ebee7f6b7292c7deb45047") - .expect(ERROR_ETH_ADDRESS); + let value = Header::default(); + let key = (1u64, H256::zero()); // PUT let tx = env.begin_mut_tx().expect(ERROR_INIT_TX); - tx.put::(key, value.clone()).expect(ERROR_PUT); + tx.put::(key.into(), value.clone()).expect(ERROR_PUT); tx.commit().expect(ERROR_COMMIT); // GET let tx = env.begin_tx().expect(ERROR_INIT_TX); - let result = tx.get::(key).expect(ERROR_GET); + let result = tx.get::(key.into()).expect(ERROR_GET); assert!(result.expect(ERROR_RETURN_VALUE) == value); tx.commit().expect(ERROR_COMMIT); } @@ -207,7 +214,11 @@ mod tests { fn db_closure_put_get() { let path = TempDir::new().expect(test_utils::ERROR_TEMPDIR).into_path(); - let value = vec![1, 3, 3, 7]; + let value = Account { + nonce: 18446744073709551615, + bytecode_hash: H256::random(), + balance: U256::max_value(), + }; let key = Address::from_str("0xa2c122be93b0074270ebee7f6b7292c7deb45047") .expect(ERROR_ETH_ADDRESS); @@ -216,7 +227,7 @@ mod tests { // PUT let result = env.update(|tx| { - tx.put::(key, value.clone()).expect(ERROR_PUT); + tx.put::(key, value).expect(ERROR_PUT); 200 }); assert!(result.expect(ERROR_RETURN_VALUE) == 200); diff --git a/crates/db/src/kv/models/blocks.rs b/crates/db/src/kv/models/blocks.rs new file mode 100644 index 00000000000..4e319373e86 --- /dev/null +++ b/crates/db/src/kv/models/blocks.rs @@ -0,0 +1,64 @@ +//! Block related models and types. + +use crate::kv::{ + table::{Decode, Encode}, + KVError, +}; +use bytes::Bytes; +use reth_primitives::{BlockHash, BlockNumber, H256}; + +/// Total chain number of transactions. Key for [`CumulativeTxCount`]. +pub type NumTransactions = u64; + +/// Number of transactions in the block. Value for [`BlockBodies`]. +pub type NumTxesInBlock = u16; + +/// Hash of the block header. Value for [`CanonicalHeaders`] +pub type HeaderHash = H256; + +/// BlockNumber concatenated with BlockHash. Used as a key for multiple tables. Having the first +/// element as BlockNumber, helps out with querying/sorting. +/// +/// Since it's used as a key, the `BlockNumber` is not compressed when encoding it. +#[derive(Debug)] +#[allow(non_camel_case_types)] +pub struct BlockNumHash((BlockNumber, BlockHash)); + +impl BlockNumHash { + /// Consumes `Self` and returns [`BlockNumber`], [`BlockHash`] + pub fn take(self) -> (BlockNumber, BlockHash) { + (self.0 .0, self.0 .1) + } +} + +impl From<(u64, H256)> for BlockNumHash { + fn from(tpl: (u64, H256)) -> Self { + BlockNumHash(tpl) + } +} + +impl Encode for BlockNumHash { + type Encoded = [u8; 40]; + + fn encode(self) -> Self::Encoded { + let number = self.0 .0; + let hash = self.0 .1; + + let mut rnum = [0; 40]; + + rnum[..8].copy_from_slice(&number.to_be_bytes()); + rnum[8..].copy_from_slice(&hash.encode()); + rnum + } +} + +impl Decode for BlockNumHash { + fn decode>(value: B) -> Result { + let value: bytes::Bytes = value.into(); + + let num = u64::from_be_bytes(value.as_ref().try_into().map_err(|_| KVError::InvalidValue)?); + let hash = H256::decode(value.slice(8..))?; + + Ok(BlockNumHash((num, hash))) + } +} diff --git a/crates/db/src/kv/models/mod.rs b/crates/db/src/kv/models/mod.rs new file mode 100644 index 00000000000..fee897b16bd --- /dev/null +++ b/crates/db/src/kv/models/mod.rs @@ -0,0 +1,3 @@ +//! Implements data structures specific to the database + +pub mod blocks; diff --git a/crates/db/src/kv/table.rs b/crates/db/src/kv/table.rs index 5f760c135d4..70f0f70ae2b 100644 --- a/crates/db/src/kv/table.rs +++ b/crates/db/src/kv/table.rs @@ -2,9 +2,10 @@ use super::KVError; use bytes::Bytes; -use reth_primitives::{Address, U256}; -use std::fmt::Debug; - +use std::{ + fmt::Debug, + marker::{Send, Sync}, +}; /// Trait that will transform the data to be saved in the DB. pub trait Encode: Send + Sync + Sized + Debug { /// Encoded type. @@ -17,7 +18,7 @@ pub trait Encode: Send + Sync + Sized + Debug { /// Trait that will transform the data to be read from the DB. pub trait Decode: Send + Sync + Sized + Debug { /// Decodes data coming from the database. - fn decode(value: &[u8]) -> Result; + fn decode>(value: B) -> Result; } /// Generic trait that enforces the database value to implement [`Encode`] and [`Decode`]. @@ -26,10 +27,19 @@ pub trait Object: Encode + Decode {} impl Object for T where T: Encode + Decode {} /// Generic trait that a database table should follow. +/// +/// [`Table::Key`], [`Table::Value`], [`Table::SeekKey`] types should implement [`Encode`] and +/// [`Decode`] when appropriate. These traits define how the data is stored and read from the +/// database. +/// +/// It allows for the use of codecs. See [`crate::kv::models::blocks::BlockNumHash`] for a custom +/// implementation, and [`crate::kv::codecs::scale`] for the use of an external codec. pub trait Table: Send + Sync + Debug + 'static { /// Return table name as it is present inside the MDBX. const NAME: &'static str; /// Key element of `Table`. + /// + /// Sorting should be taken into account when encoding this. type Key: Encode; /// Value element of `Table`. type Value: Object; @@ -41,93 +51,7 @@ pub trait Table: Send + Sync + Debug + 'static { /// for more check: https://libmdbx.dqdkfa.ru/usage.html#autotoc_md48 pub trait DupSort: Table { /// Subkey type. For more check https://libmdbx.dqdkfa.ru/usage.html#autotoc_md48 + /// + /// Sorting should be taken into account when encoding this. type SubKey: Object; } - -impl Encode for Vec { - type Encoded = Self; - - fn encode(self) -> Self::Encoded { - self - } -} - -impl Decode for Vec { - fn decode(value: &[u8]) -> Result { - Ok(value.to_vec()) - } -} - -impl Encode for Bytes { - type Encoded = Self; - - fn encode(self) -> Self::Encoded { - self - } -} - -impl Decode for Bytes { - fn decode(value: &[u8]) -> Result { - Ok(value.to_vec().into()) - } -} - -impl Encode for Address { - type Encoded = [u8; 20]; - - fn encode(self) -> Self::Encoded { - self.0 - } -} - -impl Decode for Address { - fn decode(value: &[u8]) -> Result { - Ok(Address::from_slice(value)) - } -} - -impl Encode for u16 { - type Encoded = [u8; 2]; - - fn encode(self) -> Self::Encoded { - self.to_be_bytes() - } -} - -impl Decode for u16 { - fn decode(value: &[u8]) -> Result { - unsafe { Ok(u16::from_be_bytes(*(value.as_ptr() as *const [_; 2]))) } - } -} - -impl Encode for u64 { - type Encoded = [u8; 8]; - - fn encode(self) -> Self::Encoded { - self.to_be_bytes() - } -} - -impl Decode for u64 { - fn decode(value: &[u8]) -> Result { - unsafe { Ok(u64::from_be_bytes(*(value.as_ptr() as *const [_; 8]))) } - } -} - -impl Encode for U256 { - type Encoded = [u8; 32]; - - fn encode(self) -> Self::Encoded { - let mut result = [0; 32]; - self.to_big_endian(&mut result); - result - } -} - -impl Decode for U256 { - fn decode(value: &[u8]) -> Result { - let mut result = [0; 32]; - result.copy_from_slice(value); - Ok(Self::from_big_endian(&result)) - } -} diff --git a/crates/db/src/kv/tables.rs b/crates/db/src/kv/tables.rs index e1652c45daa..515376d59ec 100644 --- a/crates/db/src/kv/tables.rs +++ b/crates/db/src/kv/tables.rs @@ -1,7 +1,9 @@ //! Declaration of all MDBX tables. - -use crate::utils::TableType; -use reth_primitives::{Address, BlockNumber, U256}; +use crate::{ + kv::blocks::{BlockNumHash, HeaderHash, NumTransactions, NumTxesInBlock}, + utils::TableType, +}; +use reth_primitives::{Account, Address, BlockNumber, Header, Receipt}; /// Default tables that should be present inside database. pub const TABLES: [(TableType, &str); 18] = [ @@ -63,28 +65,28 @@ macro_rules! table { // TABLE DEFINITIONS // -table!(CanonicalHeaders => BNum => HeaderHash); -table!(HeaderTD => BNum_BHash => RlpTotalDifficulty); -table!(HeaderNumbers => BNum_BHash => BNum); -table!(Headers => BNum_BHash => RlpHeader); +table!(CanonicalHeaders => BlockNumber => HeaderHash); +table!(HeaderTD => BlockNumHash => RlpTotalDifficulty); +table!(HeaderNumbers => BlockNumHash => BlockNumber); +table!(Headers => BlockNumHash => Header); -table!(BlockBodies => BNum_BHash => NumTxesInBlock); -table!(CumulativeTxCount => BNum_BHash => u64); // TODO U256? +table!(BlockBodies => BlockNumHash => NumTxesInBlock); +table!(CumulativeTxCount => BlockNumHash => NumTransactions); // TODO U256? -table!(NonCanonicalTransactions => BNum_BHash_TxId => RlpTxBody); -table!(Transactions => TxId => RlpTxBody); // Canonical only -table!(Receipts => TxId => Receipt); // Canonical only -table!(Logs => TxId => Receipt); // Canonical only +table!(NonCanonicalTransactions => BlockNumHashTxNumber => RlpTxBody); +table!(Transactions => TxNumber => RlpTxBody); // Canonical only +table!(Receipts => TxNumber => Receipt); // Canonical only +table!(Logs => TxNumber => Receipt); // Canonical only -table!(PlainState => PlainStateKey => Vec); +table!(PlainState => PlainStateKey => Account); -table!(AccountHistory => Address => TxIdList); -table!(StorageHistory => Address_StorageKey => TxIdList); +table!(AccountHistory => Address => TxNumberList); +table!(StorageHistory => Address_StorageKey => TxNumberList); -table!(AccountChangeSet => TxId => AccountBeforeTx); -table!(StorageChangeSet => TxId => StorageKeyBeforeTx); +table!(AccountChangeSet => TxNumber => AccountBeforeTx); +table!(StorageChangeSet => TxNumber => StorageKeyBeforeTx); -table!(TxSenders => TxId => Address); // Is it necessary? +table!(TxSenders => TxNumber => Address); // Is it necessary? table!(Config => ConfigKey => ConfigValue); table!(SyncStage => StageId => BlockNumber); @@ -96,19 +98,12 @@ table!(SyncStage => StageId => BlockNumber); type ConfigKey = Vec; type ConfigValue = Vec; #[allow(non_camel_case_types)] -type BNum_BHash = Vec; -#[allow(non_camel_case_types)] -type BNum_BHash_TxId = Vec; -type RlpHeader = Vec; +type BlockNumHashTxNumber = Vec; type RlpTotalDifficulty = Vec; type RlpTxBody = Vec; -type Receipt = Vec; -type NumTxesInBlock = u16; // TODO can it be u16 -type BNum = u64; // TODO check size -type TxId = u64; // TODO check size -type HeaderHash = U256; +type TxNumber = u64; // TODO check size type PlainStateKey = Address; // TODO new type will have to account for address_incarna_skey as well -type TxIdList = Vec; +type TxNumberList = Vec; #[allow(non_camel_case_types)] type Address_StorageKey = Vec; type AccountBeforeTx = Vec; diff --git a/crates/db/src/utils.rs b/crates/db/src/utils.rs index fdcfeb578e9..e1313584394 100644 --- a/crates/db/src/utils.rs +++ b/crates/db/src/utils.rs @@ -4,6 +4,7 @@ use crate::kv::{ table::{Decode, Table}, KVError, }; +use bytes::Bytes; use std::borrow::Cow; /// Enum for the type of table present in libmdbx. @@ -35,7 +36,10 @@ where T: Table, T::Key: Decode, { - Ok((Decode::decode(&kv.0)?, Decode::decode(&kv.1)?)) + Ok(( + Decode::decode(Bytes::from(kv.0.into_owned()))?, + Decode::decode(Bytes::from(kv.1.into_owned()))?, + )) } /// Helper function to decode only a value from a `(key, value)` pair. @@ -43,7 +47,7 @@ pub(crate) fn decode_value<'a, T>(kv: (Cow<'a, [u8]>, Cow<'a, [u8]>)) -> Result< where T: Table, { - Decode::decode(&kv.1) + Decode::decode(Bytes::from(kv.1.into_owned())) } /// Helper function to decode a value. It can be a key or subkey. @@ -51,5 +55,5 @@ pub(crate) fn decode_one(value: Cow<'_, [u8]>) -> Result where T: Table, { - Decode::decode(&value) + Decode::decode(Bytes::from(value.into_owned())) } diff --git a/crates/primitives/Cargo.toml b/crates/primitives/Cargo.toml index 966683de542..d2ad32e33de 100644 --- a/crates/primitives/Cargo.toml +++ b/crates/primitives/Cargo.toml @@ -10,9 +10,13 @@ description = "Commonly used types in reth." [dependencies] ethers-core = { git = "https://github.com/gakonst/ethers-rs", default-features = false } bytes = "1.2" + serde = "1.0" thiserror = "1" + reth-rlp = { path = "../common/rlp", features = ["derive"]} +parity-scale-codec = { version = "3.2.1", features = ["derive", "bytes"] } +reth-codecs = { version = "0.1.0", path = "../codecs" } #used for forkid crc = "1" diff --git a/crates/primitives/src/account.rs b/crates/primitives/src/account.rs index 7542a2b6deb..4aeeedbd602 100644 --- a/crates/primitives/src/account.rs +++ b/crates/primitives/src/account.rs @@ -1,6 +1,8 @@ use crate::{H256, U256}; +use reth_codecs::main_codec; /// Account saved in database +#[main_codec] #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Account { /// Nonce. diff --git a/crates/primitives/src/header.rs b/crates/primitives/src/header.rs index 5649e7b8e99..53d8dc70a30 100644 --- a/crates/primitives/src/header.rs +++ b/crates/primitives/src/header.rs @@ -1,8 +1,10 @@ use std::ops::Deref; -use crate::{BlockNumber, Bytes, H160, H256, U256}; +use crate::{BlockNumber, H160, H256, U256}; +use reth_codecs::main_codec; /// Block header +#[main_codec] #[derive(Debug, Clone, PartialEq, Eq, Default)] pub struct Header { /// The Keccak 256-bit hash of the parent @@ -43,7 +45,7 @@ pub struct Header { pub timestamp: u64, /// An arbitrary byte array containing data relevant to this block. This must be 32 bytes or /// fewer; formally Hx. - pub extra_data: Bytes, + pub extra_data: bytes::Bytes, /// A 256-bit hash which, combined with the /// nonce, proves that a sufficient amount of computation has been carried out on this block; /// formally Hm. diff --git a/crates/primitives/src/jsonu256.rs b/crates/primitives/src/jsonu256.rs index f01be550dca..70d2f57a59f 100644 --- a/crates/primitives/src/jsonu256.rs +++ b/crates/primitives/src/jsonu256.rs @@ -73,7 +73,6 @@ impl<'a> Visitor<'a> for JsonU256Visitor { mod test { use super::JsonU256; use crate::U256; - use serde_json; #[test] fn jsonu256_deserialize() { diff --git a/crates/primitives/src/lib.rs b/crates/primitives/src/lib.rs index db5eed2166e..2a9800b3d3c 100644 --- a/crates/primitives/src/lib.rs +++ b/crates/primitives/src/lib.rs @@ -25,6 +25,8 @@ pub use log::Log; pub use receipt::Receipt; pub use transaction::{AccessList, AccessListItem, Transaction, TransactionSigned, TxType}; +/// Block hash. +pub type BlockHash = H256; /// Block Number is height of chain pub type BlockNumber = u64; /// Ethereum address diff --git a/crates/primitives/src/log.rs b/crates/primitives/src/log.rs index 78b963180c3..27565c64f6c 100644 --- a/crates/primitives/src/log.rs +++ b/crates/primitives/src/log.rs @@ -1,6 +1,8 @@ -use crate::{Address, Bytes, H256}; +use crate::{Address, H256}; +use reth_codecs::main_codec; /// Ethereum Log +#[main_codec] #[derive(Clone, Debug, PartialEq, Eq)] pub struct Log { /// Contract that emitted this log. @@ -8,5 +10,5 @@ pub struct Log { /// Topics of the log. The number of logs depend on what `LOG` opcode is used. pub topics: Vec, /// Arbitrary length data. - pub data: Bytes, + pub data: bytes::Bytes, } diff --git a/crates/primitives/src/receipt.rs b/crates/primitives/src/receipt.rs index 25bba50cba9..d784c1dd580 100644 --- a/crates/primitives/src/receipt.rs +++ b/crates/primitives/src/receipt.rs @@ -1,6 +1,8 @@ use crate::{Log, TxType, H256}; +use reth_codecs::main_codec; /// Receipt containing result of transaction execution. +#[main_codec] #[derive(Clone, Debug, PartialEq, Eq)] pub struct Receipt { /// Receipt type. diff --git a/crates/primitives/src/transaction/tx_type.rs b/crates/primitives/src/transaction/tx_type.rs index 2f8fca27ab1..2c0419d0fac 100644 --- a/crates/primitives/src/transaction/tx_type.rs +++ b/crates/primitives/src/transaction/tx_type.rs @@ -1,10 +1,13 @@ +use reth_codecs::main_codec; + /// Transaction Type +#[main_codec] #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] pub enum TxType { /// Legacy transaction pre EIP-2929 - Legacy = 0, + Legacy = 0_isize, /// AccessList transaction - EIP2930 = 1, + EIP2930 = 1_isize, /// Transaction with Priority fee - EIP1559 = 2, + EIP1559 = 2_isize, }