From 114d450bae679d8701b6f409699d075bdd68c68c Mon Sep 17 00:00:00 2001 From: Green Baneling Date: Fri, 2 Feb 2024 09:56:33 -0500 Subject: [PATCH] Use a separate database for each data domain (#1629) Closes https://github.com/FuelLabs/fuel-core/issues/1568 The change splits the `Database` into 3 databases: - `Database` - Stores only data required for normal work of the blockchain. - `Database` - Stores only data used by the off-chain services like GraphQL. - `Database` - Stores relayer-related data like events(messages or transactions) from L1. The `Database` type has a generic `Description` that implements the `DatabaseDescription` trait: ```rust /// The description of the database that makes it unique. pub trait DatabaseDescription: 'static + Clone + Debug + Send + Sync { /// The type of the column used by the database. type Column: StorageColumn + strum::EnumCount + enum_iterator::Sequence; /// The type of the height of the database used to track commits. type Height: Copy; /// Returns the expected version of the database. fn version() -> u32; /// Returns the name of the database. fn name() -> &'static str; /// Returns the column used to store the metadata. fn metadata_column() -> Self::Column; /// Returns the prefix for the column. fn prefix(column: &Self::Column) -> Option; } ``` Each database has its folder, defined by the `DatabaseDescription::name`, where actual data is stored. image Each database has its own `Column` type that describes all columns, avoiding overlaps with other tables. The change updates a little bit `StrucutredStorage` implementation and `TableWithBlueprint` to be more flexible and use the `Column` defined by the table, instead of hardcoded `fuel_core_storage::column::Column`. Other small changes: - Unified the logic of storing the database's metadata. It will be useful for https://github.com/FuelLabs/fuel-core/issues/1589 to implement a unified `commit_chagnes` function. - The `latest_height` function now uses the height from the metadata table. - Removed relayers-related tables and columns from the `fuel-core-storage` crate. - Removed part of GraphQL tables and columns from the `fuel-core-storage`. The last part will be removed during https://github.com/FuelLabs/fuel-core/issues/1583. - Moved `tx_count` metrics from `BlockImporter` to GraphQL off-chain worker. Any statistic that requires a persistent state in the database may be done outside of the blockchain. - Remove `chain_name` from the database. The `ConsensusParameters` already contains this information. - Removed the `checkpoint` function from the `RocksDB` since it is not used. Later it will be added again back but with another implementation during https://github.com/FuelLabs/fuel-core/issues/1589. - Removed `Column::ForeignColumn`, since each database has its own `Column` type. Removed the macro rules added to handle `ForeignColumn`. --- CHANGELOG.md | 3 +- Cargo.lock | 4 + benches/benches/block_target_gas.rs | 9 +- benches/benches/transaction_throughput.rs | 1 + crates/fuel-core/Cargo.toml | 1 + crates/fuel-core/src/coins_query.rs | 13 +- crates/fuel-core/src/combined_database.rs | 92 +++++++ crates/fuel-core/src/database.rs | 183 +++++++++---- crates/fuel-core/src/database/balances.rs | 3 +- crates/fuel-core/src/database/block.rs | 34 ++- crates/fuel-core/src/database/coin.rs | 11 +- crates/fuel-core/src/database/contracts.rs | 3 +- .../src/database/database_description.rs | 48 ++++ .../database_description/off_chain.rs | 37 +++ .../database/database_description/on_chain.rs | 35 +++ .../database/database_description/relayer.rs | 58 +++++ crates/fuel-core/src/database/message.rs | 13 +- crates/fuel-core/src/database/metadata.rs | 142 +++++------ crates/fuel-core/src/database/state.rs | 3 +- crates/fuel-core/src/database/statistic.rs | 77 ++++++ crates/fuel-core/src/database/storage.rs | 39 ++- crates/fuel-core/src/database/transaction.rs | 69 +++-- crates/fuel-core/src/database/transactions.rs | 217 +--------------- crates/fuel-core/src/executor.rs | 49 ++-- crates/fuel-core/src/graphql_api.rs | 2 + crates/fuel-core/src/graphql_api/database.rs | 50 ++-- crates/fuel-core/src/graphql_api/ports.rs | 46 ++-- crates/fuel-core/src/graphql_api/storage.rs | 54 ++++ .../src/graphql_api/storage/receipts.rs | 45 ++++ .../src/graphql_api/storage/transactions.rs | 212 ++++++++++++++++ .../src/graphql_api/worker_service.rs | 50 +++- crates/fuel-core/src/lib.rs | 1 + crates/fuel-core/src/query/chain.rs | 6 - crates/fuel-core/src/query/tx.rs | 14 +- crates/fuel-core/src/schema/chain.rs | 5 +- crates/fuel-core/src/schema/dap.rs | 24 +- crates/fuel-core/src/service.rs | 55 ++-- crates/fuel-core/src/service/adapters.rs | 9 +- .../src/service/adapters/block_importer.rs | 4 - .../src/service/adapters/executor.rs | 7 +- .../service/adapters/graphql_api/off_chain.rs | 47 +--- .../service/adapters/graphql_api/on_chain.rs | 48 ++-- .../src/service/adapters/producer.rs | 6 +- crates/fuel-core/src/service/genesis.rs | 43 +--- crates/fuel-core/src/service/query.rs | 2 +- crates/fuel-core/src/service/sub_services.rs | 41 ++- crates/fuel-core/src/state.rs | 53 ++-- .../src/state/in_memory/memory_store.rs | 84 ++++-- .../src/state/in_memory/transaction.rs | 81 ++++-- crates/fuel-core/src/state/rocks_db.rs | 135 +++++----- crates/metrics/src/graphql_metrics.rs | 17 +- crates/metrics/src/importer.rs | 10 - crates/services/importer/src/importer.rs | 11 - crates/services/importer/src/importer/test.rs | 3 - crates/services/importer/src/ports.rs | 3 - crates/services/relayer/Cargo.toml | 3 + crates/services/relayer/src/ports/tests.rs | 14 +- crates/services/relayer/src/storage.rs | 80 ++++-- crates/storage/src/blueprint/plain.rs | 9 +- crates/storage/src/blueprint/sparse.rs | 12 +- crates/storage/src/column.rs | 240 ++++++------------ crates/storage/src/kv_store.rs | 15 +- crates/storage/src/structured_storage.rs | 87 ++++--- .../src/structured_storage/balances.rs | 1 + .../storage/src/structured_storage/blocks.rs | 1 + .../storage/src/structured_storage/coins.rs | 1 + .../src/structured_storage/contracts.rs | 3 + .../src/structured_storage/merkle_data.rs | 1 + .../src/structured_storage/messages.rs | 2 + .../src/structured_storage/receipts.rs | 32 --- .../src/structured_storage/sealed_block.rs | 1 + .../storage/src/structured_storage/state.rs | 1 + .../src/structured_storage/transactions.rs | 2 + crates/storage/src/tables.rs | 14 - 74 files changed, 1745 insertions(+), 1086 deletions(-) create mode 100644 crates/fuel-core/src/combined_database.rs create mode 100644 crates/fuel-core/src/database/database_description.rs create mode 100644 crates/fuel-core/src/database/database_description/off_chain.rs create mode 100644 crates/fuel-core/src/database/database_description/on_chain.rs create mode 100644 crates/fuel-core/src/database/database_description/relayer.rs create mode 100644 crates/fuel-core/src/database/statistic.rs create mode 100644 crates/fuel-core/src/graphql_api/storage.rs create mode 100644 crates/fuel-core/src/graphql_api/storage/receipts.rs create mode 100644 crates/fuel-core/src/graphql_api/storage/transactions.rs delete mode 100644 crates/storage/src/structured_storage/receipts.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index a73d8c55e1..7937dda47c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,7 +27,8 @@ Description of the upcoming release here. #### Breaking - [#1639](https://github.com/FuelLabs/fuel-core/pull/1639): Make Merkle metadata, i.e. `SparseMerkleMetadata` and `DenseMerkleMetadata` type version-able enums -- [#16232](https://github.com/FuelLabs/fuel-core/pull/1632): Make `Message` type a version-able enum +- [#1632](https://github.com/FuelLabs/fuel-core/pull/1632): Make `Message` type a version-able enum +- [#1629](https://github.com/FuelLabs/fuel-core/pull/1629): Use a separate database for each data domain. Each database has its own folder where data is stored. - [#1628](https://github.com/FuelLabs/fuel-core/pull/1628): Make `CompressedCoin` type a version-able enum - [#1616](https://github.com/FuelLabs/fuel-core/pull/1616): Make `BlockHeader` type a version-able enum - [#1614](https://github.com/FuelLabs/fuel-core/pull/1614): Use the default consensus key regardless of trigger mode. The change is breaking because it removes the `--dev-keys` argument. If the `debug` flag is set, the default consensus key will be used, regardless of the trigger mode. diff --git a/Cargo.lock b/Cargo.lock index c5539af562..a4d5d1261f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2637,6 +2637,7 @@ dependencies = [ "proptest", "rand", "rocksdb", + "serde", "serde_json", "strum 0.25.0", "strum_macros 0.25.3", @@ -2960,6 +2961,7 @@ dependencies = [ "anyhow", "async-trait", "bytes", + "enum-iterator", "ethers-contract", "ethers-core", "ethers-providers", @@ -2975,6 +2977,8 @@ dependencies = [ "rand", "serde", "serde_json", + "strum 0.25.0", + "strum_macros 0.25.3", "test-case", "thiserror", "tokio", diff --git a/benches/benches/block_target_gas.rs b/benches/benches/block_target_gas.rs index 3a18e53aab..7e1e4f0134 100644 --- a/benches/benches/block_target_gas.rs +++ b/benches/benches/block_target_gas.rs @@ -16,6 +16,7 @@ use criterion::{ use ed25519_dalek::Signer; use ethnum::U256; use fuel_core::{ + combined_database::CombinedDatabase, service::{ config::Trigger, Config, @@ -325,8 +326,11 @@ fn service_with_many_contracts( .unwrap(); } - let service = fuel_core::service::FuelService::new(database, config.clone()) - .expect("Unable to start a FuelService"); + let service = FuelService::new( + CombinedDatabase::new(database, Default::default(), Default::default()), + config.clone(), + ) + .expect("Unable to start a FuelService"); service.start().expect("Unable to start the service"); (service, rt) } @@ -456,6 +460,7 @@ fn replace_contract_in_service( service .shared .database + .on_chain_mut() .storage_as_mut::() .insert(contract_id, &contract_bytecode) .unwrap(); diff --git a/benches/benches/transaction_throughput.rs b/benches/benches/transaction_throughput.rs index 820e8fc397..3e983f3d5f 100644 --- a/benches/benches/transaction_throughput.rs +++ b/benches/benches/transaction_throughput.rs @@ -112,6 +112,7 @@ where let block = srv .shared .database + .on_chain() .get_sealed_block_by_height(&1.into()) .unwrap() .unwrap(); diff --git a/crates/fuel-core/Cargo.toml b/crates/fuel-core/Cargo.toml index 7a54e142f0..56d4db0d6a 100644 --- a/crates/fuel-core/Cargo.toml +++ b/crates/fuel-core/Cargo.toml @@ -46,6 +46,7 @@ rocksdb = { version = "0.21", default-features = false, features = [ "lz4", "multi-threaded-cf", ], optional = true } +serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true, features = ["raw_value"] } strum = { workspace = true } strum_macros = { workspace = true } diff --git a/crates/fuel-core/src/coins_query.rs b/crates/fuel-core/src/coins_query.rs index a1b537a5c1..d219b925c4 100644 --- a/crates/fuel-core/src/coins_query.rs +++ b/crates/fuel-core/src/coins_query.rs @@ -228,7 +228,7 @@ mod tests { CoinsQueryError, SpendQuery, }, - database::Database, + combined_database::CombinedDatabase, fuel_core_graphql_api::api_service::ReadDatabase as ServiceDatabase, query::asset_query::{ AssetQuery, @@ -922,7 +922,7 @@ mod tests { } pub struct TestDatabase { - database: Database, + database: CombinedDatabase, last_coin_index: u64, last_message_index: u64, } @@ -937,8 +937,9 @@ mod tests { } fn service_database(&self) -> ServiceDatabase { - let database = self.database.clone(); - ServiceDatabase::new(database.clone(), database) + let on_chain = self.database.on_chain().clone(); + let off_chain = self.database.off_chain().clone(); + ServiceDatabase::new(on_chain, off_chain) } } @@ -958,7 +959,7 @@ mod tests { coin.set_amount(amount); coin.set_asset_id(asset_id); - let db = &mut self.database; + let db = self.database.on_chain_mut(); StorageMutate::::insert(db, &id, &coin).unwrap(); coin.uncompress(id) @@ -978,7 +979,7 @@ mod tests { } .into(); - let db = &mut self.database; + let db = self.database.on_chain_mut(); StorageMutate::::insert(db, message.id(), &message).unwrap(); message diff --git a/crates/fuel-core/src/combined_database.rs b/crates/fuel-core/src/combined_database.rs new file mode 100644 index 0000000000..e31f0367e1 --- /dev/null +++ b/crates/fuel-core/src/combined_database.rs @@ -0,0 +1,92 @@ +use crate::database::{ + database_description::{ + off_chain::OffChain, + on_chain::OnChain, + relayer::Relayer, + }, + Database, + Result as DatabaseResult, +}; +use fuel_core_storage::Result as StorageResult; +use fuel_core_types::{ + blockchain::primitives::DaBlockHeight, + fuel_types::BlockHeight, +}; + +/// A database that combines the on-chain, off-chain and relayer databases into one entity. +#[derive(Default, Clone)] +pub struct CombinedDatabase { + on_chain: Database, + off_chain: Database, + relayer: Database, +} + +impl CombinedDatabase { + pub fn new( + on_chain: Database, + off_chain: Database, + relayer: Database, + ) -> Self { + Self { + on_chain, + off_chain, + relayer, + } + } + + #[cfg(feature = "rocksdb")] + pub fn open(path: &std::path::Path, capacity: usize) -> DatabaseResult { + // TODO: Use different cache sizes for different databases + let on_chain = Database::open(path, capacity)?; + let off_chain = Database::open(path, capacity)?; + let relayer = Database::open(path, capacity)?; + Ok(Self { + on_chain, + off_chain, + relayer, + }) + } + + pub fn in_memory() -> Self { + Self::new( + Database::in_memory(), + Database::in_memory(), + Database::in_memory(), + ) + } + + pub fn init( + &mut self, + block_height: &BlockHeight, + da_block_height: &DaBlockHeight, + ) -> StorageResult<()> { + self.on_chain.init(block_height)?; + self.off_chain.init(block_height)?; + self.relayer.init(da_block_height)?; + Ok(()) + } + + pub fn on_chain(&self) -> &Database { + &self.on_chain + } + + #[cfg(any(feature = "test-helpers", test))] + pub fn on_chain_mut(&mut self) -> &mut Database { + &mut self.on_chain + } + + pub fn off_chain(&self) -> &Database { + &self.off_chain + } + + pub fn relayer(&self) -> &Database { + &self.relayer + } + + pub fn flush(self) -> DatabaseResult<()> { + self.on_chain.flush()?; + self.off_chain.flush()?; + self.relayer.flush()?; + Ok(()) + } +} diff --git a/crates/fuel-core/src/database.rs b/crates/fuel-core/src/database.rs index 1cf59114ba..96d99bc326 100644 --- a/crates/fuel-core/src/database.rs +++ b/crates/fuel-core/src/database.rs @@ -1,5 +1,13 @@ use crate::{ - database::transaction::DatabaseTransaction, + database::{ + database_description::{ + off_chain::OffChain, + on_chain::OnChain, + relayer::Relayer, + DatabaseDescription, + }, + transaction::DatabaseTransaction, + }, state::{ in_memory::memory_store::MemoryStore, DataSource, @@ -66,25 +74,26 @@ use std::path::Path; use tempfile::TempDir; // Storages implementation -mod block; -mod contracts; -mod message; -mod sealed_block; -mod state; - -pub(crate) mod coin; - pub mod balances; +pub mod block; +pub mod coin; +pub mod contracts; +pub mod database_description; +pub mod message; pub mod metadata; +pub mod sealed_block; +pub mod state; +pub mod statistic; pub mod storage; pub mod transaction; pub mod transactions; -pub type Column = fuel_core_storage::column::Column; - #[derive(Clone, Debug)] -pub struct Database { - data: StructuredStorage, +pub struct Database +where + Description: DatabaseDescription, +{ + data: StructuredStorage>, // used for RAII _drop: Arc, } @@ -118,10 +127,13 @@ impl Drop for DropResources { } } -impl Database { +impl Database +where + Description: DatabaseDescription, +{ pub fn new(data_source: D) -> Self where - D: Into, + D: Into>, { Self { data: StructuredStorage::new(data_source.into()), @@ -137,7 +149,7 @@ impl Database { #[cfg(feature = "rocksdb")] pub fn open(path: &Path, capacity: impl Into>) -> DatabaseResult { use anyhow::Context; - let db = RocksDb::default_open(path, capacity.into()).map_err(Into::::into).context("Failed to open rocksdb, you may need to wipe a pre-existing incompatible db `rm -rf ~/.fuel/db`")?; + let db = RocksDb::::default_open(path, capacity.into()).map_err(Into::::into).context("Failed to open rocksdb, you may need to wipe a pre-existing incompatible db `rm -rf ~/.fuel/db`")?; Ok(Database { data: StructuredStorage::new(Arc::new(db).into()), @@ -155,7 +167,7 @@ impl Database { #[cfg(feature = "rocksdb")] pub fn rocksdb() -> Self { let tmp_dir = TempDir::new().unwrap(); - let db = RocksDb::default_open(tmp_dir.path(), None).unwrap(); + let db = RocksDb::::default_open(tmp_dir.path(), None).unwrap(); Self { data: StructuredStorage::new(Arc::new(db).into()), _drop: Arc::new( @@ -170,21 +182,20 @@ impl Database { } } - pub fn transaction(&self) -> DatabaseTransaction { + pub fn transaction(&self) -> DatabaseTransaction { self.into() } - pub fn checkpoint(&self) -> DatabaseResult { - self.data.as_ref().checkpoint() - } - pub fn flush(self) -> DatabaseResult<()> { self.data.as_ref().flush() } } -impl KeyValueStore for DataSource { - type Column = Column; +impl KeyValueStore for DataSource +where + Description: DatabaseDescription, +{ + type Column = Description::Column; fn put(&self, key: &[u8], column: Self::Column, value: Value) -> StorageResult<()> { self.as_ref().put(key, column, value) @@ -242,7 +253,10 @@ impl KeyValueStore for DataSource { } } -impl BatchOperations for DataSource { +impl BatchOperations for DataSource +where + Description: DatabaseDescription, +{ fn batch_write( &self, entries: &mut dyn Iterator, Self::Column, WriteOperation)>, @@ -252,13 +266,16 @@ impl BatchOperations for DataSource { } /// Read-only methods. -impl Database { +impl Database +where + Description: DatabaseDescription, +{ pub(crate) fn iter_all( &self, direction: Option, ) -> impl Iterator> + '_ where - M: Mappable + TableWithBlueprint, + M: Mappable + TableWithBlueprint, M::Blueprint: Blueprint, { self.iter_all_filtered::(None, None, direction) @@ -269,7 +286,7 @@ impl Database { prefix: Option

, ) -> impl Iterator> + '_ where - M: Mappable + TableWithBlueprint, + M: Mappable + TableWithBlueprint, M::Blueprint: Blueprint, P: AsRef<[u8]>, { @@ -282,7 +299,7 @@ impl Database { direction: Option, ) -> impl Iterator> + '_ where - M: Mappable + TableWithBlueprint, + M: Mappable + TableWithBlueprint, M::Blueprint: Blueprint, { self.iter_all_filtered::(None, start, direction) @@ -295,7 +312,7 @@ impl Database { direction: Option, ) -> impl Iterator> + '_ where - M: Mappable + TableWithBlueprint, + M: Mappable + TableWithBlueprint, M::Blueprint: Blueprint, P: AsRef<[u8]>, { @@ -331,22 +348,31 @@ impl Database { } } -impl Transactional for Database { - type Storage = Database; +impl Transactional for Database +where + Description: DatabaseDescription, +{ + type Storage = Database; - fn transaction(&self) -> StorageTransaction { + fn transaction(&self) -> StorageTransaction> { StorageTransaction::new(self.transaction()) } } -impl AsRef for Database { - fn as_ref(&self) -> &Database { +impl AsRef> for Database +where + Description: DatabaseDescription, +{ + fn as_ref(&self) -> &Database { self } } -impl AsMut for Database { - fn as_mut(&mut self) -> &mut Database { +impl AsMut> for Database +where + Description: DatabaseDescription, +{ + fn as_mut(&mut self) -> &mut Database { self } } @@ -354,7 +380,10 @@ impl AsMut for Database { /// Construct an ephemeral database /// uses rocksdb when rocksdb features are enabled /// uses in-memory when rocksdb features are disabled -impl Default for Database { +impl Default for Database +where + Description: DatabaseDescription, +{ fn default() -> Self { #[cfg(not(feature = "rocksdb"))] { @@ -387,8 +416,8 @@ impl ChainConfigDb for Database { } } -impl AtomicView for Database { - type View = Database; +impl AtomicView for Database { + type View = Self; type Height = BlockHeight; @@ -410,16 +439,31 @@ impl AtomicView for Database { } } -pub struct RelayerReadDatabase(Database); +impl AtomicView for Database { + type View = Self; + + type Height = BlockHeight; + + fn latest_height(&self) -> BlockHeight { + // TODO: The database should track the latest height inside of the database object + // instead of fetching it from the `FuelBlocks` table. As a temporary solution, + // fetch it from the table for now. + self.latest_height().unwrap_or_default() + } + + fn view_at(&self, _: &BlockHeight) -> StorageResult { + // TODO: Unimplemented until of the https://github.com/FuelLabs/fuel-core/issues/451 + Ok(self.latest_view()) + } -impl RelayerReadDatabase { - pub fn new(database: Database) -> Self { - Self(database) + fn latest_view(&self) -> Self::View { + // TODO: https://github.com/FuelLabs/fuel-core/issues/1581 + self.clone() } } -impl AtomicView for RelayerReadDatabase { - type View = Database; +impl AtomicView for Database { + type View = Self; type Height = DaBlockHeight; fn latest_height(&self) -> Self::Height { @@ -430,7 +474,7 @@ impl AtomicView for RelayerReadDatabase { // instead of fetching it from the `RelayerMetadata` table. As a temporary solution, // fetch it from the table for now. // https://github.com/FuelLabs/fuel-core/issues/1589 - self.0.get_finalized_da_height().unwrap_or_default() + self.get_finalized_da_height().unwrap_or_default() } #[cfg(not(feature = "relayer"))] { @@ -443,24 +487,51 @@ impl AtomicView for RelayerReadDatabase { } fn latest_view(&self) -> Self::View { - self.0.clone() + self.clone() } } #[cfg(feature = "rocksdb")] -pub fn convert_to_rocksdb_direction( - direction: fuel_core_storage::iter::IterDirection, -) -> rocksdb::Direction { +pub fn convert_to_rocksdb_direction(direction: IterDirection) -> rocksdb::Direction { match direction { IterDirection::Forward => rocksdb::Direction::Forward, IterDirection::Reverse => rocksdb::Direction::Reverse, } } -#[test] -fn column_keys_not_exceed_count() { - use enum_iterator::all; - for column in all::() { - assert!(column.as_usize() < Column::COUNT); +#[cfg(test)] +mod tests { + use crate::database::database_description::{ + off_chain::OffChain, + on_chain::OnChain, + relayer::Relayer, + DatabaseDescription, + }; + + fn column_keys_not_exceed_count() + where + Description: DatabaseDescription, + { + use enum_iterator::all; + use fuel_core_storage::kv_store::StorageColumn; + use strum::EnumCount; + for column in all::() { + assert!(column.as_usize() < Description::Column::COUNT); + } + } + + #[test] + fn column_keys_not_exceed_count_test_on_chain() { + column_keys_not_exceed_count::(); + } + + #[test] + fn column_keys_not_exceed_count_test_off_chain() { + column_keys_not_exceed_count::(); + } + + #[test] + fn column_keys_not_exceed_count_test_relayer() { + column_keys_not_exceed_count::(); } } diff --git a/crates/fuel-core/src/database/balances.rs b/crates/fuel-core/src/database/balances.rs index c9f7783db0..746272f6d8 100644 --- a/crates/fuel-core/src/database/balances.rs +++ b/crates/fuel-core/src/database/balances.rs @@ -41,6 +41,7 @@ impl Database { #[cfg(test)] mod tests { use super::*; + use crate::database::database_description::on_chain::OnChain; use fuel_core_storage::StorageAsMut; use fuel_core_types::fuel_types::AssetId; use rand::Rng; @@ -77,7 +78,7 @@ mod tests { .root(&contract_id) .expect("Should get root"); - let seq_database = &mut Database::default(); + let seq_database = &mut Database::::default(); for (asset, value) in data.iter() { seq_database .storage::() diff --git a/crates/fuel-core/src/database/block.rs b/crates/fuel-core/src/database/block.rs index 277cf1da57..a7189e8c5a 100644 --- a/crates/fuel-core/src/database/block.rs +++ b/crates/fuel-core/src/database/block.rs @@ -1,5 +1,10 @@ use crate::database::{ - Column, + database_description::{ + on_chain::OnChain, + DatabaseDescription, + DatabaseMetadata, + }, + metadata::MetadataTable, Database, }; use fuel_core_storage::{ @@ -62,9 +67,10 @@ impl Mappable for FuelBlockSecondaryKeyBlockHeights { impl TableWithBlueprint for FuelBlockSecondaryKeyBlockHeights { type Blueprint = Plain>; + type Column = fuel_core_storage::column::Column; - fn column() -> Column { - Column::FuelBlockSecondaryKeyBlockHeights + fn column() -> Self::Column { + Self::Column::FuelBlockSecondaryKeyBlockHeights } } @@ -130,6 +136,17 @@ impl StorageMutate for Database { self.storage::() .insert(height, &metadata)?; + // TODO: Temporary solution to store the block height in the database manually here. + // Later it will be controlled by the `commit_changes` function on the `Database` side. + // https://github.com/FuelLabs/fuel-core/issues/1589 + self.storage::>().insert( + &(), + &DatabaseMetadata::V1 { + version: OnChain::version(), + height: *height, + }, + )?; + Ok(prev) } @@ -156,17 +173,6 @@ impl StorageMutate for Database { } impl Database { - pub fn latest_height(&self) -> StorageResult { - let pair = self - .iter_all::(Some(IterDirection::Reverse)) - .next() - .transpose()?; - - let (block_height, _) = pair.ok_or(not_found!("BlockHeight"))?; - - Ok(block_height) - } - pub fn latest_compressed_block(&self) -> StorageResult> { let pair = self .iter_all::(Some(IterDirection::Reverse)) diff --git a/crates/fuel-core/src/database/coin.rs b/crates/fuel-core/src/database/coin.rs index 04b262592e..5215127b90 100644 --- a/crates/fuel-core/src/database/coin.rs +++ b/crates/fuel-core/src/database/coin.rs @@ -1,5 +1,5 @@ use crate::database::{ - Column, + database_description::on_chain::OnChain, Database, }; use fuel_core_chain_config::CoinConfig; @@ -54,9 +54,10 @@ impl Mappable for OwnedCoins { impl TableWithBlueprint for OwnedCoins { type Blueprint = Plain; + type Column = fuel_core_storage::column::Column; - fn column() -> Column { - Column::OwnedCoins + fn column() -> Self::Column { + Self::Column::OwnedCoins } } @@ -100,7 +101,7 @@ impl StorageMutate for Database { } } -impl Database { +impl Database { pub fn owned_coins_ids( &self, owner: &Address, @@ -122,7 +123,9 @@ impl Database { }) }) } +} +impl Database { pub fn coin(&self, utxo_id: &UtxoId) -> StorageResult { let coin = self .storage_as_ref::() diff --git a/crates/fuel-core/src/database/contracts.rs b/crates/fuel-core/src/database/contracts.rs index 2dd4418ea5..f6bb2cbee7 100644 --- a/crates/fuel-core/src/database/contracts.rs +++ b/crates/fuel-core/src/database/contracts.rs @@ -119,6 +119,7 @@ impl Database { #[cfg(test)] mod tests { use super::*; + use crate::database::database_description::on_chain::OnChain; use fuel_core_storage::StorageAsMut; use fuel_core_types::fuel_tx::Contract; use rand::{ @@ -134,7 +135,7 @@ mod tests { rng.fill_bytes(bytes.as_mut()); let contract: Contract = Contract::from(bytes); - let database = &mut Database::default(); + let database = &mut Database::::default(); database .storage::() .insert(&contract_id, contract.as_ref()) diff --git a/crates/fuel-core/src/database/database_description.rs b/crates/fuel-core/src/database/database_description.rs new file mode 100644 index 0000000000..8f2aefec46 --- /dev/null +++ b/crates/fuel-core/src/database/database_description.rs @@ -0,0 +1,48 @@ +use core::fmt::Debug; +use fuel_core_storage::kv_store::StorageColumn; + +pub mod off_chain; +pub mod on_chain; +pub mod relayer; + +/// The description of the database that makes it unique. +pub trait DatabaseDescription: 'static + Clone + Debug + Send + Sync { + /// The type of the column used by the database. + type Column: StorageColumn + strum::EnumCount + enum_iterator::Sequence; + /// The type of the height of the database used to track commits. + type Height: Copy; + + /// Returns the expected version of the database. + fn version() -> u32; + + /// Returns the name of the database. + fn name() -> &'static str; + + /// Returns the column used to store the metadata. + fn metadata_column() -> Self::Column; + + /// Returns the prefix for the column. + fn prefix(column: &Self::Column) -> Option; +} + +/// The metadata of the database contains information about the version and its height. +#[derive(Copy, Clone, Debug, serde::Serialize, serde::Deserialize)] +pub enum DatabaseMetadata { + V1 { version: u32, height: Height }, +} + +impl DatabaseMetadata { + /// Returns the version of the database. + pub fn version(&self) -> u32 { + match self { + Self::V1 { version, .. } => *version, + } + } + + /// Returns the height of the database. + pub fn height(&self) -> &Height { + match self { + Self::V1 { height, .. } => height, + } + } +} diff --git a/crates/fuel-core/src/database/database_description/off_chain.rs b/crates/fuel-core/src/database/database_description/off_chain.rs new file mode 100644 index 0000000000..b0f5b37f6d --- /dev/null +++ b/crates/fuel-core/src/database/database_description/off_chain.rs @@ -0,0 +1,37 @@ +use crate::{ + database::database_description::DatabaseDescription, + fuel_core_graphql_api, +}; +use fuel_core_types::fuel_types::BlockHeight; + +#[derive(Clone, Debug)] +pub struct OffChain; + +impl DatabaseDescription for OffChain { + type Column = fuel_core_graphql_api::storage::Column; + type Height = BlockHeight; + + fn version() -> u32 { + 0 + } + + fn name() -> &'static str { + "off_chain" + } + + fn metadata_column() -> Self::Column { + Self::Column::Metadata + } + + fn prefix(column: &Self::Column) -> Option { + match column { + Self::Column::OwnedCoins + | Self::Column::TransactionsByOwnerBlockIdx + | Self::Column::OwnedMessageIds => { + // prefix is address length + Some(32) + } + _ => None, + } + } +} diff --git a/crates/fuel-core/src/database/database_description/on_chain.rs b/crates/fuel-core/src/database/database_description/on_chain.rs new file mode 100644 index 0000000000..2eb3f17269 --- /dev/null +++ b/crates/fuel-core/src/database/database_description/on_chain.rs @@ -0,0 +1,35 @@ +use crate::database::database_description::DatabaseDescription; +use fuel_core_types::fuel_types::BlockHeight; + +#[derive(Clone, Debug)] +pub struct OnChain; + +impl DatabaseDescription for OnChain { + type Column = fuel_core_storage::column::Column; + type Height = BlockHeight; + + fn version() -> u32 { + 0 + } + + fn name() -> &'static str { + "on_chain" + } + + fn metadata_column() -> Self::Column { + Self::Column::Metadata + } + + fn prefix(column: &Self::Column) -> Option { + match column { + Self::Column::OwnedCoins + | Self::Column::OwnedMessageIds + | Self::Column::ContractsAssets + | Self::Column::ContractsState => { + // prefix is address length + Some(32) + } + _ => None, + } + } +} diff --git a/crates/fuel-core/src/database/database_description/relayer.rs b/crates/fuel-core/src/database/database_description/relayer.rs new file mode 100644 index 0000000000..8074af3009 --- /dev/null +++ b/crates/fuel-core/src/database/database_description/relayer.rs @@ -0,0 +1,58 @@ +use crate::database::database_description::DatabaseDescription; +use fuel_core_storage::kv_store::StorageColumn; +use fuel_core_types::blockchain::primitives::DaBlockHeight; + +/// The column used by the relayer database in the case if the relayer is disabled. +#[derive( + Debug, + Copy, + Clone, + strum_macros::EnumCount, + strum_macros::IntoStaticStr, + PartialEq, + Eq, + enum_iterator::Sequence, + Hash, +)] +pub enum DummyColumn { + Metadata, +} + +impl StorageColumn for DummyColumn { + fn name(&self) -> &'static str { + self.into() + } + + fn id(&self) -> u32 { + *self as u32 + } +} + +#[derive(Clone, Debug)] +pub struct Relayer; + +impl DatabaseDescription for Relayer { + #[cfg(feature = "relayer")] + type Column = fuel_core_relayer::storage::Column; + + #[cfg(not(feature = "relayer"))] + type Column = DummyColumn; + + type Height = DaBlockHeight; + + fn version() -> u32 { + 0 + } + + fn name() -> &'static str { + "relayer" + } + + fn metadata_column() -> Self::Column { + Self::Column::Metadata + } + + fn prefix(_: &Self::Column) -> Option { + None + } +} diff --git a/crates/fuel-core/src/database/message.rs b/crates/fuel-core/src/database/message.rs index c797942ed8..6f1c5a8fe2 100644 --- a/crates/fuel-core/src/database/message.rs +++ b/crates/fuel-core/src/database/message.rs @@ -1,5 +1,5 @@ use crate::database::{ - Column, + database_description::on_chain::OnChain, Database, }; use fuel_core_chain_config::MessageConfig; @@ -63,9 +63,10 @@ impl Decode for Manual { impl TableWithBlueprint for OwnedMessageIds { type Blueprint = Plain, Postcard>; + type Column = fuel_core_storage::column::Column; - fn column() -> fuel_core_storage::column::Column { - Column::OwnedMessageIds + fn column() -> Self::Column { + Self::Column::OwnedMessageIds } } @@ -110,7 +111,7 @@ impl StorageMutate for Database { } } -impl Database { +impl Database { pub fn owned_message_ids( &self, owner: &Address, @@ -126,7 +127,9 @@ impl Database { ) .map(|res| res.map(|(key, _)| *key.nonce())) } +} +impl Database { pub fn all_messages( &self, start: Option, @@ -183,7 +186,7 @@ mod tests { #[test] fn owned_message_ids() { - let mut db = Database::default(); + let mut db = Database::::default(); let message = Message::default(); // insert a message with the first id diff --git a/crates/fuel-core/src/database/metadata.rs b/crates/fuel-core/src/database/metadata.rs index 665b72e42f..ef9a70c130 100644 --- a/crates/fuel-core/src/database/metadata.rs +++ b/crates/fuel-core/src/database/metadata.rs @@ -1,127 +1,109 @@ use crate::{ database::{ + database_description::{ + DatabaseDescription, + DatabaseMetadata, + }, storage::UseStructuredImplementation, - Column, Database, Error as DatabaseError, }, state::DataSource, }; -use fuel_core_chain_config::ChainConfig; use fuel_core_storage::{ blueprint::plain::Plain, codec::postcard::Postcard, + not_found, structured_storage::{ StructuredStorage, TableWithBlueprint, }, + Error as StorageError, Mappable, Result as StorageResult, - StorageMutate, + StorageAsRef, }; +use fuel_core_types::fuel_merkle::storage::StorageMutate; -/// The table that stores all metadata. Each key is a string, while the value depends on the context. -/// The tables mostly used to store metadata for correct work of the `fuel-core`. -pub struct MetadataTable(core::marker::PhantomData); +/// The table that stores all metadata about the database. +pub struct MetadataTable(core::marker::PhantomData); -impl Mappable for MetadataTable +impl Mappable for MetadataTable where - V: Clone, + Description: DatabaseDescription, { - type Key = str; - type OwnedKey = String; - type Value = V; - type OwnedValue = V; + type Key = (); + type OwnedKey = (); + type Value = DatabaseMetadata; + type OwnedValue = Self::Value; } -impl TableWithBlueprint for MetadataTable +impl TableWithBlueprint for MetadataTable where - V: Clone, + Description: DatabaseDescription, { type Blueprint = Plain; + type Column = Description::Column; - fn column() -> Column { - Column::Metadata + fn column() -> Self::Column { + Description::metadata_column() } } -impl UseStructuredImplementation> for StructuredStorage where - V: Clone +impl UseStructuredImplementation> + for StructuredStorage> +where + Description: DatabaseDescription, { } -pub(crate) const DB_VERSION_KEY: &str = "version"; -pub(crate) const CHAIN_NAME_KEY: &str = "chain_name"; -/// Tracks the total number of transactions written to the chain -/// It's useful for analyzing TPS or other metrics. -pub(crate) const TX_COUNT: &str = "total_tx_count"; - -/// Can be used to perform migrations in the future. -pub(crate) const DB_VERSION: u32 = 0x00; - -impl Database { +impl Database +where + Description: DatabaseDescription, + Self: StorageMutate, Error = StorageError>, +{ /// Ensures the database is initialized and that the database version is correct - pub fn init(&mut self, config: &ChainConfig) -> StorageResult<()> { + pub fn init(&mut self, height: &Description::Height) -> StorageResult<()> { use fuel_core_storage::StorageAsMut; - // initialize chain name if not set - if self.get_chain_name()?.is_none() { - self.storage::>() - .insert(CHAIN_NAME_KEY, &config.chain_name) - .and_then(|v| { - if v.is_some() { - Err(DatabaseError::ChainAlreadyInitialized.into()) - } else { - Ok(()) - } - })?; + + if !self + .storage::>() + .contains_key(&())? + { + let old = self.storage::>().insert( + &(), + &DatabaseMetadata::V1 { + version: Description::version(), + height: *height, + }, + )?; + + if old.is_some() { + return Err(DatabaseError::ChainAlreadyInitialized.into()) + } } - // Ensure the database version is correct - if let Some(version) = self.storage::>().get(DB_VERSION_KEY)? { - let version = version.into_owned(); - if version != DB_VERSION { - return Err(DatabaseError::InvalidDatabaseVersion { - found: version, - expected: DB_VERSION, - })? + let metadata = self + .storage::>() + .get(&())? + .expect("We checked its existence above"); + + if metadata.version() != Description::version() { + return Err(DatabaseError::InvalidDatabaseVersion { + found: metadata.version(), + expected: Description::version(), } - } else { - self.storage::>() - .insert(DB_VERSION_KEY, &DB_VERSION)?; + .into()) } + Ok(()) } - pub fn get_chain_name(&self) -> StorageResult> { - use fuel_core_storage::StorageAsRef; - self.storage::>() - .get(CHAIN_NAME_KEY) - .map(|v| v.map(|v| v.into_owned())) - } + pub fn latest_height(&self) -> StorageResult { + let metadata = self.storage::>().get(&())?; - pub fn increase_tx_count(&self, new_txs: u64) -> StorageResult { - use fuel_core_storage::StorageAsRef; - // TODO: how should tx count be initialized after regenesis? - let current_tx_count: u64 = self - .storage::>() - .get(TX_COUNT)? - .unwrap_or_default() - .into_owned(); - // Using saturating_add because this value doesn't significantly impact the correctness of execution. - let new_tx_count = current_tx_count.saturating_add(new_txs); - <_ as StorageMutate>>::insert( - // TODO: Workaround to avoid a mutable borrow of self - &mut StructuredStorage::new(self.data.as_ref()), - TX_COUNT, - &new_tx_count, - )?; - Ok(new_tx_count) - } + let metadata = metadata.ok_or(not_found!(MetadataTable))?; - pub fn get_tx_count(&self) -> StorageResult { - use fuel_core_storage::StorageAsRef; - self.storage::>() - .get(TX_COUNT) - .map(|v| v.unwrap_or_default().into_owned()) + Ok(*metadata.height()) } } diff --git a/crates/fuel-core/src/database/state.rs b/crates/fuel-core/src/database/state.rs index efa6be36b5..01c3971e1d 100644 --- a/crates/fuel-core/src/database/state.rs +++ b/crates/fuel-core/src/database/state.rs @@ -36,6 +36,7 @@ impl Database { #[cfg(test)] mod tests { use super::*; + use crate::database::database_description::on_chain::OnChain; use fuel_core_storage::StorageAsMut; use fuel_core_types::fuel_types::Bytes32; use rand::Rng; @@ -71,7 +72,7 @@ mod tests { .root(&contract_id) .expect("Should get root"); - let seq_database = &mut Database::default(); + let seq_database = &mut Database::::default(); for (key, value) in data.iter() { seq_database .storage::() diff --git a/crates/fuel-core/src/database/statistic.rs b/crates/fuel-core/src/database/statistic.rs new file mode 100644 index 0000000000..59a1802127 --- /dev/null +++ b/crates/fuel-core/src/database/statistic.rs @@ -0,0 +1,77 @@ +use crate::{ + database::{ + database_description::off_chain::OffChain, + storage::UseStructuredImplementation, + Database, + }, + fuel_core_graphql_api, + state::DataSource, +}; +use fuel_core_storage::{ + blueprint::plain::Plain, + codec::postcard::Postcard, + structured_storage::{ + StructuredStorage, + TableWithBlueprint, + }, + Mappable, + Result as StorageResult, + StorageMutate, +}; + +/// The table that stores all statistic about blockchain. Each key is a string, while the value +/// depends on the context. +pub struct StatisticTable(core::marker::PhantomData); + +impl Mappable for StatisticTable +where + V: Clone, +{ + type Key = str; + type OwnedKey = String; + type Value = V; + type OwnedValue = V; +} + +impl TableWithBlueprint for StatisticTable +where + V: Clone, +{ + type Blueprint = Plain; + type Column = fuel_core_graphql_api::storage::Column; + + fn column() -> Self::Column { + Self::Column::Statistic + } +} + +impl UseStructuredImplementation> + for StructuredStorage> +where + V: Clone, +{ +} + +/// Tracks the total number of transactions written to the chain +/// It's useful for analyzing TPS or other metrics. +pub(crate) const TX_COUNT: &str = "total_tx_count"; + +impl Database { + pub fn increase_tx_count(&mut self, new_txs: u64) -> StorageResult { + use fuel_core_storage::StorageAsRef; + // TODO: how should tx count be initialized after regenesis? + let current_tx_count: u64 = self + .storage::>() + .get(TX_COUNT)? + .unwrap_or_default() + .into_owned(); + // Using saturating_add because this value doesn't significantly impact the correctness of execution. + let new_tx_count = current_tx_count.saturating_add(new_txs); + <_ as StorageMutate>>::insert( + &mut self.data, + TX_COUNT, + &new_tx_count, + )?; + Ok(new_tx_count) + } +} diff --git a/crates/fuel-core/src/database/storage.rs b/crates/fuel-core/src/database/storage.rs index 62c9385b27..0e5b1f1f48 100644 --- a/crates/fuel-core/src/database/storage.rs +++ b/crates/fuel-core/src/database/storage.rs @@ -2,12 +2,16 @@ use crate::{ database::{ block::FuelBlockSecondaryKeyBlockHeights, coin::OwnedCoins, + database_description::DatabaseDescription, message::OwnedMessageIds, + Database, + }, + fuel_core_graphql_api::storage::{ + receipts::Receipts, transactions::{ OwnedTransactions, TransactionStatuses, }, - Database, }, state::DataSource, }; @@ -28,7 +32,6 @@ use fuel_core_storage::{ ContractsRawCode, ContractsState, ProcessedTransactions, - Receipts, SealedBlockConsensus, SpentMessages, Transactions, @@ -63,7 +66,10 @@ where macro_rules! use_structured_implementation { ($($m:ty),*) => { $( - impl UseStructuredImplementation<$m> for StructuredStorage {} + impl UseStructuredImplementation<$m> for StructuredStorage> + where + Description: DatabaseDescription, + {} )* }; } @@ -93,14 +99,15 @@ use_structured_implementation!( ); #[cfg(feature = "relayer")] use_structured_implementation!( - fuel_core_relayer::storage::RelayerMetadata, + fuel_core_relayer::storage::DaHeightTable, fuel_core_relayer::storage::EventsHistory ); -impl StorageInspect for Database +impl StorageInspect for Database where + Description: DatabaseDescription, M: Mappable, - StructuredStorage: + StructuredStorage>: StorageInspect + UseStructuredImplementation, { type Error = StorageError; @@ -114,10 +121,11 @@ where } } -impl StorageMutate for Database +impl StorageMutate for Database where + Description: DatabaseDescription, M: Mappable, - StructuredStorage: + StructuredStorage>: StorageMutate + UseStructuredImplementation, { fn insert( @@ -133,10 +141,11 @@ where } } -impl MerkleRootStorage for Database +impl MerkleRootStorage for Database where + Description: DatabaseDescription, M: Mappable, - StructuredStorage: + StructuredStorage>: MerkleRootStorage + UseStructuredImplementation, { fn root(&self, key: &Key) -> StorageResult { @@ -144,10 +153,11 @@ where } } -impl StorageSize for Database +impl StorageSize for Database where + Description: DatabaseDescription, M: Mappable, - StructuredStorage: + StructuredStorage>: StorageSize + UseStructuredImplementation, { fn size_of_value(&self, key: &M::Key) -> StorageResult> { @@ -155,10 +165,11 @@ where } } -impl StorageRead for Database +impl StorageRead for Database where + Description: DatabaseDescription, M: Mappable, - StructuredStorage: + StructuredStorage>: StorageRead + UseStructuredImplementation, { fn read(&self, key: &M::Key, buf: &mut [u8]) -> StorageResult> { diff --git a/crates/fuel-core/src/database/transaction.rs b/crates/fuel-core/src/database/transaction.rs index ec3f3de67d..26fd488fd6 100644 --- a/crates/fuel-core/src/database/transaction.rs +++ b/crates/fuel-core/src/database/transaction.rs @@ -1,6 +1,12 @@ use crate::{ - database::Database, - state::in_memory::transaction::MemoryTransactionView, + database::{ + database_description::DatabaseDescription, + Database, + }, + state::{ + in_memory::transaction::MemoryTransactionView, + DataSource, + }, }; use fuel_core_storage::{ transactional::Transaction, @@ -16,58 +22,83 @@ use std::{ }; #[derive(Clone, Debug)] -pub struct DatabaseTransaction { +pub struct DatabaseTransaction +where + Description: DatabaseDescription, +{ // The primary datastores - changes: Arc, + changes: Arc>, // The inner db impl using these stores - database: Database, + database: Database, } -impl AsRef for DatabaseTransaction { - fn as_ref(&self) -> &Database { +impl AsRef> for DatabaseTransaction +where + Description: DatabaseDescription, +{ + fn as_ref(&self) -> &Database { &self.database } } -impl AsMut for DatabaseTransaction { - fn as_mut(&mut self) -> &mut Database { +impl AsMut> for DatabaseTransaction +where + Description: DatabaseDescription, +{ + fn as_mut(&mut self) -> &mut Database { &mut self.database } } -impl Deref for DatabaseTransaction { - type Target = Database; +impl Deref for DatabaseTransaction +where + Description: DatabaseDescription, +{ + type Target = Database; fn deref(&self) -> &Self::Target { &self.database } } -impl DerefMut for DatabaseTransaction { +impl DerefMut for DatabaseTransaction +where + Description: DatabaseDescription, +{ fn deref_mut(&mut self) -> &mut Self::Target { &mut self.database } } -impl Default for DatabaseTransaction { +impl Default for DatabaseTransaction +where + Description: DatabaseDescription, +{ fn default() -> Self { - Database::default().transaction() + Database::::default().transaction() } } -impl Transaction for DatabaseTransaction { +impl Transaction> for DatabaseTransaction +where + Description: DatabaseDescription, +{ fn commit(&mut self) -> StorageResult<()> { // TODO: should commit be fallible if this api is meant to be atomic? self.changes.commit() } } -impl From<&Database> for DatabaseTransaction { - fn from(source: &Database) -> Self { - let data = Arc::new(MemoryTransactionView::new(source.data.as_ref().clone())); +impl From<&Database> for DatabaseTransaction +where + Description: DatabaseDescription, +{ + fn from(source: &Database) -> Self { + let database: &DataSource = source.data.as_ref(); + let data = Arc::new(MemoryTransactionView::new(database.clone())); Self { changes: data.clone(), - database: Database::new(data), + database: Database::::new(data), } } } diff --git a/crates/fuel-core/src/database/transactions.rs b/crates/fuel-core/src/database/transactions.rs index 2f977e4848..db5c82ee41 100644 --- a/crates/fuel-core/src/database/transactions.rs +++ b/crates/fuel-core/src/database/transactions.rs @@ -1,24 +1,19 @@ -use crate::database::{ - Column, - Database, -}; -use core::{ - array::TryFromSliceError, - mem::size_of, +use crate::{ + database::{ + database_description::off_chain::OffChain, + Database, + }, + fuel_core_graphql_api::storage::transactions::{ + OwnedTransactionIndexCursor, + OwnedTransactionIndexKey, + OwnedTransactions, + TransactionIndex, + TransactionStatuses, + }, }; use fuel_core_storage::{ - blueprint::plain::Plain, - codec::{ - manual::Manual, - postcard::Postcard, - raw::Raw, - Decode, - Encode, - }, iter::IterDirection, - structured_storage::TableWithBlueprint, tables::Transactions, - Mappable, Result as StorageResult, }; use fuel_core_types::{ @@ -35,69 +30,6 @@ use fuel_core_types::{ services::txpool::TransactionStatus, }; -/// These tables allow iteration over all transactions owned by an address. -pub struct OwnedTransactions; - -impl Mappable for OwnedTransactions { - type Key = OwnedTransactionIndexKey; - type OwnedKey = Self::Key; - type Value = Bytes32; - type OwnedValue = Self::Value; -} - -impl TableWithBlueprint for OwnedTransactions { - type Blueprint = Plain, Raw>; - - fn column() -> Column { - Column::TransactionsByOwnerBlockIdx - } -} - -/// The table stores the status of each transaction. -pub struct TransactionStatuses; - -impl Mappable for TransactionStatuses { - type Key = Bytes32; - type OwnedKey = Self::Key; - type Value = TransactionStatus; - type OwnedValue = Self::Value; -} - -impl TableWithBlueprint for TransactionStatuses { - type Blueprint = Plain; - - fn column() -> Column { - Column::TransactionStatus - } -} - -#[cfg(test)] -mod test { - use super::*; - - fn generate_key(rng: &mut impl rand::Rng) -> ::Key { - let mut bytes = [0u8; INDEX_SIZE]; - rng.fill(bytes.as_mut()); - bytes.into() - } - - fuel_core_storage::basic_storage_tests!( - OwnedTransactions, - [1u8; INDEX_SIZE].into(), - ::Value::default(), - ::Value::default(), - generate_key - ); - - fuel_core_storage::basic_storage_tests!( - TransactionStatuses, - ::Key::default(), - TransactionStatus::Submitted { - time: fuel_core_types::tai64::Tai64::UNIX_EPOCH, - } - ); -} - impl Database { pub fn all_transactions( &self, @@ -107,7 +39,9 @@ impl Database { self.iter_all_by_start::(start, direction) .map(|res| res.map(|(_, tx)| tx)) } +} +impl Database { /// Iterates over a KV mapping of `[address + block height + tx idx] => transaction id`. This /// allows for efficient lookup of transaction ids associated with an address, sorted by /// block age and ordering within a block. The cursor tracks the `[block height + tx idx]` for @@ -164,126 +98,3 @@ impl Database { .map(|v| v.map(|v| v.into_owned())) } } - -const TX_INDEX_SIZE: usize = size_of::(); -const BLOCK_HEIGHT: usize = size_of::(); -const INDEX_SIZE: usize = Address::LEN + BLOCK_HEIGHT + TX_INDEX_SIZE; - -fn owned_tx_index_key( - owner: &Address, - height: BlockHeight, - tx_idx: TransactionIndex, -) -> [u8; INDEX_SIZE] { - let mut default = [0u8; INDEX_SIZE]; - // generate prefix to enable sorted indexing of transactions by owner - // owner + block_height + tx_idx - default[0..Address::LEN].copy_from_slice(owner.as_ref()); - default[Address::LEN..Address::LEN + BLOCK_HEIGHT] - .copy_from_slice(height.to_bytes().as_ref()); - default[Address::LEN + BLOCK_HEIGHT..].copy_from_slice(tx_idx.to_be_bytes().as_ref()); - default -} - -////////////////////////////////////// Not storage part ////////////////////////////////////// - -pub type TransactionIndex = u16; - -#[derive(Clone)] -pub struct OwnedTransactionIndexKey { - owner: Address, - block_height: BlockHeight, - tx_idx: TransactionIndex, -} - -impl OwnedTransactionIndexKey { - pub fn new( - owner: &Address, - block_height: BlockHeight, - tx_idx: TransactionIndex, - ) -> Self { - Self { - owner: *owner, - block_height, - tx_idx, - } - } -} - -impl From<[u8; INDEX_SIZE]> for OwnedTransactionIndexKey { - fn from(bytes: [u8; INDEX_SIZE]) -> Self { - let owner: [u8; 32] = bytes[..32].try_into().expect("It's an array of 32 bytes"); - // the first 32 bytes are the owner, which is already known when querying - let mut block_height_bytes: [u8; 4] = Default::default(); - block_height_bytes.copy_from_slice(&bytes[32..36]); - let mut tx_idx_bytes: [u8; 2] = Default::default(); - tx_idx_bytes.copy_from_slice(&bytes.as_ref()[36..38]); - - Self { - owner: Address::from(owner), - block_height: u32::from_be_bytes(block_height_bytes).into(), - tx_idx: u16::from_be_bytes(tx_idx_bytes), - } - } -} - -impl TryFrom<&[u8]> for OwnedTransactionIndexKey { - type Error = TryFromSliceError; - - fn try_from(bytes: &[u8]) -> Result { - let bytes: [u8; INDEX_SIZE] = bytes.try_into()?; - Ok(Self::from(bytes)) - } -} - -impl Encode for Manual { - type Encoder<'a> = [u8; INDEX_SIZE]; - - fn encode(t: &OwnedTransactionIndexKey) -> Self::Encoder<'_> { - owned_tx_index_key(&t.owner, t.block_height, t.tx_idx) - } -} - -impl Decode for Manual { - fn decode(bytes: &[u8]) -> anyhow::Result { - OwnedTransactionIndexKey::try_from(bytes) - .map_err(|_| anyhow::anyhow!("Unable to decode bytes")) - } -} - -#[derive(Clone, Debug, PartialOrd, Eq, PartialEq)] -pub struct OwnedTransactionIndexCursor { - pub block_height: BlockHeight, - pub tx_idx: TransactionIndex, -} - -impl From for OwnedTransactionIndexCursor { - fn from(key: OwnedTransactionIndexKey) -> Self { - OwnedTransactionIndexCursor { - block_height: key.block_height, - tx_idx: key.tx_idx, - } - } -} - -impl From> for OwnedTransactionIndexCursor { - fn from(bytes: Vec) -> Self { - let mut block_height_bytes: [u8; 4] = Default::default(); - block_height_bytes.copy_from_slice(&bytes[..4]); - let mut tx_idx_bytes: [u8; 2] = Default::default(); - tx_idx_bytes.copy_from_slice(&bytes[4..6]); - - Self { - block_height: u32::from_be_bytes(block_height_bytes).into(), - tx_idx: u16::from_be_bytes(tx_idx_bytes), - } - } -} - -impl From for Vec { - fn from(cursor: OwnedTransactionIndexCursor) -> Self { - let mut bytes = Vec::with_capacity(8); - bytes.extend(cursor.block_height.to_bytes()); - bytes.extend(cursor.tx_idx.to_be_bytes()); - bytes - } -} diff --git a/crates/fuel-core/src/executor.rs b/crates/fuel-core/src/executor.rs index 643ce2c7bc..d96370041c 100644 --- a/crates/fuel-core/src/executor.rs +++ b/crates/fuel-core/src/executor.rs @@ -2808,7 +2808,10 @@ mod tests { #[cfg(feature = "relayer")] mod relayer { use super::*; - use crate::database::RelayerReadDatabase; + use crate::database::database_description::{ + on_chain::OnChain, + relayer::Relayer, + }; use fuel_core_relayer::storage::EventsHistory; use fuel_core_storage::{ tables::{ @@ -2819,7 +2822,7 @@ mod tests { StorageAsMut, }; - fn database_with_genesis_block(da_block_height: u64) -> Database { + fn database_with_genesis_block(da_block_height: u64) -> Database { let db = Database::default(); let mut block = Block::default(); block.header_mut().set_da_height(da_block_height.into()); @@ -2835,7 +2838,7 @@ mod tests { db } - fn add_message_to_relayer(db: &mut Database, message: Message) { + fn add_message_to_relayer(db: &mut Database, message: Message) { let mut db_transaction = db.transaction(); let da_height = message.da_height(); db.storage::() @@ -2844,7 +2847,7 @@ mod tests { db_transaction.commit().expect("Should commit events"); } - fn add_messages_to_relayer(db: &mut Database, relayer_da_height: u64) { + fn add_messages_to_relayer(db: &mut Database, relayer_da_height: u64) { for da_height in 0..=relayer_da_height { let mut message = Message::default(); message.set_da_height(da_height.into()); @@ -2855,11 +2858,12 @@ mod tests { } fn create_relayer_executor( - database: Database, - ) -> Executor { + on_chain: Database, + relayer: Database, + ) -> Executor, Database> { Executor { - database_view_provider: database.clone(), - relayer_view_provider: RelayerReadDatabase::new(database), + database_view_provider: on_chain, + relayer_view_provider: relayer, config: Arc::new(Default::default()), } } @@ -2923,21 +2927,22 @@ mod tests { input: Input, ) -> Result<(), ExecutorError> { let genesis_da_height = input.genesis_da_height.unwrap_or_default(); - let mut db = if let Some(genesis_da_height) = input.genesis_da_height { + let on_chain_db = if let Some(genesis_da_height) = input.genesis_da_height { database_with_genesis_block(genesis_da_height) } else { Database::default() }; + let mut relayer_db = Database::::default(); // Given let relayer_da_height = input.relayer_da_height; let block_height = input.block_height; let block_da_height = input.block_da_height; - add_messages_to_relayer(&mut db, relayer_da_height); - assert_eq!(db.iter_all::(None).count(), 0); + add_messages_to_relayer(&mut relayer_db, relayer_da_height); + assert_eq!(on_chain_db.iter_all::(None).count(), 0); // When - let producer = create_relayer_executor(db); + let producer = create_relayer_executor(on_chain_db, relayer_db); let block = test_block(block_height.into(), block_da_height.into(), 10); let result = producer.execute_and_commit( ExecutionTypes::Production(block.into()), @@ -2964,17 +2969,18 @@ mod tests { #[test] fn block_producer_does_not_take_messages_for_the_same_height() { let genesis_da_height = 1u64; - let mut db = database_with_genesis_block(genesis_da_height); + let on_chain_db = database_with_genesis_block(genesis_da_height); + let mut relayer_db = Database::::default(); // Given let relayer_da_height = 10u64; let block_height = 1u32; let block_da_height = 1u64; - add_messages_to_relayer(&mut db, relayer_da_height); - assert_eq!(db.iter_all::(None).count(), 0); + add_messages_to_relayer(&mut relayer_db, relayer_da_height); + assert_eq!(on_chain_db.iter_all::(None).count(), 0); // When - let producer = create_relayer_executor(db); + let producer = create_relayer_executor(on_chain_db, relayer_db); let block = test_block(block_height.into(), block_da_height.into(), 10); let result = producer .execute_and_commit( @@ -2992,7 +2998,8 @@ mod tests { #[test] fn block_producer_can_use_just_added_message_in_the_transaction() { let genesis_da_height = 1u64; - let mut db = database_with_genesis_block(genesis_da_height); + let on_chain_db = database_with_genesis_block(genesis_da_height); + let mut relayer_db = Database::::default(); let block_height = 1u32; let block_da_height = 2u64; @@ -3000,11 +3007,11 @@ mod tests { let mut message = Message::default(); message.set_da_height(block_da_height.into()); message.set_nonce(nonce); - add_message_to_relayer(&mut db, message); + add_message_to_relayer(&mut relayer_db, message); // Given - assert_eq!(db.iter_all::(None).count(), 0); - assert_eq!(db.iter_all::(None).count(), 0); + assert_eq!(on_chain_db.iter_all::(None).count(), 0); + assert_eq!(on_chain_db.iter_all::(None).count(), 0); let tx = TransactionBuilder::script(vec![], vec![]) .script_gas_limit(10) .add_unsigned_message_input( @@ -3019,7 +3026,7 @@ mod tests { // When let mut block = test_block(block_height.into(), block_da_height.into(), 0); *block.transactions_mut() = vec![tx]; - let producer = create_relayer_executor(db); + let producer = create_relayer_executor(on_chain_db, relayer_db); let result = producer .execute_and_commit( ExecutionTypes::Production(block.into()), diff --git a/crates/fuel-core/src/graphql_api.rs b/crates/fuel-core/src/graphql_api.rs index 12603d964a..34eb81e2c2 100644 --- a/crates/fuel-core/src/graphql_api.rs +++ b/crates/fuel-core/src/graphql_api.rs @@ -13,6 +13,7 @@ pub mod api_service; pub mod database; pub(crate) mod metrics_extension; pub mod ports; +pub mod storage; pub(crate) mod view_extension; pub mod worker_service; @@ -25,6 +26,7 @@ pub struct Config { pub min_gas_price: u64, pub max_tx: usize, pub max_depth: usize, + pub chain_name: String, pub consensus_parameters: ConsensusParameters, pub consensus_key: Option>, } diff --git a/crates/fuel-core/src/graphql_api/database.rs b/crates/fuel-core/src/graphql_api/database.rs index c2a2ecff51..a2d0ad145b 100644 --- a/crates/fuel-core/src/graphql_api/database.rs +++ b/crates/fuel-core/src/graphql_api/database.rs @@ -1,5 +1,3 @@ -mod arc_wrapper; - use crate::fuel_core_graphql_api::{ database::arc_wrapper::ArcWrapper, ports::{ @@ -11,13 +9,13 @@ use crate::fuel_core_graphql_api::{ OffChainDatabase, OnChainDatabase, }, + storage::receipts::Receipts, }; use fuel_core_storage::{ iter::{ BoxedIter, IterDirection, }, - tables::Receipts, transactional::AtomicView, Error as StorageError, Mappable, @@ -60,6 +58,8 @@ use std::{ sync::Arc, }; +mod arc_wrapper; + /// The on-chain view of the database used by the [`ReadView`] to fetch on-chain data. pub type OnChainView = Arc; /// The off-chain view of the database used by the [`ReadView`] to fetch off-chain data. @@ -171,10 +171,6 @@ impl DatabaseContracts for ReadView { } impl DatabaseChain for ReadView { - fn chain_name(&self) -> StorageResult { - self.on_chain.chain_name() - } - fn da_height(&self) -> StorageResult { self.on_chain.da_height() } @@ -191,31 +187,14 @@ impl DatabaseMessageProof for ReadView { } } -impl OnChainDatabase for ReadView {} - -impl StorageInspect for ReadView { - type Error = StorageError; - - fn get( - &self, - key: &::Key, - ) -> StorageResult::OwnedValue>>> { - self.off_chain.get(key) - } - - fn contains_key(&self, key: &::Key) -> StorageResult { - self.off_chain.contains_key(key) - } -} - -impl OffChainDatabase for ReadView { +impl OnChainDatabase for ReadView { fn owned_message_ids( &self, owner: &Address, start_message_id: Option, direction: IterDirection, ) -> BoxedIter<'_, StorageResult> { - self.off_chain + self.on_chain .owned_message_ids(owner, start_message_id, direction) } @@ -225,9 +204,26 @@ impl OffChainDatabase for ReadView { start_coin: Option, direction: IterDirection, ) -> BoxedIter<'_, StorageResult> { - self.off_chain.owned_coins_ids(owner, start_coin, direction) + self.on_chain.owned_coins_ids(owner, start_coin, direction) } +} +impl StorageInspect for ReadView { + type Error = StorageError; + + fn get( + &self, + key: &::Key, + ) -> StorageResult::OwnedValue>>> { + self.off_chain.get(key) + } + + fn contains_key(&self, key: &::Key) -> StorageResult { + self.off_chain.contains_key(key) + } +} + +impl OffChainDatabase for ReadView { fn tx_status(&self, tx_id: &TxId) -> StorageResult { self.off_chain.tx_status(tx_id) } diff --git a/crates/fuel-core/src/graphql_api/ports.rs b/crates/fuel-core/src/graphql_api/ports.rs index 3e63781a3a..1f74b84c28 100644 --- a/crates/fuel-core/src/graphql_api/ports.rs +++ b/crates/fuel-core/src/graphql_api/ports.rs @@ -1,3 +1,4 @@ +use crate::fuel_core_graphql_api::storage::receipts::Receipts; use async_trait::async_trait; use fuel_core_services::stream::BoxStream; use fuel_core_storage::{ @@ -12,7 +13,6 @@ use fuel_core_storage::{ ContractsRawCode, FuelBlocks, Messages, - Receipts, SealedBlockConsensus, Transactions, }, @@ -62,20 +62,6 @@ use std::sync::Arc; pub trait OffChainDatabase: Send + Sync + StorageInspect { - fn owned_message_ids( - &self, - owner: &Address, - start_message_id: Option, - direction: IterDirection, - ) -> BoxedIter<'_, StorageResult>; - - fn owned_coins_ids( - &self, - owner: &Address, - start_coin: Option, - direction: IterDirection, - ) -> BoxedIter<'_, StorageResult>; - fn tx_status(&self, tx_id: &TxId) -> StorageResult; fn owned_transactions_ids( @@ -87,6 +73,8 @@ pub trait OffChainDatabase: } /// The on chain database port expected by GraphQL API service. +// TODO: Move `owned_message_ids` and `owned_coins_ids`` to `OffChainDatabase` +// https://github.com/FuelLabs/fuel-core/issues/1583 pub trait OnChainDatabase: Send + Sync @@ -98,6 +86,19 @@ pub trait OnChainDatabase: + DatabaseChain + DatabaseMessageProof { + fn owned_message_ids( + &self, + owner: &Address, + start_message_id: Option, + direction: IterDirection, + ) -> BoxedIter<'_, StorageResult>; + + fn owned_coins_ids( + &self, + owner: &Address, + start_coin: Option, + direction: IterDirection, + ) -> BoxedIter<'_, StorageResult>; } /// Trait that specifies all the getters required for blocks. @@ -145,8 +146,6 @@ pub trait DatabaseContracts: /// Trait that specifies all the getters required for chain metadata. pub trait DatabaseChain { - fn chain_name(&self) -> StorageResult; - fn da_height(&self) -> StorageResult; } @@ -203,9 +202,15 @@ pub trait P2pPort: Send + Sync { } pub mod worker { + use crate::{ + database::{ + database_description::off_chain::OffChain, + metadata::MetadataTable, + }, + fuel_core_graphql_api::storage::receipts::Receipts, + }; use fuel_core_services::stream::BoxStream; use fuel_core_storage::{ - tables::Receipts, transactional::Transactional, Error as StorageError, Result as StorageResult, @@ -227,6 +232,7 @@ pub mod worker { Send + Sync + StorageMutate + + StorageMutate, Error = StorageError> + Transactional { fn record_tx_id_owner( @@ -242,6 +248,10 @@ pub mod worker { id: &Bytes32, status: TransactionStatus, ) -> StorageResult>; + + /// Update metadata about the total number of transactions on the chain. + /// Returns the total count after the update. + fn increase_tx_count(&mut self, new_txs_count: u64) -> StorageResult; } pub trait BlockImporter { diff --git a/crates/fuel-core/src/graphql_api/storage.rs b/crates/fuel-core/src/graphql_api/storage.rs new file mode 100644 index 0000000000..c155c66c71 --- /dev/null +++ b/crates/fuel-core/src/graphql_api/storage.rs @@ -0,0 +1,54 @@ +use fuel_core_storage::kv_store::StorageColumn; + +pub mod receipts; +pub mod transactions; + +/// GraphQL database tables column ids to the corresponding [`fuel_core_storage::Mappable`] table. +#[repr(u32)] +#[derive( + Copy, + Clone, + Debug, + strum_macros::EnumCount, + strum_macros::IntoStaticStr, + PartialEq, + Eq, + enum_iterator::Sequence, + Hash, +)] +pub enum Column { + /// The column id of metadata about the blockchain + Metadata = 0, + /// See [`Receipts`](receipts::Receipts) + Receipts = 1, + /// The column of the table that stores `true` if `owner` owns `Coin` with `coin_id` + OwnedCoins = 2, + /// Transaction id to current status + TransactionStatus = 3, + /// The column of the table of all `owner`'s transactions + TransactionsByOwnerBlockIdx = 4, + /// The column of the table that stores `true` if `owner` owns `Message` with `message_id` + OwnedMessageIds = 5, + /// The column of the table that stores statistic about the blockchain. + Statistic = 6, +} + +impl Column { + /// The total count of variants in the enum. + pub const COUNT: usize = ::COUNT; + + /// Returns the `usize` representation of the `Column`. + pub fn as_u32(&self) -> u32 { + *self as u32 + } +} + +impl StorageColumn for Column { + fn name(&self) -> &'static str { + self.into() + } + + fn id(&self) -> u32 { + self.as_u32() + } +} diff --git a/crates/fuel-core/src/graphql_api/storage/receipts.rs b/crates/fuel-core/src/graphql_api/storage/receipts.rs new file mode 100644 index 0000000000..c3a922da41 --- /dev/null +++ b/crates/fuel-core/src/graphql_api/storage/receipts.rs @@ -0,0 +1,45 @@ +use fuel_core_storage::{ + blueprint::plain::Plain, + codec::{ + postcard::Postcard, + raw::Raw, + }, + structured_storage::TableWithBlueprint, + Mappable, +}; +use fuel_core_types::fuel_tx::{ + Bytes32, + Receipt, +}; + +/// Receipts of different hidden internal operations. +pub struct Receipts; + +impl Mappable for Receipts { + /// Unique identifier of the transaction. + type Key = Self::OwnedKey; + type OwnedKey = Bytes32; + type Value = [Receipt]; + type OwnedValue = Vec; +} + +impl TableWithBlueprint for Receipts { + type Blueprint = Plain; + type Column = super::Column; + + fn column() -> Self::Column { + Self::Column::Receipts + } +} + +#[cfg(test)] +fuel_core_storage::basic_storage_tests!( + Receipts, + ::Key::from([1u8; 32]), + vec![Receipt::ret( + Default::default(), + Default::default(), + Default::default(), + Default::default() + )] +); diff --git a/crates/fuel-core/src/graphql_api/storage/transactions.rs b/crates/fuel-core/src/graphql_api/storage/transactions.rs new file mode 100644 index 0000000000..757f73bd94 --- /dev/null +++ b/crates/fuel-core/src/graphql_api/storage/transactions.rs @@ -0,0 +1,212 @@ +use fuel_core_storage::{ + blueprint::plain::Plain, + codec::{ + manual::Manual, + postcard::Postcard, + raw::Raw, + Decode, + Encode, + }, + structured_storage::TableWithBlueprint, + Mappable, +}; +use fuel_core_types::{ + fuel_tx::{ + Address, + Bytes32, + }, + fuel_types::BlockHeight, + services::txpool::TransactionStatus, +}; +use std::{ + array::TryFromSliceError, + mem::size_of, +}; + +/// These tables allow iteration over all transactions owned by an address. +pub struct OwnedTransactions; + +impl Mappable for OwnedTransactions { + type Key = OwnedTransactionIndexKey; + type OwnedKey = Self::Key; + type Value = Bytes32; + type OwnedValue = Self::Value; +} + +impl TableWithBlueprint for OwnedTransactions { + type Blueprint = Plain, Raw>; + type Column = super::Column; + + fn column() -> Self::Column { + Self::Column::TransactionsByOwnerBlockIdx + } +} + +/// The table stores the status of each transaction. +pub struct TransactionStatuses; + +impl Mappable for TransactionStatuses { + type Key = Bytes32; + type OwnedKey = Self::Key; + type Value = TransactionStatus; + type OwnedValue = Self::Value; +} + +impl TableWithBlueprint for TransactionStatuses { + type Blueprint = Plain; + type Column = super::Column; + + fn column() -> Self::Column { + Self::Column::TransactionStatus + } +} + +const TX_INDEX_SIZE: usize = size_of::(); +const BLOCK_HEIGHT: usize = size_of::(); +const INDEX_SIZE: usize = Address::LEN + BLOCK_HEIGHT + TX_INDEX_SIZE; + +fn owned_tx_index_key( + owner: &Address, + height: BlockHeight, + tx_idx: TransactionIndex, +) -> [u8; INDEX_SIZE] { + let mut default = [0u8; INDEX_SIZE]; + // generate prefix to enable sorted indexing of transactions by owner + // owner + block_height + tx_idx + default[0..Address::LEN].copy_from_slice(owner.as_ref()); + default[Address::LEN..Address::LEN + BLOCK_HEIGHT] + .copy_from_slice(height.to_bytes().as_ref()); + default[Address::LEN + BLOCK_HEIGHT..].copy_from_slice(tx_idx.to_be_bytes().as_ref()); + default +} + +////////////////////////////////////// Not storage part ////////////////////////////////////// + +pub type TransactionIndex = u16; + +#[derive(Clone)] +pub struct OwnedTransactionIndexKey { + pub owner: Address, + pub block_height: BlockHeight, + pub tx_idx: TransactionIndex, +} + +impl OwnedTransactionIndexKey { + pub fn new( + owner: &Address, + block_height: BlockHeight, + tx_idx: TransactionIndex, + ) -> Self { + Self { + owner: *owner, + block_height, + tx_idx, + } + } +} + +impl From<[u8; INDEX_SIZE]> for OwnedTransactionIndexKey { + fn from(bytes: [u8; INDEX_SIZE]) -> Self { + let owner: [u8; 32] = bytes[..32].try_into().expect("It's an array of 32 bytes"); + // the first 32 bytes are the owner, which is already known when querying + let mut block_height_bytes: [u8; 4] = Default::default(); + block_height_bytes.copy_from_slice(&bytes[32..36]); + let mut tx_idx_bytes: [u8; 2] = Default::default(); + tx_idx_bytes.copy_from_slice(&bytes.as_ref()[36..38]); + + Self { + owner: Address::from(owner), + block_height: u32::from_be_bytes(block_height_bytes).into(), + tx_idx: u16::from_be_bytes(tx_idx_bytes), + } + } +} + +impl TryFrom<&[u8]> for OwnedTransactionIndexKey { + type Error = TryFromSliceError; + + fn try_from(bytes: &[u8]) -> Result { + let bytes: [u8; INDEX_SIZE] = bytes.try_into()?; + Ok(Self::from(bytes)) + } +} + +impl Encode for Manual { + type Encoder<'a> = [u8; INDEX_SIZE]; + + fn encode(t: &OwnedTransactionIndexKey) -> Self::Encoder<'_> { + owned_tx_index_key(&t.owner, t.block_height, t.tx_idx) + } +} + +impl Decode for Manual { + fn decode(bytes: &[u8]) -> anyhow::Result { + OwnedTransactionIndexKey::try_from(bytes) + .map_err(|_| anyhow::anyhow!("Unable to decode bytes")) + } +} + +#[derive(Clone, Debug, PartialOrd, Eq, PartialEq)] +pub struct OwnedTransactionIndexCursor { + pub block_height: BlockHeight, + pub tx_idx: TransactionIndex, +} + +impl From for OwnedTransactionIndexCursor { + fn from(key: OwnedTransactionIndexKey) -> Self { + OwnedTransactionIndexCursor { + block_height: key.block_height, + tx_idx: key.tx_idx, + } + } +} + +impl From> for OwnedTransactionIndexCursor { + fn from(bytes: Vec) -> Self { + let mut block_height_bytes: [u8; 4] = Default::default(); + block_height_bytes.copy_from_slice(&bytes[..4]); + let mut tx_idx_bytes: [u8; 2] = Default::default(); + tx_idx_bytes.copy_from_slice(&bytes[4..6]); + + Self { + block_height: u32::from_be_bytes(block_height_bytes).into(), + tx_idx: u16::from_be_bytes(tx_idx_bytes), + } + } +} + +impl From for Vec { + fn from(cursor: OwnedTransactionIndexCursor) -> Self { + let mut bytes = Vec::with_capacity(8); + bytes.extend(cursor.block_height.to_bytes()); + bytes.extend(cursor.tx_idx.to_be_bytes()); + bytes + } +} + +#[cfg(test)] +mod test { + use super::*; + + fn generate_key(rng: &mut impl rand::Rng) -> ::Key { + let mut bytes = [0u8; INDEX_SIZE]; + rng.fill(bytes.as_mut()); + bytes.into() + } + + fuel_core_storage::basic_storage_tests!( + OwnedTransactions, + [1u8; INDEX_SIZE].into(), + ::Value::default(), + ::Value::default(), + generate_key + ); + + fuel_core_storage::basic_storage_tests!( + TransactionStatuses, + ::Key::default(), + TransactionStatus::Submitted { + time: fuel_core_types::tai64::Tai64::UNIX_EPOCH, + } + ); +} diff --git a/crates/fuel-core/src/graphql_api/worker_service.rs b/crates/fuel-core/src/graphql_api/worker_service.rs index 22f5471922..2fa074c09f 100644 --- a/crates/fuel-core/src/graphql_api/worker_service.rs +++ b/crates/fuel-core/src/graphql_api/worker_service.rs @@ -1,4 +1,18 @@ -use crate::fuel_core_graphql_api::ports; +use crate::{ + database::{ + database_description::{ + off_chain::OffChain, + DatabaseDescription, + DatabaseMetadata, + }, + metadata::MetadataTable, + }, + fuel_core_graphql_api::{ + ports, + storage::receipts::Receipts, + }, +}; +use fuel_core_metrics::graphql_metrics::graphql_metrics; use fuel_core_services::{ stream::BoxStream, EmptyShared, @@ -8,7 +22,6 @@ use fuel_core_services::{ StateWatcher, }; use fuel_core_storage::{ - tables::Receipts, Result as StorageResult, StorageAsMut, }; @@ -63,17 +76,37 @@ where // TODO: Implement the creation of indexes for the messages and coins. // Implement table `BlockId -> BlockHeight` to get the block height by block id. // https://github.com/FuelLabs/fuel-core/issues/1583 + let block = &result.sealed_block.entity; let mut transaction = self.database.transaction(); // save the status for every transaction using the finalized block id self.persist_transaction_status(&result, transaction.as_mut())?; // save the associated owner for each transaction in the block - self.index_tx_owners_for_block( - &result.sealed_block.entity, - transaction.as_mut(), - )?; + self.index_tx_owners_for_block(block, transaction.as_mut())?; + let total_tx_count = transaction + .as_mut() + .increase_tx_count(block.transactions().len() as u64) + .unwrap_or_default(); + + // TODO: Temporary solution to store the block height in the database manually here. + // Later it will be controlled by the `commit_changes` function on the `Database` side. + // https://github.com/FuelLabs/fuel-core/issues/1589 + transaction + .as_mut() + .storage::>() + .insert( + &(), + &DatabaseMetadata::V1 { + version: OffChain::version(), + height: *block.header().height(), + }, + )?; + transaction.commit()?; + // update the importer metrics after the block is successfully committed + graphql_metrics().total_txs_count.set(total_tx_count as i64); + Ok(()) } @@ -214,10 +247,13 @@ where } async fn into_task( - self, + mut self, _: &StateWatcher, _: Self::TaskParams, ) -> anyhow::Result { + let total_tx_count = self.database.increase_tx_count(0).unwrap_or_default(); + graphql_metrics().total_txs_count.set(total_tx_count as i64); + // TODO: It is possible that the node was shut down before we processed all imported blocks. // It could lead to some missed blocks and the database's inconsistent state. // Because the result of block execution is not stored on the chain, it is impossible diff --git a/crates/fuel-core/src/lib.rs b/crates/fuel-core/src/lib.rs index ed4c86cfa7..81535b5600 100644 --- a/crates/fuel-core/src/lib.rs +++ b/crates/fuel-core/src/lib.rs @@ -24,6 +24,7 @@ pub use fuel_core_txpool as txpool; pub use fuel_core_types as types; pub mod coins_query; +pub mod combined_database; pub mod database; pub mod executor; pub mod model; diff --git a/crates/fuel-core/src/query/chain.rs b/crates/fuel-core/src/query/chain.rs index b9408ddfcd..aebf442cf3 100644 --- a/crates/fuel-core/src/query/chain.rs +++ b/crates/fuel-core/src/query/chain.rs @@ -3,16 +3,10 @@ use fuel_core_storage::Result as StorageResult; use fuel_core_types::blockchain::primitives::DaBlockHeight; pub trait ChainQueryData: Send + Sync { - fn name(&self) -> StorageResult; - fn da_height(&self) -> StorageResult; } impl ChainQueryData for D { - fn name(&self) -> StorageResult { - self.chain_name() - } - fn da_height(&self) -> StorageResult { self.da_height() } diff --git a/crates/fuel-core/src/query/tx.rs b/crates/fuel-core/src/query/tx.rs index ebc2531f27..e103cc3d85 100644 --- a/crates/fuel-core/src/query/tx.rs +++ b/crates/fuel-core/src/query/tx.rs @@ -1,6 +1,9 @@ -use crate::fuel_core_graphql_api::ports::{ - OffChainDatabase, - OnChainDatabase, +use crate::fuel_core_graphql_api::{ + ports::{ + OffChainDatabase, + OnChainDatabase, + }, + storage::receipts::Receipts, }; use fuel_core_storage::{ iter::{ @@ -9,10 +12,7 @@ use fuel_core_storage::{ IterDirection, }, not_found, - tables::{ - Receipts, - Transactions, - }, + tables::Transactions, Result as StorageResult, StorageAsRef, }; diff --git a/crates/fuel-core/src/schema/chain.rs b/crates/fuel-core/src/schema/chain.rs index 7c8bb918aa..535f1c11ae 100644 --- a/crates/fuel-core/src/schema/chain.rs +++ b/crates/fuel-core/src/schema/chain.rs @@ -3,6 +3,7 @@ use crate::{ database::ReadView, Config as GraphQLConfig, }, + graphql_api::Config, query::{ BlockQueryData, ChainQueryData, @@ -683,8 +684,8 @@ impl HeavyOperation { #[Object] impl ChainInfo { async fn name(&self, ctx: &Context<'_>) -> async_graphql::Result { - let query: &ReadView = ctx.data_unchecked(); - Ok(query.name()?) + let config: &Config = ctx.data_unchecked(); + Ok(config.chain_name.clone()) } async fn latest_block(&self, ctx: &Context<'_>) -> async_graphql::Result { diff --git a/crates/fuel-core/src/schema/dap.rs b/crates/fuel-core/src/schema/dap.rs index 3e0c6600bc..c54f668060 100644 --- a/crates/fuel-core/src/schema/dap.rs +++ b/crates/fuel-core/src/schema/dap.rs @@ -1,9 +1,13 @@ use crate::{ database::{ + database_description::on_chain::OnChain, transaction::DatabaseTransaction, Database, }, - schema::scalars::U64, + schema::scalars::{ + U32, + U64, + }, }; use async_graphql::{ Context, @@ -35,6 +39,7 @@ use fuel_core_types::{ IntoChecked, }, consts, + state::DebugEval, Interpreter, InterpreterError, }, @@ -51,9 +56,6 @@ use tracing::{ }; use uuid::Uuid; -use crate::schema::scalars::U32; -use fuel_core_types::fuel_vm::state::DebugEval; - pub struct Config { /// `true` means that debugger functionality is enabled. debug_enabled: bool, @@ -63,7 +65,7 @@ pub struct Config { pub struct ConcreteStorage { vm: HashMap, Script>>, tx: HashMap>, - db: HashMap, + db: HashMap>, params: ConsensusParameters, } @@ -93,7 +95,7 @@ impl ConcreteStorage { pub fn init( &mut self, txs: &[Script], - storage: DatabaseTransaction, + storage: DatabaseTransaction, ) -> anyhow::Result { let id = Uuid::new_v4(); let id = ID::from(id); @@ -124,7 +126,11 @@ impl ConcreteStorage { self.db.remove(id).is_some() } - pub fn reset(&mut self, id: &ID, storage: DatabaseTransaction) -> anyhow::Result<()> { + pub fn reset( + &mut self, + id: &ID, + storage: DatabaseTransaction, + ) -> anyhow::Result<()> { let vm_database = Self::vm_database(&storage)?; let tx = self .tx @@ -156,7 +162,9 @@ impl ConcreteStorage { .ok_or_else(|| anyhow::anyhow!("The VM instance was not found")) } - fn vm_database(storage: &DatabaseTransaction) -> anyhow::Result> { + fn vm_database( + storage: &DatabaseTransaction, + ) -> anyhow::Result> { let block = storage .get_current_block()? .ok_or(not_found!("Block for VMDatabase"))?; diff --git a/crates/fuel-core/src/service.rs b/crates/fuel-core/src/service.rs index 316eeaebfe..3030fa1cfa 100644 --- a/crates/fuel-core/src/service.rs +++ b/crates/fuel-core/src/service.rs @@ -1,5 +1,6 @@ use self::adapters::BlockImporterAdapter; use crate::{ + combined_database::CombinedDatabase, database::Database, service::{ adapters::{ @@ -50,11 +51,15 @@ pub struct SharedState { pub network: Option, #[cfg(feature = "relayer")] /// The Relayer shared state. - pub relayer: Option>, + pub relayer: Option< + fuel_core_relayer::SharedState< + Database, + >, + >, /// The GraphQL shared state. pub graph_ql: crate::fuel_core_graphql_api::api_service::SharedState, /// The underlying database. - pub database: Database, + pub database: CombinedDatabase, /// Subscribe to new block production. pub block_importer: BlockImporterAdapter, /// The config of the service. @@ -77,7 +82,7 @@ pub struct FuelService { impl FuelService { /// Creates a `FuelService` instance from service config #[tracing::instrument(skip_all, fields(name = %config.name))] - pub fn new(database: Database, config: Config) -> anyhow::Result { + pub fn new(database: CombinedDatabase, config: Config) -> anyhow::Result { let config = config.make_config_consistent(); let task = Task::new(database, config)?; let runner = ServiceRunner::new(task); @@ -93,7 +98,7 @@ impl FuelService { /// Creates and starts fuel node instance from service config pub async fn new_node(config: Config) -> anyhow::Result { // initialize database - let database = match config.database_type { + let combined_database = match config.database_type { #[cfg(feature = "rocksdb")] DbType::RocksDb => { // use a default tmp rocksdb if no path is provided @@ -101,30 +106,43 @@ impl FuelService { warn!( "No RocksDB path configured, initializing database with a tmp directory" ); - Database::default() + CombinedDatabase::default() } else { tracing::info!( "Opening database {:?} with cache size \"{}\"", config.database_path, config.max_database_cache_size ); - Database::open(&config.database_path, config.max_database_cache_size)? + CombinedDatabase::open( + &config.database_path, + config.max_database_cache_size, + )? } } - DbType::InMemory => Database::in_memory(), + DbType::InMemory => CombinedDatabase::in_memory(), #[cfg(not(feature = "rocksdb"))] - _ => Database::in_memory(), + _ => CombinedDatabase::in_memory(), }; - Self::from_database(database, config).await + Self::from_combined_database(combined_database, config).await } - /// Creates and starts fuel node instance from service config and a pre-existing database + /// Creates and starts fuel node instance from service config and a pre-existing on-chain database pub async fn from_database( database: Database, config: Config, ) -> anyhow::Result { - let service = Self::new(database, config)?; + let combined_database = + CombinedDatabase::new(database, Default::default(), Default::default()); + Self::from_combined_database(combined_database, config).await + } + + /// Creates and starts fuel node instance from service config and a pre-existing combined database + pub async fn from_combined_database( + combined_database: CombinedDatabase, + config: Config, + ) -> anyhow::Result { + let service = Self::new(combined_database, config)?; service.runner.start_and_await().await?; Ok(service) } @@ -195,14 +213,21 @@ pub struct Task { impl Task { /// Private inner method for initializing the fuel service task - pub fn new(mut database: Database, config: Config) -> anyhow::Result { + pub fn new(mut database: CombinedDatabase, config: Config) -> anyhow::Result { // initialize state tracing::info!("Initializing database"); - database.init(&config.chain_conf)?; + let block_height = config + .chain_conf + .initial_state + .as_ref() + .and_then(|state| state.height) + .unwrap_or_default(); + let da_block_height = 0u64.into(); + database.init(&block_height, &da_block_height)?; // initialize sub services tracing::info!("Initializing sub services"); - let (services, shared) = sub_services::init_sub_services(&config, &database)?; + let (services, shared) = sub_services::init_sub_services(&config, database)?; Ok(Task { services, shared }) } @@ -228,7 +253,7 @@ impl RunnableService for Task { _: &StateWatcher, _: Self::TaskParams, ) -> anyhow::Result { - let view = self.shared.database.latest_view(); + let view = self.shared.database.on_chain().latest_view(); // check if chain is initialized if let Err(err) = view.get_genesis() { if err.is_not_found() { diff --git a/crates/fuel-core/src/service/adapters.rs b/crates/fuel-core/src/service/adapters.rs index f2d451c235..19d042338b 100644 --- a/crates/fuel-core/src/service/adapters.rs +++ b/crates/fuel-core/src/service/adapters.rs @@ -1,7 +1,7 @@ use crate::{ database::{ + database_description::relayer::Relayer, Database, - RelayerReadDatabase, }, service::sub_services::BlockProducerService, }; @@ -67,13 +67,13 @@ impl TransactionsSource { #[derive(Clone)] pub struct ExecutorAdapter { - pub executor: Arc>, + pub executor: Arc>>, } impl ExecutorAdapter { pub fn new( database: Database, - relayer_database: RelayerReadDatabase, + relayer_database: Database, config: fuel_core_executor::Config, ) -> Self { let executor = Executor { @@ -115,9 +115,8 @@ impl ConsensusAdapter { #[derive(Clone)] pub struct MaybeRelayerAdapter { - pub database: Database, #[cfg(feature = "relayer")] - pub relayer_synced: Option>, + pub relayer_synced: Option>>, #[cfg(feature = "relayer")] pub da_deploy_height: fuel_core_types::blockchain::primitives::DaBlockHeight, } diff --git a/crates/fuel-core/src/service/adapters/block_importer.rs b/crates/fuel-core/src/service/adapters/block_importer.rs index f02856e844..62d9968a7f 100644 --- a/crates/fuel-core/src/service/adapters/block_importer.rs +++ b/crates/fuel-core/src/service/adapters/block_importer.rs @@ -88,10 +88,6 @@ impl ImporterDatabase for Database { .transpose()? .map(|(height, _)| height)) } - - fn increase_tx_count(&self, new_txs_count: u64) -> StorageResult { - self.increase_tx_count(new_txs_count).map_err(Into::into) - } } impl ExecutorDatabase for Database { diff --git a/crates/fuel-core/src/service/adapters/executor.rs b/crates/fuel-core/src/service/adapters/executor.rs index ef591508d2..c316ebfc15 100644 --- a/crates/fuel-core/src/service/adapters/executor.rs +++ b/crates/fuel-core/src/service/adapters/executor.rs @@ -1,5 +1,8 @@ use crate::{ - database::Database, + database::{ + database_description::relayer::Relayer, + Database, + }, service::adapters::{ ExecutorAdapter, TransactionsSource, @@ -64,7 +67,7 @@ impl fuel_core_executor::refs::ContractStorageTrait for Database { impl fuel_core_executor::ports::ExecutorDatabaseTrait for Database {} -impl fuel_core_executor::ports::RelayerPort for Database { +impl fuel_core_executor::ports::RelayerPort for Database { fn enabled(&self) -> bool { #[cfg(feature = "relayer")] { diff --git a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs index a892b84c2b..00ce75838c 100644 --- a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs +++ b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs @@ -1,11 +1,14 @@ use crate::{ database::{ - transactions::OwnedTransactionIndexCursor, + database_description::off_chain::OffChain, Database, }, - fuel_core_graphql_api::ports::{ - worker, - OffChainDatabase, + fuel_core_graphql_api::{ + ports::{ + worker, + OffChainDatabase, + }, + storage::transactions::OwnedTransactionIndexCursor, }, }; use fuel_core_storage::{ @@ -24,38 +27,12 @@ use fuel_core_types::{ Address, Bytes32, TxPointer, - UtxoId, - }, - fuel_types::{ - BlockHeight, - Nonce, }, + fuel_types::BlockHeight, services::txpool::TransactionStatus, }; -impl OffChainDatabase for Database { - fn owned_message_ids( - &self, - owner: &Address, - start_message_id: Option, - direction: IterDirection, - ) -> BoxedIter<'_, StorageResult> { - self.owned_message_ids(owner, start_message_id, Some(direction)) - .map(|result| result.map_err(StorageError::from)) - .into_boxed() - } - - fn owned_coins_ids( - &self, - owner: &Address, - start_coin: Option, - direction: IterDirection, - ) -> BoxedIter<'_, StorageResult> { - self.owned_coins_ids(owner, start_coin, Some(direction)) - .map(|res| res.map_err(StorageError::from)) - .into_boxed() - } - +impl OffChainDatabase for Database { fn tx_status(&self, tx_id: &TxId) -> StorageResult { self.get_tx_status(tx_id) .transpose() @@ -78,7 +55,7 @@ impl OffChainDatabase for Database { } } -impl worker::OffChainDatabase for Database { +impl worker::OffChainDatabase for Database { fn record_tx_id_owner( &mut self, owner: &Address, @@ -96,4 +73,8 @@ impl worker::OffChainDatabase for Database { ) -> StorageResult> { Database::update_tx_status(self, id, status) } + + fn increase_tx_count(&mut self, new_txs_count: u64) -> StorageResult { + Database::increase_tx_count(self, new_txs_count) + } } diff --git a/crates/fuel-core/src/service/adapters/graphql_api/on_chain.rs b/crates/fuel-core/src/service/adapters/graphql_api/on_chain.rs index 09ec40a989..c0b1416a0d 100644 --- a/crates/fuel-core/src/service/adapters/graphql_api/on_chain.rs +++ b/crates/fuel-core/src/service/adapters/graphql_api/on_chain.rs @@ -30,7 +30,11 @@ use fuel_core_types::{ }, }, entities::message::Message, - fuel_tx::AssetId, + fuel_tx::{ + Address, + AssetId, + UtxoId, + }, fuel_types::{ BlockHeight, Nonce, @@ -103,25 +107,33 @@ impl DatabaseContracts for Database { } impl DatabaseChain for Database { - fn chain_name(&self) -> StorageResult { - pub const DEFAULT_NAME: &str = "Fuel.testnet"; + fn da_height(&self) -> StorageResult { + self.latest_compressed_block()? + .map(|block| block.header().da_height) + .ok_or(not_found!("DaBlockHeight")) + } +} - Ok(self - .get_chain_name()? - .unwrap_or_else(|| DEFAULT_NAME.to_string())) +impl OnChainDatabase for Database { + fn owned_message_ids( + &self, + owner: &Address, + start_message_id: Option, + direction: IterDirection, + ) -> BoxedIter<'_, StorageResult> { + self.owned_message_ids(owner, start_message_id, Some(direction)) + .map(|result| result.map_err(StorageError::from)) + .into_boxed() } - fn da_height(&self) -> StorageResult { - #[cfg(feature = "relayer")] - { - use fuel_core_relayer::ports::RelayerDb; - self.get_finalized_da_height() - } - #[cfg(not(feature = "relayer"))] - { - Ok(0u64.into()) - } + fn owned_coins_ids( + &self, + owner: &Address, + start_coin: Option, + direction: IterDirection, + ) -> BoxedIter<'_, StorageResult> { + self.owned_coins_ids(owner, start_coin, Some(direction)) + .map(|res| res.map_err(StorageError::from)) + .into_boxed() } } - -impl OnChainDatabase for Database {} diff --git a/crates/fuel-core/src/service/adapters/producer.rs b/crates/fuel-core/src/service/adapters/producer.rs index 8a851ddca4..957c494fcb 100644 --- a/crates/fuel-core/src/service/adapters/producer.rs +++ b/crates/fuel-core/src/service/adapters/producer.rs @@ -114,12 +114,12 @@ impl fuel_core_producer::ports::Relayer for MaybeRelayerAdapter { ) -> anyhow::Result { #[cfg(feature = "relayer")] { - use fuel_core_relayer::ports::RelayerDb; if let Some(sync) = self.relayer_synced.as_ref() { sync.await_at_least_synced(height).await?; + sync.get_finalized_da_height() + } else { + Ok(0u64.into()) } - - Ok(self.database.get_finalized_da_height().unwrap_or_default()) } #[cfg(not(feature = "relayer"))] { diff --git a/crates/fuel-core/src/service/genesis.rs b/crates/fuel-core/src/service/genesis.rs index fe642cc7fb..6bcbc50b87 100644 --- a/crates/fuel-core/src/service/genesis.rs +++ b/crates/fuel-core/src/service/genesis.rs @@ -375,10 +375,13 @@ fn init_contract_balance( mod tests { use super::*; - use crate::service::{ - config::Config, - FuelService, - Task, + use crate::{ + combined_database::CombinedDatabase, + service::{ + config::Config, + FuelService, + Task, + }, }; use fuel_core_chain_config::{ ChainConfig, @@ -412,30 +415,6 @@ mod tests { }; use std::vec; - #[tokio::test] - async fn config_initializes_chain_name() { - let test_name = "test_net_123".to_string(); - let service_config = Config { - chain_conf: ChainConfig { - chain_name: test_name.clone(), - ..ChainConfig::local_testnet() - }, - ..Config::local_node() - }; - - let db = Database::default(); - FuelService::from_database(db.clone(), service_config) - .await - .unwrap(); - - assert_eq!( - test_name, - db.get_chain_name() - .unwrap() - .expect("Expected a chain name to be set") - ) - } - #[tokio::test] async fn config_initializes_block_height() { let test_height = BlockHeight::from(99u32); @@ -717,8 +696,8 @@ mod tests { ..Config::local_node() }; - let db = Database::default(); - let task = Task::new(db.clone(), service_config).unwrap(); + let db = CombinedDatabase::default(); + let task = Task::new(db, service_config).unwrap(); let init_result = task.into_task(&Default::default(), ()).await; assert!(init_result.is_err()) @@ -757,8 +736,8 @@ mod tests { ..Config::local_node() }; - let db = Database::default(); - let task = Task::new(db.clone(), service_config).unwrap(); + let db = CombinedDatabase::default(); + let task = Task::new(db, service_config).unwrap(); let init_result = task.into_task(&Default::default(), ()).await; assert!(init_result.is_err()) diff --git a/crates/fuel-core/src/service/query.rs b/crates/fuel-core/src/service/query.rs index b53d2db0fe..c538cc6528 100644 --- a/crates/fuel-core/src/service/query.rs +++ b/crates/fuel-core/src/service/query.rs @@ -71,7 +71,7 @@ impl FuelService { id: Bytes32, ) -> anyhow::Result>> { let txpool = self.shared.txpool.clone(); - let db = self.shared.database.clone(); + let db = self.shared.database.off_chain().clone(); let rx = txpool.tx_update_subscribe(id)?; Ok(transaction_status_change( move |id| match db.get_tx_status(&id)? { diff --git a/crates/fuel-core/src/service/sub_services.rs b/crates/fuel-core/src/service/sub_services.rs index a08d7f4d0b..8adfafef88 100644 --- a/crates/fuel-core/src/service/sub_services.rs +++ b/crates/fuel-core/src/service/sub_services.rs @@ -1,10 +1,8 @@ #![allow(clippy::let_unit_value)] use super::adapters::P2PAdapter; use crate::{ - database::{ - Database, - RelayerReadDatabase, - }, + combined_database::CombinedDatabase, + database::Database, fuel_core_graphql_api, fuel_core_graphql_api::Config as GraphQLConfig, schema::build_schema, @@ -35,8 +33,6 @@ use fuel_core_types::blockchain::primitives::DaBlockHeight; pub type PoAService = fuel_core_poa::Service; -#[cfg(feature = "relayer")] -pub type RelayerService = fuel_core_relayer::Service; #[cfg(feature = "p2p")] pub type P2PService = fuel_core_p2p::service::Service; pub type TxPoolService = fuel_core_txpool::Service; @@ -45,13 +41,14 @@ pub type BlockProducerService = fuel_core_producer::block_producer::Producer< TxPoolAdapter, ExecutorAdapter, >; -pub type GraphQL = crate::fuel_core_graphql_api::api_service::Service; +pub type GraphQL = fuel_core_graphql_api::api_service::Service; pub fn init_sub_services( config: &Config, - database: &Database, + database: CombinedDatabase, ) -> anyhow::Result<(SubServices, SharedState)> { let last_block_header = database + .on_chain() .get_current_block()? .map(|block| block.header().clone()) .unwrap_or({ @@ -61,8 +58,8 @@ pub fn init_sub_services( let last_height = *last_block_header.height(); let executor = ExecutorAdapter::new( - database.clone(), - RelayerReadDatabase::new(database.clone()), + database.on_chain().clone(), + database.relayer().clone(), fuel_core_executor::Config { consensus_parameters: config.chain_conf.consensus_parameters.clone(), coinbase_recipient: config @@ -74,11 +71,11 @@ pub fn init_sub_services( }, ); - let verifier = VerifierAdapter::new(config, database.clone()); + let verifier = VerifierAdapter::new(config, database.on_chain().clone()); let importer_adapter = BlockImporterAdapter::new( config.block_importer.clone(), - database.clone(), + database.on_chain().clone(), executor.clone(), verifier.clone(), ); @@ -86,7 +83,7 @@ pub fn init_sub_services( #[cfg(feature = "relayer")] let relayer_service = if let Some(config) = &config.relayer { Some(fuel_core_relayer::new_service( - database.clone(), + database.relayer().clone(), config.clone(), )?) } else { @@ -94,7 +91,6 @@ pub fn init_sub_services( }; let relayer_adapter = MaybeRelayerAdapter { - database: database.clone(), #[cfg(feature = "relayer")] relayer_synced: relayer_service.as_ref().map(|r| r.shared.clone()), #[cfg(feature = "relayer")] @@ -109,7 +105,7 @@ pub fn init_sub_services( fuel_core_p2p::service::new_service( config.chain_conf.consensus_parameters.chain_id, p2p_config, - database.clone(), + database.on_chain().clone(), importer_adapter.clone(), ) }); @@ -138,7 +134,7 @@ pub fn init_sub_services( let txpool = fuel_core_txpool::new_service( config.txpool.clone(), - database.clone(), + database.on_chain().clone(), importer_adapter.clone(), p2p_adapter.clone(), last_height, @@ -147,7 +143,7 @@ pub fn init_sub_services( let block_producer = fuel_core_producer::Producer { config: config.block_producer.clone(), - view_provider: database.clone(), + view_provider: database.on_chain().clone(), txpool: tx_pool_adapter.clone(), executor: Arc::new(executor), relayer: Box::new(relayer_adapter.clone()), @@ -194,11 +190,11 @@ pub fn init_sub_services( config.chain_conf.consensus_parameters.clone(), config.debug, ) - .data(database.clone()); + .data(database.on_chain().clone()); let graphql_worker = fuel_core_graphql_api::worker_service::new_service( importer_adapter.clone(), - database.clone(), + database.off_chain().clone(), ); let graphql_config = GraphQLConfig { @@ -209,6 +205,7 @@ pub fn init_sub_services( min_gas_price: config.txpool.min_gas_price, max_tx: config.txpool.max_tx, max_depth: config.txpool.max_depth, + chain_name: config.chain_conf.chain_name.clone(), consensus_parameters: config.chain_conf.consensus_parameters.clone(), consensus_key: config.consensus_key.clone(), }; @@ -216,8 +213,8 @@ pub fn init_sub_services( let graph_ql = fuel_core_graphql_api::api_service::new_service( graphql_config, schema, - database.clone(), - database.clone(), + database.on_chain().clone(), + database.off_chain().clone(), Box::new(tx_pool_adapter), Box::new(producer_adapter), Box::new(poa_adapter.clone()), @@ -234,7 +231,7 @@ pub fn init_sub_services( #[cfg(feature = "relayer")] relayer: relayer_service.as_ref().map(|r| r.shared.clone()), graph_ql: graph_ql.shared.clone(), - database: database.clone(), + database, block_importer: importer_adapter, config: config.clone(), }; diff --git a/crates/fuel-core/src/state.rs b/crates/fuel-core/src/state.rs index 83c93851df..b35055071c 100644 --- a/crates/fuel-core/src/state.rs +++ b/crates/fuel-core/src/state.rs @@ -1,8 +1,9 @@ use crate::{ database::{ - Column, - Database, - Error as DatabaseError, + database_description::{ + on_chain::OnChain, + DatabaseDescription, + }, Result as DatabaseResult, }, state::in_memory::{ @@ -26,39 +27,57 @@ pub mod in_memory; #[cfg(feature = "rocksdb")] pub mod rocks_db; -type DataSourceInner = Arc>; +type DataSourceInner = Arc>; #[derive(Clone, Debug)] -pub struct DataSource(DataSourceInner); +pub struct DataSource(DataSourceInner) +where + Description: DatabaseDescription; -impl From> for DataSource { - fn from(inner: Arc) -> Self { +impl From>> + for DataSource +where + Description: DatabaseDescription, +{ + fn from(inner: Arc>) -> Self { Self(inner) } } #[cfg(feature = "rocksdb")] -impl From> for DataSource { - fn from(inner: Arc) -> Self { +impl From>> for DataSource +where + Description: DatabaseDescription, +{ + fn from(inner: Arc>) -> Self { Self(inner) } } -impl From> for DataSource { - fn from(inner: Arc) -> Self { +impl From>> for DataSource +where + Description: DatabaseDescription, +{ + fn from(inner: Arc>) -> Self { Self(inner) } } -impl core::ops::Deref for DataSource { - type Target = DataSourceInner; +impl core::ops::Deref for DataSource +where + Description: DatabaseDescription, +{ + type Target = DataSourceInner; fn deref(&self) -> &Self::Target { &self.0 } } -impl core::ops::DerefMut for DataSource { +impl core::ops::DerefMut for DataSource +where + Description: DatabaseDescription, +{ fn deref_mut(&mut self) -> &mut Self::Target { &mut self.0 } @@ -67,11 +86,5 @@ impl core::ops::DerefMut for DataSource { pub trait TransactableStorage: IteratorableStore + BatchOperations + Debug + Send + Sync { - fn checkpoint(&self) -> DatabaseResult { - Err(DatabaseError::Other(anyhow::anyhow!( - "Checkpoint is not supported" - ))) - } - fn flush(&self) -> DatabaseResult<()>; } diff --git a/crates/fuel-core/src/state/in_memory/memory_store.rs b/crates/fuel-core/src/state/in_memory/memory_store.rs index bcab81cb7f..6141a0f1ad 100644 --- a/crates/fuel-core/src/state/in_memory/memory_store.rs +++ b/crates/fuel-core/src/state/in_memory/memory_store.rs @@ -1,6 +1,9 @@ use crate::{ database::{ - Column, + database_description::{ + on_chain::OnChain, + DatabaseDescription, + }, Result as DatabaseResult, }, state::{ @@ -18,6 +21,7 @@ use fuel_core_storage::{ kv_store::{ KVItem, KeyValueStore, + StorageColumn, Value, }, Result as StorageResult, @@ -31,16 +35,38 @@ use std::{ }, }; -#[derive(Default, Debug)] -pub struct MemoryStore { +#[derive(Debug)] +pub struct MemoryStore +where + Description: DatabaseDescription, +{ // TODO: Remove `Mutex`. - inner: [Mutex, Value>>; Column::COUNT], + inner: Vec, Value>>>, + _marker: core::marker::PhantomData, } -impl MemoryStore { +impl Default for MemoryStore +where + Description: DatabaseDescription, +{ + fn default() -> Self { + use strum::EnumCount; + Self { + inner: (0..Description::Column::COUNT) + .map(|_| Mutex::new(BTreeMap::new())) + .collect(), + _marker: Default::default(), + } + } +} + +impl MemoryStore +where + Description: DatabaseDescription, +{ pub fn iter_all( &self, - column: Column, + column: Description::Column, prefix: Option<&[u8]>, start: Option<&[u8]>, direction: IterDirection, @@ -104,13 +130,16 @@ impl MemoryStore { } } -impl KeyValueStore for MemoryStore { - type Column = Column; +impl KeyValueStore for MemoryStore +where + Description: DatabaseDescription, +{ + type Column = Description::Column; fn replace( &self, key: &[u8], - column: Column, + column: Self::Column, value: Value, ) -> StorageResult> { Ok(self.inner[column.as_usize()] @@ -119,7 +148,12 @@ impl KeyValueStore for MemoryStore { .insert(key.to_vec(), value)) } - fn write(&self, key: &[u8], column: Column, buf: &[u8]) -> StorageResult { + fn write( + &self, + key: &[u8], + column: Self::Column, + buf: &[u8], + ) -> StorageResult { let len = buf.len(); self.inner[column.as_usize()] .lock() @@ -128,18 +162,18 @@ impl KeyValueStore for MemoryStore { Ok(len) } - fn take(&self, key: &[u8], column: Column) -> StorageResult> { + fn take(&self, key: &[u8], column: Self::Column) -> StorageResult> { Ok(self.inner[column.as_usize()] .lock() .expect("poisoned") .remove(&key.to_vec())) } - fn delete(&self, key: &[u8], column: Column) -> StorageResult<()> { + fn delete(&self, key: &[u8], column: Self::Column) -> StorageResult<()> { self.take(key, column).map(|_| ()) } - fn get(&self, key: &[u8], column: Column) -> StorageResult> { + fn get(&self, key: &[u8], column: Self::Column) -> StorageResult> { Ok(self.inner[column.as_usize()] .lock() .expect("poisoned") @@ -148,10 +182,13 @@ impl KeyValueStore for MemoryStore { } } -impl IteratorableStore for MemoryStore { +impl IteratorableStore for MemoryStore +where + Description: DatabaseDescription, +{ fn iter_all( &self, - column: Column, + column: Self::Column, prefix: Option<&[u8]>, start: Option<&[u8]>, direction: IterDirection, @@ -160,9 +197,15 @@ impl IteratorableStore for MemoryStore { } } -impl BatchOperations for MemoryStore {} +impl BatchOperations for MemoryStore where + Description: DatabaseDescription +{ +} -impl TransactableStorage for MemoryStore { +impl TransactableStorage for MemoryStore +where + Description: DatabaseDescription, +{ fn flush(&self) -> DatabaseResult<()> { for lock in self.inner.iter() { lock.lock().expect("poisoned").clear(); @@ -174,13 +217,14 @@ impl TransactableStorage for MemoryStore { #[cfg(test)] mod tests { use super::*; + use fuel_core_storage::column::Column; use std::sync::Arc; #[test] fn can_use_unit_value() { let key = vec![0x00]; - let db = MemoryStore::default(); + let db = MemoryStore::::default(); let expected = Arc::new(vec![]); db.put(&key.to_vec(), Column::Metadata, expected.clone()) .unwrap(); @@ -205,7 +249,7 @@ mod tests { fn can_use_unit_key() { let key: Vec = Vec::with_capacity(0); - let db = MemoryStore::default(); + let db = MemoryStore::::default(); let expected = Arc::new(vec![1, 2, 3]); db.put(&key, Column::Metadata, expected.clone()).unwrap(); @@ -229,7 +273,7 @@ mod tests { fn can_use_unit_key_and_value() { let key: Vec = Vec::with_capacity(0); - let db = MemoryStore::default(); + let db = MemoryStore::::default(); let expected = Arc::new(vec![]); db.put(&key, Column::Metadata, expected.clone()).unwrap(); diff --git a/crates/fuel-core/src/state/in_memory/transaction.rs b/crates/fuel-core/src/state/in_memory/transaction.rs index 7dcb96d827..ad31b8c88e 100644 --- a/crates/fuel-core/src/state/in_memory/transaction.rs +++ b/crates/fuel-core/src/state/in_memory/transaction.rs @@ -1,6 +1,9 @@ use crate::{ database::{ - Column, + database_description::{ + on_chain::OnChain, + DatabaseDescription, + }, Result as DatabaseResult, }, state::{ @@ -20,6 +23,7 @@ use fuel_core_storage::{ kv_store::{ KVItem, KeyValueStore, + StorageColumn, Value, WriteOperation, }, @@ -41,22 +45,31 @@ use std::{ }; #[derive(Debug)] -pub struct MemoryTransactionView { - view_layer: MemoryStore, +pub struct MemoryTransactionView +where + Description: DatabaseDescription, +{ + view_layer: MemoryStore, // TODO: Remove `Mutex`. // use hashmap to collapse changes (e.g. insert then remove the same key) - changes: [Mutex, WriteOperation>>; Column::COUNT], - data_source: DataSource, + changes: Vec, WriteOperation>>>, + data_source: DataSource, } -impl MemoryTransactionView { +impl MemoryTransactionView +where + Description: DatabaseDescription, +{ pub fn new(source: D) -> Self where - D: Into, + D: Into>, { + use strum::EnumCount; Self { view_layer: MemoryStore::default(), - changes: Default::default(), + changes: (0..Description::Column::COUNT) + .map(|_| Mutex::new(HashMap::new())) + .collect(), data_source: source.into(), } } @@ -65,7 +78,7 @@ impl MemoryTransactionView { let mut iter = self .changes .iter() - .zip(enum_iterator::all::()) + .zip(enum_iterator::all::()) .flat_map(|(column_map, column)| { let mut map = column_map.lock().expect("poisoned lock"); let changes = core::mem::take(map.deref_mut()); @@ -77,13 +90,16 @@ impl MemoryTransactionView { } } -impl KeyValueStore for MemoryTransactionView { - type Column = Column; +impl KeyValueStore for MemoryTransactionView +where + Description: DatabaseDescription, +{ + type Column = Description::Column; fn replace( &self, key: &[u8], - column: Column, + column: Self::Column, value: Value, ) -> StorageResult> { let key_vec = key.to_vec(); @@ -100,7 +116,12 @@ impl KeyValueStore for MemoryTransactionView { } } - fn write(&self, key: &[u8], column: Column, buf: &[u8]) -> StorageResult { + fn write( + &self, + key: &[u8], + column: Self::Column, + buf: &[u8], + ) -> StorageResult { let k = key.to_vec(); self.changes[column.as_usize()] .lock() @@ -109,7 +130,7 @@ impl KeyValueStore for MemoryTransactionView { self.view_layer.write(key, column, buf) } - fn take(&self, key: &[u8], column: Column) -> StorageResult> { + fn take(&self, key: &[u8], column: Self::Column) -> StorageResult> { let k = key.to_vec(); let contained_key = { let mut lock = self.changes[column.as_usize()] @@ -125,7 +146,7 @@ impl KeyValueStore for MemoryTransactionView { } } - fn delete(&self, key: &[u8], column: Column) -> StorageResult<()> { + fn delete(&self, key: &[u8], column: Self::Column) -> StorageResult<()> { let k = key.to_vec(); self.changes[column.as_usize()] .lock() @@ -134,7 +155,11 @@ impl KeyValueStore for MemoryTransactionView { self.view_layer.delete(key, column) } - fn size_of_value(&self, key: &[u8], column: Column) -> StorageResult> { + fn size_of_value( + &self, + key: &[u8], + column: Self::Column, + ) -> StorageResult> { // try to fetch data from View layer if any changes to the key if self.changes[column.as_usize()] .lock() @@ -149,7 +174,7 @@ impl KeyValueStore for MemoryTransactionView { } } - fn get(&self, key: &[u8], column: Column) -> StorageResult> { + fn get(&self, key: &[u8], column: Self::Column) -> StorageResult> { // try to fetch data from View layer if any changes to the key if self.changes[column.as_usize()] .lock() @@ -166,7 +191,7 @@ impl KeyValueStore for MemoryTransactionView { fn read( &self, key: &[u8], - column: Column, + column: Self::Column, buf: &mut [u8], ) -> StorageResult> { // try to fetch data from View layer if any changes to the key @@ -184,10 +209,13 @@ impl KeyValueStore for MemoryTransactionView { } } -impl IteratorableStore for MemoryTransactionView { +impl IteratorableStore for MemoryTransactionView +where + Description: DatabaseDescription, +{ fn iter_all( &self, - column: Column, + column: Self::Column, prefix: Option<&[u8]>, start: Option<&[u8]>, direction: IterDirection, @@ -242,9 +270,15 @@ impl IteratorableStore for MemoryTransactionView { } } -impl BatchOperations for MemoryTransactionView {} +impl BatchOperations for MemoryTransactionView where + Description: DatabaseDescription +{ +} -impl TransactableStorage for MemoryTransactionView { +impl TransactableStorage for MemoryTransactionView +where + Description: DatabaseDescription, +{ fn flush(&self) -> DatabaseResult<()> { for lock in self.changes.iter() { lock.lock().expect("poisoned lock").clear(); @@ -257,8 +291,11 @@ impl TransactableStorage for MemoryTransactionView { #[cfg(test)] mod tests { use super::*; + use fuel_core_storage::column::Column; use std::sync::Arc; + type MemoryTransactionView = super::MemoryTransactionView; + #[test] fn get_returns_from_view() { // setup diff --git a/crates/fuel-core/src/state/rocks_db.rs b/crates/fuel-core/src/state/rocks_db.rs index 85b37faab3..748305974a 100644 --- a/crates/fuel-core/src/state/rocks_db.rs +++ b/crates/fuel-core/src/state/rocks_db.rs @@ -1,8 +1,7 @@ use crate::{ database::{ convert_to_rocksdb_direction, - Column, - Database, + database_description::DatabaseDescription, Error as DatabaseError, Result as DatabaseResult, }, @@ -22,6 +21,7 @@ use fuel_core_storage::{ kv_store::{ KVItem, KeyValueStore, + StorageColumn, Value, WriteOperation, }, @@ -29,7 +29,6 @@ use fuel_core_storage::{ }; use rand::RngCore; use rocksdb::{ - checkpoint::Checkpoint, BlockBasedOptions, BoundColumnFamily, Cache, @@ -45,6 +44,7 @@ use rocksdb::{ }; use std::{ env, + fmt::Debug, iter, path::{ Path, @@ -91,28 +91,32 @@ impl Drop for ShallowTempDir { } #[derive(Debug)] -pub struct RocksDb { +pub struct RocksDb { db: DB, - capacity: Option, + _marker: core::marker::PhantomData, } -impl RocksDb { +impl RocksDb +where + Description: DatabaseDescription, +{ pub fn default_open>( path: P, capacity: Option, - ) -> DatabaseResult { + ) -> DatabaseResult { Self::open( path, - enum_iterator::all::().collect::>(), + enum_iterator::all::().collect::>(), capacity, ) } pub fn open>( path: P, - columns: Vec, + columns: Vec, capacity: Option, - ) -> DatabaseResult { + ) -> DatabaseResult { + let path = path.as_ref().join(Description::name()); let mut block_opts = BlockBasedOptions::default(); // See https://github.com/facebook/rocksdb/blob/a1523efcdf2f0e8133b9a9f6e170a0dad49f928f/include/rocksdb/table.h#L246-L271 for details on what the format versions are/do. block_opts.set_format_version(5); @@ -134,10 +138,7 @@ impl RocksDb { block_opts.set_bloom_filter(10.0, true); let cf_descriptors = columns.clone().into_iter().map(|i| { - ColumnFamilyDescriptor::new( - RocksDb::col_name(i), - Self::cf_opts(i, &block_opts), - ) + ColumnFamilyDescriptor::new(Self::col_name(i), Self::cf_opts(i, &block_opts)) }); let mut opts = Options::default(); @@ -160,7 +161,7 @@ impl RocksDb { Ok(db) => { for i in columns { let opts = Self::cf_opts(i, &block_opts); - db.create_cf(RocksDb::col_name(i), &opts) + db.create_cf(Self::col_name(i), &opts) .map_err(|e| DatabaseError::Other(e.into()))?; } Ok(db) @@ -172,7 +173,7 @@ impl RocksDb { let cf_descriptors = columns.clone().into_iter().map(|i| { ColumnFamilyDescriptor::new( - RocksDb::col_name(i), + Self::col_name(i), Self::cf_opts(i, &block_opts), ) }); @@ -183,49 +184,33 @@ impl RocksDb { ok => ok, } .map_err(|e| DatabaseError::Other(e.into()))?; - let rocks_db = RocksDb { db, capacity }; + let rocks_db = RocksDb { + db, + _marker: Default::default(), + }; Ok(rocks_db) } - pub fn checkpoint>(&self, path: P) -> DatabaseResult<()> { - Checkpoint::new(&self.db) - .and_then(|checkpoint| checkpoint.create_checkpoint(path)) - .map_err(|e| { - DatabaseError::Other(anyhow::anyhow!( - "Failed to create a checkpoint: {}", - e - )) - }) - } - - fn cf(&self, column: Column) -> Arc { + fn cf(&self, column: Description::Column) -> Arc { self.db - .cf_handle(&RocksDb::col_name(column)) + .cf_handle(&Self::col_name(column)) .expect("invalid column state") } - fn col_name(column: Column) -> String { + fn col_name(column: Description::Column) -> String { format!("col-{}", column.as_usize()) } - fn cf_opts(column: Column, block_opts: &BlockBasedOptions) -> Options { + fn cf_opts(column: Description::Column, block_opts: &BlockBasedOptions) -> Options { let mut opts = Options::default(); opts.create_if_missing(true); opts.set_compression_type(DBCompressionType::Lz4); opts.set_block_based_table_factory(block_opts); // All double-keys should be configured here - match column { - Column::OwnedCoins - | Column::TransactionsByOwnerBlockIdx - | Column::OwnedMessageIds - | Column::ContractsAssets - | Column::ContractsState => { - // prefix is address length - opts.set_prefix_extractor(SliceTransform::create_fixed_prefix(32)) - } - _ => {} - }; + if let Some(size) = Description::prefix(&column) { + opts.set_prefix_extractor(SliceTransform::create_fixed_prefix(size)) + } opts } @@ -240,7 +225,7 @@ impl RocksDb { fn reverse_prefix_iter( &self, prefix: &[u8], - column: Column, + column: Description::Column, ) -> impl Iterator + '_ { let maybe_next_item = next_prefix(prefix.to_vec()) .and_then(|next_prefix| { @@ -289,7 +274,7 @@ impl RocksDb { fn _iter_all( &self, - column: Column, + column: Description::Column, opts: ReadOptions, iter_mode: IteratorMode, ) -> impl Iterator + '_ { @@ -312,10 +297,18 @@ impl RocksDb { } } -impl KeyValueStore for RocksDb { - type Column = Column; +impl KeyValueStore for RocksDb +where + Description: DatabaseDescription, +{ + type Column = Description::Column; - fn write(&self, key: &[u8], column: Column, buf: &[u8]) -> StorageResult { + fn write( + &self, + key: &[u8], + column: Self::Column, + buf: &[u8], + ) -> StorageResult { let r = buf.len(); self.db .put_cf(&self.cf(column), key, buf) @@ -327,13 +320,17 @@ impl KeyValueStore for RocksDb { Ok(r) } - fn delete(&self, key: &[u8], column: Column) -> StorageResult<()> { + fn delete(&self, key: &[u8], column: Self::Column) -> StorageResult<()> { self.db .delete_cf(&self.cf(column), key) .map_err(|e| DatabaseError::Other(e.into()).into()) } - fn size_of_value(&self, key: &[u8], column: Column) -> StorageResult> { + fn size_of_value( + &self, + key: &[u8], + column: Self::Column, + ) -> StorageResult> { database_metrics().read_meter.inc(); Ok(self @@ -343,7 +340,7 @@ impl KeyValueStore for RocksDb { .map(|value| value.len())) } - fn get(&self, key: &[u8], column: Column) -> StorageResult> { + fn get(&self, key: &[u8], column: Self::Column) -> StorageResult> { database_metrics().read_meter.inc(); let value = self @@ -361,7 +358,7 @@ impl KeyValueStore for RocksDb { fn read( &self, key: &[u8], - column: Column, + column: Self::Column, mut buf: &mut [u8], ) -> StorageResult> { database_metrics().read_meter.inc(); @@ -386,10 +383,13 @@ impl KeyValueStore for RocksDb { } } -impl IteratorableStore for RocksDb { +impl IteratorableStore for RocksDb +where + Description: DatabaseDescription, +{ fn iter_all( &self, - column: Column, + column: Self::Column, prefix: Option<&[u8]>, start: Option<&[u8]>, direction: IterDirection, @@ -454,10 +454,13 @@ impl IteratorableStore for RocksDb { } } -impl BatchOperations for RocksDb { +impl BatchOperations for RocksDb +where + Description: DatabaseDescription, +{ fn batch_write( &self, - entries: &mut dyn Iterator, Column, WriteOperation)>, + entries: &mut dyn Iterator, Self::Column, WriteOperation)>, ) -> StorageResult<()> { let mut batch = WriteBatch::default(); @@ -483,18 +486,10 @@ impl BatchOperations for RocksDb { } } -impl TransactableStorage for RocksDb { - fn checkpoint(&self) -> DatabaseResult { - let tmp_dir = ShallowTempDir::new(); - self.checkpoint(&tmp_dir.path)?; - let db = RocksDb::default_open(&tmp_dir.path, self.capacity)?; - let database = Database::new(Arc::new(db)).with_drop(Box::new(move || { - drop(tmp_dir); - })); - - Ok(database) - } - +impl TransactableStorage for RocksDb +where + Description: DatabaseDescription, +{ fn flush(&self) -> DatabaseResult<()> { self.db .flush_wal(true) @@ -520,9 +515,11 @@ fn next_prefix(mut prefix: Vec) -> Option> { #[cfg(test)] mod tests { use super::*; + use crate::database::database_description::on_chain::OnChain; + use fuel_core_storage::column::Column; use tempfile::TempDir; - fn create_db() -> (RocksDb, TempDir) { + fn create_db() -> (RocksDb, TempDir) { let tmp_dir = TempDir::new().unwrap(); ( RocksDb::default_open(tmp_dir.path(), None).unwrap(), diff --git a/crates/metrics/src/graphql_metrics.rs b/crates/metrics/src/graphql_metrics.rs index 508c18dc1c..050156a358 100644 --- a/crates/metrics/src/graphql_metrics.rs +++ b/crates/metrics/src/graphql_metrics.rs @@ -3,6 +3,7 @@ use prometheus_client::{ encoding::EncodeLabelSet, metrics::{ family::Family, + gauge::Gauge, histogram::Histogram, }, registry::Registry, @@ -17,17 +18,31 @@ pub struct Label { pub struct GraphqlMetrics { pub registry: Registry, + // using gauges in case blocks are rolled back for any reason + pub total_txs_count: Gauge, requests: Family, } impl GraphqlMetrics { fn new() -> Self { let mut registry = Registry::default(); + let tx_count_gauge = Gauge::default(); let requests = Family::::new_with_constructor(|| { Histogram::new(timing_buckets().iter().cloned()) }); registry.register("graphql_request_duration_seconds", "", requests.clone()); - Self { registry, requests } + + registry.register( + "importer_tx_count", + "the total amount of transactions that have been imported on chain", + tx_count_gauge.clone(), + ); + + Self { + registry, + total_txs_count: tx_count_gauge, + requests, + } } pub fn graphql_observe(&self, query: &str, time: f64) { diff --git a/crates/metrics/src/importer.rs b/crates/metrics/src/importer.rs index 41e75d6d22..25d8e1bd91 100644 --- a/crates/metrics/src/importer.rs +++ b/crates/metrics/src/importer.rs @@ -13,8 +13,6 @@ use std::sync::{ pub struct ImporterMetrics { pub registry: Registry, - // using gauges in case blocks are rolled back for any reason - pub total_txs_count: Gauge, pub block_height: Gauge, pub latest_block_import_timestamp: Gauge, pub execute_and_commit_duration: Histogram, @@ -24,18 +22,11 @@ impl Default for ImporterMetrics { fn default() -> Self { let mut registry = Registry::default(); - let tx_count_gauge = Gauge::default(); let block_height_gauge = Gauge::default(); let latest_block_import_ms = Gauge::default(); let execute_and_commit_duration = Histogram::new(timing_buckets().iter().cloned()); - registry.register( - "importer_tx_count", - "the total amount of transactions that have been imported on chain", - tx_count_gauge.clone(), - ); - registry.register( "importer_block_height", "the current height of the chain", @@ -56,7 +47,6 @@ impl Default for ImporterMetrics { Self { registry, - total_txs_count: tx_count_gauge, block_height: block_height_gauge, latest_block_import_timestamp: latest_block_import_ms, execute_and_commit_duration, diff --git a/crates/services/importer/src/importer.rs b/crates/services/importer/src/importer.rs index 6d442210a7..885f59ceb7 100644 --- a/crates/services/importer/src/importer.rs +++ b/crates/services/importer/src/importer.rs @@ -279,15 +279,9 @@ where return Err(Error::NotUnique(expected_next_height)) } - // Update the total tx count in chain metadata - let total_txs = db_after_execution - // Safety: casting len to u64 since it's impossible to execute a block with more than 2^64 txs - .increase_tx_count(result.sealed_block.entity.transactions().len() as u64)?; - db_tx.commit()?; // update the importer metrics after the block is successfully committed - importer_metrics().total_txs_count.set(total_txs as i64); importer_metrics() .block_height .set(*actual_next_height.deref() as i64); @@ -322,11 +316,6 @@ where .latest_block_height() .unwrap_or_default() .unwrap_or_default(); - let total_tx_count = self.database.increase_tx_count(0).unwrap_or_default(); - - importer_metrics() - .total_txs_count - .set(total_tx_count as i64); importer_metrics() .block_height .set(*current_block_height.deref() as i64); diff --git a/crates/services/importer/src/importer/test.rs b/crates/services/importer/src/importer/test.rs index 595d80159b..889fe07a22 100644 --- a/crates/services/importer/src/importer/test.rs +++ b/crates/services/importer/src/importer/test.rs @@ -52,7 +52,6 @@ mockall::mock! { impl ImporterDatabase for Database { fn latest_block_height(&self) -> StorageResult>; - fn increase_tx_count(&self, new_txs_count: u64) -> StorageResult; } impl ExecutorDatabase for Database { @@ -117,7 +116,6 @@ where let mut db = MockDatabase::default(); db.expect_latest_block_height() .returning(move || result().map(|v| v.map(Into::into))); - db.expect_increase_tx_count().returning(Ok); db } } @@ -140,7 +138,6 @@ where db.expect_store_new_block() .returning(move |_, _| store_block()); db.expect_commit().times(commits).returning(|| Ok(())); - db.expect_increase_tx_count().returning(Ok); db } } diff --git a/crates/services/importer/src/ports.rs b/crates/services/importer/src/ports.rs index 99f097fefe..a8947ab8c3 100644 --- a/crates/services/importer/src/ports.rs +++ b/crates/services/importer/src/ports.rs @@ -36,9 +36,6 @@ pub trait Executor: Send + Sync { pub trait ImporterDatabase: Send + Sync { /// Returns the latest block height. fn latest_block_height(&self) -> StorageResult>; - /// Update metadata about the total number of transactions on the chain. - /// Returns the total count after the update. - fn increase_tx_count(&self, new_txs_count: u64) -> StorageResult; } /// The port for returned database from the executor. diff --git a/crates/services/relayer/Cargo.toml b/crates/services/relayer/Cargo.toml index 0d9ea134ab..086ae8ecc9 100644 --- a/crates/services/relayer/Cargo.toml +++ b/crates/services/relayer/Cargo.toml @@ -13,6 +13,7 @@ description = "Fuel Relayer" anyhow = { workspace = true } async-trait = { workspace = true } bytes = { version = "1.1", optional = true } +enum-iterator = { workspace = true } ethers-contract = { version = "2", default-features = false, features = [ "abigen", ] } @@ -29,6 +30,8 @@ once_cell = { workspace = true } parking_lot = { workspace = true, optional = true } serde = { workspace = true, optional = true } serde_json = { workspace = true, optional = true } +strum = { workspace = true } +strum_macros = { workspace = true } thiserror = { workspace = true, optional = true } tokio = { workspace = true, features = ["macros"] } tracing = { workspace = true } diff --git a/crates/services/relayer/src/ports/tests.rs b/crates/services/relayer/src/ports/tests.rs index 5e30ceacae..50c9f11af5 100644 --- a/crates/services/relayer/src/ports/tests.rs +++ b/crates/services/relayer/src/ports/tests.rs @@ -1,8 +1,8 @@ use crate::{ ports::RelayerDb, storage::{ + DaHeightTable, EventsHistory, - RelayerMetadata, }, }; use fuel_core_storage::test_helpers::MockStorage; @@ -17,12 +17,12 @@ fn test_insert_events() { db.expect_insert::() .times(1) .returning(|_, _| Ok(None)); - db.expect_insert::() + db.expect_insert::() .times(1) .withf(move |_, v| **v == same_height) .returning(|_, _| Ok(None)); db.expect_commit().returning(|| Ok(())); - db.expect_get::() + db.expect_get::() .once() .returning(|_| Ok(Some(std::borrow::Cow::Owned(9u64.into())))); let mut db = db.into_transactional(); @@ -55,12 +55,12 @@ fn insert_always_raises_da_height_monotonically() { let mut db = MockStorage::default(); db.expect_insert::() .returning(|_, _| Ok(None)); - db.expect_insert::() + db.expect_insert::() .once() .withf(move |_, v| *v == same_height) .returning(|_, _| Ok(None)); db.expect_commit().returning(|| Ok(())); - db.expect_get::() + db.expect_get::() .once() .returning(|_| Ok(None)); @@ -138,13 +138,13 @@ fn set_raises_da_height_monotonically( ) { let mut db = MockStorage::default(); if let Some(h) = inserts.into() { - db.expect_insert::() + db.expect_insert::() .once() .withf(move |_, v| **v == h) .returning(|_, _| Ok(None)); } let get = get.into().map(|g| Cow::Owned(g.into())); - db.expect_get::() + db.expect_get::() .once() .returning(move |_| Ok(get.clone())); db.expect_commit().returning(|| Ok(())); diff --git a/crates/services/relayer/src/storage.rs b/crates/services/relayer/src/storage.rs index c5aede5671..bbe50e530d 100644 --- a/crates/services/relayer/src/storage.rs +++ b/crates/services/relayer/src/storage.rs @@ -7,7 +7,7 @@ use fuel_core_storage::{ postcard::Postcard, primitive::Primitive, }, - column::Column, + kv_store::StorageColumn, structured_storage::TableWithBlueprint, transactional::Transactional, Error as StorageError, @@ -22,9 +22,51 @@ use fuel_core_types::{ services::relayer::Event, }; -/// Metadata for relayer. -pub struct RelayerMetadata; -impl Mappable for RelayerMetadata { +/// GraphQL database tables column ids to the corresponding [`fuel_core_storage::Mappable`] table. +#[repr(u32)] +#[derive( + Copy, + Clone, + Debug, + strum_macros::EnumCount, + strum_macros::IntoStaticStr, + PartialEq, + Eq, + enum_iterator::Sequence, + Hash, +)] +pub enum Column { + /// The column id of metadata about the relayer storage. + Metadata = 0, + /// The column of the table that stores history of the relayer. + History = 1, + /// The column that tracks the da height of the relayer. + RelayerHeight = 2, +} + +impl Column { + /// The total count of variants in the enum. + pub const COUNT: usize = ::COUNT; + + /// Returns the `usize` representation of the `Column`. + pub fn as_u32(&self) -> u32 { + *self as u32 + } +} + +impl StorageColumn for Column { + fn name(&self) -> &'static str { + self.into() + } + + fn id(&self) -> u32 { + self.as_u32() + } +} + +/// Teh table to track the relayer's da height. +pub struct DaHeightTable; +impl Mappable for DaHeightTable { type Key = Self::OwnedKey; type OwnedKey = (); type Value = Self::OwnedValue; @@ -36,11 +78,12 @@ impl Mappable for RelayerMetadata { /// changed from a unit value. const METADATA_KEY: () = (); -impl TableWithBlueprint for RelayerMetadata { +impl TableWithBlueprint for DaHeightTable { type Blueprint = Plain>; + type Column = Column; fn column() -> Column { - Column::RelayerMetadata + Column::RelayerHeight } } @@ -58,9 +101,10 @@ impl Mappable for EventsHistory { impl TableWithBlueprint for EventsHistory { type Blueprint = Plain, Postcard>; + type Column = Column; fn column() -> Column { - Column::RelayerHistory + Column::History } } @@ -68,9 +112,9 @@ impl RelayerDb for T where T: Send + Sync, T: Transactional, - T: StorageMutate, + T: StorageMutate, Storage: StorageMutate - + StorageMutate, + + StorageMutate, { fn insert_events( &mut self, @@ -116,7 +160,7 @@ where } fn get_finalized_da_height(&self) -> StorageResult { - Ok(*StorageAsRef::storage::(&self) + Ok(*StorageAsRef::storage::(&self) .get(&METADATA_KEY)? .unwrap_or_default()) } @@ -127,22 +171,20 @@ fn grow_monotonically( height: &DaBlockHeight, ) -> StorageResult<()> where - Storage: StorageMutate, + Storage: StorageMutate, { let current = (&s) - .storage::() + .storage::() .get(&METADATA_KEY)? .map(|cow| cow.as_u64()); match current { Some(current) => { if **height > current { - s.storage::() - .insert(&METADATA_KEY, height)?; + s.storage::().insert(&METADATA_KEY, height)?; } } None => { - s.storage::() - .insert(&METADATA_KEY, height)?; + s.storage::().insert(&METADATA_KEY, height)?; } } Ok(()) @@ -153,9 +195,9 @@ mod tests { use super::*; fuel_core_storage::basic_storage_tests!( - RelayerMetadata, - ::Key::default(), - ::Value::default() + DaHeightTable, + ::Key::default(), + ::Value::default() ); fuel_core_storage::basic_storage_tests!( diff --git a/crates/storage/src/blueprint/plain.rs b/crates/storage/src/blueprint/plain.rs index 7a9e696e81..22d02a771e 100644 --- a/crates/storage/src/blueprint/plain.rs +++ b/crates/storage/src/blueprint/plain.rs @@ -13,10 +13,10 @@ use crate::{ Encode, Encoder, }, - column::Column, kv_store::{ BatchOperations, KeyValueStore, + StorageColumn, WriteOperation, }, structured_storage::TableWithBlueprint, @@ -92,10 +92,13 @@ where } } -impl SupportsBatching for Plain +impl SupportsBatching + for Plain where + Column: StorageColumn, S: BatchOperations, - M: Mappable + TableWithBlueprint>, + M: Mappable + + TableWithBlueprint, Column = Column>, M::Blueprint: Blueprint, { fn init<'a, Iter>(storage: &mut S, column: S::Column, set: Iter) -> StorageResult<()> diff --git a/crates/storage/src/blueprint/sparse.rs b/crates/storage/src/blueprint/sparse.rs index ed0db6555a..9e0deb6310 100644 --- a/crates/storage/src/blueprint/sparse.rs +++ b/crates/storage/src/blueprint/sparse.rs @@ -13,7 +13,6 @@ use crate::{ Encode, Encoder, }, - column::Column, kv_store::{ BatchOperations, KeyValueStore, @@ -241,13 +240,14 @@ where } } -impl +impl MerkleRootStorage for StructuredStorage where S: KeyValueStore, M: Mappable + TableWithBlueprint< Blueprint = Sparse, + Column = Column, >, Self: StorageMutate + StorageInspect, @@ -270,13 +270,15 @@ type NodeKeyCodec = type NodeValueCodec = <::Blueprint as Blueprint>::ValueCodec; -impl SupportsBatching - for Sparse +impl + SupportsBatching for Sparse where + Column: StorageColumn, S: BatchOperations, M: Mappable + TableWithBlueprint< Blueprint = Sparse, + Column = Column, >, KeyCodec: Encode + Decode, ValueCodec: Encode + Decode, @@ -285,7 +287,7 @@ where Key = MerkleRoot, Value = sparse::Primitive, OwnedValue = sparse::Primitive, - > + TableWithBlueprint, + > + TableWithBlueprint, KeyConverter: PrimaryKey, Nodes::Blueprint: Blueprint, for<'a> StructuredStorage<&'a mut S>: StorageMutate diff --git a/crates/storage/src/column.rs b/crates/storage/src/column.rs index aaac725657..d277ea2a0b 100644 --- a/crates/storage/src/column.rs +++ b/crates/storage/src/column.rs @@ -4,118 +4,74 @@ use crate::kv_store::StorageColumn; -/// Helper macro to generate the `Column` enum and its implementation for `as_u32` method. -macro_rules! column_definition { - ($(#[$meta:meta])* $vis:vis enum $name:ident { - $(#[$complex_meta:meta])* $complex_variants:ident($body:ident), - $($(#[$const_meta:meta])* $const_variants:ident = $const_number:expr,)* - }) => { - $(#[$meta])* - $vis enum $name { - $($(#[$const_meta])* $const_variants = $const_number,)* - $(#[$complex_meta])* $complex_variants($body), - } - - impl $name { - /// Returns the `u32` representation of the `Self`. - pub fn as_u32(&self) -> u32 { - match self { - $($name::$const_variants => $const_number,)* - $name::$complex_variants(foreign) => foreign.id, - } - } - } - } -} - -column_definition! { - /// Database tables column ids to the corresponding [`crate::Mappable`] table. - #[repr(u32)] - #[derive( - Copy, - Clone, - Debug, - strum_macros::EnumCount, - strum_macros::IntoStaticStr, - PartialEq, - Eq, - enum_iterator::Sequence, - Hash, - )] - pub enum Column { - /// The foreign column is not related to the required tables. - ForeignColumn(ForeignColumn), - - // Tables that are required for the state transition and fraud proving. - - /// See [`ContractsRawCode`](crate::tables::ContractsRawCode) - ContractsRawCode = 0, - /// See [`ContractsInfo`](crate::tables::ContractsInfo) - ContractsInfo = 1, - /// See [`ContractsState`](crate::tables::ContractsState) - ContractsState = 2, - /// See [`ContractsLatestUtxo`](crate::tables::ContractsLatestUtxo) - ContractsLatestUtxo = 3, - /// See [`ContractsAssets`](crate::tables::ContractsAssets) - ContractsAssets = 4, - /// See [`Coins`](crate::tables::Coins) - Coins = 5, - /// See [`Transactions`](crate::tables::Transactions) - Transactions = 6, - /// See [`FuelBlocks`](crate::tables::FuelBlocks) - FuelBlocks = 7, - /// See [`FuelBlockMerkleData`](crate::tables::merkle::FuelBlockMerkleData) - FuelBlockMerkleData = 8, - /// See [`FuelBlockMerkleMetadata`](crate::tables::merkle::FuelBlockMerkleMetadata) - FuelBlockMerkleMetadata = 9, - /// Messages that have been spent. - /// Existence of a key in this column means that the message has been spent. - /// See [`SpentMessages`](crate::tables::SpentMessages) - SpentMessages = 10, - /// See [`ContractsAssetsMerkleData`](crate::tables::merkle::ContractsAssetsMerkleData) - ContractsAssetsMerkleData = 11, - /// See [`ContractsAssetsMerkleMetadata`](crate::tables::merkle::ContractsAssetsMerkleMetadata) - ContractsAssetsMerkleMetadata = 12, - /// See [`ContractsStateMerkleData`](crate::tables::merkle::ContractsStateMerkleData) - ContractsStateMerkleData = 13, - /// See [`ContractsStateMerkleMetadata`](crate::tables::merkle::ContractsStateMerkleMetadata) - ContractsStateMerkleMetadata = 14, - /// See [`Messages`](crate::tables::Messages) - Messages = 15, - /// See [`ProcessedTransactions`](crate::tables::ProcessedTransactions) - ProcessedTransactions = 16, - - // TODO: Extract the columns below into a separate enum to not mix - // required columns and non-required columns. It will break `MemoryStore` - // and `MemoryTransactionView` because they rely on linear index incrementation. - - // Below are the tables used for p2p, block production, starting the node. - - /// The column id of metadata about the blockchain - Metadata = 17, - /// See [`Receipts`](crate::tables::Receipts) - Receipts = 18, - /// See `FuelBlockSecondaryKeyBlockHeights` - FuelBlockSecondaryKeyBlockHeights = 19, - /// See [`SealedBlockConsensus`](crate::tables::SealedBlockConsensus) - FuelBlockConsensus = 20, - /// Metadata for the relayer - /// See `RelayerMetadata` - RelayerMetadata = 21, - /// The history for the relayer - RelayerHistory = 22, - - // Below are not required tables. They are used for API and may be removed or moved to another place in the future. - - /// The column of the table that stores `true` if `owner` owns `Coin` with `coin_id` - OwnedCoins = 23, - /// Transaction id to current status - TransactionStatus = 24, - /// The column of the table of all `owner`'s transactions - TransactionsByOwnerBlockIdx = 25, - /// The column of the table that stores `true` if `owner` owns `Message` with `message_id` - OwnedMessageIds = 26, - } +/// Database tables column ids to the corresponding [`crate::Mappable`] table. +#[repr(u32)] +#[derive( + Copy, + Clone, + Debug, + strum_macros::EnumCount, + strum_macros::IntoStaticStr, + PartialEq, + Eq, + enum_iterator::Sequence, + Hash, +)] +pub enum Column { + /// See [`ContractsRawCode`](crate::tables::ContractsRawCode) + ContractsRawCode = 0, + /// See [`ContractsInfo`](crate::tables::ContractsInfo) + ContractsInfo = 1, + /// See [`ContractsState`](crate::tables::ContractsState) + ContractsState = 2, + /// See [`ContractsLatestUtxo`](crate::tables::ContractsLatestUtxo) + ContractsLatestUtxo = 3, + /// See [`ContractsAssets`](crate::tables::ContractsAssets) + ContractsAssets = 4, + /// See [`Coins`](crate::tables::Coins) + Coins = 5, + /// See [`Transactions`](crate::tables::Transactions) + Transactions = 6, + /// See [`FuelBlocks`](crate::tables::FuelBlocks) + FuelBlocks = 7, + /// See [`FuelBlockMerkleData`](crate::tables::merkle::FuelBlockMerkleData) + FuelBlockMerkleData = 8, + /// See [`FuelBlockMerkleMetadata`](crate::tables::merkle::FuelBlockMerkleMetadata) + FuelBlockMerkleMetadata = 9, + /// Messages that have been spent. + /// Existence of a key in this column means that the message has been spent. + /// See [`SpentMessages`](crate::tables::SpentMessages) + SpentMessages = 10, + /// See [`ContractsAssetsMerkleData`](crate::tables::merkle::ContractsAssetsMerkleData) + ContractsAssetsMerkleData = 11, + /// See [`ContractsAssetsMerkleMetadata`](crate::tables::merkle::ContractsAssetsMerkleMetadata) + ContractsAssetsMerkleMetadata = 12, + /// See [`ContractsStateMerkleData`](crate::tables::merkle::ContractsStateMerkleData) + ContractsStateMerkleData = 13, + /// See [`ContractsStateMerkleMetadata`](crate::tables::merkle::ContractsStateMerkleMetadata) + ContractsStateMerkleMetadata = 14, + /// See [`Messages`](crate::tables::Messages) + Messages = 15, + /// See [`ProcessedTransactions`](crate::tables::ProcessedTransactions) + ProcessedTransactions = 16, + + // TODO: Extract the columns below into a separate enum to not mix + // required columns and non-required columns. It will break `MemoryStore` + // and `MemoryTransactionView` because they rely on linear index incrementation. + + // Below are the tables used for p2p, block production, starting the node. + /// The column id of metadata about the blockchain + Metadata = 17, + /// See `FuelBlockSecondaryKeyBlockHeights` + FuelBlockSecondaryKeyBlockHeights = 18, + /// See [`SealedBlockConsensus`](crate::tables::SealedBlockConsensus) + FuelBlockConsensus = 19, + + // Below are not required tables. They are used for API and may be removed or moved to another place in the future. + /// The column of the table that stores `true` if `owner` owns `Coin` with `coin_id` + OwnedCoins = 20, + /// The column of the table that stores `true` if `owner` owns `Message` with `message_id` + OwnedMessageIds = 21, } impl Column { @@ -123,71 +79,17 @@ impl Column { pub const COUNT: usize = ::COUNT; /// Returns the `usize` representation of the `Column`. - pub fn as_usize(&self) -> usize { - self.as_u32() as usize + pub fn as_u32(&self) -> u32 { + *self as u32 } } impl StorageColumn for Column { fn name(&self) -> &'static str { - match self { - Column::ForeignColumn(foreign) => foreign.name, - variant => variant.into(), - } + self.into() } fn id(&self) -> u32 { self.as_u32() } } - -/// The foreign column is not related to the required tables. -/// It can be used to extend the database with additional tables. -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct ForeignColumn { - id: u32, - name: &'static str, -} - -impl ForeignColumn { - /// Creates the foreign column ensuring that the id and name - /// are not already used by the [`Column`] required tables. - pub fn new(id: u32, name: &'static str) -> anyhow::Result { - for column in enum_iterator::all::() { - if column.id() == id { - anyhow::bail!("Column id {} is already used by {}", id, column.name()); - } - if column.name() == name { - anyhow::bail!( - "Column name {} is already used by {}", - name, - column.name() - ); - } - } - Ok(Self { id, name }) - } -} - -/// It is required to implement iteration over the variants of the enum. -/// The `ForeignColumn` is not iterable, so we implement the `Sequence` trait -/// to do nothing. -impl enum_iterator::Sequence for ForeignColumn { - const CARDINALITY: usize = 0; - - fn next(&self) -> Option { - None - } - - fn previous(&self) -> Option { - None - } - - fn first() -> Option { - None - } - - fn last() -> Option { - None - } -} diff --git a/crates/storage/src/kv_store.rs b/crates/storage/src/kv_store.rs index 5d6154684d..19166ec269 100644 --- a/crates/storage/src/kv_store.rs +++ b/crates/storage/src/kv_store.rs @@ -12,12 +12,17 @@ pub type Value = Arc>; pub type KVItem = StorageResult<(Vec, Value)>; /// A column of the storage. -pub trait StorageColumn: Clone { +pub trait StorageColumn: Copy + core::fmt::Debug { /// Returns the name of the column. fn name(&self) -> &'static str; /// Returns the id of the column. fn id(&self) -> u32; + + /// Returns the id of the column as an `usize`. + fn as_usize(&self) -> usize { + self.id() as usize + } } // TODO: Use `&mut self` for all mutable methods. @@ -41,7 +46,7 @@ pub trait KeyValueStore { value: Value, ) -> StorageResult> { // FIXME: This is a race condition. We should use a transaction. - let old_value = self.get(key, column.clone())?; + let old_value = self.get(key, column)?; self.put(key, column, value)?; Ok(old_value) } @@ -53,7 +58,7 @@ pub trait KeyValueStore { /// Removes the value from the storage and returns it. fn take(&self, key: &[u8], column: Self::Column) -> StorageResult> { // FIXME: This is a race condition. We should use a transaction. - let old_value = self.get(key, column.clone())?; + let old_value = self.get(key, column)?; self.delete(key, column)?; Ok(old_value) } @@ -72,7 +77,7 @@ pub trait KeyValueStore { key: &[u8], column: Self::Column, ) -> StorageResult> { - Ok(self.get(key, column.clone())?.map(|value| value.len())) + Ok(self.get(key, column)?.map(|value| value.len())) } /// Returns the value from the storage. @@ -85,7 +90,7 @@ pub trait KeyValueStore { column: Self::Column, buf: &mut [u8], ) -> StorageResult> { - self.get(key, column.clone())? + self.get(key, column)? .map(|value| { let read = value.len(); if read != buf.len() { diff --git a/crates/storage/src/structured_storage.rs b/crates/storage/src/structured_storage.rs index 04076644ce..4ca74ac6b0 100644 --- a/crates/storage/src/structured_storage.rs +++ b/crates/storage/src/structured_storage.rs @@ -6,10 +6,10 @@ use crate::{ Blueprint, SupportsBatching, }, - column::Column, kv_store::{ BatchOperations, KeyValueStore, + StorageColumn, }, Error as StorageError, Mappable, @@ -26,7 +26,6 @@ pub mod coins; pub mod contracts; pub mod merkle_data; pub mod messages; -pub mod receipts; pub mod sealed_block; pub mod state; pub mod transactions; @@ -37,9 +36,11 @@ pub mod transactions; pub trait TableWithBlueprint: Mappable + Sized { /// The type of the blueprint used by the table. type Blueprint; + /// The column type used by the table. + type Column: StorageColumn; /// The column occupied by the table. - fn column() -> Column; + fn column() -> Self::Column; } /// The wrapper around the key-value storage that implements the storage traits for the tables @@ -68,10 +69,10 @@ impl AsMut for StructuredStorage { } } -impl StorageInspect for StructuredStorage +impl StorageInspect for StructuredStorage where S: KeyValueStore, - M: Mappable + TableWithBlueprint, + M: Mappable + TableWithBlueprint, M::Blueprint: Blueprint, { type Error = StorageError; @@ -86,10 +87,10 @@ where } } -impl StorageMutate for StructuredStorage +impl StorageMutate for StructuredStorage where S: KeyValueStore, - M: Mappable + TableWithBlueprint, + M: Mappable + TableWithBlueprint, M::Blueprint: Blueprint, { fn insert( @@ -110,10 +111,10 @@ where } } -impl StorageSize for StructuredStorage +impl StorageSize for StructuredStorage where S: KeyValueStore, - M: Mappable + TableWithBlueprint, + M: Mappable + TableWithBlueprint, M::Blueprint: Blueprint, { fn size_of_value(&self, key: &M::Key) -> Result, Self::Error> { @@ -125,10 +126,10 @@ where } } -impl StorageBatchMutate for StructuredStorage +impl StorageBatchMutate for StructuredStorage where S: BatchOperations, - M: Mappable + TableWithBlueprint, + M: Mappable + TableWithBlueprint, M::Blueprint: SupportsBatching, { fn init_storage<'a, Iter>(&mut self, set: Iter) -> Result<(), Self::Error> @@ -162,8 +163,8 @@ where #[cfg(feature = "test-helpers")] pub mod test { use crate as fuel_core_storage; + use crate::kv_store::StorageColumn; use fuel_core_storage::{ - column::Column, kv_store::{ BatchOperations, KeyValueStore, @@ -176,15 +177,28 @@ pub mod test { collections::HashMap, }; - type Storage = RefCell), Vec>>; + type Storage = RefCell), Vec>>; /// The in-memory storage for testing purposes. - #[derive(Default, Debug, PartialEq, Eq)] - pub struct InMemoryStorage { + #[derive(Debug, PartialEq, Eq)] + pub struct InMemoryStorage { storage: Storage, + _marker: core::marker::PhantomData, } - impl KeyValueStore for InMemoryStorage { + impl Default for InMemoryStorage { + fn default() -> Self { + Self { + storage: Storage::default(), + _marker: Default::default(), + } + } + } + + impl KeyValueStore for InMemoryStorage + where + Column: StorageColumn, + { type Column = Column; fn write( @@ -196,12 +210,14 @@ pub mod test { let write = buf.len(); self.storage .borrow_mut() - .insert((column, key.to_vec()), buf.to_vec()); + .insert((column.id(), key.to_vec()), buf.to_vec()); Ok(write) } fn delete(&self, key: &[u8], column: Self::Column) -> StorageResult<()> { - self.storage.borrow_mut().remove(&(column, key.to_vec())); + self.storage + .borrow_mut() + .remove(&(column.id(), key.to_vec())); Ok(()) } @@ -209,12 +225,12 @@ pub mod test { Ok(self .storage .borrow_mut() - .get(&(column, key.to_vec())) + .get(&(column.id(), key.to_vec())) .map(|v| v.clone().into())) } } - impl BatchOperations for InMemoryStorage {} + impl BatchOperations for InMemoryStorage where Column: StorageColumn {} /// The macro that generates basic storage tests for the table with [`InMemoryStorage`]. #[macro_export] @@ -229,6 +245,7 @@ pub mod test { structured_storage::{ test::InMemoryStorage, StructuredStorage, + TableWithBlueprint, }, StorageAsMut, }; @@ -248,7 +265,7 @@ pub mod test { #[test] fn get() { - let mut storage = InMemoryStorage::default(); + let mut storage = InMemoryStorage::<<$table as TableWithBlueprint>::Column>::default(); let mut structured_storage = StructuredStorage::new(&mut storage); let key = $key; @@ -270,7 +287,7 @@ pub mod test { #[test] fn insert() { - let mut storage = InMemoryStorage::default(); + let mut storage = InMemoryStorage::<<$table as TableWithBlueprint>::Column>::default(); let mut structured_storage = StructuredStorage::new(&mut storage); let key = $key; @@ -290,7 +307,7 @@ pub mod test { #[test] fn remove() { - let mut storage = InMemoryStorage::default(); + let mut storage = InMemoryStorage::<<$table as TableWithBlueprint>::Column>::default(); let mut structured_storage = StructuredStorage::new(&mut storage); let key = $key; @@ -309,7 +326,7 @@ pub mod test { #[test] fn exists() { - let mut storage = InMemoryStorage::default(); + let mut storage = InMemoryStorage::<<$table as TableWithBlueprint>::Column>::default(); let mut structured_storage = StructuredStorage::new(&mut storage); let key = $key; @@ -334,7 +351,7 @@ pub mod test { #[test] fn exists_false_after_removing() { - let mut storage = InMemoryStorage::default(); + let mut storage = InMemoryStorage::<<$table as TableWithBlueprint>::Column>::default(); let mut structured_storage = StructuredStorage::new(&mut storage); let key = $key; @@ -366,9 +383,9 @@ pub mod test { SeedableRng, }; - let empty_storage = InMemoryStorage::default(); + let empty_storage = InMemoryStorage::<<$table as TableWithBlueprint>::Column>::default(); - let mut init_storage = InMemoryStorage::default(); + let mut init_storage = InMemoryStorage::<<$table as TableWithBlueprint>::Column>::default(); let mut init_structured_storage = StructuredStorage::new(&mut init_storage); let mut rng = &mut StdRng::seed_from_u64(1234); @@ -384,7 +401,7 @@ pub mod test { }) ).expect("Should initialize the storage successfully"); - let mut insert_storage = InMemoryStorage::default(); + let mut insert_storage = InMemoryStorage::<<$table as TableWithBlueprint>::Column>::default(); let mut insert_structured_storage = StructuredStorage::new(&mut insert_storage); <_ as $crate::StorageBatchMutate<$table>>::insert_batch( @@ -447,7 +464,7 @@ pub mod test { #[test] fn root() { - let mut storage = InMemoryStorage::default(); + let mut storage = InMemoryStorage::<<$table as TableWithBlueprint>::Column>::default(); let mut structured_storage = StructuredStorage::new(&mut storage); let rng = &mut StdRng::seed_from_u64(1234); @@ -462,7 +479,7 @@ pub mod test { #[test] fn root_returns_empty_root_for_empty_metadata() { - let mut storage = InMemoryStorage::default(); + let mut storage = InMemoryStorage::<<$table as TableWithBlueprint>::Column>::default(); let mut structured_storage = StructuredStorage::new(&mut storage); let empty_root = fuel_core_types::fuel_merkle::sparse::in_memory::MerkleTree::new().root(); @@ -475,7 +492,7 @@ pub mod test { #[test] fn put_updates_the_state_merkle_root_for_the_given_metadata() { - let mut storage = InMemoryStorage::default(); + let mut storage = InMemoryStorage::<<$table as TableWithBlueprint>::Column>::default(); let mut structured_storage = StructuredStorage::new(&mut storage); let rng = &mut StdRng::seed_from_u64(1234); @@ -513,7 +530,7 @@ pub mod test { #[test] fn remove_updates_the_state_merkle_root_for_the_given_metadata() { - let mut storage = InMemoryStorage::default(); + let mut storage = InMemoryStorage::<<$table as TableWithBlueprint>::Column>::default(); let mut structured_storage = StructuredStorage::new(&mut storage); let rng = &mut StdRng::seed_from_u64(1234); @@ -562,7 +579,7 @@ pub mod test { let given_primary_key = $current_key; let foreign_primary_key = $foreign_key; - let mut storage = InMemoryStorage::default(); + let mut storage = InMemoryStorage::<<$table as TableWithBlueprint>::Column>::default(); let mut structured_storage = StructuredStorage::new(&mut storage); let rng = &mut StdRng::seed_from_u64(1234); @@ -598,7 +615,7 @@ pub mod test { #[test] fn put_creates_merkle_metadata_when_empty() { - let mut storage = InMemoryStorage::default(); + let mut storage = InMemoryStorage::<<$table as TableWithBlueprint>::Column>::default(); let mut structured_storage = StructuredStorage::new(&mut storage); let rng = &mut StdRng::seed_from_u64(1234); @@ -624,7 +641,7 @@ pub mod test { #[test] fn remove_deletes_merkle_metadata_when_empty() { - let mut storage = InMemoryStorage::default(); + let mut storage = InMemoryStorage::<<$table as TableWithBlueprint>::Column>::default(); let mut structured_storage = StructuredStorage::new(&mut storage); let rng = &mut StdRng::seed_from_u64(1234); diff --git a/crates/storage/src/structured_storage/balances.rs b/crates/storage/src/structured_storage/balances.rs index 2bd9019e9c..4d26b0e19f 100644 --- a/crates/storage/src/structured_storage/balances.rs +++ b/crates/storage/src/structured_storage/balances.rs @@ -43,6 +43,7 @@ impl TableWithBlueprint for ContractsAssets { ContractsAssetsMerkleData, KeyConverter, >; + type Column = Column; fn column() -> Column { Column::ContractsAssets diff --git a/crates/storage/src/structured_storage/blocks.rs b/crates/storage/src/structured_storage/blocks.rs index 22f033c688..d09259255b 100644 --- a/crates/storage/src/structured_storage/blocks.rs +++ b/crates/storage/src/structured_storage/blocks.rs @@ -13,6 +13,7 @@ use crate::{ impl TableWithBlueprint for FuelBlocks { type Blueprint = Plain, Postcard>; + type Column = Column; fn column() -> Column { Column::FuelBlocks diff --git a/crates/storage/src/structured_storage/coins.rs b/crates/storage/src/structured_storage/coins.rs index 53d45f6ca6..759f2c774a 100644 --- a/crates/storage/src/structured_storage/coins.rs +++ b/crates/storage/src/structured_storage/coins.rs @@ -13,6 +13,7 @@ use crate::{ impl TableWithBlueprint for Coins { type Blueprint = Plain, Postcard>; + type Column = Column; fn column() -> Column { Column::Coins diff --git a/crates/storage/src/structured_storage/contracts.rs b/crates/storage/src/structured_storage/contracts.rs index 5e935a2f07..58bb9c2b88 100644 --- a/crates/storage/src/structured_storage/contracts.rs +++ b/crates/storage/src/structured_storage/contracts.rs @@ -28,6 +28,7 @@ use fuel_core_types::fuel_tx::ContractId; // because we don't need to store the size of the contract. We store/load raw bytes. impl TableWithBlueprint for ContractsRawCode { type Blueprint = Plain; + type Column = Column; fn column() -> Column { Column::ContractsRawCode @@ -56,6 +57,7 @@ where impl TableWithBlueprint for ContractsInfo { type Blueprint = Plain; + type Column = Column; fn column() -> Column { Column::ContractsInfo @@ -64,6 +66,7 @@ impl TableWithBlueprint for ContractsInfo { impl TableWithBlueprint for ContractsLatestUtxo { type Blueprint = Plain; + type Column = Column; fn column() -> Column { Column::ContractsLatestUtxo diff --git a/crates/storage/src/structured_storage/merkle_data.rs b/crates/storage/src/structured_storage/merkle_data.rs index b597be35f8..23bb0865be 100644 --- a/crates/storage/src/structured_storage/merkle_data.rs +++ b/crates/storage/src/structured_storage/merkle_data.rs @@ -26,6 +26,7 @@ macro_rules! merkle_table { ($table:ident, $key_codec:ident) => { impl TableWithBlueprint for $table { type Blueprint = Plain<$key_codec, Postcard>; + type Column = Column; fn column() -> Column { Column::$table diff --git a/crates/storage/src/structured_storage/messages.rs b/crates/storage/src/structured_storage/messages.rs index 08addab8ea..78d92c66a1 100644 --- a/crates/storage/src/structured_storage/messages.rs +++ b/crates/storage/src/structured_storage/messages.rs @@ -16,6 +16,7 @@ use crate::{ impl TableWithBlueprint for Messages { type Blueprint = Plain; + type Column = Column; fn column() -> Column { Column::Messages @@ -24,6 +25,7 @@ impl TableWithBlueprint for Messages { impl TableWithBlueprint for SpentMessages { type Blueprint = Plain; + type Column = Column; fn column() -> Column { Column::SpentMessages diff --git a/crates/storage/src/structured_storage/receipts.rs b/crates/storage/src/structured_storage/receipts.rs deleted file mode 100644 index 5e40cd2e4d..0000000000 --- a/crates/storage/src/structured_storage/receipts.rs +++ /dev/null @@ -1,32 +0,0 @@ -//! The module contains implementations and tests for the `Receipts` table. - -use crate::{ - blueprint::plain::Plain, - codec::{ - postcard::Postcard, - raw::Raw, - }, - column::Column, - structured_storage::TableWithBlueprint, - tables::Receipts, -}; - -impl TableWithBlueprint for Receipts { - type Blueprint = Plain; - - fn column() -> Column { - Column::Receipts - } -} - -#[cfg(test)] -crate::basic_storage_tests!( - Receipts, - ::Key::from([1u8; 32]), - vec![fuel_core_types::fuel_tx::Receipt::ret( - Default::default(), - Default::default(), - Default::default(), - Default::default() - )] -); diff --git a/crates/storage/src/structured_storage/sealed_block.rs b/crates/storage/src/structured_storage/sealed_block.rs index 4d4b9c56d1..d170b85b50 100644 --- a/crates/storage/src/structured_storage/sealed_block.rs +++ b/crates/storage/src/structured_storage/sealed_block.rs @@ -13,6 +13,7 @@ use crate::{ impl TableWithBlueprint for SealedBlockConsensus { type Blueprint = Plain, Postcard>; + type Column = Column; fn column() -> Column { Column::FuelBlockConsensus diff --git a/crates/storage/src/structured_storage/state.rs b/crates/storage/src/structured_storage/state.rs index c28b8c2a30..31c5672483 100644 --- a/crates/storage/src/structured_storage/state.rs +++ b/crates/storage/src/structured_storage/state.rs @@ -43,6 +43,7 @@ impl TableWithBlueprint for ContractsState { ContractsStateMerkleData, KeyConverter, >; + type Column = Column; fn column() -> Column { Column::ContractsState diff --git a/crates/storage/src/structured_storage/transactions.rs b/crates/storage/src/structured_storage/transactions.rs index 5605ecdbe1..d68dfd42c5 100644 --- a/crates/storage/src/structured_storage/transactions.rs +++ b/crates/storage/src/structured_storage/transactions.rs @@ -16,6 +16,7 @@ use crate::{ impl TableWithBlueprint for Transactions { type Blueprint = Plain; + type Column = Column; fn column() -> Column { Column::Transactions @@ -31,6 +32,7 @@ crate::basic_storage_tests!( impl TableWithBlueprint for ProcessedTransactions { type Blueprint = Plain; + type Column = Column; fn column() -> Column { Column::ProcessedTransactions diff --git a/crates/storage/src/tables.rs b/crates/storage/src/tables.rs index 92e29d6981..ce8d98233e 100644 --- a/crates/storage/src/tables.rs +++ b/crates/storage/src/tables.rs @@ -13,14 +13,12 @@ use fuel_core_types::{ message::Message, }, fuel_tx::{ - Receipt, Transaction, TxId, UtxoId, }, fuel_types::{ BlockHeight, - Bytes32, ContractId, Nonce, }, @@ -57,18 +55,6 @@ impl Mappable for ContractsLatestUtxo { type OwnedValue = ContractUtxoInfo; } -// TODO: Move definition to the service that is responsible for its usage. -/// Receipts of different hidden internal operations. -pub struct Receipts; - -impl Mappable for Receipts { - /// Unique identifier of the transaction. - type Key = Self::OwnedKey; - type OwnedKey = Bytes32; - type Value = [Receipt]; - type OwnedValue = Vec; -} - /// The table of consensus metadata associated with sealed (finalized) blocks pub struct SealedBlockConsensus;