feat(trie): parallel storage roots #6903
Conversation
rkrasiuk
added
C-enhancement
rkrasiuk
force-pushed
the
rkrasiuk/trie-parallel
branch
from
28d1a6f
to
3290361
Compare
rkrasiuk
requested review from
rakita,
joshieDo,
shekhirin and
gakonst
as code owners
| #[tokio::test] | ||
| async fn random_async_root() { | ||
| let manager = TaskManager::new(Handle::current()); | ||
| let task_executor = Arc::new(manager.executor()); | ||
|
|
||
| let factory = create_test_provider_factory(); | ||
| let consistent_view = ConsistentDbView::new(factory.clone()); | ||
|
|
||
| let mut rng = rand::thread_rng(); | ||
| let mut state = (0..100) | ||
| .map(|_| { | ||
| let address = Address::random(); | ||
| let account = | ||
| Account { balance: U256::from(rng.gen::<u64>()), ..Default::default() }; | ||
| let mut storage = HashMap::<B256, U256>::default(); | ||
| let has_storage = rng.gen_bool(0.7); | ||
| if has_storage { | ||
| for _ in 0..100 { | ||
| storage.insert( | ||
| B256::from(U256::from(rng.gen::<u64>())), | ||
| U256::from(rng.gen::<u64>()), | ||
| ); | ||
| } | ||
| } | ||
| (address, (account, storage)) | ||
| }) | ||
| .collect::<HashMap<_, _>>(); |
There was a problem hiding this comment.
this should be a fuzztest with the techniques we used in the serial processor?
| let provider_ro = self.view.provider_ro()?; | ||
| let hashed_cursor_factory = | ||
| HashedPostStateCursorFactory::new(provider_ro.tx_ref(), &hashed_state_sorted); | ||
| let trie_cursor_factory = provider_ro.tx_ref(); | ||
|
|
||
| let hashed_account_cursor = | ||
| hashed_cursor_factory.hashed_account_cursor().map_err(ProviderError::Database)?; | ||
| let trie_cursor = | ||
| trie_cursor_factory.account_trie_cursor().map_err(ProviderError::Database)?; | ||
|
|
||
| let walker = TrieWalker::new(trie_cursor, prefix_sets.account_prefix_set) | ||
| .with_updates(retain_updates); | ||
| let mut account_node_iter = AccountNodeIter::new(walker, hashed_account_cursor); | ||
| let mut hash_builder = HashBuilder::default().with_updates(retain_updates); |
There was a problem hiding this comment.
makes me think we may want helpers for setting these things up cuz now it's a bit verbose
| let mut storage_roots = storage_root_targets | ||
| .into_par_iter() |
There was a problem hiding this comment.
how large would this be usually?
and how long does one iteration take?
| let mut storage_roots = storage_root_targets | ||
| .into_par_iter() | ||
| .map(|(hashed_address, prefix_set)| { | ||
| let provider_ro = self.view.provider_ro()?; |
There was a problem hiding this comment.
is this even necessary?
since Tx: Send+ Sync you should be able to access this in this scope so you only need to create it once
although the tx ops will sync via mutex
There was a problem hiding this comment.
absolutely, because the calculations will choke on that mutex
| // Pre-calculate storage roots in parallel for accounts which were changed. | ||
| debug!(target: "trie::parallel_state_root", len = storage_root_targets.len(), "pre-calculating storage roots"); | ||
| let mut storage_roots = storage_root_targets | ||
| .into_par_iter() |
There was a problem hiding this comment.
probably a good idea to use chunks here instead to reduce ro txs and rayon overhead
There was a problem hiding this comment.
the aforementioned overhead is smaller than the overhead of consecutive blocking ops imo
rkrasiuk
force-pushed
the
rkrasiuk/trie-parallel
branch
from
f42475d
to
76d8f08
Compare
|
Benchmarks for state size State size: State size: State size: State size: |
|
Updated numbers with intermediate nodes committed to database: State size: State size: State size: State size: |
|
|
||
| #[cfg(feature = "parallel")] | ||
| /// Implementation of parallel state root computation. | ||
| pub mod parallel_root; |
|
Beta is cut. Should we merge and ship to benchmarkooors? |
rkrasiuk
force-pushed
the
rkrasiuk/trie-parallel
branch
from
d937966
to
3811e2a
Compare
rkrasiuk
force-pushed
the
rkrasiuk/trie-parallel
branch
from
3811e2a
to
5b7b911
Compare
Description
Supersedes #6576.
Builds on top of #6896.
Creates
ParallelStateRootandAsyncStateRootincremental root calculators. See respective docs for details & differences.Intended usage:
ParallelStateRoot- for internal use in the blockchain tree (integration will be done in a follow up) and externally is sync environmentsAsyncStateRoot- for external use in async environments