Skip to content

Commit

Permalink
Disables disk usage from the unchecked_map class.
Browse files Browse the repository at this point in the history 
With the ascending bootstrap process the number of blocks needing to be tracked can be limited and remain in memory.

The maximum number of blocks is set to 64k though typically far fewer are actually needed.

Co-authored-by: Piotr Wójcik <3044353+pwojcikdev@users.noreply.github.com>
Co-authored-by: gr0vity-dev <85646666+gr0vity-dev@users.noreply.github.com>
  • Loading branch information
@clemahieu @pwojcikdev @gr0vity-dev
3 people committed Mar 14, 2023
1 parent 3004649 commit 196198e
Show file tree
Hide file tree
Showing 15 changed files with 154 additions and 256 deletions.
3 changes: 1 addition & 2 deletions nano/core_test/block_store.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -450,9 +450,8 @@ TEST (block_store, empty_bootstrap)
auto store = nano::make_store (logger, nano::unique_path (), nano::dev::constants);
nano::unchecked_map unchecked{ *store, system.stats, false };
ASSERT_TRUE (!store->init_error ());
auto transaction (store->tx_begin_read ());
size_t count = 0;
unchecked.for_each (transaction, [&count] (nano::unchecked_key const & key, nano::unchecked_info const & info) {
unchecked.for_each ([&count] (nano::unchecked_key const & key, nano::unchecked_info const & info) {
++count;
});
ASSERT_EQ (count, 0);
Expand Down
12 changes: 6 additions & 6 deletions nano/core_test/bootstrap.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,12 +127,12 @@ TEST (bulk_pull, ascending_one_hash)
auto connection = std::make_shared<nano::transport::tcp_server> (socket, system.nodes[0]);
auto req = std::make_unique<nano::bulk_pull> (nano::dev::network_params.network);
req->start = nano::dev::genesis->hash ();
req->end = nano::dev::genesis->hash ();
req->end.clear ();
req->header.flag_set (nano::message_header::bulk_pull_ascending_flag);
auto request = std::make_shared<nano::bulk_pull_server> (connection, std::move (req));
auto block_out1 = request->get_next ();
ASSERT_NE (nullptr, block_out1);
ASSERT_EQ (block_out1->hash (), nano::dev::genesis->hash ());
ASSERT_EQ (block_out1->hash (), block1->hash ());
ASSERT_EQ (nullptr, request->get_next ());
}

Expand All @@ -158,7 +158,7 @@ TEST (bulk_pull, ascending_two_account)
auto socket = std::make_shared<nano::transport::socket> (node, nano::transport::socket::endpoint_type_t::server);
auto connection = std::make_shared<nano::transport::tcp_server> (socket, system.nodes[0]);
auto req = std::make_unique<nano::bulk_pull> (nano::dev::network_params.network);
req->start = nano::dev::genesis->hash ();
req->start = nano::dev::genesis->account ();
req->end.clear ();
req->header.flag_set (nano::message_header::bulk_pull_ascending_flag);
auto request = std::make_shared<nano::bulk_pull_server> (connection, std::move (req));
Expand All @@ -172,7 +172,7 @@ TEST (bulk_pull, ascending_two_account)
}

/**
Tests that the `end' value is respected in the bulk_pull message
Tests that the `end' value is respected in the bulk_pull message when the ascending flag is used.
*/
TEST (bulk_pull, ascending_end)
{
Expand Down Expand Up @@ -1351,7 +1351,7 @@ TEST (bootstrap_processor, lazy_pruning_missing_block)
ASSERT_FALSE (node2->ledger.block_or_pruned_exists (state_open->hash ()));
{
auto transaction (node2->store.tx_begin_read ());
ASSERT_TRUE (node2->unchecked.exists (transaction, nano::unchecked_key (send2->root ().as_block_hash (), send2->hash ())));
ASSERT_TRUE (node2->unchecked.exists (nano::unchecked_key (send2->root ().as_block_hash (), send2->hash ())));
}
// Insert missing block
node2->process_active (send1);
Expand Down Expand Up @@ -2045,7 +2045,7 @@ TEST (bulk, DISABLED_genesis_pruning)
ASSERT_EQ (1, node2->ledger.cache.block_count);
{
auto transaction (node2->store.tx_begin_write ());
node2->unchecked.clear (transaction);
node2->unchecked.clear ();
}
// Insert pruned blocks
node2->process_active (send1);
Expand Down
8 changes: 4 additions & 4 deletions nano/core_test/confirmation_height.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -391,14 +391,14 @@ TEST (confirmation_height, gap_bootstrap)
node1.process_active (receive2);
// Waits for the unchecked_map to process the 4 blocks added to the block_processor, saving them in the unchecked table
auto check_block_is_listed = [&] (nano::transaction const & transaction_a, nano::block_hash const & block_hash_a) {
return !node1.unchecked.get (transaction_a, block_hash_a).empty ();
return !node1.unchecked.get (block_hash_a).empty ();
};
ASSERT_TIMELY (5s, check_block_is_listed (node1.store.tx_begin_read (), receive2->previous ()));

// Confirmation heights should not be updated
{
auto transaction (node1.store.tx_begin_read ());
auto unchecked_count (node1.unchecked.count (transaction));
auto unchecked_count (node1.unchecked.count ());
ASSERT_EQ (unchecked_count, 2);

nano::confirmation_height_info confirmation_height_info;
Expand All @@ -410,7 +410,7 @@ TEST (confirmation_height, gap_bootstrap)
// Now complete the chain where the block comes in on the bootstrap network.
node1.block_processor.add (open1);

ASSERT_TIMELY (5s, node1.unchecked.count (node1.store.tx_begin_read ()) == 0);
ASSERT_TIMELY (5s, node1.unchecked.count () == 0);
// Confirmation height should be unchanged and unchecked should now be 0
{
auto transaction = node1.store.tx_begin_read ();
Expand Down Expand Up @@ -544,7 +544,7 @@ TEST (confirmation_height, gap_live)

// This should confirm the open block and the source of the receive blocks
auto transaction = node->store.tx_begin_read ();
auto unchecked_count = node->unchecked.count (transaction);
auto unchecked_count = node->unchecked.count ();
ASSERT_EQ (unchecked_count, 0);

nano::confirmation_height_info confirmation_height_info{};
Expand Down
37 changes: 18 additions & 19 deletions nano/core_test/ledger.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3999,10 +3999,10 @@ TEST (ledger, epoch_open_pending)
ASSERT_EQ (nano::process_result::gap_epoch_open_pending, process_result.code);
node1.block_processor.add (epoch_open);
// Waits for the block to get saved in the database
ASSERT_TIMELY (10s, 1 == node1.unchecked.count (node1.store.tx_begin_read ()));
ASSERT_TIMELY (10s, 1 == node1.unchecked.count ());
ASSERT_FALSE (node1.ledger.block_or_pruned_exists (epoch_open->hash ()));
// Open block should be inserted into unchecked
auto blocks = node1.unchecked.get (node1.store.tx_begin_read (), nano::hash_or_account (epoch_open->account ()).hash);
auto blocks = node1.unchecked.get (nano::hash_or_account (epoch_open->account ()).hash);
ASSERT_EQ (blocks.size (), 1);
ASSERT_EQ (blocks[0].block->full_hash (), epoch_open->full_hash ());
// New block to process epoch open
Expand Down Expand Up @@ -4293,16 +4293,16 @@ TEST (ledger, unchecked_epoch)
node1.block_processor.add (epoch1);
{
// Waits for the epoch1 block to pass through block_processor and unchecked.put queues
ASSERT_TIMELY (10s, 1 == node1.unchecked.count (node1.store.tx_begin_read ()));
auto blocks = node1.unchecked.get (node1.store.tx_begin_read (), epoch1->previous ());
ASSERT_TIMELY (10s, 1 == node1.unchecked.count ());
auto blocks = node1.unchecked.get (epoch1->previous ());
ASSERT_EQ (blocks.size (), 1);
}
node1.block_processor.add (send1);
node1.block_processor.add (open1);
ASSERT_TIMELY (5s, node1.store.block.exists (node1.store.tx_begin_read (), epoch1->hash ()));
{
// Waits for the last blocks to pass through block_processor and unchecked.put queues
ASSERT_TIMELY (10s, 0 == node1.unchecked.count (node1.store.tx_begin_read ()));
ASSERT_TIMELY (10s, 0 == node1.unchecked.count ());
auto info = node1.ledger.account_info (node1.store.tx_begin_read (), destination.pub);
ASSERT_TRUE (info);
ASSERT_EQ (info->epoch (), nano::epoch::epoch_1);
Expand Down Expand Up @@ -4367,8 +4367,8 @@ TEST (ledger, unchecked_epoch_invalid)
node1.block_processor.add (epoch2);
{
// Waits for the last blocks to pass through block_processor and unchecked.put queues
ASSERT_TIMELY (10s, 2 == node1.unchecked.count (node1.store.tx_begin_read ()));
auto blocks = node1.unchecked.get (node1.store.tx_begin_read (), epoch1->previous ());
ASSERT_TIMELY (10s, 2 == node1.unchecked.count ());
auto blocks = node1.unchecked.get (epoch1->previous ());
ASSERT_EQ (blocks.size (), 2);
}
node1.block_processor.add (send1);
Expand All @@ -4378,9 +4378,9 @@ TEST (ledger, unchecked_epoch_invalid)
{
auto transaction = node1.store.tx_begin_read ();
ASSERT_FALSE (node1.store.block.exists (transaction, epoch1->hash ()));
auto unchecked_count = node1.unchecked.count (transaction);
auto unchecked_count = node1.unchecked.count ();
ASSERT_EQ (unchecked_count, 0);
ASSERT_EQ (unchecked_count, node1.unchecked.count (transaction));
ASSERT_EQ (unchecked_count, node1.unchecked.count ());
auto info = node1.ledger.account_info (transaction, destination.pub);
ASSERT_TRUE (info);
ASSERT_NE (info->epoch (), nano::epoch::epoch_1);
Expand Down Expand Up @@ -4434,15 +4434,15 @@ TEST (ledger, unchecked_open)
node1.block_processor.add (open1);
{
// Waits for the last blocks to pass through block_processor and unchecked.put queues
ASSERT_TIMELY (5s, 1 == node1.unchecked.count (node1.store.tx_begin_read ()));
// When open1 existists in unchecked, we know open2 has been processed.
auto blocks = node1.unchecked.get (node1.store.tx_begin_read (), open1->source ());
ASSERT_TIMELY (10s, 1 == node1.unchecked.count ());
// Get the next peer for attempting a tcp bootstrap connection
auto blocks = node1.unchecked.get (open1->source ());
ASSERT_EQ (blocks.size (), 1);
}
node1.block_processor.add (send1);
// Waits for the send1 block to pass through block_processor and unchecked.put queues
ASSERT_TIMELY (5s, node1.store.block.exists (node1.store.tx_begin_read (), open1->hash ()));
ASSERT_EQ (0, node1.unchecked.count (node1.store.tx_begin_read ()));
ASSERT_EQ (0, node1.unchecked.count ());
}

TEST (ledger, unchecked_receive)
Expand Down Expand Up @@ -4493,13 +4493,13 @@ TEST (ledger, unchecked_receive)
node1.block_processor.add (send1);
node1.block_processor.add (receive1);
auto check_block_is_listed = [&] (nano::transaction const & transaction_a, nano::block_hash const & block_hash_a) {
return !node1.unchecked.get (transaction_a, block_hash_a).empty ();
return !node1.unchecked.get (block_hash_a).empty ();
};
// Previous block for receive1 is unknown, signature cannot be validated
{
// Waits for the last blocks to pass through block_processor and unchecked.put queues
ASSERT_TIMELY (15s, check_block_is_listed (node1.store.tx_begin_read (), receive1->previous ()));
auto blocks = node1.unchecked.get (node1.store.tx_begin_read (), receive1->previous ());
auto blocks = node1.unchecked.get (receive1->previous ());
ASSERT_EQ (blocks.size (), 1);
}
// Waits for the open1 block to pass through block_processor and unchecked.put queues
Expand All @@ -4508,12 +4508,12 @@ TEST (ledger, unchecked_receive)
// Previous block for receive1 is known, signature was validated
{
auto transaction = node1.store.tx_begin_read ();
auto blocks (node1.unchecked.get (transaction, receive1->source ()));
auto blocks (node1.unchecked.get (receive1->source ()));
ASSERT_EQ (blocks.size (), 1);
}
node1.block_processor.add (send2);
ASSERT_TIMELY (10s, node1.store.block.exists (node1.store.tx_begin_read (), receive1->hash ()));
ASSERT_EQ (0, node1.unchecked.count (node1.store.tx_begin_read ()));
ASSERT_EQ (0, node1.unchecked.count ());
}

TEST (ledger, confirmation_height_not_updated)
Expand Down Expand Up @@ -5542,8 +5542,7 @@ TEST (ledger, migrate_lmdb_to_rocksdb)
uint16_t port = 100;
nano::lmdb::store store{ logger, path / "data.ldb", nano::dev::constants };
nano::unchecked_map unchecked{ store, system.stats, false };
nano::stats stats{};
nano::ledger ledger{ store, stats, nano::dev::constants };
nano::ledger ledger{ store, system.stats, nano::dev::constants };
nano::work_pool pool{ nano::dev::network_params.network, std::numeric_limits<unsigned>::max () };

std::shared_ptr<nano::block> send = nano::state_block_builder ()
Expand Down
10 changes: 5 additions & 5 deletions nano/core_test/node.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2984,7 +2984,7 @@ TEST (node, block_processor_signatures)
node1.process_active (receive2);
node1.process_active (receive3);
ASSERT_TIMELY (5s, node1.block (receive2->hash ()) != nullptr); // Implies send1, send2, send3, receive1.
ASSERT_TIMELY (5s, node1.unchecked.count (node1.store.tx_begin_read ()) == 0);
ASSERT_TIMELY (5s, node1.unchecked.count () == 0);
ASSERT_EQ (nullptr, node1.block (receive3->hash ())); // Invalid signer
ASSERT_EQ (nullptr, node1.block (send4->hash ())); // Invalid signature via process_active
ASSERT_EQ (nullptr, node1.block (send5->hash ())); // Invalid signature via unchecked
Expand Down Expand Up @@ -3289,17 +3289,17 @@ TEST (node, unchecked_cleanup)
ASSERT_FALSE (node.network.publish_filter.apply (bytes.data (), bytes.size ()));
node.process_active (open);
// Waits for the open block to get saved in the database
ASSERT_TIMELY (15s, 1 == node.unchecked.count (node.store.tx_begin_read ()));
ASSERT_TIMELY (15s, 1 == node.unchecked.count ());
node.config.unchecked_cutoff_time = std::chrono::seconds (2);
ASSERT_EQ (1, node.unchecked.count (node.store.tx_begin_read ()));
ASSERT_EQ (1, node.unchecked.count ());
std::this_thread::sleep_for (std::chrono::seconds (1));
node.unchecked_cleanup ();
ASSERT_TRUE (node.network.publish_filter.apply (bytes.data (), bytes.size ()));
ASSERT_EQ (1, node.unchecked.count (node.store.tx_begin_read ()));
ASSERT_EQ (1, node.unchecked.count ());
std::this_thread::sleep_for (std::chrono::seconds (2));
node.unchecked_cleanup ();
ASSERT_FALSE (node.network.publish_filter.apply (bytes.data (), bytes.size ()));
ASSERT_EQ (0, node.unchecked.count (node.store.tx_begin_read ()));
ASSERT_EQ (0, node.unchecked.count ());
}

/** This checks that a node can be opened (without being blocked) when a write lock is held elsewhere */
Expand Down

0 comments on commit 196198e

Please sign in to comment.