Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Bootstrap serving on dedicated thread pool #3865

Merged
merged 4 commits into from
Jul 21, 2022
Merged

Bootstrap serving on dedicated thread pool #3865

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
f5eaf49
Create dedicated bootstrap workers
pwojcikdev Jul 18, 2022
64b1cad
Use dedicated bootstrap workers in `bootstrap_bulk_pull_*` servers
pwojcikdev Jul 18, 2022
b44d6d7
Use dedicated bootstrap workers in `frontier_req` server
pwojcikdev Jul 18, 2022
7fdb573
Add config option `bootstrap_serving_threads`
pwojcikdev Jul 18, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions nano/core_test/toml.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,7 @@ TEST (toml, daemon_config_deserialize_defaults)
ASSERT_EQ (conf.node.bootstrap_connections, defaults.node.bootstrap_connections);
ASSERT_EQ (conf.node.bootstrap_connections_max, defaults.node.bootstrap_connections_max);
ASSERT_EQ (conf.node.bootstrap_initiator_threads, defaults.node.bootstrap_initiator_threads);
ASSERT_EQ (conf.node.bootstrap_serving_threads, defaults.node.bootstrap_serving_threads);
ASSERT_EQ (conf.node.bootstrap_frontier_request_count, defaults.node.bootstrap_frontier_request_count);
ASSERT_EQ (conf.node.bootstrap_fraction_numerator, defaults.node.bootstrap_fraction_numerator);
ASSERT_EQ (conf.node.conf_height_processor_batch_min_time, defaults.node.conf_height_processor_batch_min_time);
Expand Down Expand Up @@ -395,6 +396,7 @@ TEST (toml, daemon_config_deserialize_no_defaults)
bootstrap_connections = 999
bootstrap_connections_max = 999
bootstrap_initiator_threads = 999
bootstrap_serving_threads = 999
bootstrap_frontier_request_count = 9999
bootstrap_fraction_numerator = 999
conf_height_processor_batch_min_time = 999
Expand Down Expand Up @@ -559,6 +561,7 @@ TEST (toml, daemon_config_deserialize_no_defaults)
ASSERT_NE (conf.node.bootstrap_connections, defaults.node.bootstrap_connections);
ASSERT_NE (conf.node.bootstrap_connections_max, defaults.node.bootstrap_connections_max);
ASSERT_NE (conf.node.bootstrap_initiator_threads, defaults.node.bootstrap_initiator_threads);
ASSERT_NE (conf.node.bootstrap_serving_threads, defaults.node.bootstrap_serving_threads);
ASSERT_NE (conf.node.bootstrap_frontier_request_count, defaults.node.bootstrap_frontier_request_count);
ASSERT_NE (conf.node.bootstrap_fraction_numerator, defaults.node.bootstrap_fraction_numerator);
ASSERT_NE (conf.node.conf_height_processor_batch_min_time, defaults.node.conf_height_processor_batch_min_time);
Expand Down
3 changes: 3 additions & 0 deletions nano/lib/threading.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,9 @@ std::string nano::thread_role::get_string (nano::thread_role::name role)
case nano::thread_role::name::worker:
thread_role_name_string = "Worker";
break;
case nano::thread_role::name::bootstrap_worker:
thread_role_name_string = "Bootstrap work";
break;
case nano::thread_role::name::request_aggregator:
thread_role_name_string = "Req aggregator";
break;
Expand Down
1 change: 1 addition & 0 deletions nano/lib/threading.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ namespace thread_role
rpc_process_container,
confirmation_height_processing,
worker,
bootstrap_worker,
request_aggregator,
state_block_signature_verification,
epoch_upgrader,
Expand Down
8 changes: 6 additions & 2 deletions nano/node/bootstrap/bootstrap_bulk_pull.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -492,7 +492,9 @@ void nano::bulk_pull_server::sent_action (boost::system::error_code const & ec,
{
if (!ec)
{
send_next ();
connection->node->bootstrap_workers.push_task ([this_l = shared_from_this ()] () {
this_l->send_next ();
});
}
else
{
Expand Down Expand Up @@ -770,7 +772,9 @@ void nano::bulk_pull_account_server::sent_action (boost::system::error_code cons
{
if (!ec)
{
send_next_block ();
connection->node->bootstrap_workers.push_task ([this_l = shared_from_this ()] () {
this_l->send_next_block ();
});
}
else
{
Expand Down
5 changes: 4 additions & 1 deletion nano/node/bootstrap/bootstrap_frontier.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -316,7 +316,10 @@ void nano::frontier_req_server::sent_action (boost::system::error_code const & e
if (!ec)
{
count++;
send_next ();

connection->node->bootstrap_workers.push_task ([this_l = shared_from_this ()] () {
this_l->send_next ();
});
}
else
{
Expand Down
18 changes: 12 additions & 6 deletions nano/node/bootstrap/bootstrap_server.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -663,13 +663,17 @@ class request_response_visitor : public nano::message_visitor
}
void bulk_pull (nano::bulk_pull const &) override
{
auto response (std::make_shared<nano::bulk_pull_server> (connection, std::unique_ptr<nano::bulk_pull> (static_cast<nano::bulk_pull *> (connection->requests.front ().release ()))));
response->send_next ();
connection->node->bootstrap_workers.push_task ([connection = connection] () {
auto response (std::make_shared<nano::bulk_pull_server> (connection, std::unique_ptr<nano::bulk_pull> (static_cast<nano::bulk_pull *> (connection->requests.front ().release ()))));
response->send_next ();
});
}
void bulk_pull_account (nano::bulk_pull_account const &) override
{
auto response (std::make_shared<nano::bulk_pull_account_server> (connection, std::unique_ptr<nano::bulk_pull_account> (static_cast<nano::bulk_pull_account *> (connection->requests.front ().release ()))));
response->send_frontier ();
connection->node->bootstrap_workers.push_task ([connection = connection] () {
auto response (std::make_shared<nano::bulk_pull_account_server> (connection, std::unique_ptr<nano::bulk_pull_account> (static_cast<nano::bulk_pull_account *> (connection->requests.front ().release ()))));
response->send_frontier ();
});
}
void bulk_push (nano::bulk_push const &) override
{
Expand All @@ -678,8 +682,10 @@ class request_response_visitor : public nano::message_visitor
}
void frontier_req (nano::frontier_req const &) override
{
auto response (std::make_shared<nano::frontier_req_server> (connection, std::unique_ptr<nano::frontier_req> (static_cast<nano::frontier_req *> (connection->requests.front ().release ()))));
response->send_next ();
connection->node->bootstrap_workers.push_task ([connection = connection] () {
auto response (std::make_shared<nano::frontier_req_server> (connection, std::unique_ptr<nano::frontier_req> (static_cast<nano::frontier_req *> (connection->requests.front ().release ()))));
response->send_next ();
});
}
void telemetry_req (nano::telemetry_req const & message_a) override
{
Expand Down
1 change: 1 addition & 0 deletions nano/node/node.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,7 @@ nano::node::node (boost::asio::io_context & io_ctx_a, boost::filesystem::path co
network_params{ config.network_params },
stats (config.stat_config),
workers (std::max (3u, config.io_threads / 4), nano::thread_role::name::worker),
bootstrap_workers{ config.bootstrap_serving_threads, nano::thread_role::name::bootstrap_worker },
flags (flags_a),
work (work_a),
distributed_work (*this),
Expand Down
1 change: 1 addition & 0 deletions nano/node/node.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,7 @@ class node final : public std::enable_shared_from_this<nano::node>
nano::network_params & network_params;
nano::stat stats;
nano::thread_pool workers;
nano::thread_pool bootstrap_workers;
std::shared_ptr<nano::websocket::listener> websocket_server;
nano::node_flags flags;
nano::work_pool & work;
Expand Down
2 changes: 2 additions & 0 deletions nano/node/nodeconfig.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,7 @@ nano::error nano::node_config::serialize_toml (nano::tomlconfig & toml) const
toml.put ("bootstrap_connections", bootstrap_connections, "Number of outbound bootstrap connections. Must be a power of 2. Defaults to 4.\nWarning: a larger amount of connections may use substantially more system memory.\ntype:uint64");
toml.put ("bootstrap_connections_max", bootstrap_connections_max, "Maximum number of inbound bootstrap connections. Defaults to 64.\nWarning: a larger amount of connections may use additional system memory.\ntype:uint64");
toml.put ("bootstrap_initiator_threads", bootstrap_initiator_threads, "Number of threads dedicated to concurrent bootstrap attempts. Defaults to 1.\nWarning: a larger amount of attempts may use additional system memory and disk IO.\ntype:uint64");
toml.put ("bootstrap_serving_threads", bootstrap_serving_threads, "Number of threads dedicated to serving bootstrap data to other peers. Defaults to half the number of CPU threads, and at least 2.\ntype:uint64");
toml.put ("bootstrap_frontier_request_count", bootstrap_frontier_request_count, "Number frontiers per bootstrap frontier request. Defaults to 1048576.\ntype:uint32,[1024..4294967295]");
toml.put ("block_processor_batch_max_time", block_processor_batch_max_time.count (), "The maximum time the block processor can continuously process blocks for.\ntype:milliseconds");
toml.put ("allow_local_peers", allow_local_peers, "Enable or disable local host peering.\ntype:bool");
Expand Down Expand Up @@ -335,6 +336,7 @@ nano::error nano::node_config::deserialize_toml (nano::tomlconfig & toml)
toml.get<unsigned> ("bootstrap_connections", bootstrap_connections);
toml.get<unsigned> ("bootstrap_connections_max", bootstrap_connections_max);
toml.get<unsigned> ("bootstrap_initiator_threads", bootstrap_initiator_threads);
toml.get<unsigned> ("bootstrap_serving_threads", bootstrap_serving_threads);
toml.get<uint32_t> ("bootstrap_frontier_request_count", bootstrap_frontier_request_count);
toml.get<bool> ("enable_voting", enable_voting);
toml.get<bool> ("allow_local_peers", allow_local_peers);
Expand Down
1 change: 1 addition & 0 deletions nano/node/nodeconfig.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ class node_config
unsigned bootstrap_connections{ 4 };
unsigned bootstrap_connections_max{ 64 };
unsigned bootstrap_initiator_threads{ 1 };
unsigned bootstrap_serving_threads{ std::max<unsigned> (2, std::thread::hardware_concurrency () / 2) };
uint32_t bootstrap_frontier_request_count{ 1024 * 1024 };
nano::websocket::config websocket_config;
nano::diagnostics_config diagnostics_config;
Expand Down