Optimistic elections (#4111)

nano/core_test/CMakeLists.txt

@@ -33,6 +33,7 @@ add_executable(
   network.cpp
   network_filter.cpp
   node.cpp
+  optimistic_scheduler.cpp
   processing_queue.cpp
   processor_service.cpp
   peer_container.cpp

nano/core_test/active_transactions.cpp

@@ -428,6 +428,7 @@ TEST (active_transactions, inactive_votes_cache_election_start)
 	nano::test::system system;
 	nano::node_config node_config (nano::test::get_available_port (), system.logging);
 	node_config.frontiers_confirmation = nano::frontiers_confirmation_mode::disabled;
+	node_config.optimistic_scheduler.enabled = false;
 	auto & node = *system.add_node (node_config);
 	nano::block_hash latest (node.latest (nano::dev::genesis_key.pub));
 	nano::keypair key1, key2;
@@ -1418,6 +1419,7 @@ TEST (active_transactions, limit_vote_hinted_elections)
 	nano::node_config config = system.default_config ();
 	const int aec_limit = 10;
 	config.frontiers_confirmation = nano::frontiers_confirmation_mode::disabled;
+	config.optimistic_scheduler.enabled = false;
 	config.active_elections_size = aec_limit;
 	config.active_elections_hinted_limit_percentage = 10; // Should give us a limit of 1 hinted election
 	auto & node = *system.add_node (config);

nano/core_test/optimistic_scheduler.cpp

@@ -0,0 +1,105 @@
+#include <nano/node/election.hpp>
+#include <nano/test_common/chains.hpp>
+#include <nano/test_common/system.hpp>
+#include <nano/test_common/testutil.hpp>
+
+#include <gtest/gtest.h>
+
+#include <chrono>
+
+using namespace std::chrono_literals;
+
+/*
+ * Ensure account gets activated for a single unconfirmed account chain
+ */
+TEST (optimistic_scheduler, activate_one)
+{
+	nano::test::system system{};
+	auto & node = *system.add_node ();
+
+	// Needs to be greater than optimistic scheduler `gap_threshold`
+	const int howmany_blocks = 64;
+
+	auto chains = nano::test::setup_chains (system, node, /* single chain */ 1, howmany_blocks, nano::dev::genesis_key, /* do not confirm */ false);
+	auto & [account, blocks] = chains.front ();
+
+	// Confirm block towards at the beginning the chain, so gap between confirmation and account frontier is larger than `gap_threshold`
+	ASSERT_TRUE (nano::test::confirm (node, { blocks.at (11) }));
+	ASSERT_TIMELY (5s, nano::test::confirmed (node, { blocks.at (11) }));
+
+	// Ensure unconfirmed account head block gets activated
+	auto const & block = blocks.back ();
+	ASSERT_TIMELY (5s, node.active.active (block->hash ()));
+	ASSERT_TRUE (node.active.election (block->qualified_root ())->behavior () == nano::election_behavior::optimistic);
+}
+
+/*
+ * Ensure account gets activated for a single unconfirmed account chain with nothing yet confirmed
+ */
+TEST (optimistic_scheduler, activate_one_zero_conf)
+{
+	nano::test::system system{};
+	auto & node = *system.add_node ();
+
+	// Can be smaller than optimistic scheduler `gap_threshold`
+	// This is meant to activate short account chains (eg. binary tree spam leaf accounts)
+	const int howmany_blocks = 6;
+
+	auto chains = nano::test::setup_chains (system, node, /* single chain */ 1, howmany_blocks, nano::dev::genesis_key, /* do not confirm */ false);
+	auto & [account, blocks] = chains.front ();
+
+	// Ensure unconfirmed account head block gets activated
+	auto const & block = blocks.back ();
+	ASSERT_TIMELY (5s, node.active.active (block->hash ()));
+	ASSERT_TRUE (node.active.election (block->qualified_root ())->behavior () == nano::election_behavior::optimistic);
+}
+
+/*
+ * Ensure account gets activated for a multiple unconfirmed account chains
+ */
+TEST (optimistic_scheduler, activate_many)
+{
+	nano::test::system system{};
+	auto & node = *system.add_node ();
+
+	// Needs to be greater than optimistic scheduler `gap_threshold`
+	const int howmany_blocks = 64;
+	const int howmany_chains = 16;
+
+	auto chains = nano::test::setup_chains (system, node, howmany_chains, howmany_blocks, nano::dev::genesis_key, /* do not confirm */ false);
+
+	// Ensure all unconfirmed accounts head block gets activated
+	ASSERT_TIMELY (5s, std::all_of (chains.begin (), chains.end (), [&] (auto const & entry) {
+		auto const & [account, blocks] = entry;
+		auto const & block = blocks.back ();
+		return node.active.active (block->hash ()) && node.active.election (block->qualified_root ())->behavior () == nano::election_behavior::optimistic;
+	}));
+}
+
+/*
+ * Ensure accounts with some blocks already confirmed and with less than `gap_threshold` blocks do not get activated
+ */
+TEST (optimistic_scheduler, under_gap_threshold)
+{
+	nano::test::system system{};
+	nano::node_config config = system.default_config ();
+	config.frontiers_confirmation = nano::frontiers_confirmation_mode::disabled;
+	auto & node = *system.add_node (config);
+
+	// Must be smaller than optimistic scheduler `gap_threshold`
+	const int howmany_blocks = 64;
+
+	auto chains = nano::test::setup_chains (system, node, /* single chain */ 1, howmany_blocks, nano::dev::genesis_key, /* do not confirm */ false);
+	auto & [account, blocks] = chains.front ();
+
+	// Confirm block towards the end of the chain, so gap between confirmation and account frontier is less than `gap_threshold`
+	ASSERT_TRUE (nano::test::confirm (node, { blocks.at (55) }));
+	ASSERT_TIMELY (5s, nano::test::confirmed (node, { blocks.at (55) }));
+
+	// Manually trigger backlog scan
+	node.backlog.trigger ();
+
+	// Ensure unconfirmed account head block gets activated
+	auto const & block = blocks.back ();
+	ASSERT_NEVER (3s, node.active.active (block->hash ()));
+}

nano/core_test/toml.cpp

@@ -264,6 +264,10 @@ TEST (toml, daemon_config_deserialize_defaults)
 	ASSERT_EQ (conf.node.rocksdb_config.enable, defaults.node.rocksdb_config.enable);
 	ASSERT_EQ (conf.node.rocksdb_config.memory_multiplier, defaults.node.rocksdb_config.memory_multiplier);
 	ASSERT_EQ (conf.node.rocksdb_config.io_threads, defaults.node.rocksdb_config.io_threads);
+
+	ASSERT_EQ (conf.node.optimistic_scheduler.enabled, defaults.node.optimistic_scheduler.enabled);
+	ASSERT_EQ (conf.node.optimistic_scheduler.gap_threshold, defaults.node.optimistic_scheduler.gap_threshold);
+	ASSERT_EQ (conf.node.optimistic_scheduler.max_size, defaults.node.optimistic_scheduler.max_size);
 }
 
 TEST (toml, optional_child)
@@ -519,6 +523,11 @@ TEST (toml, daemon_config_deserialize_no_defaults)
 	max_databases = 999
 	map_size = 999
 
+	[node.optimistic_scheduler]
+	enabled = false
+	gap_threshold = 999
+	max_size = 999
+
 	[node.rocksdb]
 	enable = true
 	memory_multiplier = 3
@@ -682,6 +691,10 @@ TEST (toml, daemon_config_deserialize_no_defaults)
 	ASSERT_EQ (nano::rocksdb_config::using_rocksdb_in_tests (), defaults.node.rocksdb_config.enable);
 	ASSERT_NE (conf.node.rocksdb_config.memory_multiplier, defaults.node.rocksdb_config.memory_multiplier);
 	ASSERT_NE (conf.node.rocksdb_config.io_threads, defaults.node.rocksdb_config.io_threads);
+
+	ASSERT_NE (conf.node.optimistic_scheduler.enabled, defaults.node.optimistic_scheduler.enabled);
+	ASSERT_NE (conf.node.optimistic_scheduler.gap_threshold, defaults.node.optimistic_scheduler.gap_threshold);
+	ASSERT_NE (conf.node.optimistic_scheduler.max_size, defaults.node.optimistic_scheduler.max_size);
 }
 
 /** There should be no required values **/

nano/lib/config.hpp

@@ -251,6 +251,7 @@ class network_constants
 			telemetry_cache_cutoff = 2000ms;
 			telemetry_request_interval = 500ms;
 			telemetry_broadcast_interval = 500ms;
+			optimistic_activation_delay = 2s;
 		}
 	}
 
@@ -302,6 +303,9 @@ class network_constants
 	/** Telemetry data older than this value is considered stale */
 	std::chrono::milliseconds telemetry_cache_cutoff{ 1000 * 130 }; // 2 * `telemetry_broadcast_interval` + some margin
 
+	/** How much to delay activation of optimistic elections to avoid interfering with election scheduler */
+	std::chrono::seconds optimistic_activation_delay{ 30 };
+
 	/** Returns the network this object contains values for */
 	nano::networks network () const
 	{

nano/lib/stats_enums.hpp

@@ -42,6 +42,7 @@ enum class type : uint8_t
 	backlog,
 	unchecked,
 	election_scheduler,
+	optimistic_scheduler,
 
 	_last // Must be the last enum
 };
@@ -159,6 +160,7 @@ enum class detail : uint8_t
 	// election types
 	normal,
 	hinted,
+	optimistic,
 
 	// received messages
 	invalid_header,

nano/lib/threading.cpp

@@ -107,6 +107,8 @@ std::string nano::thread_role::get_string (nano::thread_role::name role)
 			break;
 		case nano::thread_role::name::telemetry:
 			thread_role_name_string = "Telemetry";
+		case nano::thread_role::name::optimistic_scheduler:
+			thread_role_name_string = "Optimistic";
 			break;
 		default:
 			debug_assert (false && "nano::thread_role::get_string unhandled thread role");

nano/lib/threading.hpp

@@ -48,6 +48,7 @@ namespace thread_role
 		vote_generator_queue,
 		bootstrap_server,
 		telemetry,
+		optimistic_scheduler,
 	};
 
 	/*

nano/lib/utility.hpp

@@ -203,4 +203,24 @@ constexpr TARGET_TYPE narrow_cast (SOURCE_TYPE const & val)
 
 // Issue #3748
 void sort_options_description (const boost::program_options::options_description & source, boost::program_options::options_description & target);
+
+using clock = std::chrono::steady_clock;
+
+/**
+ * Check whether time elapsed between `last` and `now` is greater than `duration`
+ */
+template <typename Duration>
+bool elapsed (nano::clock::time_point const & last, Duration duration, nano::clock::time_point const & now)
+{
+	return last + duration < now;
+}
+
+/**
+ * Check whether time elapsed since `last` is greater than `duration`
+ */
+template <typename Duration>
+bool elapsed (nano::clock::time_point const & last, Duration duration)
+{
+	return elapsed (last, duration, nano::clock::now ());
+}
 }

nano/node/CMakeLists.txt

@@ -143,6 +143,8 @@ add_library(
   openclconfig.cpp
   openclwork.hpp
   openclwork.cpp
+  optimistic_scheduler.hpp
+  optimistic_scheduler.cpp
   peer_exclusion.hpp
   peer_exclusion.cpp
   portmapping.hpp

nano/node/active_transactions.cpp

@@ -196,6 +196,11 @@ int64_t nano::active_transactions::limit (nano::election_behavior behavior) cons
 			const uint64_t limit = node.config.active_elections_hinted_limit_percentage * node.config.active_elections_size / 100;
 			return static_cast<int64_t> (limit);
 		}
+		case nano::election_behavior::optimistic:
+		{
+			const uint64_t limit = node.config.active_elections_optimistic_limit_percentage * node.config.active_elections_size / 100;
+			return static_cast<int64_t> (limit);
+		}
 	}
 
 	debug_assert (false, "unknown election behavior");
@@ -210,8 +215,8 @@ int64_t nano::active_transactions::vacancy (nano::election_behavior behavior) co
 		case nano::election_behavior::normal:
 			return limit () - static_cast<int64_t> (roots.size ());
 		case nano::election_behavior::hinted:
-			return limit (nano::election_behavior::hinted) - count_by_behavior[nano::election_behavior::hinted];
-			;
+		case nano::election_behavior::optimistic:
+			return limit (behavior) - count_by_behavior[behavior];
 	}
 	debug_assert (false); // Unknown enum
 	return 0;
@@ -261,6 +266,7 @@ void nano::active_transactions::request_confirm (nano::unique_lock<nano::mutex>
 
 void nano::active_transactions::cleanup_election (nano::unique_lock<nano::mutex> & lock_a, std::shared_ptr<nano::election> election)
 {
+	debug_assert (!mutex.try_lock ());
 	debug_assert (lock_a.owns_lock ());
 
 	node.stats.inc (completion_type (*election), nano::to_stat_detail (election->behavior ()));
@@ -368,6 +374,7 @@ nano::election_insertion_result nano::active_transactions::insert (const std::sh
 
 nano::election_insertion_result nano::active_transactions::insert_impl (nano::unique_lock<nano::mutex> & lock_a, std::shared_ptr<nano::block> const & block_a, nano::election_behavior election_behavior_a, std::function<void (std::shared_ptr<nano::block> const &)> const & confirmation_action_a)
 {
+	debug_assert (!mutex.try_lock ());
 	debug_assert (lock_a.owns_lock ());
 	debug_assert (block_a->has_sideband ());
 	nano::election_insertion_result result;
@@ -594,7 +601,7 @@ bool nano::active_transactions::publish (std::shared_ptr<nano::block> const & bl
 			{
 				cache->fill (election);
 			}
-			node.stats.inc (nano::stat::type::election, nano::stat::detail::election_block_conflict);
+			node.stats.inc (nano::stat::type::active, nano::stat::detail::election_block_conflict);
 		}
 	}
 	return result;
@@ -645,8 +652,6 @@ void nano::active_transactions::add_inactive_vote_cache (nano::block_hash const
 	if (node.ledger.weight (vote->account) > node.minimum_principal_weight ())
 	{
 		node.inactive_vote_cache.vote (hash, vote);
-
-		node.stats.inc (nano::stat::type::vote_cache, nano::stat::detail::vote_processed);
 	}
 }
 
@@ -676,6 +681,7 @@ std::unique_ptr<nano::container_info_component> nano::collect_container_info (ac
 	composite->add_component (std::make_unique<container_info_leaf> (container_info{ "election_winner_details", active_transactions.election_winner_details_size (), sizeof (decltype (active_transactions.election_winner_details)::value_type) }));
 	composite->add_component (std::make_unique<container_info_leaf> (container_info{ "normal", static_cast<std::size_t> (active_transactions.count_by_behavior[nano::election_behavior::normal]), 0 }));
 	composite->add_component (std::make_unique<container_info_leaf> (container_info{ "hinted", static_cast<std::size_t> (active_transactions.count_by_behavior[nano::election_behavior::hinted]), 0 }));
+	composite->add_component (std::make_unique<container_info_leaf> (container_info{ "optimistic", static_cast<std::size_t> (active_transactions.count_by_behavior[nano::election_behavior::optimistic]), 0 }));
 
 	composite->add_component (active_transactions.recently_confirmed.collect_container_info ("recently_confirmed"));
 	composite->add_component (active_transactions.recently_cemented.collect_container_info ("recently_cemented"));

nano/node/election.cpp

@@ -124,9 +124,23 @@ bool nano::election::state_change (nano::election::state_t expected_a, nano::ele
 	return result;
 }
 
+std::chrono::milliseconds nano::election::confirm_req_time () const
+{
+	switch (behavior ())
+	{
+		case election_behavior::normal:
+		case election_behavior::hinted:
+			return base_latency () * 5;
+		case election_behavior::optimistic:
+			return base_latency () * 2;
+	}
+	debug_assert (false);
+	return {};
+}
+
 void nano::election::send_confirm_req (nano::confirmation_solicitor & solicitor_a)
 {
-	if ((base_latency () * 5) < (std::chrono::steady_clock::now () - last_req))
+	if (confirm_req_time () < (std::chrono::steady_clock::now () - last_req))
 	{
 		nano::lock_guard<nano::mutex> guard{ mutex };
 		if (!solicitor_a.add (*this))
@@ -225,8 +239,10 @@ std::chrono::milliseconds nano::election::time_to_live () const
 		case election_behavior::normal:
 			return std::chrono::milliseconds (5 * 60 * 1000);
 		case election_behavior::hinted:
+		case election_behavior::optimistic:
 			return std::chrono::milliseconds (30 * 1000);
 	}
+	debug_assert (false);
 	return {};
 }
 
@@ -646,6 +662,10 @@ nano::stat::detail nano::to_stat_detail (nano::election_behavior behavior)
 		{
 			return nano::stat::detail::hinted;
 		}
+		case nano::election_behavior::optimistic:
+		{
+			return nano::stat::detail::optimistic;
+		}
 	}
 
 	debug_assert (false, "unknown election behavior");

nano/node/election.hpp

@@ -46,7 +46,19 @@ class election_vote_result final
 enum class election_behavior
 {
 	normal,
-	hinted
+	/**
+	 * Hinted elections:
+	 * - shorter timespan
+	 * - limited space inside AEC
+	 */
+	hinted,
+	/**
+	 * Optimistic elections:
+	 * - shorter timespan
+	 * - limited space inside AEC
+	 * - more frequent confirmation requests
+	 */
+	optimistic,
 };
 
 nano::stat::detail to_stat_detail (nano::election_behavior);
@@ -161,10 +173,14 @@ class election final : public std::enable_shared_from_this<nano::election>
 	void remove_block (nano::block_hash const &);
 	bool replace_by_weight (nano::unique_lock<nano::mutex> & lock_a, nano::block_hash const &);
 	std::chrono::milliseconds time_to_live () const;
-	/*
+	/**
 	 * Calculates minimum time delay between subsequent votes when processing non-final votes
 	 */
 	std::chrono::seconds cooldown_time (nano::uint128_t weight) const;
+	/**
+	 * Calculates time delay between broadcasting confirmation requests
+	 */
+	std::chrono::milliseconds confirm_req_time () const;
 
 private:
 	std::unordered_map<nano::block_hash, std::shared_ptr<nano::block>> last_blocks;