Hinted election scheduler

nano/lib/stats.cpp

@@ -545,6 +545,9 @@ std::string nano::stat::type_to_string (stat::type type)
 		case nano::stat::type::vote_cache:
 			res = "vote_cache";
 			break;
+		case nano::stat::type::hinting:
+			res = "hinting";
+			break;
 	}
 	return res;
 }
@@ -937,6 +940,15 @@ std::string nano::stat::detail_to_string (stat::detail detail)
 		case nano::stat::detail::invalid_network:
 			res = "invalid_network";
 			break;
+		case nano::stat::detail::hinted:
+			res = "hinted";
+			break;
+		case nano::stat::detail::insert_failed:
+			res = "insert_failed";
+			break;
+		case nano::stat::detail::missing_block:
+			res = "missing_block";
+			break;
 	}
 	return res;
 }

nano/lib/stats.hpp

@@ -244,7 +244,8 @@ class stat final
 		filter,
 		telemetry,
 		vote_generator,
-		vote_cache
+		vote_cache,
+		hinting
 	};
 
 	/** Optional detail type */
@@ -413,7 +414,12 @@ class stat final
 		generator_broadcasts,
 		generator_replies,
 		generator_replies_discarded,
-		generator_spacing
+		generator_spacing,
+
+		// hinting
+		hinted,
+		insert_failed,
+		missing_block,
 	};
 
 	/** Direction of the stat. If the direction is irrelevant, use in */

nano/lib/threading.cpp

@@ -96,6 +96,9 @@ std::string nano::thread_role::get_string (nano::thread_role::name role)
 		case nano::thread_role::name::backlog_population:
 			thread_role_name_string = "Backlog";
 			break;
+		case nano::thread_role::name::election_hinting:
+			thread_role_name_string = "Hinting";
+			break;
 		default:
 			debug_assert (false && "nano::thread_role::get_string unhandled thread role");
 	}

nano/lib/threading.hpp

@@ -43,7 +43,8 @@ namespace thread_role
 		db_parallel_traversal,
 		election_scheduler,
 		unchecked,
-		backlog_population
+		backlog_population,
+		election_hinting
 	};
 
 	/*

nano/node/CMakeLists.txt

@@ -68,6 +68,8 @@ add_library(
   election_scheduler.cpp
   gap_cache.hpp
   gap_cache.cpp
+  hinted_scheduler.hpp
+  hinted_scheduler.cpp
   inactive_cache_information.hpp
   inactive_cache_information.cpp
   inactive_cache_status.hpp

nano/node/active_transactions.cpp

@@ -166,10 +166,28 @@ void nano::active_transactions::block_already_cemented_callback (nano::block_has
 	remove_election_winner_details (hash_a);
 }
 
+int64_t nano::active_transactions::limit () const
+{
+	return static_cast<int64_t> (node.config.active_elections_size);
+}
+
+int64_t nano::active_transactions::hinted_limit () const
+{
+	const uint64_t limit = node.config.active_elections_hinted_limit_percentage * node.config.active_elections_size / 100;
+	return static_cast<int64_t> (limit);
+}
+
 int64_t nano::active_transactions::vacancy () const
 {
 	nano::lock_guard<nano::mutex> lock{ mutex };
-	auto result = static_cast<int64_t> (node.config.active_elections_size) - static_cast<int64_t> (roots.size ());
+	auto result = limit () - static_cast<int64_t> (roots.size ());
+	return result;
+}
+
+int64_t nano::active_transactions::vacancy_hinted () const
+{
+	nano::lock_guard<nano::mutex> lock{ mutex };
+	auto result = hinted_limit () - active_hinted_elections_count;
 	return result;
 }
 
@@ -422,15 +440,10 @@ nano::election_insertion_result nano::active_transactions::insert_impl (nano::un
 
 nano::election_insertion_result nano::active_transactions::insert_hinted (std::shared_ptr<nano::block> const & block_a)
 {
-	nano::unique_lock<nano::mutex> lock (mutex);
+	debug_assert (block_a != nullptr);
+	debug_assert (vacancy_hinted () > 0); // Should only be called when there are free hinted election slots
 
-	const std::size_t limit = node.config.active_elections_hinted_limit_percentage * node.config.active_elections_size / 100;
-	if (active_hinted_elections_count >= limit)
-	{
-		// Reached maximum number of hinted elections, drop new ones
-		node.stats.inc (nano::stat::type::election, nano::stat::detail::election_hinted_overflow);
-		return {};
-	}
+	nano::unique_lock<nano::mutex> lock{ mutex };
 
 	auto result = insert_impl (lock, block_a, nano::election_behavior::hinted);
 	if (result.inserted)
@@ -446,7 +459,10 @@ nano::vote_code nano::active_transactions::vote (std::shared_ptr<nano::vote> con
 	nano::vote_code result{ nano::vote_code::indeterminate };
 	// If all hashes were recently confirmed then it is a replay
 	unsigned recently_confirmed_counter (0);
+
 	std::vector<std::pair<std::shared_ptr<nano::election>, nano::block_hash>> process;
+	std::vector<nano::block_hash> inactive; // Hashes that should be added to inactive vote cache
+
 	{
 		nano::unique_lock<nano::mutex> lock (mutex);
 		for (auto const & hash : vote_a->hashes)
@@ -458,10 +474,7 @@ nano::vote_code nano::active_transactions::vote (std::shared_ptr<nano::vote> con
 			}
 			else if (!recently_confirmed.exists (hash))
 			{
-				lock.unlock ();
-				add_inactive_vote_cache (hash, vote_a);
-				check_inactive_vote_cache (hash);
-				lock.lock ();
+				inactive.emplace_back (hash);
 			}
 			else
 			{
@@ -470,6 +483,12 @@ nano::vote_code nano::active_transactions::vote (std::shared_ptr<nano::vote> con
 		}
 	}
 
+	// Process inactive votes outside of the critical section
+	for (auto & hash : inactive)
+	{
+		add_inactive_vote_cache (hash, vote_a);
+	}
+
 	if (!process.empty ())
 	{
 		bool replay (false);
@@ -663,48 +682,6 @@ void nano::active_transactions::add_inactive_vote_cache (nano::block_hash const
 	}
 }
 
-void nano::active_transactions::check_inactive_vote_cache (nano::block_hash const & hash)
-{
-	if (auto entry = node.inactive_vote_cache.find (hash); entry)
-	{
-		const auto min_tally = (node.online_reps.trended () / 100) * node.config.election_hint_weight_percent;
-
-		// Check that we passed minimum voting weight threshold to start a hinted election
-		if (entry->tally > min_tally)
-		{
-			auto transaction (node.store.tx_begin_read ());
-			auto block = node.store.block.get (transaction, hash);
-			// Check if we have the block in ledger
-			if (block)
-			{
-				// We have the block, check that it's not yet confirmed
-				if (!node.block_confirmed_or_being_confirmed (transaction, hash))
-				{
-					insert_hinted (block);
-				}
-			}
-			else
-			{
-				// We don't have the block yet, try to bootstrap it
-				// TODO: Details of bootstraping a block are not `active_transactions` concern, encapsulate somewhere
-				if (!node.ledger.pruning || !node.store.pruned.exists (transaction, hash))
-				{
-					node.gap_cache.bootstrap_start (hash);
-				}
-			}
-		}
-	}
-}
-
-/*
- * This is called when a new block is received from live network
- * We check if maybe we already have enough inactive votes stored for it to start an election
- */
-void nano::active_transactions::trigger_inactive_votes_cache_election (std::shared_ptr<nano::block> const & block_a)
-{
-	check_inactive_vote_cache (block_a->hash ());
-}
-
 std::size_t nano::active_transactions::election_winner_details_size ()
 {
 	nano::lock_guard<nano::mutex> guard (election_winner_details_mutex);
@@ -724,22 +701,27 @@ std::unique_ptr<nano::container_info_component> nano::collect_container_info (ac
 	std::size_t blocks_count;
 	std::size_t recently_confirmed_count;
 	std::size_t recently_cemented_count;
+	std::size_t hinted_count;
 
 	{
 		nano::lock_guard<nano::mutex> guard (active_transactions.mutex);
 		roots_count = active_transactions.roots.size ();
 		blocks_count = active_transactions.blocks.size ();
 		recently_confirmed_count = active_transactions.recently_confirmed.size ();
 		recently_cemented_count = active_transactions.recently_cemented.size ();
+		hinted_count = active_transactions.active_hinted_elections_count;
 	}
 
 	auto composite = std::make_unique<container_info_composite> (name);
 	composite->add_component (std::make_unique<container_info_leaf> (container_info{ "roots", roots_count, sizeof (decltype (active_transactions.roots)::value_type) }));
 	composite->add_component (std::make_unique<container_info_leaf> (container_info{ "blocks", blocks_count, sizeof (decltype (active_transactions.blocks)::value_type) }));
 	composite->add_component (std::make_unique<container_info_leaf> (container_info{ "election_winner_details", active_transactions.election_winner_details_size (), sizeof (decltype (active_transactions.election_winner_details)::value_type) }));
-	composite->add_component (std::make_unique<container_info_leaf> (container_info{ "recently_confirmed", recently_confirmed_count, sizeof (decltype (active_transactions.recently_confirmed.confirmed)::value_type) }));
-	composite->add_component (std::make_unique<container_info_leaf> (container_info{ "recently_cemented", recently_cemented_count, sizeof (decltype (active_transactions.recently_cemented.cemented)::value_type) }));
+	composite->add_component (std::make_unique<container_info_leaf> (container_info{ "hinted", hinted_count, 0 }));
 	composite->add_component (collect_container_info (active_transactions.generator, "generator"));
+
+	composite->add_component (active_transactions.recently_confirmed.collect_container_info ("recently_confirmed"));
+	composite->add_component (active_transactions.recently_cemented.collect_container_info ("recently_cemented"));
+
 	return composite;
 }
 
@@ -798,6 +780,15 @@ nano::recently_confirmed_cache::entry_t nano::recently_confirmed_cache::back ()
 	return confirmed.back ();
 }
 
+std::unique_ptr<nano::container_info_component> nano::recently_confirmed_cache::collect_container_info (const std::string & name)
+{
+	nano::unique_lock<nano::mutex> lock{ mutex };
+
+	auto composite = std::make_unique<container_info_composite> (name);
+	composite->add_component (std::make_unique<container_info_leaf> (container_info{ "confirmed", confirmed.size (), sizeof (decltype (confirmed)::value_type) }));
+	return composite;
+}
+
 /*
  * class recently_cemented
  */
@@ -828,3 +819,12 @@ std::size_t nano::recently_cemented_cache::size () const
 	nano::lock_guard<nano::mutex> guard{ mutex };
 	return cemented.size ();
 }
+
+std::unique_ptr<nano::container_info_component> nano::recently_cemented_cache::collect_container_info (const std::string & name)
+{
+	nano::unique_lock<nano::mutex> lock{ mutex };
+
+	auto composite = std::make_unique<container_info_composite> (name);
+	composite->add_component (std::make_unique<container_info_leaf> (container_info{ "cemented", cemented.size (), sizeof (decltype (cemented)::value_type) }));
+	return composite;
+}

nano/node/active_transactions.hpp

@@ -76,7 +76,8 @@ class recently_confirmed_cache final
 
 	mutable nano::mutex mutex;
 
-	friend std::unique_ptr<container_info_component> collect_container_info (active_transactions &, std::string const &);
+public: // Container info
+	std::unique_ptr<container_info_component> collect_container_info (std::string const &);
 };
 
 /*
@@ -99,7 +100,8 @@ class recently_cemented_cache final
 
 	mutable nano::mutex mutex;
 
-	friend std::unique_ptr<container_info_component> collect_container_info (active_transactions &, std::string const &);
+public: // Container info
+	std::unique_ptr<container_info_component> collect_container_info (std::string const &);
 };
 
 class election_insertion_result final
@@ -146,6 +148,12 @@ class active_transactions final
 
 	explicit active_transactions (nano::node &, nano::confirmation_height_processor &);
 	~active_transactions ();
+
+	/*
+	 * Starts new election with hinted behavior type
+	 * Hinted elections have shorter timespan and only can take up limited space inside active elections container
+	 */
+	nano::election_insertion_result insert_hinted (std::shared_ptr<nano::block> const & block_a);
 	// Distinguishes replay votes, cannot be determined if the block is not in any election
 	nano::vote_code vote (std::shared_ptr<nano::vote> const &);
 	// Is the root of this block in the roots container
@@ -170,13 +178,25 @@ class active_transactions final
 	void block_cemented_callback (std::shared_ptr<nano::block> const &);
 	void block_already_cemented_callback (nano::block_hash const &);
 
+	/*
+	 * Maximum number of all elections that should be present in this container.
+	 * This is only a soft limit, it is possible for this container to exceed this count.
+	 */
+	int64_t limit () const;
+	/*
+	 * Maximum number of hinted elections that should be present in this container.
+	 */
+	int64_t hinted_limit () const;
 	int64_t vacancy () const;
+	/*
+	 * How many election slots are available for hinted elections.
+	 * The limit of AEC taken up by hinted elections is controlled by `node_config::active_elections_hinted_limit_percentage`
+	 */
+	int64_t vacancy_hinted () const;
 	std::function<void ()> vacancy_update{ [] () {} };
 
 	std::unordered_map<nano::block_hash, std::shared_ptr<nano::election>> blocks;
 
-	// Inserts an election if conditions are met
-	void trigger_inactive_votes_cache_election (std::shared_ptr<nano::block> const &);
 	nano::election_scheduler & scheduler;
 	nano::confirmation_height_processor & confirmation_height_processor;
 	nano::node & node;
@@ -197,10 +217,7 @@ class active_transactions final
 	std::unordered_map<nano::block_hash, std::shared_ptr<nano::election>> election_winner_details;
 
 	// Call action with confirmed block, may be different than what we started with
-	// clang-format off
-	nano::election_insertion_result insert_impl (nano::unique_lock<nano::mutex> &, std::shared_ptr<nano::block> const&, nano::election_behavior = nano::election_behavior::normal, std::function<void(std::shared_ptr<nano::block>const&)> const & = nullptr);
-	// clang-format on
-	nano::election_insertion_result insert_hinted (std::shared_ptr<nano::block> const & block_a);
+	nano::election_insertion_result insert_impl (nano::unique_lock<nano::mutex> &, std::shared_ptr<nano::block> const &, nano::election_behavior = nano::election_behavior::normal, std::function<void (std::shared_ptr<nano::block> const &)> const & = nullptr);
 	void request_loop ();
 	void request_confirm (nano::unique_lock<nano::mutex> &);
 	void erase (nano::qualified_root const &);
@@ -209,8 +226,11 @@ class active_transactions final
 	// Returns a list of elections sorted by difficulty, mutex must be locked
 	std::vector<std::shared_ptr<nano::election>> list_active_impl (std::size_t) const;
 
+	/*
+	 * Checks if vote passes minimum representative weight threshold and adds it to inactive vote cache
+	 * TODO: Should be moved to `vote_cache` class
+	 */
 	void add_inactive_vote_cache (nano::block_hash const & hash, std::shared_ptr<nano::vote> const vote);
-	void check_inactive_vote_cache (nano::block_hash const & hash);
 
 	nano::condition_variable condition;
 	bool started{ false };

nano/node/blockprocessor.cpp

@@ -320,10 +320,9 @@ void nano::block_processor::process_live (nano::transaction const & transaction_
 		auto account = block_a->account ().is_zero () ? block_a->sideband ().account : block_a->account ();
 		node.scheduler.activate (account, transaction_a);
 	}
-	else
-	{
-		node.active.trigger_inactive_votes_cache_election (block_a);
-	}
+
+	// Notify inactive vote cache about a new live block
+	node.inactive_vote_cache.trigger (block_a->hash ());
 
 	// Announce block contents to the network
 	if (origin_a == nano::block_origin::local)

nano/node/election_scheduler.cpp

@@ -99,7 +99,12 @@ bool nano::election_scheduler::manual_queue_predicate () const
 
 bool nano::election_scheduler::overfill_predicate () const
 {
-	return node.active.vacancy () < 0;
+	/*
+	 * Both normal and hinted election schedulers are well-behaved, meaning they first check for AEC vacancy before inserting new elections.
+	 * However, it is possible that AEC will be temporarily overfilled in case it's running at full capacity and election hinting or manual queue kicks in.
+	 * That case will lead to unwanted churning of elections, so this allows for AEC to be overfilled to 125% until erasing of elections happens.
+	 */
+	return node.active.vacancy () < -(node.active.limit () / 4);
 }
 
 void nano::election_scheduler::run ()