Merge remote-tracking branch 'upstream/develop' into network/tcp_proc…

…ess_messages_threads

.github/workflows/analyzers.yml

@@ -0,0 +1,15 @@
+name: Static Analyzers
+
+on: [push, pull_request]
+
+jobs:
+  clang_format:
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@50fbc62
+      - name: Get clang-format 8
+        env: 
+          DEBIAN_FRONTEND: noninteractive
+        run: sudo update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-8 1000
+      - name: Clang Format
+        run: ci/check-commit-format.sh 

.github/workflows/release_test.yml

@@ -24,6 +24,7 @@ jobs:
 
   gcc_test:
     runs-on: ubuntu-18.04
+    timeout-minutes: 60
     steps:
       - uses: actions/checkout@722adc6
       - name: Checkout Submodules
@@ -35,6 +36,7 @@ jobs:
 
   clang_test:
     runs-on: ubuntu-18.04
+    timeout-minutes: 60
     steps:
       - uses: actions/checkout@722adc6
       - name: Checkout Submodules
@@ -46,6 +48,7 @@ jobs:
 
   windows_test:
     runs-on: windows-latest
+    timeout-minutes: 60
     steps:
       - uses: actions/checkout@722adc6
       - name: Checkout Submodules

.github/workflows/tests.yml

@@ -20,19 +20,9 @@ jobs:
       - name: Run Tests
         run: ci/build-travis.sh "/tmp/qt/lib/cmake/Qt5";
 
-  clang_format:
-    runs-on: ubuntu-18.04
-    steps:
-      - uses: actions/checkout@50fbc62
-      - name: Get clang-format 8
-        env: 
-          DEBIAN_FRONTEND: noninteractive
-        run: sudo update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-8 1000
-      - name: Clang Format
-        run: ci/check-commit-format.sh 
-
   gcc_test:
     runs-on: ubuntu-18.04
+    timeout-minutes: 60
     steps:
       - uses: actions/checkout@722adc6
       - name: Checkout Submodules
@@ -44,6 +34,7 @@ jobs:
 
   clang_test:
     runs-on: ubuntu-18.04
+    timeout-minutes: 60
     steps:
       - uses: actions/checkout@722adc6
       - name: Checkout Submodules
@@ -55,6 +46,7 @@ jobs:
 
   windows_test:
     runs-on: windows-latest
+    timeout-minutes: 60
     steps:
       - uses: actions/checkout@722adc6
       - name: Windows Defender

nano/core_test/active_transactions.cpp

@@ -5,6 +5,8 @@
 
 #include <gtest/gtest.h>
 
+#include <numeric>
+
 using namespace std::chrono_literals;
 
 namespace nano
@@ -147,12 +149,12 @@ TEST (active_transactions, adjusted_difficulty_priority)
 
 	//genesis and key1,key2 are opened
 	//start chain of 2 on each
-	auto send3 (std::make_shared<nano::state_block> (nano::test_genesis_key.pub, send2->hash (), nano::test_genesis_key.pub, 9 * nano::xrb_ratio, key3.pub, nano::test_genesis_key.prv, nano::test_genesis_key.pub, *system.work.generate (send2->hash (), nano::difficulty::from_multiplier (1500, node1.network_params.network.publish_thresholds.base))));
-	auto send4 (std::make_shared<nano::state_block> (nano::test_genesis_key.pub, send3->hash (), nano::test_genesis_key.pub, 8 * nano::xrb_ratio, key3.pub, nano::test_genesis_key.prv, nano::test_genesis_key.pub, *system.work.generate (send3->hash (), nano::difficulty::from_multiplier (1500, node1.network_params.network.publish_thresholds.base))));
-	auto send5 (std::make_shared<nano::state_block> (key1.pub, open1->hash (), key1.pub, 9 * nano::xrb_ratio, key3.pub, key1.prv, key1.pub, *system.work.generate (open1->hash (), nano::difficulty::from_multiplier (100, node1.network_params.network.publish_thresholds.base))));
-	auto send6 (std::make_shared<nano::state_block> (key1.pub, send5->hash (), key1.pub, 8 * nano::xrb_ratio, key3.pub, key1.prv, key1.pub, *system.work.generate (send5->hash (), nano::difficulty::from_multiplier (100, node1.network_params.network.publish_thresholds.base))));
-	auto send7 (std::make_shared<nano::state_block> (key2.pub, open2->hash (), key2.pub, 9 * nano::xrb_ratio, key3.pub, key2.prv, key2.pub, *system.work.generate (open2->hash (), nano::difficulty::from_multiplier (500, node1.network_params.network.publish_thresholds.base))));
-	auto send8 (std::make_shared<nano::state_block> (key2.pub, send7->hash (), key2.pub, 8 * nano::xrb_ratio, key3.pub, key2.prv, key2.pub, *system.work.generate (send7->hash (), nano::difficulty::from_multiplier (500, node1.network_params.network.publish_thresholds.base))));
+	auto send3 (std::make_shared<nano::state_block> (nano::test_genesis_key.pub, send2->hash (), nano::test_genesis_key.pub, 9 * nano::xrb_ratio, key3.pub, nano::test_genesis_key.prv, nano::test_genesis_key.pub, *system.work.generate (send2->hash (), nano::difficulty::from_multiplier (150, node1.network_params.network.publish_thresholds.base))));
+	auto send4 (std::make_shared<nano::state_block> (nano::test_genesis_key.pub, send3->hash (), nano::test_genesis_key.pub, 8 * nano::xrb_ratio, key3.pub, nano::test_genesis_key.prv, nano::test_genesis_key.pub, *system.work.generate (send3->hash (), nano::difficulty::from_multiplier (150, node1.network_params.network.publish_thresholds.base))));
+	auto send5 (std::make_shared<nano::state_block> (key1.pub, open1->hash (), key1.pub, 9 * nano::xrb_ratio, key3.pub, key1.prv, key1.pub, system.work_generate_limited (open1->hash (), nano::difficulty::from_multiplier (10, node1.network_params.network.publish_thresholds.base), nano::difficulty::from_multiplier (50, node1.network_params.network.publish_thresholds.base))));
+	auto send6 (std::make_shared<nano::state_block> (key1.pub, send5->hash (), key1.pub, 8 * nano::xrb_ratio, key3.pub, key1.prv, key1.pub, system.work_generate_limited (send5->hash (), nano::difficulty::from_multiplier (10, node1.network_params.network.publish_thresholds.base), nano::difficulty::from_multiplier (50, node1.network_params.network.publish_thresholds.base))));
+	auto send7 (std::make_shared<nano::state_block> (key2.pub, open2->hash (), key2.pub, 9 * nano::xrb_ratio, key3.pub, key2.prv, key2.pub, system.work_generate_limited (open2->hash (), nano::difficulty::from_multiplier (50, node1.network_params.network.publish_thresholds.base), nano::difficulty::from_multiplier (150, node1.network_params.network.publish_thresholds.base))));
+	auto send8 (std::make_shared<nano::state_block> (key2.pub, send7->hash (), key2.pub, 8 * nano::xrb_ratio, key3.pub, key2.prv, key2.pub, system.work_generate_limited (send7->hash (), nano::difficulty::from_multiplier (50, node1.network_params.network.publish_thresholds.base), nano::difficulty::from_multiplier (150, node1.network_params.network.publish_thresholds.base))));
 
 	node1.process_active (send3); // genesis
 	node1.process_active (send5); // key1
@@ -849,7 +851,7 @@ TEST (active_transactions, confirmation_consistency)
 		nano::lock_guard<std::mutex> guard (node.active.mutex);
 		ASSERT_EQ (i + 1, node.active.recently_confirmed.size ());
 		ASSERT_EQ (block->qualified_root (), node.active.recently_confirmed.back ().first);
-		ASSERT_EQ (i + 1, node.active.recently_cemented.size ());
+		ASSERT_TIMELY (1s, i + 1 == node.active.recently_cemented.size ()); // done after a callback
 	}
 }
 }
@@ -880,17 +882,37 @@ TEST (active_transactions, insertion_prioritization)
 		node.active.update_active_difficulty (lock);
 	};
 
-	ASSERT_TRUE (node.active.insert (blocks[2]).prioritized);
+	ASSERT_TRUE (node.active.insert (blocks[2]).election->prioritized ());
 	update_active_difficulty ();
-	ASSERT_FALSE (node.active.insert (blocks[3]).prioritized);
+	ASSERT_FALSE (node.active.insert (blocks[3]).election->prioritized ());
 	update_active_difficulty ();
-	ASSERT_TRUE (node.active.insert (blocks[1]).prioritized);
+	ASSERT_TRUE (node.active.insert (blocks[1]).election->prioritized ());
 	update_active_difficulty ();
-	ASSERT_FALSE (node.active.insert (blocks[4]).prioritized);
+	ASSERT_FALSE (node.active.insert (blocks[4]).election->prioritized ());
 	update_active_difficulty ();
-	ASSERT_TRUE (node.active.insert (blocks[0]).prioritized);
+	ASSERT_TRUE (node.active.insert (blocks[0]).election->prioritized ());
 	update_active_difficulty ();
-	ASSERT_FALSE (node.active.insert (blocks[5]).prioritized);
+	ASSERT_FALSE (node.active.insert (blocks[5]).election->prioritized ());
 	update_active_difficulty ();
-	ASSERT_FALSE (node.active.insert (blocks[6]).prioritized);
+	ASSERT_FALSE (node.active.insert (blocks[6]).election->prioritized ());
+}
+
+TEST (active_difficulty, less_than_one)
+{
+	nano::system system (1);
+	auto & node (*system.nodes[0]);
+	nano::unique_lock<std::mutex> lock (node.active.mutex);
+	auto base_active_difficulty = node.network_params.network.publish_thresholds.epoch_1;
+	auto min_active_difficulty = node.network_params.network.publish_thresholds.entry;
+	auto min_multiplier = nano::difficulty::to_multiplier (min_active_difficulty, base_active_difficulty);
+	ASSERT_EQ (node.active.trended_active_difficulty, base_active_difficulty);
+	for (int i = 0; i < node.active.multipliers_cb.size () - 1; ++i)
+	{
+		node.active.multipliers_cb.push_front (min_multiplier);
+	}
+	auto sum (std::accumulate (node.active.multipliers_cb.begin (), node.active.multipliers_cb.end (), double(0)));
+	auto difficulty = nano::difficulty::from_multiplier (sum / node.active.multipliers_cb.size (), node.network_params.network.publish_thresholds.epoch_1);
+	node.active.multipliers_cb.push_front (min_multiplier);
+	node.active.update_active_difficulty (lock);
+	ASSERT_EQ (node.active.trended_active_difficulty, difficulty);
 }

nano/core_test/bootstrap.cpp

@@ -423,8 +423,8 @@ TEST (bootstrap_processor, frontiers_unconfirmed)
 		ASSERT_NO_ERROR (system.poll ());
 	}
 	//Add single excluded peers record (2 records are required to drop peer)
-	node3->bootstrap_initiator.excluded_peers.add (nano::transport::map_endpoint_to_tcp (node1->network.endpoint ()), 0);
-	ASSERT_FALSE (node3->bootstrap_initiator.excluded_peers.check (nano::transport::map_endpoint_to_tcp (node1->network.endpoint ())));
+	node3->network.excluded_peers.add (nano::transport::map_endpoint_to_tcp (node1->network.endpoint ()), 0);
+	ASSERT_FALSE (node3->network.excluded_peers.check (nano::transport::map_endpoint_to_tcp (node1->network.endpoint ())));
 	node3->bootstrap_initiator.bootstrap (node1->network.endpoint ());
 	system.deadline_set (15s);
 	while (node3->bootstrap_initiator.in_progress ())
@@ -434,7 +434,7 @@ TEST (bootstrap_processor, frontiers_unconfirmed)
 	ASSERT_FALSE (node3->ledger.block_exists (send1->hash ()));
 	ASSERT_FALSE (node3->ledger.block_exists (open1->hash ()));
 	ASSERT_EQ (1, node3->stats.count (nano::stat::type::bootstrap, nano::stat::detail::frontier_confirmation_failed, nano::stat::dir::in)); // failed request from node1
-	ASSERT_TRUE (node3->bootstrap_initiator.excluded_peers.check (nano::transport::map_endpoint_to_tcp (node1->network.endpoint ())));
+	ASSERT_TRUE (node3->network.excluded_peers.check (nano::transport::map_endpoint_to_tcp (node1->network.endpoint ())));
 }
 
 TEST (bootstrap_processor, frontiers_confirmed)

nano/core_test/confirmation_height.cpp

@@ -1000,7 +1000,7 @@ TEST (confirmation_height, prioritize_frontiers)
 		transaction.refresh ();
 		node->active.prioritize_frontiers_for_confirmation (transaction, std::chrono::seconds (1), std::chrono::seconds (1));
 		ASSERT_TRUE (priority_orders_match (node->active.priority_wallet_cementable_frontiers, std::array<nano::account, num_accounts>{ key3.pub, nano::genesis_account, key4.pub, key1.pub, key2.pub }));
-		node->active.search_frontiers (transaction);
+		node->active.confirm_prioritized_frontiers (transaction);
 
 		// Check that the active transactions roots contains the frontiers
 		system.deadline_set (std::chrono::seconds (10));
@@ -1208,14 +1208,12 @@ TEST (confirmation_height, dependent_election)
 
 		add_callback_stats (*node);
 
-		// Start an election and vote, should confirm the block
-		node->block_confirm (send2);
-
 		{
 			// The write guard prevents the confirmation height processor doing any writes.
-			// Note: This test could still fail intermittently due to thread scheduling between active and confirmation height.
 			system.deadline_set (10s);
 			auto write_guard = node->write_database_queue.wait (nano::writer::testing);
+			// Start an election and vote, should confirm the block
+			node->block_confirm (send2);
 			while (!node->write_database_queue.contains (nano::writer::confirmation_height))
 			{
 				ASSERT_NO_ERROR (system.poll ());

nano/core_test/confirmation_solicitor.cpp

@@ -33,12 +33,12 @@ TEST (confirmation_solicitor, batches)
 		nano::lock_guard<std::mutex> guard (node2.active.mutex);
 		for (size_t i (0); i < nano::network::confirm_req_hashes_max; ++i)
 		{
-			auto election (std::make_shared<nano::election> (node2, send, nullptr));
+			auto election (std::make_shared<nano::election> (node2, send, nullptr, false));
 			ASSERT_FALSE (solicitor.add (*election));
 		}
 		ASSERT_EQ (1, solicitor.max_confirm_req_batches);
 		// Reached the maximum amount of requests for the channel
-		auto election (std::make_shared<nano::election> (node2, send, nullptr));
+		auto election (std::make_shared<nano::election> (node2, send, nullptr, false));
 		ASSERT_TRUE (solicitor.add (*election));
 		// Broadcasting should be immediate
 		ASSERT_EQ (0, node2.stats.count (nano::stat::type::message, nano::stat::detail::publish, nano::stat::dir::out));

nano/core_test/conflicts.cpp

@@ -275,12 +275,14 @@ TEST (conflicts, adjusted_difficulty)
 	ASSERT_GT (adjusted_difficulties.find (open2->hash ())->second, adjusted_difficulties.find (change1->hash ())->second);
 	// Independent elections can have higher difficulty than adjusted tree
 	nano::keypair key4;
+	auto send5 (std::make_shared<nano::state_block> (key3.pub, change1->hash (), nano::test_genesis_key.pub, 0, key4.pub, key3.prv, key3.pub, *system.work.generate (change1->hash ()))); // Pending for open epoch block
+	node1.process_active (send5);
 	auto open_epoch2 (std::make_shared<nano::state_block> (key4.pub, 0, 0, 0, node1.ledger.epoch_link (nano::epoch::epoch_1), nano::test_genesis_key.prv, nano::test_genesis_key.pub, *system.work.generate (key4.pub, adjusted_difficulties.find (send1->hash ())->second)));
 	ASSERT_GT (open_epoch2->difficulty (), adjusted_difficulties.find (send1->hash ())->second);
 	node1.process_active (open_epoch2);
 	node1.block_processor.flush ();
 	system.deadline_set (3s);
-	while (node1.active.size () != 11)
+	while (node1.active.size () != 12)
 	{
 		ASSERT_NO_ERROR (system.poll ());
 	}

nano/core_test/difficulty.cpp

@@ -1,8 +1,28 @@
+#include <nano/lib/blocks.hpp>
 #include <nano/lib/config.hpp>
+#include <nano/lib/epoch.hpp>
 #include <nano/lib/numbers.hpp>
+#include <nano/lib/work.hpp>
+#include <nano/node/testing.hpp>
 
 #include <gtest/gtest.h>
 
+TEST (system, work_generate_limited)
+{
+	nano::system system;
+	nano::block_hash key (1);
+	nano::network_constants constants;
+	auto min = constants.publish_thresholds.entry;
+	auto max = constants.publish_thresholds.base;
+	for (int i = 0; i < 5; ++i)
+	{
+		auto work = system.work_generate_limited (key, min, max);
+		auto difficulty = nano::work_difficulty (nano::work_version::work_1, key, work);
+		ASSERT_GE (difficulty, min);
+		ASSERT_LT (difficulty, max);
+	}
+}
+
 TEST (difficulty, multipliers)
 {
 	// For ASSERT_DEATH_IF_SUPPORTED
@@ -60,16 +80,6 @@ TEST (difficulty, multipliers)
 	}
 }
 
-TEST (difficulty, network_constants)
-{
-	nano::network_constants constants;
-	ASSERT_NEAR (1., nano::difficulty::to_multiplier (constants.publish_full.epoch_2_receive, constants.publish_full.entry), 1e-10);
-	ASSERT_NEAR (1., nano::difficulty::to_multiplier (constants.publish_full.epoch_2, constants.publish_full.base), 1e-10);
-	ASSERT_NEAR (8., nano::difficulty::to_multiplier (constants.publish_full.epoch_2, constants.publish_full.epoch_1), 1e-10);
-	ASSERT_NEAR (1 / 8., nano::difficulty::to_multiplier (constants.publish_full.epoch_2_receive, constants.publish_full.epoch_1), 1e-10);
-	ASSERT_NEAR (1 / 64., nano::difficulty::to_multiplier (constants.publish_beta.base, constants.publish_full.epoch_1), 1e-10);
-}
-
 TEST (difficulty, overflow)
 {
 	// Overflow max (attempt to overflow & receive lower difficulty)
@@ -110,3 +120,44 @@ TEST (difficulty, zero)
 		ASSERT_EQ (difficulty, nano::difficulty::from_multiplier (multiplier, base));
 	}
 }
+
+TEST (difficulty, network_constants)
+{
+	nano::network_constants constants;
+	auto & full_thresholds = constants.publish_full;
+	auto & beta_thresholds = constants.publish_beta;
+	auto & test_thresholds = constants.publish_test;
+
+	ASSERT_NEAR (8., nano::difficulty::to_multiplier (full_thresholds.epoch_2, full_thresholds.epoch_1), 1e-10);
+	ASSERT_NEAR (1 / 8., nano::difficulty::to_multiplier (full_thresholds.epoch_2_receive, full_thresholds.epoch_1), 1e-10);
+	ASSERT_NEAR (1., nano::difficulty::to_multiplier (full_thresholds.epoch_2_receive, full_thresholds.entry), 1e-10);
+	ASSERT_NEAR (1., nano::difficulty::to_multiplier (full_thresholds.epoch_2, full_thresholds.base), 1e-10);
+
+	ASSERT_NEAR (1 / 64., nano::difficulty::to_multiplier (beta_thresholds.epoch_1, full_thresholds.epoch_1), 1e-10);
+	ASSERT_NEAR (2., nano::difficulty::to_multiplier (beta_thresholds.epoch_2, beta_thresholds.epoch_1), 1e-10);
+	ASSERT_NEAR (1 / 2., nano::difficulty::to_multiplier (beta_thresholds.epoch_2_receive, beta_thresholds.epoch_1), 1e-10);
+	ASSERT_NEAR (1., nano::difficulty::to_multiplier (beta_thresholds.epoch_2_receive, beta_thresholds.entry), 1e-10);
+	ASSERT_NEAR (1., nano::difficulty::to_multiplier (beta_thresholds.epoch_2, beta_thresholds.base), 1e-10);
+
+	ASSERT_NEAR (8., nano::difficulty::to_multiplier (test_thresholds.epoch_2, test_thresholds.epoch_1), 1e-10);
+	ASSERT_NEAR (1 / 8., nano::difficulty::to_multiplier (test_thresholds.epoch_2_receive, test_thresholds.epoch_1), 1e-10);
+	ASSERT_NEAR (1., nano::difficulty::to_multiplier (test_thresholds.epoch_2_receive, test_thresholds.entry), 1e-10);
+	ASSERT_NEAR (1., nano::difficulty::to_multiplier (test_thresholds.epoch_2, test_thresholds.base), 1e-10);
+
+	nano::work_version version{ nano::work_version::work_1 };
+	ASSERT_EQ (constants.publish_thresholds.base, constants.publish_thresholds.epoch_2);
+	ASSERT_EQ (constants.publish_thresholds.base, nano::work_threshold_base (version));
+	ASSERT_EQ (constants.publish_thresholds.entry, nano::work_threshold_entry (version));
+	ASSERT_EQ (constants.publish_thresholds.epoch_1, nano::work_threshold (version, nano::block_details (nano::epoch::epoch_0, false, false, false)));
+	ASSERT_EQ (constants.publish_thresholds.epoch_1, nano::work_threshold (version, nano::block_details (nano::epoch::epoch_1, false, false, false)));
+	ASSERT_EQ (constants.publish_thresholds.epoch_1, nano::work_threshold (version, nano::block_details (nano::epoch::epoch_1, false, false, false)));
+
+	// Send [+ change]
+	ASSERT_EQ (constants.publish_thresholds.epoch_2, nano::work_threshold (version, nano::block_details (nano::epoch::epoch_2, true, false, false)));
+	// Change
+	ASSERT_EQ (constants.publish_thresholds.epoch_2, nano::work_threshold (version, nano::block_details (nano::epoch::epoch_2, false, false, false)));
+	// Receive [+ change] / Open
+	ASSERT_EQ (constants.publish_thresholds.epoch_2_receive, nano::work_threshold (version, nano::block_details (nano::epoch::epoch_2, false, true, false)));
+	// Epoch
+	ASSERT_EQ (constants.publish_thresholds.epoch_2_receive, nano::work_threshold (version, nano::block_details (nano::epoch::epoch_2, false, false, true)));
+}