Merge remote-tracking branch 'upstream/develop' into tcp/attempts_list

CMakeLists.txt

@@ -53,6 +53,9 @@ set (NANO_FUZZER_TEST OFF CACHE BOOL "")
 option (NANO_STACKTRACE_BACKTRACE "Use BOOST_STACKTRACE_USE_BACKTRACE in stacktraces, for POSIX" OFF)
 if (NANO_STACKTRACE_BACKTRACE)
 	add_definitions(-DNANO_STACKTRACE_BACKTRACE=1)
+	if (BACKTRACE_INCLUDE)
+		add_definitions(-DBOOST_STACKTRACE_BACKTRACE_INCLUDE_FILE=${BACKTRACE_INCLUDE})
+	endif()
 endif ()
 
 if (${NANO_TIMED_LOCKS} GREATER 0)

ci/actions/windows/build.ps1

@@ -1,5 +1,4 @@
-# Stop immediately if any error happens
-$ErrorActionPreference = "Stop"
+$ErrorActionPreference = "Continue"
 
 if (${env:artifact} -eq 1) {
     if ( ${env:BETA} -eq 1 ) {

ci/actions/windows/deploy.ps1

@@ -1,5 +1,4 @@
-# Stop immediately if any error happens
-$ErrorActionPreference = "Stop"
+$ErrorActionPreference = "Continue"
 
 if ( ${env:BETA} -eq 1 ) {
     $network_cfg="beta"

ci/actions/windows/disable_windows_defender.ps1

@@ -1,3 +1,5 @@
+$ErrorActionPreference = "Continue"
+
 Set-MpPreference -DisableArchiveScanning $true
 Set-MpPreference -DisableRealtimeMonitoring $true
 Set-MpPreference -DisableBehaviorMonitoring $true

ci/actions/windows/install_deps.ps1

@@ -1,3 +1,5 @@
+$ErrorActionPreference = "Continue"
+
 function Get-RedirectedUri {
     <#
     .SYNOPSIS

ci/actions/windows/signing.ps1

@@ -1,3 +1,5 @@
+$ErrorActionPreference = "Continue"
+
 if (Test-Path env:CSC_LINK) {
   $path = Join-Path -Path "$env:TMP" -ChildPath csc.p12
   [IO.File]::WriteAllBytes($path, [Convert]::FromBase64String($env:CSC_LINK))

ci/build-travis.sh

@@ -10,16 +10,22 @@ OS=`uname`
 
 # This is to prevent out of scope access in async_write from asio which is not picked up by static analysers
 if [[ $(grep -rl --exclude="*asio.hpp" "asio::async_write" ./nano) ]]; then
-    echo "using boost::asio::async_write directly is not permitted (except in nano/lib/asio.hpp). Use nano::async_write instead"
+    echo "Using boost::asio::async_write directly is not permitted (except in nano/lib/asio.hpp). Use nano::async_write instead"
     exit 1
 fi
 
 # prevent unsolicited use of std::lock_guard & std::unique_lock outside of allowed areas
 if [[ $(grep -rl --exclude={"*random_pool.cpp","*random_pool.hpp","*random_pool_shuffle.hpp","*locks.hpp","*locks.cpp"} "std::unique_lock\|std::lock_guard\|std::condition_variable" ./nano) ]]; then
-    echo "using std::unique_lock, std::lock_guard or std::condition_variable is not permitted (except in nano/lib/locks.hpp and non-nano dependent libraries). Use the nano::* versions instead"
+    echo "Using std::unique_lock, std::lock_guard or std::condition_variable is not permitted (except in nano/lib/locks.hpp and non-nano dependent libraries). Use the nano::* versions instead"
     exit 1
 fi
 
+if [[ $(grep -rlP "^\s*assert \(" ./nano) ]]; then
+    echo "Using assert is not permitted. Use debug_assert instead."
+    exit 1
+fi
+
+# prevent unsolicited use of std::lock_guard & std::unique_lock outside of allowed areas
 mkdir build
 pushd build
 
@@ -44,8 +50,15 @@ ulimit -S -n 8192
 if [[ "$OS" == 'Linux' ]]; then
     ROCKSDB="-DROCKSDB_LIBRARIES=/tmp/rocksdb/lib/librocksdb.a \
     -DROCKSDB_INCLUDE_DIRS=/tmp/rocksdb/include"
+    if clang --version; then
+        BACKTRACE="-DNANO_STACKTRACE_BACKTRACE=ON \
+        -DBACKTRACE_INCLUDE=</tmp/backtrace.h>"
+    else
+        BACKTRACE="-DNANO_STACKTRACE_BACKTRACE=ON"
+    fi
 else
     ROCKSDB=""
+    BACKTRACE=""
 fi
 
 cmake \
@@ -61,6 +74,7 @@ cmake \
     -DBOOST_ROOT=/tmp/boost/ \
     -DQt5_DIR=${qt_dir} \
     -DCI_TEST="1" \
+    ${BACKTRACE} \
     ${SANITIZERS} \
     ..
 

docker/ci/Dockerfile-clang

@@ -14,6 +14,11 @@ RUN update-alternatives --install /usr/bin/cc cc /usr/bin/clang 100
 RUN update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++ 100
 ENV BOOST_ROOT=/tmp/boost
 
-ADD util/build_prep/fetch_boost.sh fetch_boost.sh
+ADD util/build_prep/bootstrap_boost.sh bootstrap_boost.sh
+RUN ./bootstrap_boost.sh -m -c -B 1.70
 
-RUN ./fetch_boost.sh
+# workaround to get a path that can be easily passed into cmake for
+# BOOST_STACKTRACE_BACKTRACE_INCLUDE_FILE 
+# see https://www.boost.org/doc/libs/1_70_0/doc/html/stacktrace/configuration_and_build.html#stacktrace.configuration_and_build.f3
+
+RUN ln -s /usr/lib/gcc/x86_64-linux-gnu/5/include/backtrace.h /tmp/backtrace.h

nano/core_test/active_transactions.cpp

@@ -370,7 +370,6 @@ TEST (active_transactions, inactive_votes_cache)
 	}
 	node.process_active (send);
 	node.block_processor.flush ();
-	bool confirmed (false);
 	system.deadline_set (5s);
 	while (!node.ledger.block_confirmed (node.store.tx_begin_read (), send->hash ()))
 	{

nano/core_test/block_store.cpp

@@ -727,7 +727,7 @@ TEST (block_store, large_iteration)
 	for (auto i (store->latest_begin (transaction, 0)), n (store->latest_end ()); i != n; ++i)
 	{
 		nano::account current (i->first);
-		assert (current.number () > previous.number ());
+		ASSERT_GT (current.number (), previous.number ());
 		accounts2.insert (current);
 		previous = current;
 	}
@@ -873,8 +873,7 @@ TEST (mdb_block_store, upgrade_v2_v3)
 		auto rep_block = ledger.representative (transaction, ledger.latest (transaction, nano::test_genesis_key.pub));
 		nano::account_info_v5 info_old (info.head, rep_block, info.open_block, info.balance, info.modified);
 		auto status (mdb_put (store.env.tx (transaction), store.accounts_v0, nano::mdb_val (nano::test_genesis_key.pub), nano::mdb_val (sizeof (info_old), &info_old), 0));
-		(void)status;
-		assert (status == 0);
+		ASSERT_EQ (status, 0);
 		store.confirmation_height_del (transaction, nano::genesis_account);
 	}
 	nano::logger_mt logger;
@@ -2320,7 +2319,7 @@ void write_sideband_v15 (nano::mdb_store & store_a, nano::transaction & transact
 	auto block = store_a.block_get (transaction_a, block_a.hash (), &sideband);
 	ASSERT_NE (block, nullptr);
 
-	assert (sideband.details.epoch <= nano::epoch::max);
+	ASSERT_LE (sideband.details.epoch, nano::epoch::max);
 	// Simulated by writing 0 on every of the most significant bits, leaving out epoch only, as if pre-upgrade
 	nano::block_sideband sideband_v15 (sideband.type, sideband.account, sideband.successor, sideband.balance, sideband.timestamp, sideband.height, sideband.details.epoch, false, false, false);
 	std::vector<uint8_t> data;
@@ -2341,8 +2340,7 @@ void modify_account_info_to_v13 (nano::mdb_store & store, nano::transaction cons
 	ASSERT_FALSE (store.account_get (transaction, account, info));
 	nano::account_info_v13 account_info_v13 (info.head, rep_block, info.open_block, info.balance, info.modified, info.block_count, info.epoch ());
 	auto status (mdb_put (store.env.tx (transaction), (info.epoch () == nano::epoch::epoch_0) ? store.accounts_v0 : store.accounts_v1, nano::mdb_val (account), nano::mdb_val (account_info_v13), 0));
-	(void)status;
-	assert (status == 0);
+	ASSERT_EQ (status, 0);
 }
 
 void modify_account_info_to_v14 (nano::mdb_store & store, nano::transaction const & transaction, nano::account const & account, uint64_t confirmation_height, nano::block_hash const & rep_block)
@@ -2351,15 +2349,13 @@ void modify_account_info_to_v14 (nano::mdb_store & store, nano::transaction cons
 	ASSERT_FALSE (store.account_get (transaction, account, info));
 	nano::account_info_v14 account_info_v14 (info.head, rep_block, info.open_block, info.balance, info.modified, info.block_count, confirmation_height, info.epoch ());
 	auto status (mdb_put (store.env.tx (transaction), info.epoch () == nano::epoch::epoch_0 ? store.accounts_v0 : store.accounts_v1, nano::mdb_val (account), nano::mdb_val (account_info_v14), 0));
-	(void)status;
-	assert (status == 0);
+	ASSERT_EQ (status, 0);
 }
 
 void modify_confirmation_height_to_v15 (nano::mdb_store & store, nano::transaction const & transaction, nano::account const & account, uint64_t confirmation_height)
 {
 	auto status (mdb_put (store.env.tx (transaction), store.confirmation_height, nano::mdb_val (account), nano::mdb_val (confirmation_height), 0));
-	(void)status;
-	assert (status == 0);
+	ASSERT_EQ (status, 0);
 }
 
 void modify_genesis_account_info_to_v5 (nano::mdb_store & store, nano::transaction const & transaction)
@@ -2370,7 +2366,6 @@ void modify_genesis_account_info_to_v5 (nano::mdb_store & store, nano::transacti
 	visitor.compute (info.head);
 	nano::account_info_v5 info_old (info.head, visitor.result, info.open_block, info.balance, info.modified);
 	auto status (mdb_put (store.env.tx (transaction), store.accounts_v0, nano::mdb_val (nano::test_genesis_key.pub), nano::mdb_val (sizeof (info_old), &info_old), 0));
-	(void)status;
-	assert (status == 0);
+	ASSERT_EQ (status, 0);
 }
 }

nano/core_test/confirmation_height.cpp

@@ -10,19 +10,21 @@ using namespace std::chrono_literals;
 
 namespace
 {
-void add_callback_stats (nano::node & node, std::vector<nano::block_hash> * observer_order = nullptr)
+void add_callback_stats (nano::node & node, std::vector<nano::block_hash> * observer_order = nullptr, std::mutex * mutex = nullptr)
 {
-	node.observers.blocks.add ([& stats = node.stats, observer_order](nano::election_status const & status_a, nano::account const &, nano::amount const &, bool) {
+	node.observers.blocks.add ([& stats = node.stats, observer_order, mutex](nano::election_status const & status_a, nano::account const &, nano::amount const &, bool) {
 		stats.inc (nano::stat::type::http_callback, nano::stat::detail::http_callback, nano::stat::dir::out);
-		if (observer_order)
+		if (mutex)
 		{
+			nano::lock_guard<std::mutex> guard (*mutex);
+			debug_assert (observer_order);
 			observer_order->push_back (status_a.winner->hash ());
 		}
 	});
 }
 nano::stat::detail get_stats_detail (nano::confirmation_height_mode mode_a)
 {
-	assert (mode_a == nano::confirmation_height_mode::bounded || mode_a == nano::confirmation_height_mode::unbounded);
+	debug_assert (mode_a == nano::confirmation_height_mode::bounded || mode_a == nano::confirmation_height_mode::unbounded);
 	return (mode_a == nano::confirmation_height_mode::bounded) ? nano::stat::detail::blocks_confirmed_bounded : nano::stat::detail::blocks_confirmed_unbounded;
 }
 }
@@ -1270,7 +1272,8 @@ TEST (confirmation_height, cemented_gap_below_receive)
 		}
 
 		std::vector<nano::block_hash> observer_order;
-		add_callback_stats (*node, &observer_order);
+		std::mutex mutex;
+		add_callback_stats (*node, &observer_order, &mutex);
 
 		node->block_confirm (open1);
 		system.deadline_set (10s);
@@ -1289,6 +1292,7 @@ TEST (confirmation_height, cemented_gap_below_receive)
 
 		// Check that the order of callbacks is correct
 		std::vector<nano::block_hash> expected_order = { send.hash (), open.hash (), send1.hash (), receive1.hash (), send2.hash (), dummy_send.hash (), receive2.hash (), dummy_send1.hash (), send3.hash (), open1->hash () };
+		nano::lock_guard<std::mutex> guard (mutex);
 		ASSERT_EQ (observer_order, expected_order);
 	};
 

nano/core_test/confirmation_solicitor.cpp

@@ -12,15 +12,16 @@ TEST (confirmation_solicitor, batches)
 	nano::node_config node_config (nano::get_available_port (), system.logging);
 	node_config.enable_voting = false;
 	node_config.frontiers_confirmation = nano::frontiers_confirmation_mode::disabled;
-	auto & node1 = *system.add_node (node_config);
-	auto channel1 (node1.network.udp_channels.create (node1.network.endpoint ()));
-	// Solicitor will only solicit from this representative
-	nano::representative representative (nano::test_genesis_key.pub, nano::genesis_amount, channel1);
-	node_config.peering_port = nano::get_available_port ();
 	nano::node_flags node_flags;
+	node_flags.disable_udp = false;
+	auto & node1 = *system.add_node (node_config, node_flags);
+	node_config.peering_port = nano::get_available_port ();
 	// To prevent races on the solicitor
 	node_flags.disable_request_loop = true;
 	auto & node2 = *system.add_node (node_config, node_flags);
+	// Solicitor will only solicit from this representative
+	auto channel1 (node2.network.udp_channels.create (node1.network.endpoint ()));
+	nano::representative representative (nano::test_genesis_key.pub, nano::genesis_amount, channel1);
 	// Lock active_transactions which uses the solicitor
 	{
 		nano::lock_guard<std::mutex> active_guard (node2.active.mutex);
@@ -43,6 +44,7 @@ TEST (confirmation_solicitor, batches)
 		// Broadcasting should be immediate
 		ASSERT_EQ (0, node2.stats.count (nano::stat::type::message, nano::stat::detail::publish, nano::stat::dir::out));
 		ASSERT_FALSE (node2.active.solicitor.broadcast (*election));
+		system.deadline_set (5s);
 		while (node2.stats.count (nano::stat::type::message, nano::stat::detail::publish, nano::stat::dir::out) < 1)
 		{
 			ASSERT_NO_ERROR (system.poll ());

nano/core_test/fakes/websocket_client.hpp

@@ -45,14 +45,14 @@ class fake_websocket_client
 
 	void await_ack ()
 	{
-		assert (socket->is_open ());
+		debug_assert (socket->is_open ());
 		boost::beast::flat_buffer buffer;
 		socket->read (buffer);
 	}
 
 	boost::optional<std::string> get_response (std::chrono::seconds const deadline = 5s)
 	{
-		assert (deadline > 0s);
+		debug_assert (deadline > 0s);
 		boost::optional<std::string> result;
 		auto buffer (std::make_shared<boost::beast::flat_buffer> ());
 		socket->async_read (*buffer, [&result, &buffer, socket = this->socket](boost::beast::error_code const & ec, std::size_t const /*n*/) {

nano/core_test/memory_pool.cpp

@@ -50,7 +50,7 @@ size_t get_allocated_size ()
 	std::vector<size_t> allocated;
 	record_allocations_new_delete_allocator<T> alloc (&allocated);
 	(void)std::allocate_shared<T, record_allocations_new_delete_allocator<T>> (alloc);
-	assert (allocated.size () == 1);
+	debug_assert (allocated.size () == 1);
 	return allocated.front ();
 }
 }

nano/core_test/network_filter.cpp

@@ -10,7 +10,7 @@ TEST (network_filter, unit)
 {
 	nano::genesis genesis;
 	nano::network_filter filter (1);
-	auto one_block = [&genesis, &filter](std::shared_ptr<nano::block> const & block_a, bool expect_duplicate_a) {
+	auto one_block = [&filter](std::shared_ptr<nano::block> const & block_a, bool expect_duplicate_a) {
 		nano::publish message (block_a);
 		auto bytes (message.to_bytes ());
 		nano::bufferstream stream (bytes->data (), bytes->size ());