confirm_req_hash to reduce bandwidth usage

nano/core_test/block.cpp

@@ -332,6 +332,58 @@ TEST (block, confirm_req_serialization)
 	ASSERT_EQ (*req.block, *req2.block);
 }
 
+TEST (block, confirm_req_hash_serialization)
+{
+	nano::keypair key1;
+	nano::keypair key2;
+	nano::send_block block (1, key2.pub, 200, nano::keypair ().prv, 2, 3);
+	nano::confirm_req req (block.hash (), nano::uint512_union (block.previous (), block.root ()));
+	std::vector<uint8_t> bytes;
+	{
+		nano::vectorstream stream (bytes);
+		req.serialize (stream);
+	}
+	auto error (false);
+	nano::bufferstream stream2 (bytes.data (), bytes.size ());
+	nano::message_header header (error, stream2);
+	nano::confirm_req req2 (error, stream2, header);
+	ASSERT_FALSE (error);
+	ASSERT_EQ (req, req2);
+	ASSERT_EQ (req.roots_hashes, req2.roots_hashes);
+}
+
+TEST (block, confirm_req_hash_batch_serialization)
+{
+	nano::keypair key;
+	nano::keypair representative;
+	std::vector<std::pair<nano::block_hash, nano::uint512_union>> roots_hashes;
+	nano::state_block open (key.pub, 0, representative.pub, 2, 4, key.prv, key.pub, 5);
+	roots_hashes.push_back (std::make_pair (open.hash (), nano::uint512_union (open.previous (), open.root ())));
+	for (auto i (roots_hashes.size ()); i < 7; i++)
+	{
+		nano::keypair key1;
+		nano::keypair previous;
+		nano::state_block block (key1.pub, previous.pub, representative.pub, 2, 4, key1.prv, key1.pub, 5);
+		roots_hashes.push_back (std::make_pair (block.hash (), nano::uint512_union (block.previous (), block.root ())));
+	}
+	roots_hashes.push_back (std::make_pair (open.hash (), nano::uint512_union (0, open.root ())));
+	nano::confirm_req req (roots_hashes);
+	std::vector<uint8_t> bytes;
+	{
+		nano::vectorstream stream (bytes);
+		req.serialize (stream);
+	}
+	auto error (false);
+	nano::bufferstream stream2 (bytes.data (), bytes.size ());
+	nano::message_header header (error, stream2);
+	nano::confirm_req req2 (error, stream2, header);
+	ASSERT_FALSE (error);
+	ASSERT_EQ (req, req2);
+	ASSERT_EQ (req.roots_hashes, req2.roots_hashes);
+	ASSERT_EQ (req.roots_hashes, roots_hashes);
+	ASSERT_EQ (req2.roots_hashes, roots_hashes);
+}
+
 TEST (state_block, serialization)
 {
 	nano::keypair key1;

nano/core_test/message_parser.cpp

@@ -130,6 +130,38 @@ TEST (message_parser, exact_confirm_req_size)
 	ASSERT_NE (parser.status, nano::message_parser::parse_status::success);
 }
 
+TEST (message_parser, exact_confirm_req_hash_size)
+{
+	nano::system system (24000, 1);
+	test_visitor visitor;
+	nano::block_uniquer block_uniquer;
+	nano::vote_uniquer vote_uniquer (block_uniquer);
+	nano::message_parser parser (block_uniquer, vote_uniquer, visitor, system.work);
+	nano::send_block block (1, 1, 2, nano::keypair ().prv, 4, system.work.generate (1));
+	nano::confirm_req message (block.hash (), nano::uint512_union (block.previous (), block.root ()));
+	std::vector<uint8_t> bytes;
+	{
+		nano::vectorstream stream (bytes);
+		message.serialize (stream);
+	}
+	ASSERT_EQ (0, visitor.confirm_req_count);
+	ASSERT_EQ (parser.status, nano::message_parser::parse_status::success);
+	auto error (false);
+	nano::bufferstream stream1 (bytes.data (), bytes.size ());
+	nano::message_header header1 (error, stream1);
+	ASSERT_FALSE (error);
+	parser.deserialize_confirm_req (stream1, header1);
+	ASSERT_EQ (1, visitor.confirm_req_count);
+	ASSERT_EQ (parser.status, nano::message_parser::parse_status::success);
+	bytes.push_back (0);
+	nano::bufferstream stream2 (bytes.data (), bytes.size ());
+	nano::message_header header2 (error, stream2);
+	ASSERT_FALSE (error);
+	parser.deserialize_confirm_req (stream2, header2);
+	ASSERT_EQ (1, visitor.confirm_req_count);
+	ASSERT_NE (parser.status, nano::message_parser::parse_status::success);
+}
+
 TEST (message_parser, exact_publish_size)
 {
 	nano::system system (24000, 1);

nano/node/common.cpp

@@ -302,7 +302,7 @@ void nano::message_parser::deserialize_confirm_req (nano::stream & stream_a, nan
 	nano::confirm_req incoming (error, stream_a, header_a, &block_uniquer);
 	if (!error && at_end (stream_a))
 	{
-		if (!nano::work_validate (*incoming.block))
+		if (incoming.block == nullptr || !nano::work_validate (*incoming.block))
 		{
 			visitor.confirm_req (incoming);
 		}
@@ -484,12 +484,59 @@ block (block_a)
 {
 	header.block_type_set (block->type ());
 }
+nano::confirm_req::confirm_req (std::vector<std::pair<nano::block_hash, nano::uint512_union>> const & roots_hashes_a) :
+message (nano::message_type::confirm_req),
+roots_hashes (roots_hashes_a)
+{
+	// not_a_block (1) block type for hashes + roots request
+	header.block_type_set (nano::block_type::not_a_block);
+}
+
+nano::confirm_req::confirm_req (nano::block_hash const & hash_a, nano::uint512_union const & root_a) :
+message (nano::message_type::confirm_req),
+roots_hashes (std::vector<std::pair<nano::block_hash, nano::uint512_union>> (1, std::make_pair (hash_a, root_a)))
+{
+	assert (!roots_hashes.empty ());
+	// not_a_block (1) block type for hashes + roots request
+	header.block_type_set (nano::block_type::not_a_block);
+}
 
 bool nano::confirm_req::deserialize (nano::stream & stream_a, nano::block_uniquer * uniquer_a)
 {
+	bool result (true);
 	assert (header.type == nano::message_type::confirm_req);
-	block = nano::deserialize_block (stream_a, header.block_type (), uniquer_a);
-	auto result (block == nullptr);
+	if (header.block_type () == nano::block_type::not_a_block)
+	{
+		uint8_t count (0);
+		result = read (stream_a, count);
+		assert (count <= 32);
+		bool roots_previous[32] = { false };
+		result = read (stream_a, roots_previous);
+		for (auto i (0); i != count && !result; ++i)
+		{
+			nano::block_hash block_hash (0);
+			nano::block_hash root (0);
+			result = read (stream_a, block_hash);
+			if (!result && !block_hash.is_zero ())
+			{
+				result = read (stream_a, root);
+				if (!result && !root.is_zero ())
+				{
+					nano::uint512_union root_uint512 (roots_previous[i] ? root : 0, root);
+					roots_hashes.push_back (std::make_pair (block_hash, root_uint512));
+				}
+			}
+		}
+		if (!result)
+		{
+			result = roots_hashes.empty () || (roots_hashes.size () != count);
+		}
+	}
+	else
+	{
+		block = nano::deserialize_block (stream_a, header.block_type (), uniquer_a);
+		result = block == nullptr;
+	}
 	return result;
 }
 
@@ -500,14 +547,63 @@ void nano::confirm_req::visit (nano::message_visitor & visitor_a) const
 
 void nano::confirm_req::serialize (nano::stream & stream_a) const
 {
-	assert (block != nullptr);
 	header.serialize (stream_a);
-	block->serialize (stream_a);
+	if (header.block_type () == nano::block_type::not_a_block)
+	{
+		assert (!roots_hashes.empty ());
+		// Calculate size
+		assert (roots_hashes.size () <= 32);
+		uint8_t count (roots_hashes.size ());
+		write (stream_a, count);
+		/* Calculate uint512_union roots status
+		true = previous == root
+		false = previous == 0 */
+		bool roots_previous[32] = { false };
+		for (auto i (0); i != count; ++i)
+		{
+			roots_previous[i] = !roots_hashes[i].second.uint256s[0].is_zero ();
+			assert (roots_hashes[i].second.uint256s[0].is_zero () || roots_hashes[i].second.uint256s[0] == roots_hashes[i].second.uint256s[1]);
+		}
+		write (stream_a, roots_previous);
+		// Write hashes & roots
+		for (auto & root_hash : roots_hashes)
+		{
+			write (stream_a, root_hash.first);
+			write (stream_a, root_hash.second.uint256s[1]);
+		}
+	}
+	else
+	{
+		assert (block != nullptr);
+		block->serialize (stream_a);
+	}
 }
 
 bool nano::confirm_req::operator== (nano::confirm_req const & other_a) const
 {
-	return *block == *other_a.block;
+	bool equal (false);
+	if (block != nullptr && other_a.block != nullptr)
+	{
+		equal = *block == *other_a.block;
+	}
+	else if (!roots_hashes.empty () && !other_a.roots_hashes.empty ())
+	{
+		equal = roots_hashes == other_a.roots_hashes;
+	}
+	return equal;
+}
+
+std::string nano::confirm_req::roots_string () const
+{
+	std::string result;
+	for (auto & root_hash : roots_hashes)
+	{
+		result += root_hash.first.to_string ();
+		result += ":";
+		result += root_hash.second.to_string ();
+		result += ", ";
+	}
+	return result;
 }
 
 nano::confirm_ack::confirm_ack (bool & error_a, nano::stream & stream_a, nano::message_header const & header_a, nano::vote_uniquer * uniquer_a) :

nano/node/common.hpp

@@ -297,11 +297,15 @@ class confirm_req : public message
 public:
 	confirm_req (bool &, nano::stream &, nano::message_header const &, nano::block_uniquer * = nullptr);
 	confirm_req (std::shared_ptr<nano::block>);
+	confirm_req (std::vector<std::pair<nano::block_hash, nano::uint512_union>> const &);
+	confirm_req (nano::block_hash const &, nano::uint512_union const &);
 	bool deserialize (nano::stream &, nano::block_uniquer * = nullptr);
 	void serialize (nano::stream &) const override;
 	void visit (nano::message_visitor &) const override;
 	bool operator== (nano::confirm_req const &) const;
 	std::shared_ptr<nano::block> block;
+	std::vector<std::pair<nano::block_hash, nano::uint512_union>> roots_hashes;
+	std::string roots_string () const;
 };
 class confirm_ack : public message
 {