From 6e651ff556f2e35724468fc8d52be8b576686646 Mon Sep 17 00:00:00 2001 From: Rich Nistuk Date: Wed, 12 Sep 2018 08:53:08 -0700 Subject: [PATCH] KEP-486 Peer Can add itself to Peer List --- node/session.cpp | 2 +- options/options.cpp | 7 + options/options.hpp | 2 + options/options_base.hpp | 8 + options/simple_options.cpp | 5 +- options/simple_options.hpp | 1 + options/test/options_test.cpp | 2 + raft/raft.cpp | 243 +++++++++++++- raft/raft.hpp | 22 +- raft/test/CMakeLists.txt | 2 +- raft/test/raft_add_peers_test.cpp | 513 ++++++++++++++++++++++++++++++ raft/test/raft_test.cpp | 42 ++- swarm/main.cpp | 5 +- utils/CMakeLists.txt | 3 +- utils/container.hpp | 42 +++ utils/test/utils_test.cpp | 45 ++- 16 files changed, 919 insertions(+), 25 deletions(-) create mode 100644 raft/test/raft_add_peers_test.cpp create mode 100644 utils/container.hpp diff --git a/node/session.cpp b/node/session.cpp index c71ec203..eb3f51fd 100644 --- a/node/session.cpp +++ b/node/session.cpp @@ -98,7 +98,7 @@ session::do_read() { self->handler(msg, self); } - else if(ss.seekg(0); proto_msg.ParseFromIstream(&ss)) + else if (proto_msg.ParseFromIstream(&ss)) { self->proto_handler(proto_msg, self); } diff --git a/options/options.cpp b/options/options.cpp index 1aa72d9d..c15e8f7a 100644 --- a/options/options.cpp +++ b/options/options.cpp @@ -243,3 +243,10 @@ options::peer_validation_enabled() const //TODO: Remove this return this->raw_opts.get(PEER_VALIDATION_ENABLED); } + + +std::string +options::get_signed_key() const +{ + return this->raw_opts.get(SIGNED_KEY); +} diff --git a/options/options.hpp b/options/options.hpp index c9fb14ec..acf9d2b2 100644 --- a/options/options.hpp +++ b/options/options.hpp @@ -69,6 +69,8 @@ namespace bzn bool peer_validation_enabled() const override; + std::string get_signed_key() const override; + private: size_t parse_size(const std::string& key) const; diff --git a/options/options_base.hpp b/options/options_base.hpp index 5f53e1fe..14519f4b 100644 --- a/options/options_base.hpp +++ b/options/options_base.hpp @@ -174,11 +174,19 @@ namespace bzn * @return port */ virtual uint16_t get_http_port() const = 0; + /** * Temporary toggle for the peer validation while in QA. Defaults to false. * @return boolean if the peer_validation member is set to true. Default is false. */ virtual bool peer_validation_enabled() const = 0; + + + /** + * Signature for uuid signing verification + * @return string containing the signature + */ + virtual std::string get_signed_key() const = 0; }; } // bzn diff --git a/options/simple_options.cpp b/options/simple_options.cpp index 31748892..c8589b92 100644 --- a/options/simple_options.cpp +++ b/options/simple_options.cpp @@ -135,7 +135,10 @@ simple_options::build_options() "use pbft consensus instead of raft (experimental)") (PEER_VALIDATION_ENABLED.c_str(), po::value()->default_value(false), - "require signed key for new peers to join swarm"); + "require signed key for new peers to join swarm") + (SIGNED_KEY.c_str(), + po::value(), + "signed key for node's uuid"); this->options_root.add(experimental); po::options_description crypto("Cryptography"); diff --git a/options/simple_options.hpp b/options/simple_options.hpp index e4d98abc..57635fc9 100644 --- a/options/simple_options.hpp +++ b/options/simple_options.hpp @@ -45,6 +45,7 @@ namespace bzn::option_names const std::string STATE_DIR = "state_dir"; const std::string WS_IDLE_TIMEOUT = "ws_idle_timeout"; const std::string PEER_VALIDATION_ENABLED = "peer_validation_enabled"; + const std::string SIGNED_KEY = "signed_key"; const std::string CHAOS_ENABLED = "chaos_testing_enabled"; const std::string CHAOS_NODE_FAILURE_SHAPE = "chaos_node_failure_shape"; diff --git a/options/test/options_test.cpp b/options/test/options_test.cpp index 02e08829..4a6cfc32 100644 --- a/options/test/options_test.cpp +++ b/options/test/options_test.cpp @@ -44,6 +44,7 @@ namespace " \"logfile_rotation_size\" : \"2M\"," " \"logfile_dir\" : \".\"," " \"http_port\" : 80," + " \"signed_key\" : \"Oo8ZlDQcMlZF4hqnhN/2D...hoEgc0jRUl1b9mHSY7E4puk=\"," " \"mem_storage\" : false"; const std::string DEFAULT_CONFIG_DATA = "{" + DEFAULT_CONFIG_CONTENT + "}"; @@ -149,6 +150,7 @@ TEST_F(options_file_test, test_that_loading_of_default_config_file) EXPECT_EQ(uint16_t(80), options.get_http_port()); EXPECT_FALSE(options.peer_validation_enabled()); EXPECT_FALSE(options.get_mem_storage()); + EXPECT_EQ("Oo8ZlDQcMlZF4hqnhN/2D...hoEgc0jRUl1b9mHSY7E4puk=",options.get_signed_key()); // defaults.. { diff --git a/raft/raft.cpp b/raft/raft.cpp index 126e2c44..2468c63e 100644 --- a/raft/raft.cpp +++ b/raft/raft.cpp @@ -21,6 +21,7 @@ #include #include #include +#include namespace { @@ -46,13 +47,15 @@ raft::raft( const bzn::peers_list_t& peers, bzn::uuid_t uuid, const std::string state_dir, size_t maximum_raft_storage, - bool enable_peer_validation + bool enable_peer_validation, + const std::string& signed_key ) :timer(io_context->make_unique_steady_timer()) ,uuid(std::move(uuid)) ,node(std::move(node)) ,state_dir(std::move(state_dir)) ,enable_peer_validation(enable_peer_validation) + ,signed_key(signed_key) { // we must have a list of peers! if (peers.empty()) @@ -162,6 +165,14 @@ raft::start_heartbeat_timer() void raft::handle_election_timeout(const boost::system::error_code& ec) { + // A reason that we may be asking for an election is that we are not in the + // swarm so no one is sending us messages, if we haven't already, do a + // quick check. + if(!this->in_a_swarm) + { + this->auto_add_peer_if_required(); + } + if (ec) { LOG(debug) << "election timer was canceled: " << ec.message(); @@ -301,6 +312,14 @@ raft::handle_ws_append_entries(const bzn::json_message& msg, std::shared_ptrin_a_swarm && (msg["data"]["from"].asString() != this->get_uuid())) + { + LOG(debug) << "RAFT - just received an append entries - auto add peer is unecessary"; + this->in_a_swarm = true; + } + if ((this->current_state == bzn::raft_state::candidate || this->current_state == bzn::raft_state::leader) && this->current_term >= term) { @@ -542,6 +561,94 @@ raft::handle_remove_peer(std::shared_ptr session, const std:: } +void +raft::handle_get_peers_response_from_follower_or_candidate(const bzn::json_message& msg) +{ + LOG(debug) << "RAFT - handling get_peers_response from a follower or candidate"; + // one of the following must be true: + // 2) I'm the follower, here's the leader + // 3) I'm a candidate, there might be an election happening, try later + if (msg.isMember("message") && msg["message"].isMember("leader")) + { + LOG(debug) << "RAFT - recieved a get_peers response from a follower raft who knows the leader"; + this->leader = msg["message"]["leader"]["uuid"].asString(); + // Since the leader variable gets cleared with each timer expiration, + // we need to call auto_add_peer_if_required() immediately + // TODO RHN - pass the leader uuid as a parameter? + this->auto_add_peer_if_required(); + } + else + { + LOG(debug) << "RAFT - recieved a get_peers_response from a follower, or a candidate who doesn't know the leader - must try again"; + } + this->in_a_swarm = false; + return; +} + +void +raft::handle_get_peers_response_from_leader(const bzn::json_message& msg) +{ + LOG(debug) << "RAFT - the get_peers_response is from a leader RAFT"; + // 1) I'm the leader, here are the peers + if (msg.isMember("message")) + { + if (msg.isMember("from")) + { + this->leader = msg["from"].asString(); + LOG(debug) << "RAFT - setting the leader to [" << this->leader << "]"; + } + + if (msg["message"].isArray()) + { + auto peer = std::find_if(msg["message"].begin(), msg["message"].end(), [&](const auto& p) { return p["uuid"].asString() == this->get_uuid(); }); + if (peer == msg["message"].end()) + { + LOG(debug) << "RAFT - I am not in the peers list, need to send add_peer request"; + // we do not find ourselves in the leader's quorum! + // Time to add oursleves to the swarm! + this->add_self_to_swarm(); + return; + } + LOG(debug) << "RAFT - I am already in the peers list, no further action required"; + // if we get this far, then we know that we are in the + // leader's quorum and all is good, no need to do anything + // else, simply remember that we are in a swarm. + this->in_a_swarm = true; + return; + } + else + { + // we do not ever expect a non error response to get_peers to not have an array + // of peers. + // TODO: THROW maybe? + return; + } + } +} + + +void +raft::handle_get_peers_response(const bzn::json_message& msg) +{ + LOG(debug) << "RAFT - recieved get_peers_response"; + // This raft must have sent a get_peers request because it was + // trying to determine if it's in the swarm + // There are three options: + // 1) I'm the leader here are the peers + // 2) I'm the follower, here's the leader + // 3) I'm a candidate, there might be an election happening, try later + if (msg.isMember("error")) + { + handle_get_peers_response_from_follower_or_candidate(msg); + return; + } + else + { + this->handle_get_peers_response_from_leader(msg); + } +} + + void raft::handle_ws_raft_messages(const bzn::json_message& msg, std::shared_ptr session) { @@ -552,6 +659,20 @@ raft::handle_ws_raft_messages(const bzn::json_message& msg, std::shared_ptrin_a_swarm = true; + } + // TODO RHN we need better responses from add_peer requests. + return; + } + this->handle_add_peer(session, msg["data"]["peer"]); return; } @@ -571,6 +692,11 @@ raft::handle_ws_raft_messages(const bzn::json_message& msg, std::shared_ptrhandle_get_peers(session); return; } + else if ( msg["cmd"].asString() == "get_peers_response") + { + this->handle_get_peers_response(msg); + return; + } // check that the message is from a node in the most recent quorum if (!in_quorum(msg["data"]["from"].asString())) @@ -673,6 +799,7 @@ raft::handle_heartbeat_timeout(const boost::system::error_code& ec) this->notify_leader_status(); } + void raft::notify_leader_status() { @@ -952,16 +1079,6 @@ raft::entries_log_path() } -std::string -raft::state_path() -{ - // Refactored as the original version was resulting in double '/' occurring - boost::filesystem::path out{this->state_dir}; - out.append(this->get_uuid() + ".state"); - return out.string(); -} - - void raft::perform_commit(uint32_t& commit_index, const bzn::log_entry& log_entry) { @@ -1230,6 +1347,7 @@ raft::shutdown_on_exceeded_max_storage(bool do_throw) } } + void raft::set_audit_enabled(bool val) { @@ -1249,10 +1367,14 @@ raft::to_peer_message(const peer_address_t& address) return bzn; } + void raft::handle_get_peers(std::shared_ptr session) { bzn::json_message msg; + msg["bzn-api"] = "raft"; + msg["cmd"] = "get_peers_response"; + msg["from"] = this->get_uuid(); switch (this->current_state) { @@ -1277,3 +1399,102 @@ raft::handle_get_peers(std::shared_ptr session) } session->send_message(std::make_shared(msg), false); } + + +bzn::peers_list_t +remove_peer_from_peers_list(const bzn::peers_list_t& all_peers, const bzn::uuid_t& uuid_of_peer_to_remove) +{ + bzn::peers_list_t other_peers; + for(auto& p : all_peers) // TODO: I'd like to try std::copy_if + { + if (p.uuid != uuid_of_peer_to_remove) + { + other_peers.emplace(p); + } + } + return other_peers; +} + + +void +raft::auto_add_peer_if_required() +{ + LOG(debug) << "RAFT - may not be in a swarm - finding out"; + // NOTE: if this node is the leader, we don't need to do this test, however, we do this test during startup so + // there is no way that the node could be a leader... + // We try any peer that is not ourself, it may come back and say "i'm not the leader, but try this url", + // and that would be great, we know the leader so we ask that node for peers in the swarm, hopefully, the leader + // gives us the list. + // Or, it may come back and say, there is an election in progress. That's OK, we're just till in limbo. + + // Not that we can't trust our get_all_peers, because we are not the leader, and we may not be in the + // "official" quorum yet, so we seek out the leader and ask. + + // This may be the second time we've been here, if so that means that we've obtained the + // uuid of the leader, let's try that + + auto choose_leader=[&]() + { + // we don'know who the leader is so we should try someone else + auto other_peers = remove_peer_from_peers_list(this->get_all_peers(), this->get_uuid()); + if (other_peers.empty()) + { + throw std::runtime_error(ERROR_BOOTSTRAP_LIST_MUST_HAVE_MORE_THAN_ONE_PEER); + } + return *bzn::utils::container::choose_any_one_of(other_peers); + }; + + bzn::peer_address_t leader{((this->get_leader_unsafe().uuid.empty() && this->get_leader_unsafe().host.empty()) ? choose_leader() : this->get_leader_unsafe())}; + + auto end_point = boost::asio::ip::tcp::endpoint{boost::asio::ip::address_v4::from_string(leader.host), leader.port}; + bzn::json_message msg; + msg["bzn-api"] = "raft"; + msg["cmd"] = "get_peers"; + LOG(debug) << "RAFT - sending get_peers to [" << end_point.address().to_string() << ":" << end_point.port() << "]"; + this->node->send_message(end_point, std::make_shared(msg)); +} + + +bzn::json_message +make_add_peer_request(const bzn::peer_address_t& new_peer) +{ + bzn::json_message msg; + msg["bzn-api"] = "raft"; + msg["cmd"] = "add_peer"; + msg["data"]["peer"]["name"] = new_peer.name; + msg["data"]["peer"]["host"] = new_peer.host; + msg["data"]["peer"]["port"] = new_peer.port; + msg["data"]["peer"]["http_port"] = new_peer.http_port; + msg["data"]["peer"]["uuid"] = new_peer.uuid; + return msg; +} + + +bzn::json_message +make_secure_add_peer_request(const bzn::peer_address_t& new_peer, const std::string& signed_key) +{ + bzn::json_message msg{make_add_peer_request(new_peer)}; + msg["data"]["peer"]["signature"] = signed_key; + return msg; +} + + +void +raft::add_self_to_swarm() +{ + const auto all_peers = this->get_all_peers(); + const auto this_address = std::find_if(all_peers.begin(), all_peers.end(), + [&](const auto& peer){return peer.uuid == this->get_uuid();}); + + bzn::json_message request{make_secure_add_peer_request(*this_address, this->signed_key)}; + + bzn::peer_address_t leader{this->get_leader_unsafe()}; + auto end_point = boost::asio::ip::tcp::endpoint{boost::asio::ip::address_v4::from_string(leader.host), leader.port}; + + LOG(debug) << "RAFT sending add_peer command to the leader:\n" << request.toStyledString(); + + + + this->node->send_message(end_point, std::make_shared(request)); +} + diff --git a/raft/raft.hpp b/raft/raft.hpp index 7f6c8abc..2e0f769a 100644 --- a/raft/raft.hpp +++ b/raft/raft.hpp @@ -42,6 +42,7 @@ namespace const std::string ERROR_GET_PEERS_MUST_BE_SENT_TO_LEADER{"ERROR_GET_PEERS_MUST_BE_SENT_TO_LEADER"}; const std::string ERROR_GET_PEERS_ELECTION_IN_PROGRESS_TRY_LATER{"ERROR_GET_PEERS_ELECTION_IN_PROGRESS_TRY_LATER"}; const std::string ERROR_GET_PEERS_SELECTED_NODE_IN_UNKNOWN_STATE{"ERROR_GET_PEERS_SELECTED_NODE_IN_UNKNOWN_STATE"}; + const std::string ERROR_BOOTSTRAP_LIST_MUST_HAVE_MORE_THAN_ONE_PEER{"ERROR_BOOTSTRAP_LIST_MUST_HAVE_MORE_THAN_ONE_PEER"}; } @@ -57,7 +58,8 @@ namespace bzn bzn::uuid_t uuid, const std::string state_dir, size_t maximum_raft_storage = bzn::DEFAULT_MAX_STORAGE_SIZE, - bool enable_peer_validation = false); + bool enable_peer_validation = false, + const std::string& signed_key = ""); bzn::raft_state get_state() override; @@ -112,6 +114,12 @@ namespace bzn FRIEND_TEST(raft_test, test_that_sending_get_peers_to_follower_fails_and_provides_leader_url); FRIEND_TEST(raft_test, test_that_sending_get_peers_to_leader_responds_with_quorum); FRIEND_TEST(raft_test, test_that_sending_get_peers_to_candidate_fails); + FRIEND_TEST(raft_peers_test, test_that_raft_doesn_t_attempt_auto_add_if_already_in_quorum); + FRIEND_TEST(raft_peers_test, test_that_raft_calls_a_follower_who_knows_the_leader_raft_is_already_in_swarm); + FRIEND_TEST(raft_peers_test, test_that_raft_that_is_not_in_a_swarm_will_add_itself); + FRIEND_TEST(raft_peers_test, test_that_raft_tries_again_when_encountering_a_candidate); + FRIEND_TEST(raft_test, test_that_non_leaders_cannot_add_peers); + FRIEND_TEST(raft_test, test_that_non_leaders_cannot_remove_peers); bzn::peer_address_t get_leader_unsafe(); @@ -145,8 +153,6 @@ namespace bzn std::string entries_log_path(); - std::string state_path(); - void import_state_files(); void create_dat_file(const std::string& log_path, const bzn::peers_list_t& peers); @@ -174,6 +180,13 @@ namespace bzn bzn::json_message to_peer_message(const peer_address_t& address); + void auto_add_peer_if_required(); + void add_self_to_swarm(); + + void handle_get_peers_response_from_leader(const bzn::json_message& msg); + void handle_get_peers_response_from_follower_or_candidate(const bzn::json_message& msg); + void handle_get_peers_response(const bzn::json_message& msg); + // raft state... bzn::raft_state current_state = raft_state::follower; uint32_t current_term = 0; @@ -208,5 +221,8 @@ namespace bzn bool enable_audit = true; bool enable_peer_validation{false}; // TODO: RHN - this is only temporary, until the security functionality is tested and in use. + std::string signed_key; + + bool in_a_swarm = false; }; } // bzn diff --git a/raft/test/CMakeLists.txt b/raft/test/CMakeLists.txt index 23d77477..ee956ad6 100644 --- a/raft/test/CMakeLists.txt +++ b/raft/test/CMakeLists.txt @@ -1,4 +1,4 @@ -set(test_srcs raft_test.cpp raft_log_test.cpp) +set(test_srcs raft_test.cpp raft_log_test.cpp raft_add_peers_test.cpp) set(test_libs raft storage bootstrap proto ${Protobuf_LIBRARIES}) add_gmock_test(raft) diff --git a/raft/test/raft_add_peers_test.cpp b/raft/test/raft_add_peers_test.cpp new file mode 100644 index 00000000..f45a50ef --- /dev/null +++ b/raft/test/raft_add_peers_test.cpp @@ -0,0 +1,513 @@ +// Copyright (C) 2018 Bluzelle +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License, version 3, +// as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace ::testing; + +namespace +{ + const std::string TEST_STATE_DIR = "./.raft_test_state/"; + const bzn::uuid_t LEADER_NODE_UUID {"fb300a30-49fd-4230-8044-0e3069948e42"}; + const bzn::uuid_t TEST_NODE_UUID {"f0645cc2-476b-485d-b589-217be3ca87d5"}; + const bzn::uuid_t FOLLOWER_NODE_UUID {"8993098f-e32e-4b6f-9db9-c770c9bc2509"}; + const bzn::uuid_t CANDIDATE_NODE_UUID {"ec5a510f-24df-4026-9739-dd3744fdec6a"}; + + const bzn::peers_list_t FULL_TEST_PEER_LIST { + {"127.0.0.1", 8081, 80, "leader", LEADER_NODE_UUID}, + {"127.0.0.1", 8082, 81, "follower", FOLLOWER_NODE_UUID}, + {"127.0.0.1", 8084, 82, "sut", TEST_NODE_UUID}}; + + const bzn::peers_list_t PARTIAL_TEST_PEER_LIST{ + {"127.0.0.1", 8081, 80, "leader", LEADER_NODE_UUID}, + {"127.0.0.1", 8082, 81, "follower", FOLLOWER_NODE_UUID}}; + + const std::string signature { + "Oo8ZlDQcMlZF4hqnhN/2Dz3FYarZHrGf+87i+JUSxBu2GKFk8SYcDrwjc0DuhUCx" + "pRVQppMk5fjZtJ3r6I9066jcEpJPljU1SC1Thpy+AUEYx0r640SKRwKwmJMe6mRd" + "SJ75rcYHu5+etajOWWjMs4vYQtcwfVF3oEd9/pZjea8x6PuhnM50+FPpnvgu57L8" + "vHdeWjCqAiPyomQSLgIJPjvMJw4aHUUE3tHX1WOB8XDHdvuhi9gZODzZWbdI92JN" + "hoLbwvjmhKTeTN+FbBtdJIjC0+V0sMFmGNJQ8WIkJscN0hzRkmdlU965lHe4hqlc" + "MyEdTSnYSnC7NIHFfvJFBBYi9kcAVBwkYyALQDv6iTGMSI11/ncwdTz4/GGPodaU" + "PFxf/WVHUz6rBAtTKvn8Kg61F6cVhcFSCjiw2bWGpeWcWTL+CGbfYCvZNiAVyO7Q" + "dmfj5hoLu7KG+nxBLF8uoUl6t3BdKz9Dqg9Vf+QVtaVj/COD1nUykXXRVxfLo4dN" + "BS+aVsmOFjppKaEvmeT5SwWOSVrKZwPuTilV9jCehFbFZF6MPyiW5mcp9t4D27hM" + "oz/SiKjCqdN93YdBO4FBF/cWD5WHmD7KaaJYmnztz3W+xS7b/qk2PcN+qpZEXsfr" + "Wie4prB1umESavYLC1pLhoEgc0jRUl1b9mHSY7E4puk=" + }; + + + void + clean_state_folder() + { + try + { + if (boost::filesystem::exists(TEST_STATE_DIR)) + { + boost::filesystem::remove_all(TEST_STATE_DIR); + } + boost::filesystem::create_directory(TEST_STATE_DIR); + + if (boost::filesystem::exists("./.state")) + { + boost::filesystem::remove_all("./.state"); + } + boost::filesystem::create_directory("./.state"); + } + catch(boost::filesystem::filesystem_error const& e) + { + LOG(error) << "Error while attempting to clean the state folder:" << e.what(); + } + } + + + bzn::json_message + peers_list_to_JSON_array(const bzn::peers_list_t& list_of_peers) + { + bzn::json_message root; + for (const auto& address : list_of_peers) + { + bzn::json_message peer; + peer["port"] = address.port; + peer["http_port"] = address.http_port; + peer["host"] = address.host; + peer["uuid"] = address.uuid; + peer["name"] = address.name; + root.append(peer); + } + return root; + } + + + bzn::json_message + get_peer_response_from_leader(const bzn::peers_list_t& list_of_peers) + { + bzn::json_message response; + response["bzn-api"] = "raft"; + response["cmd"] = "get_peers_response"; + response["from"] = LEADER_NODE_UUID; + response["message"] = peers_list_to_JSON_array(list_of_peers); + return response; + } + + + bzn::json_message + get_peer_response_from_follower() + { + bzn::json_message response; + response["bzn-api"] = "raft"; + response["cmd"] = "get_peers_response"; + response["from"] = CANDIDATE_NODE_UUID; + response["error"] = ERROR_GET_PEERS_MUST_BE_SENT_TO_LEADER; + response["message"]["leader"]["uuid"] = LEADER_NODE_UUID; + return response; + } + + + bzn::json_message + get_peer_response_from_candidate() + { + bzn::json_message response; + response["bzn-api"] = "raft"; + response["cmd"] = "get_peers_response"; + response["from"] = CANDIDATE_NODE_UUID; + response["error"] = ERROR_GET_PEERS_ELECTION_IN_PROGRESS_TRY_LATER ; + return response; + } +} + + +namespace bzn +{ + class raft_peers_test : public Test + { + public: + raft_peers_test() + {} + + + void SetUp() final + { + clean_state_folder(); + this->mock_io_context = std::make_shared(); + this->mock_node = std::make_shared(); + this->mock_session = std::make_shared(); + } + + + void TearDown() final + { + clean_state_folder(); + } + + std::shared_ptr mock_node; + std::shared_ptr mock_io_context; + std::shared_ptr mock_session; + }; + + + TEST_F(raft_peers_test, test_that_raft_doesn_t_attempt_auto_add_if_already_in_quorum) + { + auto mock_steady_timer = std::make_unique(); + + // timer expectations... + EXPECT_CALL(*mock_steady_timer, expires_from_now(_)).Times(2); + EXPECT_CALL(*mock_steady_timer, cancel()).Times(2); + + // intercept the timeout callback... + bzn::asio::wait_handler wh; + EXPECT_CALL(*mock_steady_timer, async_wait(_)).WillRepeatedly(Invoke( + [&](auto handler) + { wh = handler; })); + + bzn::message_handler mh; + EXPECT_CALL(*mock_node, register_for_message("raft", _)).WillOnce(Invoke( + [&](const auto&, auto handler) + { + mh = handler; + return true; + })); + + EXPECT_CALL(*this->mock_io_context, make_unique_steady_timer()).WillOnce(Invoke( + [&]() + { return std::move(mock_steady_timer); })); + + // We need to fake the other nodes by mocking thier responses to send messages. + // There are three options: + // 1) I'm the leader here are the peers + // 2) I'm the follower, here's the leader + // 3) I'm a candidate, there might be an election happening, try later + // We will handle the first option: + EXPECT_CALL(*this->mock_node, send_message(_, _)).WillRepeatedly( Invoke( + [&](const boost::asio::ip::tcp::endpoint& /*ep*/, std::shared_ptr msg) + { + const auto request = *msg; + if (request["cmd"].asString() == "get_peers") + { + LOG(debug) << "Mock Leader Raft recieved get_peers, sending peers"; + mh(get_peer_response_from_leader(FULL_TEST_PEER_LIST), this->mock_session); + } + })); + + /////////////////////////////////////////////////////////////////////// + // create raft... + auto raft = std::make_shared( + this->mock_io_context, + mock_node, + PARTIAL_TEST_PEER_LIST, + TEST_NODE_UUID, TEST_STATE_DIR); + + // and away we go... + raft->start(); + EXPECT_FALSE(raft->in_a_swarm); + + // Get the ball rolling buy expiring the election timer... + wh(boost::system::error_code()); + + EXPECT_TRUE(raft->in_a_swarm); + } + + + TEST_F(raft_peers_test, test_that_raft_calls_a_follower_who_knows_the_leader_raft_is_already_in_swarm) + { + auto mock_steady_timer = std::make_unique(); + + // timer expectations... + EXPECT_CALL(*mock_steady_timer, expires_from_now(_)).Times(3); + EXPECT_CALL(*mock_steady_timer, cancel()).Times(3); + + // intercept the timeout callback... + bzn::asio::wait_handler wh; + EXPECT_CALL(*mock_steady_timer, async_wait(_)).Times(3).WillRepeatedly(Invoke( + [&](auto handler) + { wh = handler; })); + + bzn::message_handler mh; + EXPECT_CALL(*mock_node, register_for_message("raft", _)).WillOnce(Invoke( + [&](const auto&, auto handler) + { + mh = handler; + return true; + })); + + EXPECT_CALL(*this->mock_io_context, make_unique_steady_timer()).WillOnce(Invoke( + [&]() + { return std::move(mock_steady_timer); })); + + // We need to fake the other nodes by mocking their responses to send messages. + // There are three options: + // 1) I'm the leader here are the peers + // 2) I'm the follower, here's the leader + // 3) I'm a candidate, there might be an election happening, try later + // Handling option 2, ask follower, follower returns Leader, ask leader + bzn::json_message response; + EXPECT_CALL(*this->mock_node, send_message(_, _)).WillRepeatedly( Invoke( + [&](const boost::asio::ip::tcp::endpoint& ep, std::shared_ptr msg) + { + const auto request = *msg; + LOG(debug) << "MOCK RAFT - received cmd:[" << request["cmd"] << "] from " << ep.address().to_string() << ":" << ep.port(); + if (request["cmd"].asString() == "get_peers") + { + static size_t count; + // Let's handle option 2 + switch(count) + { + case 0: // at first raft chose the wrong node, a follower + LOG(debug) << "MOCK RAFT - follower raft responding with leader info"; + EXPECT_EQ( request["cmd"].asString(), "get_peers"); + response = get_peer_response_from_follower(); + break; + case 1: //the follower told raft who the leader was + case 2: + LOG(debug) << "MOCK RAFT - leader raft responding with peers list"; + EXPECT_EQ(request["cmd"].asString(), "get_peers"); + EXPECT_EQ(ep.port(), 8081); // raft sent msg to leader + response = get_peer_response_from_leader(FULL_TEST_PEER_LIST); + break; + default: + EXPECT_TRUE(false); // we should not get here. + break; + } + count++; + } + })); + + // create raft... + auto raft = std::make_shared(this->mock_io_context, mock_node, PARTIAL_TEST_PEER_LIST, TEST_NODE_UUID, TEST_STATE_DIR); + + // and away we go... + raft->start(); + EXPECT_FALSE(raft->in_a_swarm); + + // Get the ball rolling buy expiring the election timer... + wh(boost::system::error_code()); + + // Then raft, asked the leader, so we fake the leader sending back a + // peer list that contains raft. + mh(response, this->mock_session); + + EXPECT_FALSE(raft->in_a_swarm); + + wh(boost::system::error_code()); + + // The leader responded with the list, so + mh(response, this->mock_session); + + EXPECT_TRUE(raft->in_a_swarm); + } + + + TEST_F(raft_peers_test, test_that_raft_that_is_not_in_a_swarm_will_add_itself) + { + auto mock_steady_timer = std::make_unique(); + + // timer expectations... + EXPECT_CALL(*mock_steady_timer, expires_from_now(_)).Times(2); + EXPECT_CALL(*mock_steady_timer, cancel()).Times(2); + + // intercept the timeout callback... + bzn::asio::wait_handler wh; + EXPECT_CALL(*mock_steady_timer, async_wait(_)).Times(2).WillRepeatedly(Invoke( + [&](auto handler) + { wh = handler; })); + + + bzn::message_handler mh; + EXPECT_CALL(*mock_node, register_for_message("raft", _)).WillOnce(Invoke( + [&](const auto&, auto handler) + { + mh = handler; + return true; + })); + + + EXPECT_CALL(*this->mock_io_context, make_unique_steady_timer()).WillOnce(Invoke( + [&]() + { return std::move(mock_steady_timer); })); + + + bzn::json_message response; + // We need to fake the other nodes by mocking their responses to send messages. + // There are three options: + // 1) I'm the leader here are the peers + // 2) I'm the follower, here's the leader + // 3) I'm a candidate, there might be an election happening, try later + EXPECT_CALL(*this->mock_node, send_message(_, _)).WillRepeatedly( Invoke( + [&](const boost::asio::ip::tcp::endpoint& ep, std::shared_ptr msg) + { + static size_t count; + const auto request = *msg; + + switch(count) + { + case 0: // at first raft chose the wrong node, a follower + LOG(debug) << "MOCK RAFT - follower received get_peers from RAFT"; + EXPECT_EQ( request["cmd"].asString(), "get_peers"); + LOG(debug) << "MOCK RAFT - follower telling RAFT who the leader is via get_peers_response"; + response = get_peer_response_from_follower(); + break; + + case 1: + case 2: + LOG(debug) << "MOCK RAFT - received RequestVote from RAFT - ignoring"; + break; + + case 3: //the follower told raft who the leader was + LOG(debug) << "MOCK RAFT - leader recieved get_peers from RAFT"; + EXPECT_EQ(request["cmd"].asString(), "get_peers"); + EXPECT_EQ(ep.port(), 8081); // raft sent msg to leader + LOG(debug) << "MOCK RAFT - leader sending RAFT the peers list, but RAFT is not in it"; + response = get_peer_response_from_leader(PARTIAL_TEST_PEER_LIST); + break; + + case 4: + LOG(debug) << "MOCK RAFT - leader recieved [" << request["cmd"].asString() << "] from RAFT"; + EXPECT_EQ(request["cmd"].asString(), "add_peer"); + response["bzn-api"] = "raft"; + response["cmd"] = "add_peer"; + response["response"] = bzn::json_message(); + response["response"]["from"] = LEADER_NODE_UUID; + response["response"]["msg"] = SUCCESS_PEER_ADDED_TO_SWARM; + break; + + default: + EXPECT_TRUE(false); // we should not get here. + break; + } + count++; + })); + + // create raft... + auto raft = std::make_shared(this->mock_io_context, mock_node, FULL_TEST_PEER_LIST, TEST_NODE_UUID, TEST_STATE_DIR, bzn::DEFAULT_MAX_STORAGE_SIZE, true, signature); + + // and away we go... + raft->start(); + EXPECT_FALSE(raft->in_a_swarm); + + // Get the ball rolling buy expiring the election timer... + // this will also trigger raft to ask for the list of peers + wh(boost::system::error_code()); + + // Raft asked a follower who told raft whoe hte leader is. + mh(response, this->mock_session); + EXPECT_EQ(raft->get_leader().uuid, LEADER_NODE_UUID); + + // The leader responded with the list, but we should not be in it + mh(response, this->mock_session); + EXPECT_FALSE(raft->in_a_swarm); + + // RAFT asked for peers from leader + mh(response, this->mock_session); + EXPECT_TRUE(raft->in_a_swarm); + } + + + TEST_F(raft_peers_test, test_that_raft_tries_again_when_encountering_a_candidate) + { + auto mock_steady_timer = std::make_unique(); + + // timer expectations... + EXPECT_CALL(*mock_steady_timer, expires_from_now(_)).Times(3); + EXPECT_CALL(*mock_steady_timer, cancel()).Times(3); + + // intercept the timeout callback... + bzn::asio::wait_handler wh; + EXPECT_CALL(*mock_steady_timer, async_wait(_)).Times(3).WillRepeatedly(Invoke( + [&](auto handler) + { wh = handler; })); + + bzn::message_handler mh; + EXPECT_CALL(*mock_node, register_for_message("raft", _)).WillOnce(Invoke( + [&](const auto&, auto handler) + { + mh = handler; + return true; + })); + + EXPECT_CALL(*this->mock_io_context, make_unique_steady_timer()).WillOnce(Invoke( + [&]() + { return std::move(mock_steady_timer); })); + + bzn::json_message response; + // We need to fake the other nodes by mocking their responses to send messages. + // There are three options: + // 1) I'm the leader here are the peers + // 2) I'm the follower, here's the leader + // 3) I'm a candidate, there might be an election happening, try later + EXPECT_CALL(*this->mock_node, send_message(_, _)).WillRepeatedly( Invoke( + [&](const boost::asio::ip::tcp::endpoint& /*ep*/, std::shared_ptr msg) + { + static size_t count; + const auto request = *msg; + + switch(count) + { + case 0: // get_peers + LOG (debug) << "MOCK RAFT - received get_peers request, sending candidate error message"; + EXPECT_EQ( request["cmd"].asString(), "get_peers"); + response = get_peer_response_from_candidate(); + break; + + case 1: // RequestVote - ignore + case 2: + break; + + case 3:// get_peers + LOG (debug) << "MOCK RAFT - received get_peers request, sending candidate error message"; + EXPECT_EQ( request["cmd"].asString(), "get_peers"); + response = get_peer_response_from_candidate(); + break; + + case 4: + case 5: + break; + + default: + EXPECT_TRUE(false); // we should not get here. + break; + } + count++; + })); + + // create raft... + auto raft = std::make_shared(this->mock_io_context, mock_node, PARTIAL_TEST_PEER_LIST, TEST_NODE_UUID, TEST_STATE_DIR); + + // and away we go... + raft->start(); + EXPECT_FALSE(raft->in_a_swarm); + + // Get the ball rolling buy expiring the election timer... + wh(boost::system::error_code()); + EXPECT_FALSE(raft->in_a_swarm); + + // Then raft, asked the leader, so we fake the leader sending back a + // peer list that contains raft. + mh(response, this->mock_session); + EXPECT_FALSE(raft->in_a_swarm); + + // give it another shot, it should fail + wh(boost::system::error_code()); + EXPECT_FALSE(raft->in_a_swarm); + } +} diff --git a/raft/test/raft_test.cpp b/raft/test/raft_test.cpp index bba50177..7b978d17 100644 --- a/raft/test/raft_test.cpp +++ b/raft/test/raft_test.cpp @@ -436,7 +436,8 @@ namespace bzn raft->start(); // we should see requests for votes... and then the Append Requests - EXPECT_CALL(*this->mock_node, send_message(_, _)).Times((TEST_PEER_LIST.size() - 1) * 2); + // The "+1" is raft asking if it is in the swarm + EXPECT_CALL(*this->mock_node, send_message(_, _)).Times((TEST_PEER_LIST.size() - 1) * 2 + 1); // expire timer... wh(boost::system::error_code()); @@ -483,7 +484,7 @@ namespace bzn raft->start(); // don't care about the handler... - EXPECT_CALL(*this->mock_node, send_message(_, _)).Times(TEST_PEER_LIST.size() - 1); + EXPECT_CALL(*this->mock_node, send_message(_, _)).Times(TEST_PEER_LIST.size()); // expire timer... wh(boost::system::error_code()); @@ -532,6 +533,9 @@ namespace bzn // and away we go... raft->start(); + // ignore auto add peers for this test + raft->in_a_swarm = true; + // we should see requests for votes... std::vector mh_req; EXPECT_CALL(*this->mock_node, send_message(_, _)).Times(TEST_PEER_LIST.size() - 1).WillRepeatedly(Invoke( @@ -646,7 +650,7 @@ namespace bzn EXPECT_EQ(raft->get_status()["state"].asString(), "follower"); // we should see requests... - EXPECT_CALL(*mock_node, send_message(_, _)).Times(10); + EXPECT_CALL(*mock_node, send_message(_, _)).Times(11); // expire election timer... wh(boost::system::error_code()); @@ -1070,6 +1074,9 @@ namespace bzn // and away we go... raft->start(); + // ignore auto add peers for this test + raft->in_a_swarm = true; + // the current state must be follower EXPECT_TRUE(raft->get_state() == bzn::raft_state::follower); EXPECT_CALL(*this->mock_session, send_message(An>(),_)) @@ -1130,6 +1137,9 @@ namespace bzn // and away we go... raft->start(); + // ignore auto add peers for this test + raft->in_a_swarm = true; + // the current state must be follower EXPECT_TRUE(raft->get_state() == bzn::raft_state::follower); EXPECT_CALL(*this->mock_session, send_message(An>(),_)) @@ -1189,6 +1199,9 @@ namespace bzn // and away we go... raft->start(); + // ignore auto add peers for this test + raft->in_a_swarm = true; + // lets make this raft the leader by responding to requests for votes EXPECT_CALL(*this->mock_node, send_message(_, _)).Times((TEST_PEER_LIST.size() - 1) * 2); @@ -1254,6 +1267,9 @@ namespace bzn // and away we go... raft->start(); + // ignore auto add peers for this test + raft->in_a_swarm = true; + // lets make this raft the leader by responding to requests for votes EXPECT_CALL(*this->mock_node, send_message(_, _)).Times((TEST_PEER_LIST.size() - 1) * 2); @@ -1351,6 +1367,9 @@ namespace bzn // and away we go... raft->start(); + // ignore auto add peers for this test + raft->in_a_swarm = true; + // lets make this raft the leader by responding to requests for votes EXPECT_CALL(*this->mock_node, send_message(_, _)).Times((TEST_PEER_LIST.size() - 1) * 2); @@ -1438,6 +1457,9 @@ namespace bzn // and away we go... raft->start(); + // ignore auto add peers for this test + raft->in_a_swarm = true; + // lets make this raft the leader by responding to requests for votes EXPECT_CALL(*this->mock_node, send_message(_, _)).Times((TEST_PEER_LIST.size() - 1) * 2); @@ -1513,6 +1535,9 @@ namespace bzn // and away we go... raft->start(); + // ignore auto add peers for this test + raft->in_a_swarm = true; + // lets make this raft the leader by responding to requests for votes EXPECT_CALL(*this->mock_node, send_message(_, _)).Times((TEST_PEER_LIST.size() - 1) * 2); @@ -1565,6 +1590,9 @@ namespace bzn // and away we go... raft->start(); + // ignore auto add peers for this test + raft->in_a_swarm = true; + // lets make this raft the leader by responding to requests for votes EXPECT_CALL(*this->mock_node, send_message(_, _)).Times((TEST_PEER_LIST.size() - 1) * 2); @@ -2041,11 +2069,17 @@ namespace bzn raft->start(); + // ignore auto add peers for this test + raft->in_a_swarm = true; + EXPECT_EQ(raft->get_state(), bzn::raft_state::follower); raft->handle_get_peers(mock_session); // session will have set the value of resp via the message handler. + EXPECT_EQ(resp["bzn-api"].asString(), "raft"); + EXPECT_EQ(resp["cmd"].asString(), "get_peers_response"); + EXPECT_EQ(resp["from"].asString(), raft->get_uuid()); EXPECT_EQ(resp["error"].asString(), ERROR_GET_PEERS_MUST_BE_SENT_TO_LEADER); } @@ -2149,7 +2183,7 @@ namespace bzn raft->start(); // don't care about the handler... - EXPECT_CALL(*this->mock_node, send_message(_, _)).Times(TEST_PEER_LIST.size() - 1); + EXPECT_CALL(*this->mock_node, send_message(_, _)).Times(TEST_PEER_LIST.size() ); // expire timer... wh(boost::system::error_code()); diff --git a/swarm/main.cpp b/swarm/main.cpp index 69e2ae9d..8a0bb063 100644 --- a/swarm/main.cpp +++ b/swarm/main.cpp @@ -250,7 +250,10 @@ main(int argc, const char* argv[]) auto ep = options->get_listener(); ep.port(options->get_http_port()); - auto raft = std::make_shared(io_context, node, peers.get_peers(), options->get_uuid(), options->get_state_dir(), options->get_max_storage(), options->peer_validation_enabled()); + auto raft = std::make_shared( + io_context, node, peers.get_peers(), + options->get_uuid(), options->get_state_dir(), options->get_max_storage(), + options->peer_validation_enabled(), options->get_signed_key()); // which type of storage? std::shared_ptr storage; diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index 141674d4..975e8514 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -6,7 +6,8 @@ add_library(utils STATIC make_endpoint.hpp make_endpoint.cpp crypto.cpp - crypto.hpp) + crypto.hpp + container.hpp) target_link_libraries(utils ${CURL_LIBRARIES} ${JSONCPP_LIBRARIES} OpenSSL::SSL) diff --git a/utils/container.hpp b/utils/container.hpp new file mode 100644 index 00000000..03689daa --- /dev/null +++ b/utils/container.hpp @@ -0,0 +1,42 @@ +// Copyright (C) 2018 Bluzelle +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License, version 3, +// as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +#pragma once + +#include +#include +#include + +namespace +{ + std::mt19937 gen(std::time(0)); //Standard mersenne_twister_engine seeded with rd() +} + + +namespace bzn::utils::container +{ + // TODO: RHN - this should be templatized + bzn::peers_list_t::const_iterator + choose_any_one_of(const bzn::peers_list_t& all_peers) + { + if (all_peers.size()>0) + { + std::uniform_int_distribution<> dis(1, all_peers.size()); + auto it = all_peers.begin(); + std::advance(it, dis(gen) - 1); + return it; + } + return all_peers.end(); + } +} diff --git a/utils/test/utils_test.cpp b/utils/test/utils_test.cpp index a9369ebe..bd056706 100644 --- a/utils/test/utils_test.cpp +++ b/utils/test/utils_test.cpp @@ -17,8 +17,9 @@ #include #include #include -#include "utils/blacklist.hpp" -#include "../crypto.hpp" +#include +#include +#include using namespace::testing; @@ -328,3 +329,43 @@ TEST(util_test, test_that_verifying_asignature_with_empty_inputs_will_fail_grace EXPECT_FALSE(bzn::utils::crypto::verify_signature( public_pem, "", valid_uuid)); EXPECT_FALSE(bzn::utils::crypto::verify_signature( public_pem, signature, "")); } + + +TEST(util_test, test_that_choose_any_one_of_chooses_one_of_a_set) +{ + const bzn::uuid_t LEADER_NODE_UUID {"fb300a30-49fd-4230-8044-0e3069948e42"}; + const bzn::uuid_t TEST_NODE_UUID {"f0645cc2-476b-485d-b589-217be3ca87d5"}; + const bzn::uuid_t FOLLOWER_NODE_UUID{"8993098f-e32e-4b6f-9db9-c770c9bc2509"}; + const bzn::peers_list_t TEST_LIST { // using http_port as an index. + {"127.0.0.1", 8081, 0, "leader", LEADER_NODE_UUID}, + {"127.0.0.2", 8082, 1, "follower", FOLLOWER_NODE_UUID}, + {"127.0.0.3", 8083, 2, "sut", TEST_NODE_UUID}, + {"127.0.0.4", 8084, 3, "sut0", TEST_NODE_UUID}, + {"127.0.0.5", 8085, 4, "sut1", TEST_NODE_UUID}, + {"127.0.0.6", 8086, 5, "sut2", TEST_NODE_UUID}, + {"127.0.0.7", 8087, 6, "sut3", TEST_NODE_UUID}, + {"127.0.0.8", 8088, 7, "sut4", TEST_NODE_UUID}}; + + // How do you test a function that uses random?? Well I just want to + // make sure that every element in the array gets chosen at some point, + // and that the function is unlikely to fail. + std::array histogram{false}; + size_t count{0}; + while( static_cast(std::count_if(histogram.begin(), histogram.end(), [](bool i){return i;})) < TEST_LIST.size()) + { + auto peer = bzn::utils::container::choose_any_one_of(TEST_LIST); + EXPECT_TRUE(peer != TEST_LIST.end()); + EXPECT_TRUE( TEST_LIST.end() != find_if( TEST_LIST.begin(), TEST_LIST.end(), [peer](bzn::peer_address_t addr){ return addr.port == peer->port; })); + histogram[peer->http_port] = true; + EXPECT_TRUE(10000 > ++count); + } + // this is redundant, I know. + EXPECT_EQ(8, std::count_if(histogram.begin(), histogram.end(), [](bool i){return i;})); +} + + +TEST(util_test, test_that_choose_any_one_of_behaves_when_given_bad_input) +{ + const bzn::peers_list_t TEST_LIST {}; + EXPECT_EQ(bzn::utils::container::choose_any_one_of(TEST_LIST), TEST_LIST.end()); +}