diff --git a/common/include/loki_common.h b/common/include/loki_common.h index a4406a4b..bd6f06c5 100644 --- a/common/include/loki_common.h +++ b/common/include/loki_common.h @@ -20,7 +20,7 @@ struct sn_record_t { private: uint16_t port_; std::string sn_address_; // Snode address (pubkey plus .snode) - std::string pub_key_; // base32z + std::string pub_key_; // base32z std::string pub_key_hex_; std::string ip_; // Snode ip public: @@ -70,13 +70,11 @@ class user_pubkey_t { user_pubkey_t() {} - user_pubkey_t(std::string&& pk) : pubkey_(std::move(pk)) { - } + user_pubkey_t(std::string&& pk) : pubkey_(std::move(pk)) {} user_pubkey_t(const std::string& pk) : pubkey_(pk) {} -public: - + public: static user_pubkey_t create(std::string&& pk, bool& success) { success = true; if (pk.size() != USER_PUBKEY_SIZE) { @@ -95,9 +93,7 @@ class user_pubkey_t { return user_pubkey_t(pk); } - const std::string& str() const { - return pubkey_; - } + const std::string& str() const { return pubkey_; } }; namespace loki { diff --git a/httpserver/http_connection.cpp b/httpserver/http_connection.cpp index 06f20e9f..e9498361 100644 --- a/httpserver/http_connection.cpp +++ b/httpserver/http_connection.cpp @@ -697,11 +697,13 @@ void connection_t::process_store(const json& params) { const auto data = params["data"].get(); bool created; - auto pk = user_pubkey_t::create(params["pubKey"].get(), created); + auto pk = + user_pubkey_t::create(params["pubKey"].get(), created); if (!created) { response_.result(http::status::bad_request); - body_stream_ << fmt::format("Pubkey must be {} characters long\n", USER_PUBKEY_SIZE); + body_stream_ << fmt::format("Pubkey must be {} characters long\n", + USER_PUBKEY_SIZE); LOKI_LOG(error, "Pubkey must be {} characters long", USER_PUBKEY_SIZE); return; } @@ -805,16 +807,17 @@ void connection_t::process_snodes_by_pk(const json& params) { } bool success; - const auto pk = user_pubkey_t::create(params["pubKey"].get(), success); + const auto pk = + user_pubkey_t::create(params["pubKey"].get(), success); if (!success) { response_.result(http::status::bad_request); - body_stream_ << fmt::format("Pubkey must be {} characters long\n", USER_PUBKEY_SIZE); + body_stream_ << fmt::format("Pubkey must be {} characters long\n", + USER_PUBKEY_SIZE); LOKI_LOG(debug, "Pubkey must be {} characters long ", USER_PUBKEY_SIZE); return; } - const std::vector nodes = - service_node_.get_snodes_by_pk(pk); + const std::vector nodes = service_node_.get_snodes_by_pk(pk); const json res_body = snodes_to_json(nodes); response_.result(http::status::ok); @@ -968,11 +971,13 @@ void connection_t::process_retrieve(const json& params) { } bool success; - const auto pk = user_pubkey_t::create(params["pubKey"].get(), success); + const auto pk = + user_pubkey_t::create(params["pubKey"].get(), success); if (!success) { response_.result(http::status::bad_request); - body_stream_ << fmt::format("Pubkey must be {} characters long\n", USER_PUBKEY_SIZE); + body_stream_ << fmt::format("Pubkey must be {} characters long\n", + USER_PUBKEY_SIZE); LOKI_LOG(debug, "Pubkey must be {} characters long ", USER_PUBKEY_SIZE); return; } diff --git a/httpserver/http_connection.h b/httpserver/http_connection.h index 9b3ebd0c..74a1a457 100644 --- a/httpserver/http_connection.h +++ b/httpserver/http_connection.h @@ -278,26 +278,30 @@ void run(boost::asio::io_context& ioc, const std::string& ip, uint16_t port, namespace fmt { - template <> - struct formatter { +template <> +struct formatter { template - constexpr auto parse(ParseContext &ctx) { return ctx.begin(); } + constexpr auto parse(ParseContext& ctx) { + return ctx.begin(); + } template - auto format(const loki::SNodeError& err, FormatContext &ctx) { - - switch (err) { - case loki::SNodeError::NO_ERROR: return format_to(ctx.out(), "NO_ERROR"); - case loki::SNodeError::ERROR_OTHER: return format_to(ctx.out(), "ERROR_OTHER"); - case loki::SNodeError::NO_REACH: return format_to(ctx.out(), "NO_REACH"); - case loki::SNodeError::HTTP_ERROR: return format_to(ctx.out(), "HTTP_ERROR"); - default: return format_to(ctx.out(), "[UNKNOWN]"); - } - + auto format(const loki::SNodeError& err, FormatContext& ctx) { + + switch (err) { + case loki::SNodeError::NO_ERROR: + return format_to(ctx.out(), "NO_ERROR"); + case loki::SNodeError::ERROR_OTHER: + return format_to(ctx.out(), "ERROR_OTHER"); + case loki::SNodeError::NO_REACH: + return format_to(ctx.out(), "NO_REACH"); + case loki::SNodeError::HTTP_ERROR: + return format_to(ctx.out(), "HTTP_ERROR"); + default: + return format_to(ctx.out(), "[UNKNOWN]"); + } } +}; - }; - - -} +} // namespace fmt diff --git a/httpserver/reachability_testing.cpp b/httpserver/reachability_testing.cpp index 2d2c5121..74b8b748 100644 --- a/httpserver/reachability_testing.cpp +++ b/httpserver/reachability_testing.cpp @@ -1,6 +1,6 @@ -#include "loki_logger.h" #include "reachability_testing.h" +#include "loki_logger.h" using std::chrono::steady_clock; using namespace std::chrono_literals; @@ -60,7 +60,6 @@ bool reachability_records_t::expire(const sn_pub_key_t& sn) { if (offline_nodes_.erase(sn)) { LOKI_LOG(warn, " - removed entry for {}", sn); } - } void reachability_records_t::set_reported(const sn_pub_key_t& sn) { diff --git a/httpserver/reachability_testing.h b/httpserver/reachability_testing.h index 27b071e9..a26d6922 100644 --- a/httpserver/reachability_testing.h +++ b/httpserver/reachability_testing.h @@ -1,8 +1,8 @@ #pragma once #include "loki_common.h" -#include #include +#include namespace loki { @@ -16,14 +16,13 @@ class reach_record_t { using time_point_t = std::chrono::time_point; -public: + public: time_point_t first_failure; time_point_t last_tested; // whether it's been reported to Lokid bool reported = false; reach_record_t(); - }; } // namespace detail diff --git a/httpserver/service_node.cpp b/httpserver/service_node.cpp index a1082f90..970a2d8c 100644 --- a/httpserver/service_node.cpp +++ b/httpserver/service_node.cpp @@ -235,7 +235,21 @@ parse_swarm_update(const std::shared_ptr& response_body) { const sn_record_t sn{port, std::move(snode_address), pubkey, std::move(snode_ip)}; - swarm_map[swarm_id].push_back(sn); + const bool fully_funded = sn_json.at("funded").get(); + + /// We want to include (test) decommissioned nodes, but not + /// partially funded ones. + if (!fully_funded) { + continue; + } + + /// Storing decommissioned nodes (with dummy swarm id) in + /// a separate data structure as it seems less error prone + if (swarm_id == INVALID_SWARM_ID) { + bu.decommissioned_nodes.push_back(sn); + } else { + swarm_map[swarm_id].push_back(sn); + } } bu.height = body.at("result").at("height").get(); @@ -268,6 +282,7 @@ void ServiceNode::bootstrap_data() { fields["height"] = true; fields["block_hash"] = true; fields["hardfork"] = true; + fields["funded"] = true; params["fields"] = fields; @@ -286,6 +301,8 @@ void ServiceNode::bootstrap_data() { if (res.error_code == SNodeError::NO_ERROR) { try { const block_update_t bu = parse_swarm_update(res.body); + // TODO: this should be disabled in the "testnet" mode + // (or changed to point to testnet seeds) on_bootstrap_update(bu); } catch (const std::exception& e) { LOKI_LOG( @@ -344,7 +361,7 @@ ServiceNode::~ServiceNode() { }; void ServiceNode::relay_data_reliable(const std::shared_ptr& req, - const sn_record_t& sn) const { + const sn_record_t& sn) const { LOKI_LOG(debug, "Relaying data to: {}", sn); @@ -605,7 +622,7 @@ void ServiceNode::on_swarm_update(const block_update_t& bu) { } } - swarm_->update_state(bu.swarms, events); + swarm_->update_state(bu.swarms, bu.decommissioned_nodes, events); if (!events.new_snodes.empty()) { bootstrap_peers(events.new_snodes); @@ -615,7 +632,7 @@ void ServiceNode::on_swarm_update(const block_update_t& bu) { bootstrap_swarms(events.new_swarms); } - if (events.decommissioned) { + if (events.dissolved) { /// Go through all our PK and push them accordingly salvage_data(); } @@ -656,11 +673,12 @@ void ServiceNode::swarm_timer_tick() { fields["height"] = true; fields["block_hash"] = true; fields["hardfork"] = true; + fields["funded"] = true; params["fields"] = fields; /// TODO: include decommissioned - params["active_only"] = true; + params["active_only"] = false; lokid_client_.make_lokid_request( "get_n_service_nodes", params, [this](const sn_response_t&& res) { @@ -697,14 +715,24 @@ void ServiceNode::ping_peers_tick() { this->peer_ping_timer_.async_wait( std::bind(&ServiceNode::ping_peers_tick, this)); + + /// TODO: To be safe, let's not even test peers until we + /// have reached the right hardfork height + /// We always test one node already known to be offline /// plus one random other node (could even be the same node) - const auto other_node = swarm_->choose_other_node(); + const auto random_node = swarm_->choose_funded_node(); - if (other_node) { - LOKI_LOG(debug, "Selected random node for testing: {}", (*other_node).pub_key()); - test_reachability(*other_node); + if (random_node) { + + if (random_node == our_address_) { + LOKI_LOG(debug, "Would test our own node, skipping"); + } else { + LOKI_LOG(debug, "Selected random node for testing: {}", + (*random_node).pub_key()); + test_reachability(*random_node); + } } else { LOKI_LOG(debug, "No nodes to test for reachability"); } @@ -716,17 +744,18 @@ void ServiceNode::ping_peers_tick() { const auto offline_node = reach_records_.next_to_test(); if (offline_node) { - const boost::optional sn = swarm_->get_node_by_pk(*offline_node); + const boost::optional sn = + swarm_->get_node_by_pk(*offline_node); LOKI_LOG(debug, "No nodes offline nodes to ping test yet"); if (sn) { test_reachability(*sn); } else { - LOKI_LOG(debug, "Node does not seem to exist anymore: {}", *offline_node); + LOKI_LOG(debug, "Node does not seem to exist anymore: {}", + *offline_node); // delete its entry from test records as irrelevant reach_records_.expire(*offline_node); } } - } void ServiceNode::test_reachability(const sn_record_t& sn) { @@ -750,7 +779,6 @@ void ServiceNode::test_reachability(const sn_record_t& sn) { #endif make_sn_request(ioc_, sn, req, std::move(callback)); - } void ServiceNode::lokid_ping_timer_tick() { @@ -1004,17 +1032,19 @@ void ServiceNode::report_node_reachability(const sn_pub_key_t& sn_pk, if (success) { if (reachable) { - LOKI_LOG(debug, "Successfully reported node as reachable: {}", sn_pk); + LOKI_LOG(debug, "Successfully reported node as reachable: {}", + sn_pk); this->reach_records_.expire(sn_pk); } else { - LOKI_LOG(debug, "Successfully reported node as unreachable {}", sn_pk); + LOKI_LOG(debug, "Successfully reported node as unreachable {}", + sn_pk); this->reach_records_.set_reported(sn_pk); } } }; - lokid_client_.make_lokid_request("report_peer_storage_server_status", params, - std::move(cb)); + lokid_client_.make_lokid_request("report_peer_storage_server_status", + params, std::move(cb)); } void ServiceNode::process_reach_test_response(sn_response_t&& res, @@ -1312,7 +1342,7 @@ void ServiceNode::bootstrap_swarms( LOKI_LOG(info, "Bootstrapping swarms: {}", vec_to_string(swarms)); } - const auto& all_swarms = swarm_->all_swarms(); + const auto& all_swarms = swarm_->all_valid_swarms(); std::vector all_entries; if (!get_all_messages(all_entries)) { @@ -1554,14 +1584,15 @@ bool ServiceNode::is_pubkey_for_us(const user_pubkey_t& pk) const { return swarm_->is_pubkey_for_us(pk); } -std::vector ServiceNode::get_snodes_by_pk(const user_pubkey_t& pk) { +std::vector +ServiceNode::get_snodes_by_pk(const user_pubkey_t& pk) { if (!swarm_) { LOKI_LOG(error, "Swarm data missing"); return {}; } - const auto& all_swarms = swarm_->all_swarms(); + const auto& all_swarms = swarm_->all_valid_swarms(); swarm_id_t swarm_id = get_swarm_by_pk(all_swarms, pk); @@ -1586,17 +1617,7 @@ bool ServiceNode::is_snode_address_known(const std::string& sn_address) { return {}; } - const auto& all_swarms = swarm_->all_swarms(); - - return std::any_of(all_swarms.begin(), all_swarms.end(), - [&sn_address](const SwarmInfo& swarm_info) { - return std::any_of( - swarm_info.snodes.begin(), - swarm_info.snodes.end(), - [&sn_address](const sn_record_t& sn_record) { - return sn_record.sn_address() == sn_address; - }); - }); + return swarm_->is_fully_funded_node(sn_address); } } // namespace loki diff --git a/httpserver/service_node.h b/httpserver/service_node.h index 2bd073e1..496d2e17 100644 --- a/httpserver/service_node.h +++ b/httpserver/service_node.h @@ -201,7 +201,8 @@ class ServiceNode { /// Check if status is OK and handle failed test otherwise; note /// that we want a copy of `sn` here because of the way it is called - void process_reach_test_response(sn_response_t&& res, const sn_pub_key_t& sn); + void process_reach_test_response(sn_response_t&& res, + const sn_pub_key_t& sn); /// From a peer void process_blockchain_test_response(sn_response_t&& res, diff --git a/httpserver/swarm.cpp b/httpserver/swarm.cpp index 40eae060..68b71a6b 100644 --- a/httpserver/swarm.cpp +++ b/httpserver/swarm.cpp @@ -25,7 +25,7 @@ Swarm::~Swarm() = default; bool Swarm::is_existing_swarm(swarm_id_t sid) const { - return std::any_of(all_cur_swarms_.begin(), all_cur_swarms_.end(), + return std::any_of(all_valid_swarms_.begin(), all_valid_swarms_.end(), [sid](const SwarmInfo& cur_swarm_info) { return cur_swarm_info.swarm_id == sid; }); @@ -63,7 +63,7 @@ SwarmEvents Swarm::derive_swarm_events(const all_swarms_t& swarms) const { // Got moved to a new swarm if (!swarm_exists(swarms, cur_swarm_id_)) { // Dissolved, new to push all our data to new swarms - events.decommissioned = true; + events.dissolved = true; } // If our old swarm is still alive, there is nothing for us to do @@ -150,13 +150,14 @@ static all_swarms_t apply_ips(const all_swarms_t& swarms_to_keep, void Swarm::apply_swarm_changes(const all_swarms_t& new_swarms) { - all_cur_swarms_ = apply_ips(new_swarms, all_cur_swarms_); + all_valid_swarms_ = apply_ips(new_swarms, all_valid_swarms_); } void Swarm::update_state(const all_swarms_t& swarms, + const std::vector& decommissioned, const SwarmEvents& events) { - if (events.decommissioned) { + if (events.dissolved) { LOKI_LOG(info, "EVENT: our old swarm got DISSOLVED!"); } @@ -184,33 +185,31 @@ void Swarm::update_state(const all_swarms_t& swarms, [this](const sn_record_t& record) { return record != our_address_; }); // Store a copy of every node in a separate data structure - all_other_nodes_.clear(); + all_funded_nodes_.clear(); for (const auto& si : swarms) { for (const auto& sn : si.snodes) { - if (sn != our_address_) - all_other_nodes_.push_back(sn); + all_funded_nodes_.push_back(sn); } } } +boost::optional Swarm::choose_funded_node() const { -boost::optional Swarm::choose_other_node() const { - - // Currently just select a node from the list of active nodes, - // but include decommissioned nodes in the future too - if (all_other_nodes_.empty()) + if (all_funded_nodes_.empty()) return boost::none; const auto idx = - util::uniform_distribution_portable(all_other_nodes_.size()); + util::uniform_distribution_portable(all_funded_nodes_.size()); - return all_other_nodes_[idx]; + // Note: this can return our own node which should be fine + return all_funded_nodes_[idx]; } -boost::optional Swarm::get_node_by_pk(const sn_pub_key_t &pk) const { +boost::optional +Swarm::get_node_by_pk(const sn_pub_key_t& pk) const { - for (const auto& si : all_cur_swarms_) { + for (const auto& si : all_valid_swarms_) { for (const auto& sn : si.snodes) { if (sn.pub_key() == pk) return sn; @@ -243,7 +242,15 @@ static uint64_t hex_to_u64(const user_pubkey_t& pk) { bool Swarm::is_pubkey_for_us(const user_pubkey_t& pk) const { /// TODO: Make sure no exceptions bubble up from here! - return cur_swarm_id_ == get_swarm_by_pk(all_cur_swarms_, pk); + return cur_swarm_id_ == get_swarm_by_pk(all_valid_swarms_, pk); +} + +bool Swarm::is_fully_funded_node(const std::string& sn_address) const { + + return std::any_of(all_funded_nodes_.begin(), all_funded_nodes_.end(), + [&sn_address](const sn_record_t& sn) { + return sn.sn_address() == sn_address; + }); } swarm_id_t get_swarm_by_pk(const std::vector& all_swarms, @@ -252,19 +259,24 @@ swarm_id_t get_swarm_by_pk(const std::vector& all_swarms, const uint64_t res = hex_to_u64(pk); /// We reserve UINT64_MAX as a sentinel swarm id for unassigned snodes - constexpr swarm_id_t MAX_ID = std::numeric_limits::max() - 1; - constexpr swarm_id_t SENTINEL_ID = std::numeric_limits::max(); + constexpr swarm_id_t MAX_ID = INVALID_SWARM_ID - 1; - swarm_id_t cur_best = SENTINEL_ID; - uint64_t cur_min = SENTINEL_ID; + swarm_id_t cur_best = INVALID_SWARM_ID; + uint64_t cur_min = INVALID_SWARM_ID; /// We don't require that all_swarms is sorted, so we find /// the smallest/largest elements in the same loop - swarm_id_t leftmost_id = SENTINEL_ID; + swarm_id_t leftmost_id = INVALID_SWARM_ID; swarm_id_t rightmost_id = 0; for (const auto& si : all_swarms) { + if (si.swarm_id == INVALID_SWARM_ID) { + /// Just to be sure we check again that no decomissioned + /// node is exposed to clients + continue; + } + uint64_t dist = (si.swarm_id > res) ? (si.swarm_id - res) : (res - si.swarm_id); if (dist < cur_min) { diff --git a/httpserver/swarm.h b/httpserver/swarm.h index 8b060869..46c5979c 100644 --- a/httpserver/swarm.h +++ b/httpserver/swarm.h @@ -25,6 +25,7 @@ using all_swarms_t = std::vector; struct block_update_t { all_swarms_t swarms; + std::vector decommissioned_nodes; uint64_t height; std::string block_hash; int hardfork; @@ -37,9 +38,9 @@ struct SwarmEvents { /// our (potentially new) swarm id swarm_id_t our_swarm_id; - /// whether our swarm got decommissioned and we + /// whether our swarm got dissolved and we /// need to salvage our stale data - bool decommissioned = false; + bool dissolved = false; /// detected new swarms that need to be bootstrapped std::vector new_swarms; /// detected new snodes in our swarm @@ -51,11 +52,12 @@ struct SwarmEvents { class Swarm { swarm_id_t cur_swarm_id_ = INVALID_SWARM_ID; - std::vector all_cur_swarms_; + /// Note: this excludes the "dummy" swarm + std::vector all_valid_swarms_; sn_record_t our_address_; std::vector swarm_peers_; - - std::vector all_other_nodes_; + /// This includes decommissioned nodes + std::vector all_funded_nodes_; /// Check if `sid` is an existing (active) swarm bool is_existing_swarm(swarm_id_t sid) const; @@ -69,15 +71,23 @@ class Swarm { SwarmEvents derive_swarm_events(const all_swarms_t& swarms) const; /// Update swarm state according to `events` - void update_state(const all_swarms_t& swarms, const SwarmEvents& events); + void update_state(const all_swarms_t& swarms, + const std::vector& decommissioned, + const SwarmEvents& events); void apply_swarm_changes(const all_swarms_t& new_swarms); bool is_pubkey_for_us(const user_pubkey_t& pk) const; + /// Whether `sn_address` is found in any of the swarms, including the + /// dummy swarm with decommissioned nodes + bool is_fully_funded_node(const std::string& sn_address) const; + const std::vector& other_nodes() const; - const std::vector& all_swarms() const { return all_cur_swarms_; } + const std::vector& all_valid_swarms() const { + return all_valid_swarms_; + } swarm_id_t our_swarm_id() const { return cur_swarm_id_; } @@ -87,10 +97,10 @@ class Swarm { // Select a node from all existing nodes (excluding us); throws if there is // no other nodes - boost::optional choose_other_node() const; + boost::optional choose_funded_node() const; // Get the node with public key `pk` if exists - boost::optional get_node_by_pk(const sn_pub_key_t &pk) const; + boost::optional get_node_by_pk(const sn_pub_key_t& pk) const; }; } // namespace loki