From a6dde31f953a9ccfb738b3daba775e6e946495e6 Mon Sep 17 00:00:00 2001 From: sachaaaaa Date: Thu, 11 Jul 2019 15:01:06 +1000 Subject: [PATCH 1/4] new `print_sn_state_changes` command --- src/daemon/command_parser_executor.cpp | 35 ++++++++++- src/daemon/command_parser_executor.h | 2 + src/daemon/command_server.cpp | 7 ++- src/daemon/rpc_command_executor.cpp | 36 +++++++++++ src/daemon/rpc_command_executor.h | 2 + src/rpc/core_rpc_server.cpp | 84 +++++++++++++++++++++++++ src/rpc/core_rpc_server.h | 2 + src/rpc/core_rpc_server_commands_defs.h | 45 +++++++++++++ 8 files changed, 211 insertions(+), 2 deletions(-) diff --git a/src/daemon/command_parser_executor.cpp b/src/daemon/command_parser_executor.cpp index c5f952ee27..3d5d987bab 100644 --- a/src/daemon/command_parser_executor.cpp +++ b/src/daemon/command_parser_executor.cpp @@ -95,6 +95,39 @@ bool t_command_parser_executor::print_checkpoints(const std::vector return m_executor.print_checkpoints(start_height, end_height, print_json); } +bool t_command_parser_executor::print_sn_state_changes(const std::vector &args) +{ + uint64_t start_height; + uint64_t end_height = cryptonote::COMMAND_RPC_GET_SN_STATE_CHANGES::HEIGHT_SENTINEL_VALUE; + + if (args.empty()) { + std::cout << "Missing first argument start_height" << std::endl; + return false; + } + + std::forward_list args_list(args.begin(), args.end()); + if (!epee::string_tools::get_xtype_from_string(start_height, args_list.front())) + { + std::cout << "start_height should be a number" << std::endl; + return false; + } + + args_list.pop_front(); + + if (!parse_if_present(args_list, end_height, "end height")) + return false; + + if (!args_list.empty()) + { + std::cout << "use: print_sn_state_changes start height [end height]" + << "(omit arguments to scan until the current block)" + << std::endl; + return false; + } + + return m_executor.print_sn_state_changes(start_height, end_height); +} + bool t_command_parser_executor::print_peer_list(const std::vector& args) { if (args.size() > 3) @@ -301,7 +334,7 @@ bool t_command_parser_executor::set_log_level(const std::vector& ar } } -bool t_command_parser_executor::print_height(const std::vector& args) +bool t_command_parser_executor::print_height(const std::vector& args) { if (!args.empty()) return false; diff --git a/src/daemon/command_parser_executor.h b/src/daemon/command_parser_executor.h index 3379e650a7..ddf35dc0d5 100644 --- a/src/daemon/command_parser_executor.h +++ b/src/daemon/command_parser_executor.h @@ -166,6 +166,8 @@ class t_command_parser_executor final bool check_blockchain_pruning(const std::vector& args); bool print_net_stats(const std::vector& args); + + bool print_sn_state_changes(const std::vector &args); }; } // namespace daemonize diff --git a/src/daemon/command_server.cpp b/src/daemon/command_server.cpp index f6f1b25844..78adfe92f1 100644 --- a/src/daemon/command_server.cpp +++ b/src/daemon/command_server.cpp @@ -362,7 +362,12 @@ t_command_server::t_command_server( , "print_checkpoints [+json] [start height] [end height]" , "Query the available checkpoints between the range, omit arguments to print the last 60 checkpoints" ); - + m_command_lookup.set_handler( + "print_sn_state_changes" + , std::bind(&t_command_parser_executor::print_sn_state_changes, &m_parser, p::_1) + , "print_sn_state_changes start height [end height]" + , "Query the state changes between the range, omit the last argument to scan until the current block" + ); #if defined(LOKI_ENABLE_INTEGRATION_TEST_HOOKS) m_command_lookup.set_handler( "relay_votes_and_uptime", std::bind([rpc_server](std::vector const &args) { diff --git a/src/daemon/rpc_command_executor.cpp b/src/daemon/rpc_command_executor.cpp index 5d53dd4377..087ee2ecb5 100644 --- a/src/daemon/rpc_command_executor.cpp +++ b/src/daemon/rpc_command_executor.cpp @@ -312,6 +312,42 @@ bool t_rpc_command_executor::print_checkpoints(uint64_t start_height, uint64_t e return true; } +bool t_rpc_command_executor::print_sn_state_changes(uint64_t start_height, uint64_t end_height) +{ + cryptonote::COMMAND_RPC_GET_SN_STATE_CHANGES::request req; + cryptonote::COMMAND_RPC_GET_SN_STATE_CHANGES::response res; + epee::json_rpc::error error_resp; + + req.start_height = start_height; + req.end_height = end_height; + + if (m_is_rpc) + { + if (!m_rpc_client->json_rpc_request(req, res, "get_sn_state_changes", "Failed to query blockchain checkpoints")) + return false; + } + else + { + if (!m_rpc_server->on_get_service_nodes_state_changes(req, res, error_resp) || res.status != CORE_RPC_STATUS_OK) + { + tools::fail_msg_writer() << "Failed to query sn state changes"; + return false; + } + } + + std::stringstream output; + + output << "Service Node state Changes (blocks " << res.start_height << "-" << res.end_height << ")" << std::endl; + output << " Recommissions:\t\t" << res.total_recommission << std::endl; + output << " Unlocks:\t\t" << res.total_unlock << std::endl; + output << " Decommissions:\t\t" << res.total_decommission << std::endl; + output << " Deregistrations:\t" << res.total_deregister << std::endl; + output << " IP change penalties:\t" << res.total_ip_change_penalty << std::endl; + + tools::success_msg_writer() << output.str(); + return true; +} + bool t_rpc_command_executor::print_peer_list(bool white, bool gray, size_t limit) { cryptonote::COMMAND_RPC_GET_PEER_LIST::request req; cryptonote::COMMAND_RPC_GET_PEER_LIST::response res; diff --git a/src/daemon/rpc_command_executor.h b/src/daemon/rpc_command_executor.h index 2426260011..407125fa43 100644 --- a/src/daemon/rpc_command_executor.h +++ b/src/daemon/rpc_command_executor.h @@ -72,6 +72,8 @@ class t_rpc_command_executor final { bool print_checkpoints(uint64_t start_height, uint64_t end_height, bool print_json); + bool print_sn_state_changes(uint64_t start_height, uint64_t end_height); + bool print_peer_list(bool white = true, bool gray = true, size_t limit = 0); bool print_peer_list_stats(); diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index cc91008587..b5e8ccbbe3 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -2966,5 +2966,89 @@ namespace cryptonote res.checkpoints = db.get_checkpoints_range(req.start_height, req.end_height); return true; } + //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_get_service_nodes_state_changes(const COMMAND_RPC_GET_SN_STATE_CHANGES::request& req, COMMAND_RPC_GET_SN_STATE_CHANGES::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) + { + using blob_t = cryptonote::blobdata; + using block_pair_t = std::pair; + std::vector blocks; + std::vector txs; + + const auto& db = m_core.get_blockchain_storage(); + const uint64_t current_height = db.get_current_blockchain_height(); + + uint64_t end_height; + if (req.end_height == COMMAND_RPC_GET_SN_STATE_CHANGES::HEIGHT_SENTINEL_VALUE) { + end_height = current_height; + } else { + end_height = req.end_height; + } + + if (!db.get_blocks(req.start_height, end_height - req.start_height, blocks, txs)) { + MERROR("Could not query block at requested height: " << req.start_height); + return false; + } + + res.start_height = req.start_height; + res.end_height = end_height; + res.total_deregister = 0; + res.total_decommission = 0; + res.total_ip_change_penalty = 0; + res.total_recommission = 0; + res.total_unlock = 0; + + for (size_t i = 0; i < txs.size(); ++i) + { + const auto& blob = txs[i]; + cryptonote::transaction tx; + if (!cryptonote::parse_and_validate_tx_from_blob(blob, tx)) + { + MERROR("tx could not be validated from blob, possibly corrupt blockchain"); + continue; + } + if (tx.type == cryptonote::txtype::state_change) + { + const uint8_t hard_fork_version = blocks[i].second.major_version; + cryptonote::tx_extra_service_node_state_change state_change; + if (!cryptonote::get_service_node_state_change_from_tx_extra(tx.extra, state_change, hard_fork_version)) + { + // TODO: This seem to be triggered quite often with hf 11 blocks + // LOG_ERROR("Could not get state change from tx, possibly corrupt tx, hf_version "<< std::to_string(hard_fork_version)); + continue; + } + + switch(state_change.state) { + case service_nodes::new_state::deregister: + res.total_deregister++; + break; + + case service_nodes::new_state::decommission: + res.total_decommission++; + break; + + case service_nodes::new_state::recommission: + res.total_recommission++; + break; + + case service_nodes::new_state::ip_change_penalty: + res.total_ip_change_penalty++; + break; + + default: + MERROR("Unhandled state in on_get_service_nodes_state_changes"); + break; + } + } + + if (tx.type == cryptonote::txtype::key_image_unlock) + { + res.total_unlock++; + } + } + + res.status = CORE_RPC_STATUS_OK; + return true; + } + } // namespace cryptonote diff --git a/src/rpc/core_rpc_server.h b/src/rpc/core_rpc_server.h index 5f042d4ccd..be55e37242 100644 --- a/src/rpc/core_rpc_server.h +++ b/src/rpc/core_rpc_server.h @@ -188,6 +188,7 @@ namespace cryptonote MAP_JON_RPC_WE("get_checkpoints", on_get_checkpoints, COMMAND_RPC_GET_CHECKPOINTS) MAP_JON_RPC_WE_IF("perform_blockchain_test", on_perform_blockchain_test, COMMAND_RPC_PERFORM_BLOCKCHAIN_TEST, !m_restricted) MAP_JON_RPC_WE_IF("storage_server_ping", on_storage_server_ping, COMMAND_RPC_STORAGE_SERVER_PING, !m_restricted) + MAP_JON_RPC_WE("get_service_nodes_state_changes", on_get_service_nodes_state_changes, COMMAND_RPC_GET_SN_STATE_CHANGES) END_JSON_RPC_MAP() END_URI_MAP2() @@ -277,6 +278,7 @@ namespace cryptonote bool on_perform_blockchain_test(const COMMAND_RPC_PERFORM_BLOCKCHAIN_TEST::request& req, COMMAND_RPC_PERFORM_BLOCKCHAIN_TEST::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); bool on_storage_server_ping(const COMMAND_RPC_STORAGE_SERVER_PING::request& req, COMMAND_RPC_STORAGE_SERVER_PING::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); bool on_get_checkpoints(const COMMAND_RPC_GET_CHECKPOINTS::request& req, COMMAND_RPC_GET_CHECKPOINTS::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); + bool on_get_service_nodes_state_changes(const COMMAND_RPC_GET_SN_STATE_CHANGES::request& req, COMMAND_RPC_GET_SN_STATE_CHANGES::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); //----------------------- #if defined(LOKI_ENABLE_INTEGRATION_TEST_HOOKS) diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index cf82c3f748..c289fb2b58 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -3132,4 +3132,49 @@ namespace cryptonote }; typedef epee::misc_utils::struct_init response; }; + + LOKI_RPC_DOC_INTROSPECT + // Query hardcoded/service node checkpoints stored for the blockchain. Omit all arguments to retrieve the latest "count" checkpoints. + struct COMMAND_RPC_GET_SN_STATE_CHANGES + { + constexpr static uint32_t NUM_BLOCKS_TO_SCAN_BY_DEFAULT = 720; + constexpr static uint64_t HEIGHT_SENTINEL_VALUE = (UINT64_MAX - 1); + struct request_t + { + uint64_t start_height; + uint64_t end_height; // Optional: If omitted, the tally runs until the current block + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(start_height) + KV_SERIALIZE_OPT(end_height, HEIGHT_SENTINEL_VALUE) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init request; + + struct response_t + { + std::string status; // Generic RPC error code. "OK" is the success value. + bool untrusted; // If the result is obtained using bootstrap mode, and therefore not trusted `true`, or otherwise `false`. + + uint32_t total_deregister; + uint32_t total_ip_change_penalty; + uint32_t total_decommission; + uint32_t total_recommission; + uint32_t total_unlock; + uint64_t start_height; + uint64_t end_height; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(status) + KV_SERIALIZE(untrusted) + KV_SERIALIZE(total_deregister) + KV_SERIALIZE(total_ip_change_penalty) + KV_SERIALIZE(total_decommission) + KV_SERIALIZE(total_recommission) + KV_SERIALIZE(start_height) + KV_SERIALIZE(end_height) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init response; + }; } From 89b08a603197a7d8571140cf4f433aeda582a1d0 Mon Sep 17 00:00:00 2001 From: sachaaaaa Date: Mon, 15 Jul 2019 10:43:43 +1000 Subject: [PATCH 2/4] Address reviews --- src/daemon/command_parser_executor.cpp | 2 +- src/daemon/command_server.cpp | 2 +- src/daemon/rpc_command_executor.cpp | 2 +- src/rpc/core_rpc_server.cpp | 16 +++++++++------- 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/daemon/command_parser_executor.cpp b/src/daemon/command_parser_executor.cpp index 3d5d987bab..c43a829ea0 100644 --- a/src/daemon/command_parser_executor.cpp +++ b/src/daemon/command_parser_executor.cpp @@ -119,7 +119,7 @@ bool t_command_parser_executor::print_sn_state_changes(const std::vector [end height]" << "(omit arguments to scan until the current block)" << std::endl; return false; diff --git a/src/daemon/command_server.cpp b/src/daemon/command_server.cpp index 78adfe92f1..d5bc8fb4d5 100644 --- a/src/daemon/command_server.cpp +++ b/src/daemon/command_server.cpp @@ -365,7 +365,7 @@ t_command_server::t_command_server( m_command_lookup.set_handler( "print_sn_state_changes" , std::bind(&t_command_parser_executor::print_sn_state_changes, &m_parser, p::_1) - , "print_sn_state_changes start height [end height]" + , "print_sn_state_changes [end height]" , "Query the state changes between the range, omit the last argument to scan until the current block" ); #if defined(LOKI_ENABLE_INTEGRATION_TEST_HOOKS) diff --git a/src/daemon/rpc_command_executor.cpp b/src/daemon/rpc_command_executor.cpp index 087ee2ecb5..8010a5998a 100644 --- a/src/daemon/rpc_command_executor.cpp +++ b/src/daemon/rpc_command_executor.cpp @@ -323,7 +323,7 @@ bool t_rpc_command_executor::print_sn_state_changes(uint64_t start_height, uint6 if (m_is_rpc) { - if (!m_rpc_client->json_rpc_request(req, res, "get_sn_state_changes", "Failed to query blockchain checkpoints")) + if (!m_rpc_client->json_rpc_request(req, res, "get_service_nodes_state_changes", "Failed to query service nodes state changes")) return false; } else diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index b5e8ccbbe3..d666bc6031 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -2984,18 +2984,20 @@ namespace cryptonote end_height = req.end_height; } - if (!db.get_blocks(req.start_height, end_height - req.start_height, blocks, txs)) { - MERROR("Could not query block at requested height: " << req.start_height); + if (end_height < req.start_height){ + res.error_resp.code = CORE_RPC_ERROR_CODE_WRONG_PARAM; + res.error_resp.message = "The provided end_height needs to be higher than start_height"; + return false; + } + + if (!db.get_blocks(req.start_height, end_height - req.start_height + 1, blocks, txs)) { + res.error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; + res.error_resp.message = "Could not query block at requested height: " << req.start_height; return false; } res.start_height = req.start_height; res.end_height = end_height; - res.total_deregister = 0; - res.total_decommission = 0; - res.total_ip_change_penalty = 0; - res.total_recommission = 0; - res.total_unlock = 0; for (size_t i = 0; i < txs.size(); ++i) { From a927e7c658958c539572b4bb00e6b53948a5dcf1 Mon Sep 17 00:00:00 2001 From: sachaaaaa Date: Mon, 15 Jul 2019 11:19:55 +1000 Subject: [PATCH 3/4] Fix fetching txs for each blocks --- src/rpc/core_rpc_server.cpp | 83 ++++++++++++++++++++----------------- 1 file changed, 46 insertions(+), 37 deletions(-) diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index d666bc6031..200991d805 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -2972,7 +2972,6 @@ namespace cryptonote using blob_t = cryptonote::blobdata; using block_pair_t = std::pair; std::vector blocks; - std::vector txs; const auto& db = m_core.get_blockchain_storage(); const uint64_t current_height = db.get_current_blockchain_height(); @@ -2985,66 +2984,76 @@ namespace cryptonote } if (end_height < req.start_height){ - res.error_resp.code = CORE_RPC_ERROR_CODE_WRONG_PARAM; - res.error_resp.message = "The provided end_height needs to be higher than start_height"; + error_resp.code = CORE_RPC_ERROR_CODE_WRONG_PARAM; + error_resp.message = "The provided end_height needs to be higher than start_height"; return false; } - if (!db.get_blocks(req.start_height, end_height - req.start_height + 1, blocks, txs)) { - res.error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; - res.error_resp.message = "Could not query block at requested height: " << req.start_height; + if (!db.get_blocks(req.start_height, end_height - req.start_height + 1, blocks)) { + error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; + error_resp.message = "Could not query block at requested height: " + std::to_string(req.start_height); return false; } res.start_height = req.start_height; res.end_height = end_height; - for (size_t i = 0; i < txs.size(); ++i) + std::vector blobs; + std::vector missed_ids; + for (const auto& block : blocks) { - const auto& blob = txs[i]; - cryptonote::transaction tx; - if (!cryptonote::parse_and_validate_tx_from_blob(blob, tx)) + blobs.clear(); + if (!db.get_transactions_blobs(block.second.tx_hashes, blobs, missed_ids)) { - MERROR("tx could not be validated from blob, possibly corrupt blockchain"); + MERROR("Could not query block at requested height: " << cryptonote::get_block_height(block.second)); continue; } - if (tx.type == cryptonote::txtype::state_change) + const uint8_t hard_fork_version = block.second.major_version; + for (const auto& blob : blobs) { - const uint8_t hard_fork_version = blocks[i].second.major_version; - cryptonote::tx_extra_service_node_state_change state_change; - if (!cryptonote::get_service_node_state_change_from_tx_extra(tx.extra, state_change, hard_fork_version)) + cryptonote::transaction tx; + if (!cryptonote::parse_and_validate_tx_from_blob(blob, tx)) { - // TODO: This seem to be triggered quite often with hf 11 blocks - // LOG_ERROR("Could not get state change from tx, possibly corrupt tx, hf_version "<< std::to_string(hard_fork_version)); + MERROR("tx could not be validated from blob, possibly corrupt blockchain"); continue; } + if (tx.type == cryptonote::txtype::state_change) + { + cryptonote::tx_extra_service_node_state_change state_change; + if (!cryptonote::get_service_node_state_change_from_tx_extra(tx.extra, state_change, hard_fork_version)) + { + // TODO: This seem to be triggered quite often with hf 11 blocks + // LOG_ERROR("Could not get state change from tx, possibly corrupt tx, hf_version "<< std::to_string(hard_fork_version)); + continue; + } - switch(state_change.state) { - case service_nodes::new_state::deregister: - res.total_deregister++; - break; + switch(state_change.state) { + case service_nodes::new_state::deregister: + res.total_deregister++; + break; - case service_nodes::new_state::decommission: - res.total_decommission++; - break; + case service_nodes::new_state::decommission: + res.total_decommission++; + break; - case service_nodes::new_state::recommission: - res.total_recommission++; - break; + case service_nodes::new_state::recommission: + res.total_recommission++; + break; - case service_nodes::new_state::ip_change_penalty: - res.total_ip_change_penalty++; - break; + case service_nodes::new_state::ip_change_penalty: + res.total_ip_change_penalty++; + break; - default: - MERROR("Unhandled state in on_get_service_nodes_state_changes"); - break; + default: + MERROR("Unhandled state in on_get_service_nodes_state_changes"); + break; + } } - } - if (tx.type == cryptonote::txtype::key_image_unlock) - { - res.total_unlock++; + if (tx.type == cryptonote::txtype::key_image_unlock) + { + res.total_unlock++; + } } } From e2ca082352828426fc149cbf359be79fa69adeea Mon Sep 17 00:00:00 2001 From: sachaaaaa Date: Mon, 15 Jul 2019 13:39:32 +1000 Subject: [PATCH 4/4] sentinel value is set to current height - 1 --- src/daemon/rpc_command_executor.cpp | 2 +- src/rpc/core_rpc_server.cpp | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/daemon/rpc_command_executor.cpp b/src/daemon/rpc_command_executor.cpp index 8010a5998a..d7a89e5056 100644 --- a/src/daemon/rpc_command_executor.cpp +++ b/src/daemon/rpc_command_executor.cpp @@ -337,7 +337,7 @@ bool t_rpc_command_executor::print_sn_state_changes(uint64_t start_height, uint6 std::stringstream output; - output << "Service Node state Changes (blocks " << res.start_height << "-" << res.end_height << ")" << std::endl; + output << "Service Node State Changes (blocks " << res.start_height << "-" << res.end_height << ")" << std::endl; output << " Recommissions:\t\t" << res.total_recommission << std::endl; output << " Unlocks:\t\t" << res.total_unlock << std::endl; output << " Decommissions:\t\t" << res.total_decommission << std::endl; diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index 200991d805..c38107e105 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -2978,7 +2978,8 @@ namespace cryptonote uint64_t end_height; if (req.end_height == COMMAND_RPC_GET_SN_STATE_CHANGES::HEIGHT_SENTINEL_VALUE) { - end_height = current_height; + // current height is the block being mined, so exclude it from the results + end_height = current_height - 1; } else { end_height = req.end_height; } @@ -3022,8 +3023,7 @@ namespace cryptonote cryptonote::tx_extra_service_node_state_change state_change; if (!cryptonote::get_service_node_state_change_from_tx_extra(tx.extra, state_change, hard_fork_version)) { - // TODO: This seem to be triggered quite often with hf 11 blocks - // LOG_ERROR("Could not get state change from tx, possibly corrupt tx, hf_version "<< std::to_string(hard_fork_version)); + LOG_ERROR("Could not get state change from tx, possibly corrupt tx, hf_version "<< std::to_string(hard_fork_version)); continue; }