Permalink
Browse files

Pruning

The blockchain prunes seven eighths of prunable tx data.
This saves about two thirds of the blockchain size, while
keeping the node useful as a sync source for an eighth
of the blockchain.

No other data is currently pruned.
  • Loading branch information...
moneromooo-monero committed Apr 29, 2018
1 parent 094b0c4 commit bac84cb3b09e72ad7007688e5e8e34492d4d4a35
Showing with 3,246 additions and 423 deletions.
  1. +10 −0 contrib/epee/include/misc_log_ex.h
  2. +2 −0 contrib/epee/include/net/abstract_tcp_server2.inl
  3. +8 −2 contrib/epee/include/net/net_utils_base.h
  4. +9 −0 contrib/epee/include/string_tools.h
  5. +24 −2 src/blockchain_db/blockchain_db.cpp
  6. +62 −0 src/blockchain_db/blockchain_db.h
  7. +364 −18 src/blockchain_db/lmdb/db_lmdb.cpp
  8. +21 −0 src/blockchain_db/lmdb/db_lmdb.h
  9. +33 −0 src/blockchain_utilities/CMakeLists.txt
  10. +6 −0 src/blockchain_utilities/blockchain_export.cpp
  11. +697 −0 src/blockchain_utilities/blockchain_prune.cpp
  12. +2 −0 src/common/CMakeLists.txt
  13. +112 −0 src/common/pruning.cpp
  14. +51 −0 src/common/pruning.h
  15. +22 −1 src/cryptonote_basic/connection_context.h
  16. +11 −4 src/cryptonote_basic/cryptonote_basic.h
  17. +1 −0 src/cryptonote_basic/cryptonote_format_utils.cpp
  18. +6 −0 src/cryptonote_config.h
  19. +70 −3 src/cryptonote_core/blockchain.cpp
  20. +6 −0 src/cryptonote_core/blockchain.h
  21. +47 −0 src/cryptonote_core/cryptonote_core.cpp
  22. +39 −0 src/cryptonote_core/cryptonote_core.h
  23. +142 −47 src/cryptonote_protocol/block_queue.cpp
  24. +11 −6 src/cryptonote_protocol/block_queue.h
  25. +5 −0 src/cryptonote_protocol/cryptonote_protocol_defs.h
  26. +15 −1 src/cryptonote_protocol/cryptonote_protocol_handler.h
  27. +607 −175 src/cryptonote_protocol/cryptonote_protocol_handler.inl
  28. +23 −0 src/daemon/command_parser_executor.cpp
  29. +4 −0 src/daemon/command_parser_executor.h
  30. +10 −0 src/daemon/command_server.cpp
  31. +110 −13 src/daemon/rpc_command_executor.cpp
  32. +4 −0 src/daemon/rpc_command_executor.h
  33. +0 −2 src/debug_utilities/cn_deserialize.cpp
  34. +15 −0 src/p2p/net_node.h
  35. +149 −49 src/p2p/net_node.inl
  36. +12 −0 src/p2p/net_node_common.h
  37. +34 −2 src/p2p/net_peerlist.h
  38. +17 −0 src/p2p/net_peerlist_boost_serialization.h
  39. +7 −3 src/p2p/p2p_protocol_defs.h
  40. +73 −14 src/rpc/core_rpc_server.cpp
  41. +2 −0 src/rpc/core_rpc_server.h
  42. +42 −5 src/rpc/core_rpc_server_commands_defs.h
  43. +1 −0 src/rpc/message_data_structs.h
  44. +2 −0 src/serialization/json_object.cpp
  45. +113 −76 src/wallet/wallet2.cpp
  46. +2 −0 tests/core_proxy/core_proxy.h
  47. +1 −0 tests/unit_tests/CMakeLists.txt
  48. +2 −0 tests/unit_tests/ban.cpp
  49. +240 −0 tests/unit_tests/pruning.cpp
@@ -85,6 +85,16 @@
#define MGINFO_MAGENTA(x) MCLOG_MAGENTA(el::Level::Info, "global",x)
#define MGINFO_CYAN(x) MCLOG_CYAN(el::Level::Info, "global",x)

#define IFLOG(level, cat, type, init, x) \
do { \
if (ELPP->vRegistry()->allowed(level, cat)) { \
init; \
el::base::Writer(level, __FILE__, __LINE__, ELPP_FUNC, type).construct(cat) << x; \
} \
} while(0)
#define MIDEBUG(init, x) IFLOG(el::Level::Debug, MONERO_DEFAULT_LOG_CATEGORY, el::base::DispatchAction::NormalLog, init, x)


#define LOG_ERROR(x) MERROR(x)
#define LOG_PRINT_L0(x) MWARNING(x)
#define LOG_PRINT_L1(x) MINFO(x)
@@ -295,6 +295,7 @@ PRAGMA_WARNING_DISABLE_VS(4355)
CRITICAL_REGION_LOCAL(m_throttle_speed_in_mutex);
m_throttle_speed_in.handle_trafic_exact(bytes_transferred);
context.m_current_speed_down = m_throttle_speed_in.get_current_speed();
context.m_max_speed_down = std::max(context.m_max_speed_down, context.m_current_speed_down);
}

{
@@ -497,6 +498,7 @@ PRAGMA_WARNING_DISABLE_VS(4355)
CRITICAL_REGION_LOCAL(m_throttle_speed_out_mutex);
m_throttle_speed_out.handle_trafic_exact(cb);
context.m_current_speed_up = m_throttle_speed_out.get_current_speed();
context.m_max_speed_up = std::max(context.m_max_speed_up, context.m_current_speed_up);
}

//_info("[sock " << socket_.native_handle() << "] SEND " << cb);
@@ -228,6 +228,8 @@ namespace net_utils
uint64_t m_send_cnt;
double m_current_speed_down;
double m_current_speed_up;
double m_max_speed_down;
double m_max_speed_up;

connection_context_base(boost::uuids::uuid connection_id,
const network_address &remote_address, bool is_income,
@@ -242,7 +244,9 @@ namespace net_utils
m_recv_cnt(recv_cnt),
m_send_cnt(send_cnt),
m_current_speed_down(0),
m_current_speed_up(0)
m_current_speed_up(0),
m_max_speed_down(0),
m_max_speed_up(0)
{}

connection_context_base(): m_connection_id(),
@@ -254,7 +258,9 @@ namespace net_utils
m_recv_cnt(0),
m_send_cnt(0),
m_current_speed_down(0),
m_current_speed_up(0)
m_current_speed_up(0),
m_max_speed_down(0),
m_max_speed_up(0)
{}

connection_context_base& operator=(const connection_context_base& a)
@@ -236,6 +236,15 @@ POP_WARNINGS
return boost::lexical_cast<std::string>(val);
}
//----------------------------------------------------------------------------
template<typename T> inline std::string to_string_hex(const T &val)
{
std::stringstream ss;
ss << std::hex << val;
std::string s;
ss >> s;
return s;
}
//----------------------------------------------------------------------------

inline bool compare_no_case(const std::string& str1, const std::string& str2)
{
@@ -267,7 +267,10 @@ void BlockchainDB::pop_block(block& blk, std::vector<transaction>& txs)

for (const auto& h : boost::adaptors::reverse(blk.tx_hashes))
{
txs.push_back(get_tx(h));
cryptonote::transaction tx;
if (!get_tx(h, tx) && !get_pruned_tx(h, tx))
throw DB_ERROR("Failed to get pruned or unpruned transaction from the db");
txs.push_back(std::move(tx));
remove_transaction(h);
}
remove_transaction(get_transaction_hash(blk.miner_tx));
@@ -280,7 +283,7 @@ bool BlockchainDB::is_open() const

void BlockchainDB::remove_transaction(const crypto::hash& tx_hash)
{
transaction tx = get_tx(tx_hash);
transaction tx = get_pruned_tx(tx_hash);

for (const txin_v& tx_input : tx.vin)
{
@@ -325,6 +328,17 @@ bool BlockchainDB::get_tx(const crypto::hash& h, cryptonote::transaction &tx) co
return true;
}

bool BlockchainDB::get_pruned_tx(const crypto::hash& h, cryptonote::transaction &tx) const
{
blobdata bd;
if (!get_pruned_tx_blob(h, bd))
return false;
if (!parse_and_validate_tx_base_from_blob(bd, tx))
throw DB_ERROR("Failed to parse transaction base from blob retrieved from the db");

return true;
}

transaction BlockchainDB::get_tx(const crypto::hash& h) const
{
transaction tx;
@@ -333,6 +347,14 @@ transaction BlockchainDB::get_tx(const crypto::hash& h) const
return tx;
}

transaction BlockchainDB::get_pruned_tx(const crypto::hash& h) const
{
transaction tx;
if (!get_pruned_tx(h, tx))
throw TX_DNE(std::string("pruned tx with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str());
return tx;
}

void BlockchainDB::reset_stats()
{
num_calls = 0;
@@ -1125,6 +1125,17 @@ class BlockchainDB
*/
virtual transaction get_tx(const crypto::hash& h) const;

/**
* @brief fetches the transaction base with the given hash
*
* If the transaction does not exist, the subclass should throw TX_DNE.
*
* @param h the hash to look for
*
* @return the transaction with the given hash
*/
virtual transaction get_pruned_tx(const crypto::hash& h) const;

/**
* @brief fetches the transaction with the given hash
*
@@ -1136,6 +1147,17 @@ class BlockchainDB
*/
virtual bool get_tx(const crypto::hash& h, transaction &tx) const;

/**
* @brief fetches the transaction base with the given hash
*
* If the transaction does not exist, the subclass should return false.
*
* @param h the hash to look for
*
* @return true iff the transaction was found
*/
virtual bool get_pruned_tx(const crypto::hash& h, transaction &tx) const;

/**
* @brief fetches the transaction blob with the given hash
*
@@ -1164,6 +1186,21 @@ class BlockchainDB
*/
virtual bool get_pruned_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const = 0;

/**
* @brief fetches the prunable transaction blob with the given hash
*
* The subclass should return the prunable transaction stored which has the given
* hash.
*
* If the transaction does not exist, or if we do not have that prunable data,
* the subclass should return false.
*
* @param h the hash to look for
*
* @return true iff the transaction was found and we have its prunable data
*/
virtual bool get_prunable_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const = 0;

/**
* @brief fetches the prunable transaction hash
*
@@ -1411,6 +1448,31 @@ class BlockchainDB
*/
virtual void prune_outputs(uint64_t amount) = 0;

/**
* @brief get the blockchain pruning seed
* @return the blockchain pruning seed
*/
virtual uint32_t get_blockchain_pruning_seed() const = 0;

/**
* @brief prunes the blockchain
* @param pruning_seed the seed to use, 0 for default (highly recommended)
* @return success iff true
*/
virtual bool prune_blockchain(uint32_t pruning_seed = 0) = 0;

/**
* @brief prunes recent blockchain changes as needed, iff pruning is enabled
* @return success iff true
*/
virtual bool update_pruning() = 0;

/**
* @brief checks pruning was done correctly, iff enabled
* @return success iff true
*/
virtual bool check_pruning() = 0;

/**
* @brief runs a function over all txpool transactions
*
Oops, something went wrong.

0 comments on commit bac84cb

Please sign in to comment.