Skip to content
Permalink
Browse files

monerod can now sync from pruned blocks

If the peer (whether pruned or not itself) supports sending pruned blocks
to syncing nodes, the pruned version will be sent along with the hash
of the pruned data and the block weight. The original tx hashes can be
reconstructed from the pruned txes and theur prunable data hash. Those
hashes and the block weights are hashes and checked against the set of
precompiled hashes, ensuring the data we received is the original data.
It is currently not possible to use this system when not using the set
of precompiled hashes, since block weights can not otherwise be checked
for validity.

This is off by default for now, and is enabled by --sync-pruned-blocks
  • Loading branch information...
moneromooo-monero committed Sep 16, 2019
1 parent d0d76f7 commit 8330e772f1ed680a54833d25c4d17d09a99ab8d6
@@ -156,7 +156,8 @@ struct txpool_tx_meta_t
uint8_t relayed;
uint8_t do_not_relay;
uint8_t double_spend_seen: 1;
uint8_t bf_padding: 7;
uint8_t pruned: 1;
uint8_t bf_padding: 6;

uint8_t padding[76]; // till 192 bytes
};
@@ -2056,7 +2056,7 @@ bool BlockchainLMDB::prune_worker(int mode, uint32_t pruning_seed)
++n_prunable_records;
result = mdb_cursor_get(c_txs_prunable, &k, &v, MDB_SET);
if (result == MDB_NOTFOUND)
MWARNING("Already pruned at height " << block_height << "/" << blockchain_height);
MDEBUG("Already pruned at height " << block_height << "/" << blockchain_height);
else if (result)
throw0(DB_ERROR(lmdb_error("Failed to find transaction prunable data: ", result).c_str()));
else
@@ -2152,7 +2152,7 @@ bool BlockchainLMDB::prune_worker(int mode, uint32_t pruning_seed)
{
++n_prunable_records;
if (result == MDB_NOTFOUND)
MWARNING("Already pruned at height " << block_height << "/" << blockchain_height);
MDEBUG("Already pruned at height " << block_height << "/" << blockchain_height);
else
{
MDEBUG("Pruning at height " << block_height << "/" << blockchain_height);
@@ -2994,6 +2994,8 @@ bool BlockchainLMDB::get_tx_blob(const crypto::hash& h, cryptonote::blobdata &bd
return false;
else if (get_result)
throw0(DB_ERROR(lmdb_error("DB error attempting to fetch tx from hash", get_result).c_str()));
else if (result1.mv_size == 0)
return false;

bd.assign(reinterpret_cast<char*>(result0.mv_data), result0.mv_size);
bd.append(reinterpret_cast<char*>(result1.mv_data), result1.mv_size);
@@ -191,7 +191,7 @@ int check_flush(cryptonote::core &core, std::vector<block_complete_entry> &block
}
hashes.push_back(cryptonote::get_block_hash(block));
}
core.prevalidate_block_hashes(core.get_blockchain_storage().get_db().height(), hashes);
core.prevalidate_block_hashes(core.get_blockchain_storage().get_db().height(), hashes, {});

std::vector<block> pblocks;
if (!core.prepare_handle_incoming_blocks(blocks, pblocks))
@@ -217,7 +217,7 @@ int check_flush(cryptonote::core &core, std::vector<block_complete_entry> &block
if(tvc.m_verifivation_failed)
{
MERROR("transaction verification failed, tx_id = "
<< epee::string_tools::pod_to_hex(get_blob_hash(tx_blob)));
<< epee::string_tools::pod_to_hex(get_blob_hash(tx_blob.blob)));
core.cleanup_handle_incoming_blocks();
return 1;
}
@@ -468,13 +468,17 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path
{
cryptonote::blobdata block;
cryptonote::block_to_blob(bp.block, block);
std::vector<cryptonote::blobdata> txs;
std::vector<tx_blob_entry> txs;
for (const auto &tx: bp.txs)
{
txs.push_back(cryptonote::blobdata());
cryptonote::tx_to_blob(tx, txs.back());
txs.push_back({cryptonote::blobdata(), crypto::null_hash});
cryptonote::tx_to_blob(tx, txs.back().blob);
}
blocks.push_back({block, txs});
block_complete_entry bce;
bce.pruned = false;
bce.block = std::move(block);
bce.txs = std::move(txs);
blocks.push_back(bce);
int ret = check_flush(core, blocks, false);
if (ret)
{
@@ -99,17 +99,23 @@ bool BlocksdatFile::initialize_file(uint64_t block_stop)
return true;
}

void BlocksdatFile::write_block(const crypto::hash& block_hash)
void BlocksdatFile::write_block(const crypto::hash& block_hash, uint64_t weight)
{
m_hashes.push_back(block_hash);
m_weights.push_back(weight);
while (m_hashes.size() >= HASH_OF_HASHES_STEP)
{
crypto::hash hash;
crypto::cn_fast_hash(m_hashes.data(), HASH_OF_HASHES_STEP * sizeof(crypto::hash), hash);
memmove(m_hashes.data(), m_hashes.data() + HASH_OF_HASHES_STEP, (m_hashes.size() - HASH_OF_HASHES_STEP) * sizeof(crypto::hash));
m_hashes.resize(m_hashes.size() - HASH_OF_HASHES_STEP);
const std::string data(hash.data, sizeof(hash));
*m_raw_data_file << data;
const std::string data_hashes(hash.data, sizeof(hash));
*m_raw_data_file << data_hashes;
crypto::cn_fast_hash(m_weights.data(), HASH_OF_HASHES_STEP * sizeof(uint64_t), hash);
memmove(m_weights.data(), m_weights.data() + HASH_OF_HASHES_STEP, (m_weights.size() - HASH_OF_HASHES_STEP) * sizeof(uint64_t));
m_weights.resize(m_weights.size() - HASH_OF_HASHES_STEP);
const std::string data_weights(hash.data, sizeof(hash));
*m_raw_data_file << data_weights;
}
}

@@ -154,7 +160,8 @@ bool BlocksdatFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_mem
{
// this method's height refers to 0-based height (genesis block = height 0)
crypto::hash hash = m_blockchain_storage->get_block_id_by_height(m_cur_height);
write_block(hash);
uint64_t weight = m_blockchain_storage->get_db().get_block_weight(m_cur_height);
write_block(hash, weight);
if (m_cur_height % NUM_BLOCKS_PER_CHUNK == 0) {
num_blocks_written += NUM_BLOCKS_PER_CHUNK;
}
@@ -72,10 +72,11 @@ class BlocksdatFile
bool open_writer(const boost::filesystem::path& file_path, uint64_t block_stop);
bool initialize_file(uint64_t block_stop);
bool close();
void write_block(const crypto::hash &block_hash);
void write_block(const crypto::hash &block_hash, uint64_t weight);

private:

uint64_t m_cur_height; // tracks current height during export
std::vector<crypto::hash> m_hashes;
std::vector<uint64_t> m_weights;
};
BIN -32 Bytes (100%) src/blocks/checkpoints.dat
Binary file not shown.
@@ -55,7 +55,7 @@ namespace cryptonote
};

state m_state;
std::vector<crypto::hash> m_needed_objects;
std::vector<std::pair<crypto::hash, uint64_t>> m_needed_objects;
std::unordered_set<crypto::hash> m_requested_objects;
uint64_t m_remote_blockchain_height;
uint64_t m_last_response_height;
@@ -195,6 +195,7 @@ namespace cryptonote
private:
// hash cash
mutable std::atomic<bool> hash_valid;
mutable std::atomic<bool> prunable_hash_valid;
mutable std::atomic<bool> blob_size_valid;

public:
@@ -203,6 +204,7 @@ namespace cryptonote

// hash cash
mutable crypto::hash hash;
mutable crypto::hash prunable_hash;
mutable size_t blob_size;

bool pruned;
@@ -211,22 +213,26 @@ namespace cryptonote
std::atomic<unsigned int> prefix_size;

transaction();
transaction(const transaction &t): transaction_prefix(t), hash_valid(false), blob_size_valid(false), signatures(t.signatures), rct_signatures(t.rct_signatures), pruned(t.pruned), unprunable_size(t.unprunable_size.load()), prefix_size(t.prefix_size.load()) { if (t.is_hash_valid()) { hash = t.hash; set_hash_valid(true); } if (t.is_blob_size_valid()) { blob_size = t.blob_size; set_blob_size_valid(true); } }
transaction &operator=(const transaction &t) { transaction_prefix::operator=(t); set_hash_valid(false); set_blob_size_valid(false); signatures = t.signatures; rct_signatures = t.rct_signatures; if (t.is_hash_valid()) { hash = t.hash; set_hash_valid(true); } if (t.is_blob_size_valid()) { blob_size = t.blob_size; set_blob_size_valid(true); } pruned = t.pruned; unprunable_size = t.unprunable_size.load(); prefix_size = t.prefix_size.load(); return *this; }
transaction(const transaction &t);
transaction &operator=(const transaction &t);
virtual ~transaction();
void set_null();
void invalidate_hashes();
bool is_hash_valid() const { return hash_valid.load(std::memory_order_acquire); }
void set_hash_valid(bool v) const { hash_valid.store(v,std::memory_order_release); }
bool is_prunable_hash_valid() const { return prunable_hash_valid.load(std::memory_order_acquire); }
void set_prunable_hash_valid(bool v) const { prunable_hash_valid.store(v,std::memory_order_release); }
bool is_blob_size_valid() const { return blob_size_valid.load(std::memory_order_acquire); }
void set_blob_size_valid(bool v) const { blob_size_valid.store(v,std::memory_order_release); }
void set_hash(const crypto::hash &h) { hash = h; set_hash_valid(true); }
void set_blob_size(size_t sz) { blob_size = sz; set_blob_size_valid(true); }
void set_hash(const crypto::hash &h) const { hash = h; set_hash_valid(true); }
void set_prunable_hash(const crypto::hash &h) const { prunable_hash = h; set_prunable_hash_valid(true); }
void set_blob_size(size_t sz) const { blob_size = sz; set_blob_size_valid(true); }

BEGIN_SERIALIZE_OBJECT()
if (!typename Archive<W>::is_saving())
{
set_hash_valid(false);
set_prunable_hash_valid(false);
set_blob_size_valid(false);
}

@@ -327,6 +333,63 @@ namespace cryptonote
static size_t get_signature_size(const txin_v& tx_in);
};

inline transaction::transaction(const transaction &t):
transaction_prefix(t),
hash_valid(false),
prunable_hash_valid(false),
blob_size_valid(false),
signatures(t.signatures),
rct_signatures(t.rct_signatures),
pruned(t.pruned),
unprunable_size(t.unprunable_size.load()),
prefix_size(t.prefix_size.load())
{
if (t.is_hash_valid())
{
hash = t.hash;
set_hash_valid(true);
}
if (t.is_blob_size_valid())
{
blob_size = t.blob_size;
set_blob_size_valid(true);
}
if (t.is_prunable_hash_valid())
{
prunable_hash = t.prunable_hash;
set_prunable_hash_valid(true);
}
}

inline transaction &transaction::operator=(const transaction &t)
{
transaction_prefix::operator=(t);

set_hash_valid(false);
set_prunable_hash_valid(false);
set_blob_size_valid(false);
signatures = t.signatures;
rct_signatures = t.rct_signatures;
if (t.is_hash_valid())
{
hash = t.hash;
set_hash_valid(true);
}
if (t.is_prunable_hash_valid())
{
prunable_hash = t.prunable_hash;
set_prunable_hash_valid(true);
}
if (t.is_blob_size_valid())
{
blob_size = t.blob_size;
set_blob_size_valid(true);
}
pruned = t.pruned;
unprunable_size = t.unprunable_size.load();
prefix_size = t.prefix_size.load();
return *this;
}

inline
transaction::transaction()
@@ -346,6 +409,7 @@ namespace cryptonote
signatures.clear();
rct_signatures.type = rct::RCTTypeNull;
set_hash_valid(false);
set_prunable_hash_valid(false);
set_blob_size_valid(false);
pruned = false;
unprunable_size = 0;
@@ -356,6 +420,7 @@ namespace cryptonote
void transaction::invalidate_hashes()
{
set_hash_valid(false);
set_prunable_hash_valid(false);
set_blob_size_valid(false);
}

@@ -408,6 +473,7 @@ namespace cryptonote
void invalidate_hashes() { set_hash_valid(false); }
bool is_hash_valid() const { return hash_valid.load(std::memory_order_acquire); }
void set_hash_valid(bool v) const { hash_valid.store(v,std::memory_order_release); }
void set_hash(const crypto::hash &h) const { hash = h; set_hash_valid(true); }

transaction miner_tx;
std::vector<crypto::hash> tx_hashes;
@@ -1011,7 +1011,19 @@ namespace cryptonote
crypto::hash get_transaction_prunable_hash(const transaction& t, const cryptonote::blobdata *blobdata)
{
crypto::hash res;
if (t.is_prunable_hash_valid())
{
#ifdef ENABLE_HASH_CASH_INTEGRITY_CHECK
CHECK_AND_ASSERT_THROW_MES(!calculate_transaction_prunable_hash(t, blobdata, res) || t.hash == res, "tx hash cash integrity failure");
#endif
res = t.prunable_hash;
++tx_hashes_cached_count;
return res;
}

++tx_hashes_calculated_count;
CHECK_AND_ASSERT_THROW_MES(calculate_transaction_prunable_hash(t, blobdata, res), "Failed to calculate tx prunable hash");
t.set_prunable_hash(res);
return res;
}
//---------------------------------------------------------------
@@ -1047,11 +1059,14 @@ namespace cryptonote

// the tx hash is the hash of the 3 hashes
crypto::hash res = cn_fast_hash(hashes, sizeof(hashes));
t.set_hash(res);
return res;
}
//---------------------------------------------------------------
bool calculate_transaction_hash(const transaction& t, crypto::hash& res, size_t* blob_size)
{
CHECK_AND_ASSERT_MES(!t.pruned, false, "Cannot calculate the hash of a pruned transaction");

// v1 transactions hash the entire blob
if (t.version == 1)
{
@@ -1091,8 +1106,7 @@ namespace cryptonote
{
if (!t.is_blob_size_valid())
{
t.blob_size = blob.size();
t.set_blob_size_valid(true);
t.set_blob_size(blob.size());
}
*blob_size = t.blob_size;
}
@@ -1112,8 +1126,7 @@ namespace cryptonote
{
if (!t.is_blob_size_valid())
{
t.blob_size = get_object_blobsize(t);
t.set_blob_size_valid(true);
t.set_blob_size(get_object_blobsize(t));
}
*blob_size = t.blob_size;
}
@@ -1124,12 +1137,10 @@ namespace cryptonote
bool ret = calculate_transaction_hash(t, res, blob_size);
if (!ret)
return false;
t.hash = res;
t.set_hash_valid(true);
t.set_hash(res);
if (blob_size)
{
t.blob_size = *blob_size;
t.set_blob_size_valid(true);
t.set_blob_size(*blob_size);
}
return true;
}
@@ -1206,8 +1217,7 @@ namespace cryptonote
bool ret = calculate_block_hash(b, res);
if (!ret)
return false;
b.hash = res;
b.set_hash_valid(true);
b.set_hash(res);
return true;
}
//---------------------------------------------------------------
@@ -1273,8 +1283,7 @@ namespace cryptonote
{
calculate_block_hash(b, *block_hash, &b_blob);
++block_hashes_calculated_count;
b.hash = *block_hash;
b.set_hash_valid(true);
b.set_hash(*block_hash);
}
return true;
}
@@ -163,7 +163,7 @@

#define PER_KB_FEE_QUANTIZATION_DECIMALS 8

#define HASH_OF_HASHES_STEP 256
#define HASH_OF_HASHES_STEP 512

#define DEFAULT_TXPOOL_MAX_WEIGHT 648000000ull // 3 days at 300000, in bytes

0 comments on commit 8330e77

Please sign in to comment.
You can’t perform that action at this time.