Skip to content
Permalink
Browse files

Merge pull request #5915

8330e77 monerod can now sync from pruned blocks (moneromooo-monero)
  • Loading branch information...
luigi1111 committed Oct 8, 2019
2 parents 0e4ab60 + 8330e77 commit bf525793c724f62d4716aaca59345ba2e13a2c3c
@@ -155,7 +155,8 @@ struct txpool_tx_meta_t
uint8_t relayed;
uint8_t do_not_relay;
uint8_t double_spend_seen: 1;
uint8_t bf_padding: 7;
uint8_t pruned: 1;
uint8_t bf_padding: 6;

uint8_t padding[76]; // till 192 bytes
};
@@ -2056,7 +2056,7 @@ bool BlockchainLMDB::prune_worker(int mode, uint32_t pruning_seed)
++n_prunable_records;
result = mdb_cursor_get(c_txs_prunable, &k, &v, MDB_SET);
if (result == MDB_NOTFOUND)
MWARNING("Already pruned at height " << block_height << "/" << blockchain_height);
MDEBUG("Already pruned at height " << block_height << "/" << blockchain_height);
else if (result)
throw0(DB_ERROR(lmdb_error("Failed to find transaction prunable data: ", result).c_str()));
else
@@ -2152,7 +2152,7 @@ bool BlockchainLMDB::prune_worker(int mode, uint32_t pruning_seed)
{
++n_prunable_records;
if (result == MDB_NOTFOUND)
MWARNING("Already pruned at height " << block_height << "/" << blockchain_height);
MDEBUG("Already pruned at height " << block_height << "/" << blockchain_height);
else
{
MDEBUG("Pruning at height " << block_height << "/" << blockchain_height);
@@ -2994,6 +2994,8 @@ bool BlockchainLMDB::get_tx_blob(const crypto::hash& h, cryptonote::blobdata &bd
return false;
else if (get_result)
throw0(DB_ERROR(lmdb_error("DB error attempting to fetch tx from hash", get_result).c_str()));
else if (result1.mv_size == 0)
return false;

bd.assign(reinterpret_cast<char*>(result0.mv_data), result0.mv_size);
bd.append(reinterpret_cast<char*>(result1.mv_data), result1.mv_size);
@@ -152,7 +152,7 @@ int check_flush(cryptonote::core &core, std::vector<block_complete_entry> &block
}
hashes.push_back(cryptonote::get_block_hash(block));
}
core.prevalidate_block_hashes(core.get_blockchain_storage().get_db().height(), hashes);
core.prevalidate_block_hashes(core.get_blockchain_storage().get_db().height(), hashes, {});

std::vector<block> pblocks;
if (!core.prepare_handle_incoming_blocks(blocks, pblocks))
@@ -178,7 +178,7 @@ int check_flush(cryptonote::core &core, std::vector<block_complete_entry> &block
if(tvc.m_verifivation_failed)
{
MERROR("transaction verification failed, tx_id = "
<< epee::string_tools::pod_to_hex(get_blob_hash(tx_blob)));
<< epee::string_tools::pod_to_hex(get_blob_hash(tx_blob.blob)));
core.cleanup_handle_incoming_blocks();
return 1;
}
@@ -429,13 +429,17 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path
{
cryptonote::blobdata block;
cryptonote::block_to_blob(bp.block, block);
std::vector<cryptonote::blobdata> txs;
std::vector<tx_blob_entry> txs;
for (const auto &tx: bp.txs)
{
txs.push_back(cryptonote::blobdata());
cryptonote::tx_to_blob(tx, txs.back());
txs.push_back({cryptonote::blobdata(), crypto::null_hash});
cryptonote::tx_to_blob(tx, txs.back().blob);
}
blocks.push_back({block, txs});
block_complete_entry bce;
bce.pruned = false;
bce.block = std::move(block);
bce.txs = std::move(txs);
blocks.push_back(bce);
int ret = check_flush(core, blocks, false);
if (ret)
{
@@ -99,17 +99,23 @@ bool BlocksdatFile::initialize_file(uint64_t block_stop)
return true;
}

void BlocksdatFile::write_block(const crypto::hash& block_hash)
void BlocksdatFile::write_block(const crypto::hash& block_hash, uint64_t weight)
{
m_hashes.push_back(block_hash);
m_weights.push_back(weight);
while (m_hashes.size() >= HASH_OF_HASHES_STEP)
{
crypto::hash hash;
crypto::cn_fast_hash(m_hashes.data(), HASH_OF_HASHES_STEP * sizeof(crypto::hash), hash);
memmove(m_hashes.data(), m_hashes.data() + HASH_OF_HASHES_STEP, (m_hashes.size() - HASH_OF_HASHES_STEP) * sizeof(crypto::hash));
m_hashes.resize(m_hashes.size() - HASH_OF_HASHES_STEP);
const std::string data(hash.data, sizeof(hash));
*m_raw_data_file << data;
const std::string data_hashes(hash.data, sizeof(hash));
*m_raw_data_file << data_hashes;
crypto::cn_fast_hash(m_weights.data(), HASH_OF_HASHES_STEP * sizeof(uint64_t), hash);
memmove(m_weights.data(), m_weights.data() + HASH_OF_HASHES_STEP, (m_weights.size() - HASH_OF_HASHES_STEP) * sizeof(uint64_t));
m_weights.resize(m_weights.size() - HASH_OF_HASHES_STEP);
const std::string data_weights(hash.data, sizeof(hash));
*m_raw_data_file << data_weights;
}
}

@@ -154,7 +160,8 @@ bool BlocksdatFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_mem
{
// this method's height refers to 0-based height (genesis block = height 0)
crypto::hash hash = m_blockchain_storage->get_block_id_by_height(m_cur_height);
write_block(hash);
uint64_t weight = m_blockchain_storage->get_db().get_block_weight(m_cur_height);
write_block(hash, weight);
if (m_cur_height % NUM_BLOCKS_PER_CHUNK == 0) {
num_blocks_written += NUM_BLOCKS_PER_CHUNK;
}
@@ -72,10 +72,11 @@ class BlocksdatFile
bool open_writer(const boost::filesystem::path& file_path, uint64_t block_stop);
bool initialize_file(uint64_t block_stop);
bool close();
void write_block(const crypto::hash &block_hash);
void write_block(const crypto::hash &block_hash, uint64_t weight);

private:

uint64_t m_cur_height; // tracks current height during export
std::vector<crypto::hash> m_hashes;
std::vector<uint64_t> m_weights;
};
BIN -32 Bytes (100%) src/blocks/checkpoints.dat
Binary file not shown.
@@ -55,7 +55,7 @@ namespace cryptonote
};

state m_state;
std::vector<crypto::hash> m_needed_objects;
std::vector<std::pair<crypto::hash, uint64_t>> m_needed_objects;
std::unordered_set<crypto::hash> m_requested_objects;
uint64_t m_remote_blockchain_height;
uint64_t m_last_response_height;
@@ -195,6 +195,7 @@ namespace cryptonote
private:
// hash cash
mutable std::atomic<bool> hash_valid;
mutable std::atomic<bool> prunable_hash_valid;
mutable std::atomic<bool> blob_size_valid;

public:
@@ -203,6 +204,7 @@ namespace cryptonote

// hash cash
mutable crypto::hash hash;
mutable crypto::hash prunable_hash;
mutable size_t blob_size;

bool pruned;
@@ -211,22 +213,26 @@ namespace cryptonote
std::atomic<unsigned int> prefix_size;

transaction();
transaction(const transaction &t): transaction_prefix(t), hash_valid(false), blob_size_valid(false), signatures(t.signatures), rct_signatures(t.rct_signatures), pruned(t.pruned), unprunable_size(t.unprunable_size.load()), prefix_size(t.prefix_size.load()) { if (t.is_hash_valid()) { hash = t.hash; set_hash_valid(true); } if (t.is_blob_size_valid()) { blob_size = t.blob_size; set_blob_size_valid(true); } }
transaction &operator=(const transaction &t) { transaction_prefix::operator=(t); set_hash_valid(false); set_blob_size_valid(false); signatures = t.signatures; rct_signatures = t.rct_signatures; if (t.is_hash_valid()) { hash = t.hash; set_hash_valid(true); } if (t.is_blob_size_valid()) { blob_size = t.blob_size; set_blob_size_valid(true); } pruned = t.pruned; unprunable_size = t.unprunable_size.load(); prefix_size = t.prefix_size.load(); return *this; }
transaction(const transaction &t);
transaction &operator=(const transaction &t);
virtual ~transaction();
void set_null();
void invalidate_hashes();
bool is_hash_valid() const { return hash_valid.load(std::memory_order_acquire); }
void set_hash_valid(bool v) const { hash_valid.store(v,std::memory_order_release); }
bool is_prunable_hash_valid() const { return prunable_hash_valid.load(std::memory_order_acquire); }
void set_prunable_hash_valid(bool v) const { prunable_hash_valid.store(v,std::memory_order_release); }
bool is_blob_size_valid() const { return blob_size_valid.load(std::memory_order_acquire); }
void set_blob_size_valid(bool v) const { blob_size_valid.store(v,std::memory_order_release); }
void set_hash(const crypto::hash &h) { hash = h; set_hash_valid(true); }
void set_blob_size(size_t sz) { blob_size = sz; set_blob_size_valid(true); }
void set_hash(const crypto::hash &h) const { hash = h; set_hash_valid(true); }
void set_prunable_hash(const crypto::hash &h) const { prunable_hash = h; set_prunable_hash_valid(true); }
void set_blob_size(size_t sz) const { blob_size = sz; set_blob_size_valid(true); }

BEGIN_SERIALIZE_OBJECT()
if (!typename Archive<W>::is_saving())
{
set_hash_valid(false);
set_prunable_hash_valid(false);
set_blob_size_valid(false);
}

@@ -327,6 +333,63 @@ namespace cryptonote
static size_t get_signature_size(const txin_v& tx_in);
};

inline transaction::transaction(const transaction &t):
transaction_prefix(t),
hash_valid(false),
prunable_hash_valid(false),
blob_size_valid(false),
signatures(t.signatures),
rct_signatures(t.rct_signatures),
pruned(t.pruned),
unprunable_size(t.unprunable_size.load()),
prefix_size(t.prefix_size.load())
{
if (t.is_hash_valid())
{
hash = t.hash;
set_hash_valid(true);
}
if (t.is_blob_size_valid())
{
blob_size = t.blob_size;
set_blob_size_valid(true);
}
if (t.is_prunable_hash_valid())
{
prunable_hash = t.prunable_hash;
set_prunable_hash_valid(true);
}
}

inline transaction &transaction::operator=(const transaction &t)
{
transaction_prefix::operator=(t);

set_hash_valid(false);
set_prunable_hash_valid(false);
set_blob_size_valid(false);
signatures = t.signatures;
rct_signatures = t.rct_signatures;
if (t.is_hash_valid())
{
hash = t.hash;
set_hash_valid(true);
}
if (t.is_prunable_hash_valid())
{
prunable_hash = t.prunable_hash;
set_prunable_hash_valid(true);
}
if (t.is_blob_size_valid())
{
blob_size = t.blob_size;
set_blob_size_valid(true);
}
pruned = t.pruned;
unprunable_size = t.unprunable_size.load();
prefix_size = t.prefix_size.load();
return *this;
}

inline
transaction::transaction()
@@ -346,6 +409,7 @@ namespace cryptonote
signatures.clear();
rct_signatures.type = rct::RCTTypeNull;
set_hash_valid(false);
set_prunable_hash_valid(false);
set_blob_size_valid(false);
pruned = false;
unprunable_size = 0;
@@ -356,6 +420,7 @@ namespace cryptonote
void transaction::invalidate_hashes()
{
set_hash_valid(false);
set_prunable_hash_valid(false);
set_blob_size_valid(false);
}

@@ -408,6 +473,7 @@ namespace cryptonote
void invalidate_hashes() { set_hash_valid(false); }
bool is_hash_valid() const { return hash_valid.load(std::memory_order_acquire); }
void set_hash_valid(bool v) const { hash_valid.store(v,std::memory_order_release); }
void set_hash(const crypto::hash &h) const { hash = h; set_hash_valid(true); }

transaction miner_tx;
std::vector<crypto::hash> tx_hashes;
@@ -1011,7 +1011,19 @@ namespace cryptonote
crypto::hash get_transaction_prunable_hash(const transaction& t, const cryptonote::blobdata *blobdata)
{
crypto::hash res;
if (t.is_prunable_hash_valid())
{
#ifdef ENABLE_HASH_CASH_INTEGRITY_CHECK
CHECK_AND_ASSERT_THROW_MES(!calculate_transaction_prunable_hash(t, blobdata, res) || t.hash == res, "tx hash cash integrity failure");
#endif
res = t.prunable_hash;
++tx_hashes_cached_count;
return res;
}

++tx_hashes_calculated_count;
CHECK_AND_ASSERT_THROW_MES(calculate_transaction_prunable_hash(t, blobdata, res), "Failed to calculate tx prunable hash");
t.set_prunable_hash(res);
return res;
}
//---------------------------------------------------------------
@@ -1047,11 +1059,14 @@ namespace cryptonote

// the tx hash is the hash of the 3 hashes
crypto::hash res = cn_fast_hash(hashes, sizeof(hashes));
t.set_hash(res);
return res;
}
//---------------------------------------------------------------
bool calculate_transaction_hash(const transaction& t, crypto::hash& res, size_t* blob_size)
{
CHECK_AND_ASSERT_MES(!t.pruned, false, "Cannot calculate the hash of a pruned transaction");

// v1 transactions hash the entire blob
if (t.version == 1)
{
@@ -1091,8 +1106,7 @@ namespace cryptonote
{
if (!t.is_blob_size_valid())
{
t.blob_size = blob.size();
t.set_blob_size_valid(true);
t.set_blob_size(blob.size());
}
*blob_size = t.blob_size;
}
@@ -1112,8 +1126,7 @@ namespace cryptonote
{
if (!t.is_blob_size_valid())
{
t.blob_size = get_object_blobsize(t);
t.set_blob_size_valid(true);
t.set_blob_size(get_object_blobsize(t));
}
*blob_size = t.blob_size;
}
@@ -1124,12 +1137,10 @@ namespace cryptonote
bool ret = calculate_transaction_hash(t, res, blob_size);
if (!ret)
return false;
t.hash = res;
t.set_hash_valid(true);
t.set_hash(res);
if (blob_size)
{
t.blob_size = *blob_size;
t.set_blob_size_valid(true);
t.set_blob_size(*blob_size);
}
return true;
}
@@ -1206,8 +1217,7 @@ namespace cryptonote
bool ret = calculate_block_hash(b, res);
if (!ret)
return false;
b.hash = res;
b.set_hash_valid(true);
b.set_hash(res);
return true;
}
//---------------------------------------------------------------
@@ -1251,8 +1261,7 @@ namespace cryptonote
{
calculate_block_hash(b, *block_hash, &b_blob);
++block_hashes_calculated_count;
b.hash = *block_hash;
b.set_hash_valid(true);
b.set_hash(*block_hash);
}
return true;
}
@@ -168,7 +168,7 @@

#define PER_KB_FEE_QUANTIZATION_DECIMALS 8

#define HASH_OF_HASHES_STEP 256
#define HASH_OF_HASHES_STEP 512

#define DEFAULT_TXPOOL_MAX_WEIGHT 648000000ull // 3 days at 300000, in bytes

0 comments on commit bf52579

Please sign in to comment.
You can’t perform that action at this time.