Skip to content
Permalink
Browse files

Adding support for hidden (anonymity) txpool

  • Loading branch information
vtnerd committed Aug 3, 2019
1 parent 31714d5 commit 52f6dc27eed0b096ee78b5db301e935e026ecab5
Showing with 1,326 additions and 479 deletions.
  1. +2 −1 contrib/epee/include/span.h
  2. +84 −0 src/blockchain_db/blockchain_db.cpp
  3. +41 −8 src/blockchain_db/blockchain_db.h
  4. +36 −12 src/blockchain_db/lmdb/db_lmdb.cpp
  5. +5 −5 src/blockchain_db/lmdb/db_lmdb.h
  6. +5 −5 src/blockchain_db/testdb.h
  7. +1 −1 src/blockchain_utilities/blockchain_import.cpp
  8. +18 −13 src/cryptonote_core/blockchain.cpp
  9. +5 −4 src/cryptonote_core/blockchain.h
  10. +70 −50 src/cryptonote_core/cryptonote_core.cpp
  11. +47 −26 src/cryptonote_core/cryptonote_core.h
  12. +12 −28 tests/core_tests/txpool.h → src/cryptonote_core/i_core_events.h
  13. +118 −87 src/cryptonote_core/tx_pool.cpp
  14. +30 −23 src/cryptonote_core/tx_pool.h
  15. +1 −1 src/cryptonote_protocol/cryptonote_protocol_handler.h
  16. +17 −17 src/cryptonote_protocol/cryptonote_protocol_handler.inl
  17. +2 −2 src/cryptonote_protocol/cryptonote_protocol_handler_common.h
  18. +43 −0 src/cryptonote_protocol/enums.h
  19. +37 −0 src/cryptonote_protocol/fwd.h
  20. +1 −5 src/cryptonote_protocol/levin_notify.h
  21. +2 −1 src/p2p/net_node.h
  22. +8 −3 src/p2p/net_node.inl
  23. +4 −2 src/p2p/net_node_common.h
  24. +7 −9 src/rpc/core_rpc_server.cpp
  25. +3 −3 src/rpc/daemon_handler.cpp
  26. +4 −4 tests/core_proxy/core_proxy.cpp
  27. +6 −5 tests/core_proxy/core_proxy.h
  28. +2 −2 tests/core_tests/CMakeLists.txt
  29. +20 −19 tests/core_tests/chaingen.h
  30. +3 −1 tests/core_tests/chaingen_main.cpp
  31. +2 −1 tests/core_tests/double_spend.cpp
  32. +5 −5 tests/core_tests/double_spend.inl
  33. +561 −0 tests/core_tests/tx_pool.cpp
  34. +118 −0 tests/core_tests/tx_pool.h
  35. +0 −131 tests/core_tests/txpool.cpp
  36. +6 −5 tests/unit_tests/node_server.cpp
@@ -110,7 +110,8 @@ namespace epee
constexpr std::size_t size() const noexcept { return len; }
constexpr std::size_t size_bytes() const noexcept { return size() * sizeof(value_type); }

const T &operator[](size_t idx) const { return ptr[idx]; }
T &operator[](size_t idx) noexcept { return ptr[idx]; }
const T &operator[](size_t idx) const noexcept { return ptr[idx]; }

private:
T* ptr;
@@ -44,6 +44,71 @@ using epee::string_tools::pod_to_hex;
namespace cryptonote
{

bool matches_category(relay_method method, relay_category category) noexcept
{
switch (category)
{
default:
return false;
case relay_category::all:
return true;
case relay_category::relayable:
if (method == relay_method::none)
return false;
return true;
case relay_category::broadcasted:
case relay_category::legacy:
break;
}
// check for "broadcasted" or "legacy" methods:
switch (method)
{
default:
case relay_method::local:
return false;
case relay_method::block:
case relay_method::flood:
return true;
case relay_method::none:
break;
}
return category == relay_category::legacy;
}

void txpool_tx_meta_t::set_relay_method(relay_method method) noexcept
{
kept_by_block = 0;
do_not_relay = 0;
is_local = 0;

switch (method)
{
case relay_method::none:
do_not_relay = 1;
break;
case relay_method::local:
is_local = 1;
break;
default:
case relay_method::flood:
break;
case relay_method::block:
kept_by_block = 1;
break;
}
}

relay_method txpool_tx_meta_t::get_relay_method() const noexcept
{
if (kept_by_block)
return relay_method::block;
if (do_not_relay)
return relay_method::none;
if (is_local)
return relay_method::local;
return relay_method::flood;
}

const command_line::arg_descriptor<std::string> arg_db_sync_mode = {
"db-sync-mode"
, "Specify sync option, using format [safe|fast|fastest]:[sync|async]:[<nblocks_per_sync>[blocks]|<nbytes_per_sync>[bytes]]."
@@ -924,4 +989,23 @@ void BlockchainDB::fixup()
batch_stop();
}

bool BlockchainDB::txpool_tx_matches_category(const crypto::hash& tx_hash, relay_category category)
{
try
{
txpool_tx_meta_t meta{};
if (!get_txpool_tx_meta(tx_hash, meta))
{
MERROR("Failed to get tx meta from txpool");
return false;
}
return meta.matches(category);
}
catch (const std::exception &e)
{
MERROR("Failed to get tx meta from txpool: " << e.what());
}
return false;
}

} // namespace cryptonote
@@ -39,6 +39,7 @@
#include "cryptonote_basic/cryptonote_basic.h"
#include "cryptonote_basic/difficulty.h"
#include "cryptonote_basic/hardfork.h"
#include "cryptonote_protocol/enums.h"

/** \file
* Cryptonote Blockchain Database Interface
@@ -105,6 +106,16 @@ typedef std::pair<crypto::hash, uint64_t> tx_out_index;
extern const command_line::arg_descriptor<std::string> arg_db_sync_mode;
extern const command_line::arg_descriptor<bool, false> arg_db_salvage;

enum class relay_category : uint8_t
{
broadcasted = 0,//!< Public txes received via block/flooding/fluff
relayable, //!< Every tx not marked `relay_method::none`
legacy, //!< `relay_category::broadcasted` + `relay_method::none` for rpc relay requests or historical reasons
all //!< Everything in the db
};

bool matches_category(relay_method method, relay_category category) noexcept;

#pragma pack(push, 1)

/**
@@ -156,11 +167,22 @@ struct txpool_tx_meta_t
uint8_t do_not_relay;
uint8_t double_spend_seen: 1;
uint8_t pruned: 1;
uint8_t bf_padding: 6;
uint8_t is_local: 1;
uint8_t bf_padding: 5;

uint8_t padding[76]; // till 192 bytes

void set_relay_method(relay_method method) noexcept;
relay_method get_relay_method() const noexcept;

//! See `relay_category` description
bool matches(const relay_category category) const noexcept
{
return matches_category(get_relay_method(), category);
}
};


#define DBF_SAFE 1
#define DBF_FAST 2
#define DBF_FASTEST 4
@@ -1465,12 +1487,12 @@ class BlockchainDB
/**
* @brief get the number of transactions in the txpool
*/
virtual uint64_t get_txpool_tx_count(bool include_unrelayed_txes = true) const = 0;
virtual uint64_t get_txpool_tx_count(relay_category tx_category = relay_category::broadcasted) const = 0;

/**
* @brief check whether a txid is in the txpool
* @brief check whether a txid is in the txpool and meets tx_category requirements
*/
virtual bool txpool_has_tx(const crypto::hash &txid) const = 0;
virtual bool txpool_has_tx(const crypto::hash &txid, relay_category tx_category) const = 0;

/**
* @brief remove a txpool transaction
@@ -1494,10 +1516,11 @@ class BlockchainDB
*
* @param txid the transaction id of the transation to lookup
* @param bd the blob to return
* @param tx_category for filtering out hidden/private txes
*
* @return true if the txid was in the txpool, false otherwise
* @return True iff `txid` is in the pool and meets `tx_category` requirements
*/
virtual bool get_txpool_tx_blob(const crypto::hash& txid, cryptonote::blobdata &bd) const = 0;
virtual bool get_txpool_tx_blob(const crypto::hash& txid, cryptonote::blobdata &bd, relay_category tx_category) const = 0;

/**
* @brief get a txpool transaction's blob
@@ -1506,7 +1529,17 @@ class BlockchainDB
*
* @return the blob for that transaction
*/
virtual cryptonote::blobdata get_txpool_tx_blob(const crypto::hash& txid) const = 0;
virtual cryptonote::blobdata get_txpool_tx_blob(const crypto::hash& txid, relay_category tx_category) const = 0;

/**
* @brief Check if `tx_hash` relay status is in `category`.
*
* @param tx_hash hash of the transaction to lookup
* @param category relay status category to test against
*
* @return True if `tx_hash` latest relay status is in `category`.
*/
bool txpool_tx_matches_category(const crypto::hash& tx_hash, relay_category category);

/**
* @brief prune output data for the given amount
@@ -1604,7 +1637,7 @@ class BlockchainDB
*
* @return false if the function returns false for any transaction, otherwise true
*/
virtual bool for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const cryptonote::blobdata*)>, bool include_blob = false, bool include_unrelayed_txes = true) const = 0;
virtual bool for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const cryptonote::blobdata*)>, bool include_blob = false, relay_category category = relay_category::broadcasted) const = 0;

/**
* @brief runs a function over all key images stored
@@ -1771,7 +1771,7 @@ void BlockchainLMDB::update_txpool_tx(const crypto::hash &txid, const txpool_tx_
}
}

uint64_t BlockchainLMDB::get_txpool_tx_count(bool include_unrelayed_txes) const
uint64_t BlockchainLMDB::get_txpool_tx_count(relay_category category) const
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open();
@@ -1781,7 +1781,7 @@ uint64_t BlockchainLMDB::get_txpool_tx_count(bool include_unrelayed_txes) const

TXN_PREFIX_RDONLY();

if (include_unrelayed_txes)
if (category == relay_category::all)
{
// No filtering, we can get the number of tx the "fast" way
MDB_stat db_stats;
@@ -1807,7 +1807,7 @@ uint64_t BlockchainLMDB::get_txpool_tx_count(bool include_unrelayed_txes) const
if (result)
throw0(DB_ERROR(lmdb_error("Failed to enumerate txpool tx metadata: ", result).c_str()));
const txpool_tx_meta_t &meta = *(const txpool_tx_meta_t*)v.mv_data;
if (!meta.do_not_relay)
if (meta.matches(category))
++num_entries;
}
}
@@ -1816,7 +1816,7 @@ uint64_t BlockchainLMDB::get_txpool_tx_count(bool include_unrelayed_txes) const
return num_entries;
}

bool BlockchainLMDB::txpool_has_tx(const crypto::hash& txid) const
bool BlockchainLMDB::txpool_has_tx(const crypto::hash& txid, relay_category tx_category) const
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open();
@@ -1825,11 +1825,21 @@ bool BlockchainLMDB::txpool_has_tx(const crypto::hash& txid) const
RCURSOR(txpool_meta)

MDB_val k = {sizeof(txid), (void *)&txid};
auto result = mdb_cursor_get(m_cur_txpool_meta, &k, NULL, MDB_SET);
MDB_val v;
auto result = mdb_cursor_get(m_cur_txpool_meta, &k, &v, MDB_SET);
if (result != 0 && result != MDB_NOTFOUND)
throw1(DB_ERROR(lmdb_error("Error finding txpool tx meta: ", result).c_str()));
if (result == MDB_NOTFOUND)
return false;

bool found = true;
if (tx_category != relay_category::all)
{
const txpool_tx_meta_t &meta = *(const txpool_tx_meta_t*)v.mv_data;
found = meta.matches(tx_category);
}
TXN_POSTFIX_RDONLY();
return result != MDB_NOTFOUND;
return found;
}

void BlockchainLMDB::remove_txpool_tx(const crypto::hash& txid)
@@ -1883,7 +1893,7 @@ bool BlockchainLMDB::get_txpool_tx_meta(const crypto::hash& txid, txpool_tx_meta
return true;
}

bool BlockchainLMDB::get_txpool_tx_blob(const crypto::hash& txid, cryptonote::blobdata &bd) const
bool BlockchainLMDB::get_txpool_tx_blob(const crypto::hash& txid, cryptonote::blobdata &bd, relay_category tx_category) const
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open();
@@ -1893,6 +1903,21 @@ bool BlockchainLMDB::get_txpool_tx_blob(const crypto::hash& txid, cryptonote::bl

MDB_val k = {sizeof(txid), (void *)&txid};
MDB_val v;

// if filtering, make sure those requirements are met before copying blob
if (tx_category != relay_category::all)
{
auto result = mdb_cursor_get(m_cur_txpool_meta, &k, &v, MDB_SET);
if (result == MDB_NOTFOUND)
return false;
if (result != 0)
throw1(DB_ERROR(lmdb_error("Error finding txpool tx meta: ", result).c_str()));

const txpool_tx_meta_t& meta = *(const txpool_tx_meta_t*)v.mv_data;
if (!meta.matches(tx_category))
return false;
}

auto result = mdb_cursor_get(m_cur_txpool_blob, &k, &v, MDB_SET);
if (result == MDB_NOTFOUND)
return false;
@@ -1904,10 +1929,10 @@ bool BlockchainLMDB::get_txpool_tx_blob(const crypto::hash& txid, cryptonote::bl
return true;
}

cryptonote::blobdata BlockchainLMDB::get_txpool_tx_blob(const crypto::hash& txid) const
cryptonote::blobdata BlockchainLMDB::get_txpool_tx_blob(const crypto::hash& txid, relay_category tx_category) const
{
cryptonote::blobdata bd;
if (!get_txpool_tx_blob(txid, bd))
if (!get_txpool_tx_blob(txid, bd, tx_category))
throw1(DB_ERROR("Tx not found in txpool: "));
return bd;
}
@@ -2245,7 +2270,7 @@ bool BlockchainLMDB::check_pruning()
return prune_worker(prune_mode_check, 0);
}

bool BlockchainLMDB::for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const cryptonote::blobdata*)> f, bool include_blob, bool include_unrelayed_txes) const
bool BlockchainLMDB::for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const cryptonote::blobdata*)> f, bool include_blob, relay_category category) const
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open();
@@ -2269,8 +2294,7 @@ bool BlockchainLMDB::for_all_txpool_txes(std::function<bool(const crypto::hash&,
throw0(DB_ERROR(lmdb_error("Failed to enumerate txpool tx metadata: ", result).c_str()));
const crypto::hash txid = *(const crypto::hash*)k.mv_data;
const txpool_tx_meta_t &meta = *(const txpool_tx_meta_t*)v.mv_data;
if (!include_unrelayed_txes && meta.do_not_relay)
// Skipping that tx
if (!meta.matches(category))
continue;
const cryptonote::blobdata *passed_bd = NULL;
cryptonote::blobdata bd;
@@ -281,12 +281,12 @@ class BlockchainLMDB : public BlockchainDB

virtual void add_txpool_tx(const crypto::hash &txid, const cryptonote::blobdata &blob, const txpool_tx_meta_t& meta);
virtual void update_txpool_tx(const crypto::hash &txid, const txpool_tx_meta_t& meta);
virtual uint64_t get_txpool_tx_count(bool include_unrelayed_txes = true) const;
virtual bool txpool_has_tx(const crypto::hash &txid) const;
virtual uint64_t get_txpool_tx_count(relay_category category = relay_category::broadcasted) const;
virtual bool txpool_has_tx(const crypto::hash &txid, relay_category tx_category) const;
virtual void remove_txpool_tx(const crypto::hash& txid);
virtual bool get_txpool_tx_meta(const crypto::hash& txid, txpool_tx_meta_t &meta) const;
virtual bool get_txpool_tx_blob(const crypto::hash& txid, cryptonote::blobdata &bd) const;
virtual cryptonote::blobdata get_txpool_tx_blob(const crypto::hash& txid) const;
virtual bool get_txpool_tx_blob(const crypto::hash& txid, cryptonote::blobdata& bd, relay_category tx_category) const;
virtual cryptonote::blobdata get_txpool_tx_blob(const crypto::hash& txid, relay_category tx_category) const;
virtual uint32_t get_blockchain_pruning_seed() const;
virtual bool prune_blockchain(uint32_t pruning_seed = 0);
virtual bool update_pruning();
@@ -298,7 +298,7 @@ class BlockchainLMDB : public BlockchainDB
virtual uint64_t get_alt_block_count();
virtual void drop_alt_blocks();

virtual bool for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const cryptonote::blobdata*)> f, bool include_blob = false, bool include_unrelayed_txes = true) const;
virtual bool for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const cryptonote::blobdata*)> f, bool include_blob = false, relay_category category = relay_category::broadcasted) const;

virtual bool for_all_key_images(std::function<bool(const crypto::key_image&)>) const;
virtual bool for_blocks_range(const uint64_t& h1, const uint64_t& h2, std::function<bool(uint64_t, const crypto::hash&, const cryptonote::block&)>) const;
@@ -126,14 +126,14 @@ class BaseTestDB: public cryptonote::BlockchainDB {

virtual void add_txpool_tx(const crypto::hash &txid, const cryptonote::blobdata &blob, const cryptonote::txpool_tx_meta_t& details) override {}
virtual void update_txpool_tx(const crypto::hash &txid, const cryptonote::txpool_tx_meta_t& details) override {}
virtual uint64_t get_txpool_tx_count(bool include_unrelayed_txes = true) const override { return 0; }
virtual bool txpool_has_tx(const crypto::hash &txid) const override { return false; }
virtual uint64_t get_txpool_tx_count(relay_category tx_relay = relay_category::broadcasted) const override { return 0; }
virtual bool txpool_has_tx(const crypto::hash &txid, relay_category tx_category) const override { return false; }
virtual void remove_txpool_tx(const crypto::hash& txid) override {}
virtual bool get_txpool_tx_meta(const crypto::hash& txid, cryptonote::txpool_tx_meta_t &meta) const override { return false; }
virtual bool get_txpool_tx_blob(const crypto::hash& txid, cryptonote::blobdata &bd) const override { return false; }
virtual bool get_txpool_tx_blob(const crypto::hash& txid, cryptonote::blobdata &bd, relay_category tx_category) const override { return false; }
virtual uint64_t get_database_size() const override { return 0; }
virtual cryptonote::blobdata get_txpool_tx_blob(const crypto::hash& txid) const override { return ""; }
virtual bool for_all_txpool_txes(std::function<bool(const crypto::hash&, const cryptonote::txpool_tx_meta_t&, const cryptonote::blobdata*)>, bool include_blob = false, bool include_unrelayed_txes = false) const override { return false; }
virtual cryptonote::blobdata get_txpool_tx_blob(const crypto::hash& txid, relay_category tx_category) const override { return ""; }
virtual bool for_all_txpool_txes(std::function<bool(const crypto::hash&, const cryptonote::txpool_tx_meta_t&, const cryptonote::blobdata*)>, bool include_blob = false, relay_category category = relay_category::broadcasted) const override { return false; }

virtual void add_block( const cryptonote::block& blk
, size_t block_weight
@@ -174,7 +174,7 @@ int check_flush(cryptonote::core &core, std::vector<block_complete_entry> &block
for(auto& tx_blob: block_entry.txs)
{
tx_verification_context tvc = AUTO_VAL_INIT(tvc);
core.handle_incoming_tx(tx_blob, tvc, true, true, false);
core.handle_incoming_tx(tx_blob, tvc, relay_method::block, true);
if(tvc.m_verifivation_failed)
{
MERROR("transaction verification failed, tx_id = "

0 comments on commit 52f6dc2

Please sign in to comment.
You can’t perform that action at this time.