Permalink
Browse files

Add expedited handling

Move some thinblock handling to thinblock.cpp
from main and rewrite the reconstruction of a block.
  • Loading branch information...
zander committed Feb 8, 2017
1 parent 2a45e3c commit f7234cb106d0bd7da9f900b1d3b7d1273919b764
Showing with 478 additions and 148 deletions.
  1. +1 −0 src/allowed_args.cpp
  2. +66 −137 src/main.cpp
  3. +3 −0 src/net.cpp
  4. +2 −0 src/net.h
  5. +12 −2 src/protocol.cpp
  6. +27 −0 src/protocol.h
  7. +334 −6 src/thinblock.cpp
  8. +29 −2 src/thinblock.h
  9. +4 −1 src/version.h
View
@@ -87,6 +87,7 @@ const std::set<std::string> BITCOIND_ARGS
"enforcenodebloom",
"excessiveblocksize",
"externalip",
+ "expeditedblock",
"flextrans",
"flushwallet",
"forcednsseed",
View
@@ -4458,6 +4458,10 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv,
if (!IsThinBlocksEnabled())
pfrom->PushMessage(NetMsgType::SENDHEADERS);
}
+
+ // send our listening port in a separate version message
+ if (pfrom->nVersion >= EXPEDITED_VERSION)
+ pfrom->PushMessage(NetMsgType::VERSION2, GetListenPort());
}
@@ -5011,137 +5015,33 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv,
CXThinBlock thinBlock;
vRecv >> thinBlock;
+ // Send expedited ASAP
+ CValidationState state;
+ if (!CheckBlockHeader(thinBlock.header, state, true)) { // block header is bad
+ LogPrint("thin", "Thinblock %s received with bad header from peer %s (%d)\n", thinBlock.header.GetHash().ToString(), pfrom->addrName.c_str(), pfrom->id);
+ Misbehaving(pfrom->id, 20);
+ return false;
+ }
+ else if (!IsRecentlyExpeditedAndStore(thinBlock.header.GetHash()))
+ SendExpeditedBlock(thinBlock, 0, pfrom);
+
CInv inv(MSG_BLOCK, thinBlock.header.GetHash());
#ifdef LOG_XTHINBLOCKS
int nSizeThinBlock = ::GetSerializeSize(thinBlock, SER_NETWORK, PROTOCOL_VERSION);
- LogPrint("thin", "Received thinblock %s from peer %s (%d). Size %d bytes.\n", inv.hash.ToString(), pfrom->addrName.c_str(),pfrom->id, nSizeThinBlock);
- pfrom->nSizeThinBlock = nSizeThinBlock;
+ LogPrint("thin", "Received thinblock %s from peer %s (%d). Size %d bytes.\n", inv.hash.ToString(), pfrom->addrName.c_str(), pfrom->id, nSizeThinBlock);
#endif
- if (!pfrom->mapThinBlocksInFlight.count(inv.hash)) {
- LogPrint("thin", "Thinblock received but not requested %s from peer %s (%d)\n",inv.hash.ToString(),
- pfrom->addrName.c_str(), pfrom->addrName.c_str(), pfrom->id);
- Misbehaving(pfrom->GetId(), 20);
- }
- pfrom->thinBlock.SetNull();
- pfrom->thinBlock.nVersion = thinBlock.header.nVersion;
- pfrom->thinBlock.nBits = thinBlock.header.nBits;
- pfrom->thinBlock.nNonce = thinBlock.header.nNonce;
- pfrom->thinBlock.nTime = thinBlock.header.nTime;
- pfrom->thinBlock.hashMerkleRoot = thinBlock.header.hashMerkleRoot;
- pfrom->thinBlock.hashPrevBlock = thinBlock.header.hashPrevBlock;
- pfrom->xThinBlockHashes = thinBlock.vTxHashes;
-
- // Create the mapMissingTx from all the supplied tx's in the xthinblock
- std::map<uint256, CTransaction> mapMissingTx;
- BOOST_FOREACH(CTransaction tx, thinBlock.vMissingTx)
- mapMissingTx[tx.GetHash()] = tx;
-
- // Create a map of all 8 bytes tx hashes pointing to their full tx hash counterpart
- // We need to check all transaction sources (orphan list, mempool, and new (incoming) transactions in this block) for a collision.
- bool collision = false;
- std::map<uint64_t, uint256> mapPartialTxHash;
- LOCK(cs_main);
- std::vector<uint256> memPoolHashes;
- mempool.queryHashes(memPoolHashes);
- for (uint64_t i = 0; i < memPoolHashes.size(); i++) {
- uint64_t cheapHash = memPoolHashes[i].GetCheapHash();
- if(mapPartialTxHash.count(cheapHash)) //Check for collisions
- collision = true;
- mapPartialTxHash[cheapHash] = memPoolHashes[i];
- }
- for (map<uint256, CTransaction>::iterator mi = mapMissingTx.begin(); mi != mapMissingTx.end(); ++mi) {
- uint64_t cheapHash = (*mi).first.GetCheapHash();
- // Check for cheap hash collision. Only mark as collision if the full hash is not the same,
- // because the same tx could have been received into the mempool during the request of the xthinblock.
- // In that case we would have the same transaction twice, so it is not a real cheap hash collision and we continue normally.
- const uint256 existingHash = mapPartialTxHash[cheapHash];
- if( (!existingHash.IsNull()) ) { // Check if we already have the cheap hash
- if ((existingHash != (*mi).first)) { // Check if it really is a cheap hash collision and not just the same transaction
- collision = true;
- }
- }
- mapPartialTxHash[cheapHash] = (*mi).first;
- }
-
- // There is a remote possiblity of a Tx hash collision therefore if it occurs we re-request a normal
- // thinblock which has the full Tx hash data rather than just the truncated hash.
- if (collision) {
- vector<CInv> vGetData;
- vGetData.push_back(CInv(MSG_THINBLOCK, thinBlock.header.GetHash()));
- pfrom->PushMessage("getdata", vGetData);
- LogPrintf("TX HASH COLLISION for xthinblock: re-requesting a thinblock\n");
- return true;
+ bool fAlreadyHave = false;
+ // An expedited block or re-requested xthin can arrive and beat the original thin block request/response
+ if (!pfrom->mapThinBlocksInFlight.count(inv.hash)) {
+ LogPrint("thin", "Thinblock %s from peer %s (%d) received but we already have it\n", inv.hash.ToString(), pfrom->addrName.c_str(), pfrom->id);
+ LOCK(cs_main);
+ fAlreadyHave = AlreadyHave(inv); // I'll still continue processing if we don't have an accepted block yet
}
- int missingCount = 0;
- int unnecessaryCount = 0;
- // Xpress Validation - only perform xval if the chaintip matches the last blockhash in the thinblock
- bool fXVal = (thinBlock.header.hashPrevBlock == chainActive.Tip()->GetBlockHash()) ? true : false;
-
- // Look for each transaction in our various pools and buffers.
- // With xThinBlocks the vTxHashes contains only the first 8 bytes of the tx hash.
- BOOST_FOREACH(uint64_t &cheapHash, thinBlock.vTxHashes)
- {
- // Replace the truncated hash with the full hash value if it exists
- const uint256 hash = mapPartialTxHash[cheapHash];
- CTransaction tx;
- if (!hash.IsNull())
- {
- bool inMemPool = mempool.lookup(hash, tx);
- bool inMissingTx = mapMissingTx.count(hash) > 0;
- bool inOrphanCache = false;
-
- if ((inMemPool && inMissingTx) || (inOrphanCache && inMissingTx))
- unnecessaryCount++;
-
- if (inOrphanCache) {
- setUnVerifiedOrphanTxHash.insert(hash);
- }
- else if (inMemPool && fXVal)
- setPreVerifiedTxHash.insert(hash);
- else if (inMissingTx)
- tx = mapMissingTx[hash];
- }
- if (tx.IsNull())
- missingCount++;
- // This will push an empty/invalid transaction if we don't have it yet
- pfrom->thinBlock.vtx.push_back(tx);
- }
- pfrom->thinBlockWaitingForTxns = missingCount;
- LogPrint("thin", "thinblock waiting for: %d, unnecessary: %d, txs: %d full: %d\n", pfrom->thinBlockWaitingForTxns, unnecessaryCount, pfrom->thinBlock.vtx.size(), mapMissingTx.size());
-
- if (pfrom->thinBlockWaitingForTxns == 0) {
- // We have all the transactions now that are in this block: try to reassemble and process.
- pfrom->thinBlockWaitingForTxns = -1;
- pfrom->AddInventoryKnown(inv);
-#ifdef LOG_XTHINBLOCKS
- int blockSize = pfrom->thinBlock.GetSerializeSize(SER_NETWORK, CBlock::CURRENT_VERSION);
- LogPrint("thin", "Reassembled thin block for %s (%d bytes). Message was %d bytes, compression ratio %3.2f\n",
- pfrom->thinBlock.GetHash().ToString(),
- blockSize,
- nSizeThinBlock,
- ((float) blockSize) / ((float) nSizeThinBlock)
- );
-#endif
-
- HandleBlockMessage(pfrom, strCommand, pfrom->thinBlock, inv); // clears the thin block
- }
- else if (pfrom->thinBlockWaitingForTxns > 0) {
- // This marks the end of the transactions we've received. If we get this and we have NOT been able to
- // finish reassembling the block, we need to re-request the transactions we're missing:
- std::set<uint64_t> setHashesToRequest;
- for (size_t i = 0; i < pfrom->thinBlock.vtx.size(); i++) {
- if (pfrom->thinBlock.vtx[i].IsNull()) {
- setHashesToRequest.insert(pfrom->xThinBlockHashes[i]);
- LogPrint("thin", "Re-requesting tx ==> 8 byte hash %d\n", pfrom->xThinBlockHashes[i]);
- }
- }
- // Re-request transactions that we are still missing
- CXRequestThinBlockTx thinBlockTx(thinBlock.header.GetHash(), setHashesToRequest);
- pfrom->PushMessage(NetMsgType::GET_XBLOCKTX, thinBlockTx);
- LogPrint("thin", "Missing %d transactions for xthinblock, re-requesting\n",
- pfrom->thinBlockWaitingForTxns);
+ if (!fAlreadyHave) {
+ if (thinBlock.process(pfrom))
+ HandleBlockMessage(pfrom, strCommand, pfrom->thinBlock, thinBlock.GetInv()); // clears the thin block
}
}
@@ -5153,23 +5053,29 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv,
CInv inv(MSG_XTHINBLOCK, thinBlockTx.blockhash);
LogPrint("net", "received blocktxs for %s peer=%d\n", inv.hash.ToString(), pfrom->id);
if (!pfrom->mapThinBlocksInFlight.count(inv.hash)) {
- LogPrint("thin", "ThinblockTx received but not requested %s peer=%d\n",inv.hash.ToString(), pfrom->id);
- LOCK(cs_main);
- Misbehaving(pfrom->GetId(), 20);
+ LogPrint("thin", "xblocktx received but it was either not requested or it was beaten by another block %s peer=%d\n", inv.hash.ToString(), pfrom->id);
+ return true;
}
// Create the mapMissingTx from all the supplied tx's in the xthinblock
std::map<uint64_t, CTransaction> mapMissingTx;
- BOOST_FOREACH(CTransaction tx, thinBlockTx.vMissingTx)
+ BOOST_FOREACH(CTransaction tx, thinBlockTx.vMissingTx) {
mapMissingTx[tx.GetHash().GetCheapHash()] = tx;
+ }
- for (size_t i = 0; i < pfrom->thinBlock.vtx.size(); i++) {
- if (pfrom->thinBlock.vtx[i].IsNull()) {
- pfrom->thinBlock.vtx[i] = mapMissingTx[pfrom->xThinBlockHashes[i]];
- pfrom->thinBlockWaitingForTxns--;
- LogPrint("thin", "Got Re-requested tx ==> 8 byte hash %d\n", pfrom->xThinBlockHashes[i]);
- }
+ int count=0;
+ for (size_t i = 0; i < pfrom->thinBlock.vtx.size(); ++i) {
+ if (pfrom->thinBlock.vtx[i].IsNull()) {
+ auto val = mapMissingTx.find(pfrom->xThinBlockHashes[i]);
+ if (val != mapMissingTx.end()) {
+ pfrom->thinBlock.vtx[i] = val->second;
+ --pfrom->thinBlockWaitingForTxns;
+ }
+ count++;
+ }
}
+ LogPrint("thin", "Got %d Re-requested txs, needed %d of them\n", thinBlockTx.vMissingTx.size(), count);
+
if (pfrom->thinBlockWaitingForTxns == 0) {
// We have all the transactions now that are in this block: try to reassemble and process.
pfrom->thinBlockWaitingForTxns = -1;
@@ -5198,9 +5104,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv,
CTxOrphanCache::instance()->EraseOrphans(orphans);
}
else {
- LogPrint("thin", "Failed to retrieve all transactions for block - DOS Banned\n");
- LOCK(cs_main);
- Misbehaving(pfrom->GetId(), 100);
+ LogPrint("thin", "Failed to retrieve all transactions for block\n");
}
}
@@ -5483,6 +5387,31 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv,
}
}
}
+ else if (strCommand == NetMsgType::XPEDITEDREQUEST)
+ {
+ HandleExpeditedRequest(vRecv,pfrom);
+ }
+ else if (strCommand == NetMsgType::XPEDITEDBLK)
+ {
+ HandleExpeditedBlock(vRecv,pfrom);
+ }
+ else if (strCommand == NetMsgType::VERSION2)
+ {
+ // Each connection can only send one version message
+ if (pfrom->addrFromPort != 0) {
+ pfrom->PushMessage(NetMsgType::REJECT, strCommand, REJECT_DUPLICATE, string("Duplicate version2 message"));
+ LOCK(cs_main);
+ Misbehaving(pfrom->GetId(), 15);
+ return false;
+ }
+
+ vRecv >> pfrom->addrFromPort;
+ pfrom->PushMessage(NetMsgType::VERACK2);
+ }
+ else if (strCommand == NetMsgType::VERACK2)
+ {
+ CheckAndRequestExpeditedBlocks(pfrom);
+ }
else
{
View
@@ -2341,6 +2341,7 @@ unsigned int SendBufferSize() { return 1000*GetArg("-maxsendbuffer", DEFAULT_MAX
CNode::CNode(SOCKET hSocketIn, const CAddress& addrIn, const std::string& addrNameIn, bool fInboundIn) :
ssSend(SER_NETWORK, INIT_PROTO_VERSION),
+ addrFromPort(0),
addrKnown(5000, 0.001),
filterInventoryKnown(50000, 0.000001)
{
@@ -2415,6 +2416,8 @@ CNode::~CNode()
delete pfilter;
delete pThinBlockFilter;
+ addrFromPort = 0;
+
GetNodeSignals().FinalizeNode(GetId());
}
View
@@ -375,6 +375,8 @@ class CNode
uint64_t nGetXBlockTxLastTime; // The last time a get_xblocktx request was made
// Xtreme Thinblocks: end section
+ uint16_t addrFromPort;
+
protected:
// Denial-of-service detection/prevention
View
@@ -41,7 +41,12 @@ const char *XBLOCKTX="xblocktx";
const char *GET_XBLOCKTX="get_xblocktx";
const char *GET_XTHIN="get_xthin";
// BUIP010 Xtreme Thinblocks - end section
-};
+const char *VERSION2="buversion"; // unfortunately the unlimited team wasn't very creative with naming.
+const char *VERACK2="buverack";
+const char *XPEDITEDREQUEST="req_xpedited";
+const char *XPEDITEDBLK="Xb";
+const char *XPEDITEDTxn="Xt";
+}
static const char* ppszTypeName[] =
{
@@ -87,8 +92,13 @@ const static std::string allNetMessageTypes[] = {
NetMsgType::XTHINBLOCK,
NetMsgType::XBLOCKTX,
NetMsgType::GET_XBLOCKTX,
- NetMsgType::GET_XTHIN
+ NetMsgType::GET_XTHIN,
// BUIP010 Xtreme Thinbocks - end section
+ NetMsgType::VERSION2,
+ NetMsgType::VERACK2,
+ NetMsgType::XPEDITEDREQUEST,
+ NetMsgType::XPEDITEDBLK,
+ NetMsgType::XPEDITEDTxn,
};
const static std::vector<std::string> allNetMessageTypesVec(allNetMessageTypes, allNetMessageTypes+ARRAYLEN(allNetMessageTypes));
View
@@ -233,6 +233,33 @@ extern const char *REJECT;
*/
extern const char *SENDHEADERS;
+/**
+ * Version2, request a verack2.
+ * @since protocol version 80002
+ */
+extern const char *VERSION2;
+/**
+ * verack2, a followup message to verack since we can't extend or change old
+ * messages. This one is meant to exchange our port number.
+ * @since protocol version 80002
+ */
+extern const char *VERACK2;
+/**
+ * Send this if your node prefers to receive new block announcements
+ * and transactions directly without INVs
+ * @since protocol version 80000
+ */
+extern const char *XPEDITEDREQUEST;
+/**
+ * Block or transactions sent without explicit solicitation
+ * @since protocol version 80000
+ */
+extern const char *XPEDITEDBLK;
+/**
+ * Block or transactions sent without explicit solicitation
+ * @since protocol version 80000
+ */
+extern const char *XPEDITEDTXN;
};
/* Get a vector of all valid message types (see above) */
Oops, something went wrong.

0 comments on commit f7234cb

Please sign in to comment.