Merge #8068: Compact Blocks

48efec8 Fix some minor compact block issues that came up in review (Matt Corallo)
ccd06b9 Elaborate bucket size math (Pieter Wuille)
0d4cb48 Use vTxHashes to optimize InitData significantly (Matt Corallo)
8119026 Provide a flat list of txid/terators to txn in CTxMemPool (Matt Corallo)
678ee97 Add BIP 152 to implemented BIPs list (Matt Corallo)
56ba516 Add reconstruction debug logging (Matt Corallo)
2f34a2e Get our "best three" peers to announce blocks using cmpctblocks (Matt Corallo)
927f8ee Add ability to fetch CNode by NodeId (Matt Corallo)
d25cd3e Add receiver-side protocol implementation for CMPCTBLOCK stuff (Matt Corallo)
9c837d5 Add sender-side protocol implementation for CMPCTBLOCK stuff (Matt Corallo)
00c4078 Add protocol messages for short-ids blocks (Matt Corallo)
e3b2222 Add some blockencodings tests (Matt Corallo)
f4f8f14 Add TestMemPoolEntryHelper::FromTx version for CTransaction (Matt Corallo)
85ad31e Add partial-block block encodings API (Matt Corallo)
5249dac Add COMPACTSIZE wrapper similar to VARINT for serialization (Matt Corallo)
cbda71c Move context-required checks from CheckBlockHeader to Contextual... (Matt Corallo)
7c29ec9 If AcceptBlockHeader returns true, pindex will be set. (Matt Corallo)
96806c3 Stop trimming when mapTx is empty (Pieter Wuille)

doc/bips.md

@@ -26,3 +26,4 @@ BIPs that are implemented by Bitcoin Core (up-to-date up to **v0.13.0**):
 * [`BIP 125`](https://github.com/bitcoin/bips/blob/master/bip-0125.mediawiki): Opt-in full replace-by-fee signaling honoured in mempool and mining as of **v0.12.0** ([PR 6871](https://github.com/bitcoin/bitcoin/pull/6871)).
 * [`BIP 130`](https://github.com/bitcoin/bips/blob/master/bip-0130.mediawiki): direct headers announcement is negotiated with peer versions `>=70012` as of **v0.12.0** ([PR 6494](https://github.com/bitcoin/bitcoin/pull/6494)).
 * [`BIP 133`](https://github.com/bitcoin/bips/blob/master/bip-0133.mediawiki): feefilter messages are respected and sent for peer versions `>=70013` as of **v0.13.0** ([PR 7542](https://github.com/bitcoin/bitcoin/pull/7542)).
+* [`BIP 152`](https://github.com/bitcoin/bips/blob/master/bip-0152.mediawiki): Compact block transfer and related optimizations are used as of **v0.13.0** ([PR 8068](https://github.com/bitcoin/bitcoin/pull/8068)).

src/Makefile.am

@@ -74,6 +74,7 @@ BITCOIN_CORE_H = \
   addrman.h \
   base58.h \
   bloom.h \
+  blockencodings.h \
   chain.h \
   chainparams.h \
   chainparamsbase.h \
@@ -163,6 +164,7 @@ libbitcoin_server_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS)
 libbitcoin_server_a_SOURCES = \
   addrman.cpp \
   bloom.cpp \
+  blockencodings.cpp \
   chain.cpp \
   checkpoints.cpp \
   httprpc.cpp \

src/Makefile.test.include

@@ -45,6 +45,7 @@ BITCOIN_TESTS =\
   test/base58_tests.cpp \
   test/base64_tests.cpp \
   test/bip32_tests.cpp \
+  test/blockencodings_tests.cpp \
   test/bloom_tests.cpp \
   test/Checkpoints_tests.cpp \
   test/coins_tests.cpp \

src/blockencodings.cpp

@@ -0,0 +1,180 @@
+// Copyright (c) 2016 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include "blockencodings.h"
+#include "consensus/consensus.h"
+#include "consensus/validation.h"
+#include "chainparams.h"
+#include "hash.h"
+#include "random.h"
+#include "streams.h"
+#include "txmempool.h"
+#include "main.h"
+#include "util.h"
+
+#include <unordered_map>
+
+#define MIN_TRANSACTION_SIZE (::GetSerializeSize(CTransaction(), SER_NETWORK, PROTOCOL_VERSION))
+
+CBlockHeaderAndShortTxIDs::CBlockHeaderAndShortTxIDs(const CBlock& block) :
+        nonce(GetRand(std::numeric_limits<uint64_t>::max())),
+        shorttxids(block.vtx.size() - 1), prefilledtxn(1), header(block) {
+    FillShortTxIDSelector();
+    //TODO: Use our mempool prior to block acceptance to predictively fill more than just the coinbase
+    prefilledtxn[0] = {0, block.vtx[0]};
+    for (size_t i = 1; i < block.vtx.size(); i++) {
+        const CTransaction& tx = block.vtx[i];
+        shorttxids[i - 1] = GetShortID(tx.GetHash());
+    }
+}
+
+void CBlockHeaderAndShortTxIDs::FillShortTxIDSelector() const {
+    CDataStream stream(SER_NETWORK, PROTOCOL_VERSION);
+    stream << header << nonce;
+    CSHA256 hasher;
+    hasher.Write((unsigned char*)&(*stream.begin()), stream.end() - stream.begin());
+    uint256 shorttxidhash;
+    hasher.Finalize(shorttxidhash.begin());
+    shorttxidk0 = shorttxidhash.GetUint64(0);
+    shorttxidk1 = shorttxidhash.GetUint64(1);
+}
+
+uint64_t CBlockHeaderAndShortTxIDs::GetShortID(const uint256& txhash) const {
+    static_assert(SHORTTXIDS_LENGTH == 6, "shorttxids calculation assumes 6-byte shorttxids");
+    return SipHashUint256(shorttxidk0, shorttxidk1, txhash) & 0xffffffffffffL;
+}
+
+
+
+ReadStatus PartiallyDownloadedBlock::InitData(const CBlockHeaderAndShortTxIDs& cmpctblock) {
+    if (cmpctblock.header.IsNull() || (cmpctblock.shorttxids.empty() && cmpctblock.prefilledtxn.empty()))
+        return READ_STATUS_INVALID;
+    if (cmpctblock.shorttxids.size() + cmpctblock.prefilledtxn.size() > MAX_BLOCK_SIZE / MIN_TRANSACTION_SIZE)
+        return READ_STATUS_INVALID;
+
+    assert(header.IsNull() && txn_available.empty());
+    header = cmpctblock.header;
+    txn_available.resize(cmpctblock.BlockTxCount());
+
+    int32_t lastprefilledindex = -1;
+    for (size_t i = 0; i < cmpctblock.prefilledtxn.size(); i++) {
+        if (cmpctblock.prefilledtxn[i].tx.IsNull())
+            return READ_STATUS_INVALID;
+
+        lastprefilledindex += cmpctblock.prefilledtxn[i].index + 1; //index is a uint16_t, so cant overflow here
+        if (lastprefilledindex > std::numeric_limits<uint16_t>::max())
+            return READ_STATUS_INVALID;
+        if ((uint32_t)lastprefilledindex > cmpctblock.shorttxids.size() + i) {
+            // If we are inserting a tx at an index greater than our full list of shorttxids
+            // plus the number of prefilled txn we've inserted, then we have txn for which we
+            // have neither a prefilled txn or a shorttxid!
+            return READ_STATUS_INVALID;
+        }
+        txn_available[lastprefilledindex] = std::make_shared<CTransaction>(cmpctblock.prefilledtxn[i].tx);
+    }
+    prefilled_count = cmpctblock.prefilledtxn.size();
+
+    // Calculate map of txids -> positions and check mempool to see what we have (or dont)
+    // Because well-formed cmpctblock messages will have a (relatively) uniform distribution
+    // of short IDs, any highly-uneven distribution of elements can be safely treated as a
+    // READ_STATUS_FAILED.
+    std::unordered_map<uint64_t, uint16_t> shorttxids(cmpctblock.shorttxids.size());
+    uint16_t index_offset = 0;
+    for (size_t i = 0; i < cmpctblock.shorttxids.size(); i++) {
+        while (txn_available[i + index_offset])
+            index_offset++;
+        shorttxids[cmpctblock.shorttxids[i]] = i + index_offset;
+        // To determine the chance that the number of entries in a bucket exceeds N,
+        // we use the fact that the number of elements in a single bucket is
+        // binomially distributed (with n = the number of shorttxids S, and p =
+        // 1 / the number of buckets), that in the worst case the number of buckets is
+        // equal to S (due to std::unordered_map having a default load factor of 1.0),
+        // and that the chance for any bucket to exceed N elements is at most
+        // buckets * (the chance that any given bucket is above N elements).
+        // Thus: P(max_elements_per_bucket > N) <= S * (1 - cdf(binomial(n=S,p=1/S), N)).
+        // If we assume blocks of up to 16000, allowing 12 elements per bucket should
+        // only fail once per ~1 million block transfers (per peer and connection).
+        if (shorttxids.bucket_size(shorttxids.bucket(cmpctblock.shorttxids[i])) > 12)
+            return READ_STATUS_FAILED;
+    }
+    // TODO: in the shortid-collision case, we should instead request both transactions
+    // which collided. Falling back to full-block-request here is overkill.
+    if (shorttxids.size() != cmpctblock.shorttxids.size())
+        return READ_STATUS_FAILED; // Short ID collision
+
+    std::vector<bool> have_txn(txn_available.size());
+    LOCK(pool->cs);
+    const std::vector<std::pair<uint256, CTxMemPool::txiter> >& vTxHashes = pool->vTxHashes;
+    for (size_t i = 0; i < vTxHashes.size(); i++) {
+        uint64_t shortid = cmpctblock.GetShortID(vTxHashes[i].first);
+        std::unordered_map<uint64_t, uint16_t>::iterator idit = shorttxids.find(shortid);
+        if (idit != shorttxids.end()) {
+            if (!have_txn[idit->second]) {
+                txn_available[idit->second] = vTxHashes[i].second->GetSharedTx();
+                have_txn[idit->second]  = true;
+                mempool_count++;
+            } else {
+                // If we find two mempool txn that match the short id, just request it.
+                // This should be rare enough that the extra bandwidth doesn't matter,
+                // but eating a round-trip due to FillBlock failure would be annoying
+                if (txn_available[idit->second]) {
+                    txn_available[idit->second].reset();
+                    mempool_count--;
+                }
+            }
+        }
+        // Though ideally we'd continue scanning for the two-txn-match-shortid case,
+        // the performance win of an early exit here is too good to pass up and worth
+        // the extra risk.
+        if (mempool_count == shorttxids.size())
+            break;
+    }
+
+    LogPrint("cmpctblock", "Initialized PartiallyDownloadedBlock for block %s using a cmpctblock of size %lu\n", cmpctblock.header.GetHash().ToString(), cmpctblock.GetSerializeSize(SER_NETWORK, PROTOCOL_VERSION));
+
+    return READ_STATUS_OK;
+}
+
+bool PartiallyDownloadedBlock::IsTxAvailable(size_t index) const {
+    assert(!header.IsNull());
+    assert(index < txn_available.size());
+    return txn_available[index] ? true : false;
+}
+
+ReadStatus PartiallyDownloadedBlock::FillBlock(CBlock& block, const std::vector<CTransaction>& vtx_missing) const {
+    assert(!header.IsNull());
+    block = header;
+    block.vtx.resize(txn_available.size());
+
+    size_t tx_missing_offset = 0;
+    for (size_t i = 0; i < txn_available.size(); i++) {
+        if (!txn_available[i]) {
+            if (vtx_missing.size() <= tx_missing_offset)
+                return READ_STATUS_INVALID;
+            block.vtx[i] = vtx_missing[tx_missing_offset++];
+        } else
+            block.vtx[i] = *txn_available[i];
+    }
+    if (vtx_missing.size() != tx_missing_offset)
+        return READ_STATUS_INVALID;
+
+    CValidationState state;
+    if (!CheckBlock(block, state, Params().GetConsensus())) {
+        // TODO: We really want to just check merkle tree manually here,
+        // but that is expensive, and CheckBlock caches a block's
+        // "checked-status" (in the CBlock?). CBlock should be able to
+        // check its own merkle root and cache that check.
+        if (state.CorruptionPossible())
+            return READ_STATUS_FAILED; // Possible Short ID collision
+        return READ_STATUS_INVALID;
+    }
+
+    LogPrint("cmpctblock", "Successfully reconstructed block %s with %lu txn prefilled, %lu txn from mempool and %lu txn requested\n", header.GetHash().ToString(), prefilled_count, mempool_count, vtx_missing.size());
+    if (vtx_missing.size() < 5) {
+        for(const CTransaction& tx : vtx_missing)
+            LogPrint("cmpctblock", "Reconstructed block %s required tx %s\n", header.GetHash().ToString(), tx.GetHash().ToString());
+    }
+
+    return READ_STATUS_OK;
+}