Permalink
Browse files

Merge pull request #6508

eece63f Switch blocks to a constant-space Merkle root/branch algorithm. (Pieter Wuille)
ee60e56 Add merkle.{h,cpp}, generic merkle root/branch algorithm (Pieter Wuille)
  • Loading branch information...
sipa committed Nov 28, 2015
2 parents 8d26289 + eece63f commit 61457c29d735b77182b5fbd45e86d7e3db343857
View
@@ -100,6 +100,7 @@ BITCOIN_CORE_H = \
compat/sanity.h \
compressor.h \
consensus/consensus.h \
+ consensus/merkle.h \
consensus/params.h \
consensus/validation.h \
core_io.h \
@@ -268,6 +269,7 @@ libbitcoin_common_a_SOURCES = \
chainparams.cpp \
coins.cpp \
compressor.cpp \
+ consensus/merkle.cpp \
core_read.cpp \
core_write.cpp \
hash.cpp \
@@ -57,6 +57,7 @@ BITCOIN_TESTS =\
test/dbwrapper_tests.cpp \
test/main_tests.cpp \
test/mempool_tests.cpp \
+ test/merkle_tests.cpp \
test/miner_tests.cpp \
test/mruset_tests.cpp \
test/multisig_tests.cpp \
View
@@ -4,6 +4,7 @@
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#include "chainparams.h"
+#include "consensus/merkle.h"
#include "tinyformat.h"
#include "util.h"
@@ -32,7 +33,7 @@ static CBlock CreateGenesisBlock(const char* pszTimestamp, const CScript& genesi
genesis.nVersion = nVersion;
genesis.vtx.push_back(txNew);
genesis.hashPrevBlock.SetNull();
- genesis.hashMerkleRoot = genesis.ComputeMerkleRoot();
+ genesis.hashMerkleRoot = BlockMerkleRoot(genesis);
return genesis;
}
View
@@ -0,0 +1,172 @@
+#include "merkle.h"
+#include "hash.h"
+#include "utilstrencodings.h"
+
+/* WARNING! If you're reading this because you're learning about crypto
+ and/or designing a new system that will use merkle trees, keep in mind
+ that the following merkle tree algorithm has a serious flaw related to
+ duplicate txids, resulting in a vulnerability (CVE-2012-2459).
+
+ The reason is that if the number of hashes in the list at a given time
+ is odd, the last one is duplicated before computing the next level (which
+ is unusual in Merkle trees). This results in certain sequences of
+ transactions leading to the same merkle root. For example, these two
+ trees:
+
+ A A
+ / \ / \
+ B C B C
+ / \ | / \ / \
+ D E F D E F F
+ / \ / \ / \ / \ / \ / \ / \
+ 1 2 3 4 5 6 1 2 3 4 5 6 5 6
+
+ for transaction lists [1,2,3,4,5,6] and [1,2,3,4,5,6,5,6] (where 5 and
+ 6 are repeated) result in the same root hash A (because the hash of both
+ of (F) and (F,F) is C).
+
+ The vulnerability results from being able to send a block with such a
+ transaction list, with the same merkle root, and the same block hash as
+ the original without duplication, resulting in failed validation. If the
+ receiving node proceeds to mark that block as permanently invalid
+ however, it will fail to accept further unmodified (and thus potentially
+ valid) versions of the same block. We defend against this by detecting
+ the case where we would hash two identical hashes at the end of the list
+ together, and treating that identically to the block having an invalid
+ merkle root. Assuming no double-SHA256 collisions, this will detect all
+ known ways of changing the transactions without affecting the merkle
+ root.
+*/
+
+/* This implements a constant-space merkle root/path calculator, limited to 2^32 leaves. */
+static void MerkleComputation(const std::vector<uint256>& leaves, uint256* proot, bool* pmutated, uint32_t branchpos, std::vector<uint256>* pbranch) {
+ if (pbranch) pbranch->clear();
+ if (leaves.size() == 0) {
+ if (pmutated) *pmutated = false;
+ if (proot) *proot = uint256();
+ return;
+ }
+ bool mutated = false;
+ // count is the number of leaves processed so far.
+ uint32_t count = 0;
+ // inner is an array of eagerly computed subtree hashes, indexed by tree
+ // level (0 being the leaves).
+ // For example, when count is 25 (11001 in binary), inner[4] is the hash of
+ // the first 16 leaves, inner[3] of the next 8 leaves, and inner[0] equal to
+ // the last leaf. The other inner entries are undefined.
+ uint256 inner[32];
+ // Which position in inner is a hash that depends on the matching leaf.
+ int matchlevel = -1;
+ // First process all leaves into 'inner' values.
+ while (count < leaves.size()) {
+ uint256 h = leaves[count];
+ bool matchh = count == branchpos;
+ count++;
+ int level;
+ // For each of the lower bits in count that are 0, do 1 step. Each
+ // corresponds to an inner value that existed before processing the
+ // current leaf, and each needs a hash to combine it.
+ for (level = 0; !(count & (((uint32_t)1) << level)); level++) {
+ if (pbranch) {
+ if (matchh) {
+ pbranch->push_back(inner[level]);
+ } else if (matchlevel == level) {
+ pbranch->push_back(h);
+ matchh = true;
+ }
+ }
+ mutated |= (inner[level] == h);
+ CHash256().Write(inner[level].begin(), 32).Write(h.begin(), 32).Finalize(h.begin());
+ }
+ // Store the resulting hash at inner position level.
+ inner[level] = h;
+ if (matchh) {
+ matchlevel = level;
+ }
+ }
+ // Do a final 'sweep' over the rightmost branch of the tree to process
+ // odd levels, and reduce everything to a single top value.
+ // Level is the level (counted from the bottom) up to which we've sweeped.
+ int level = 0;
+ // As long as bit number level in count is zero, skip it. It means there
+ // is nothing left at this level.
+ while (!(count & (((uint32_t)1) << level))) {
+ level++;
+ }
+ uint256 h = inner[level];
+ bool matchh = matchlevel == level;
+ while (count != (((uint32_t)1) << level)) {
+ // If we reach this point, h is an inner value that is not the top.
+ // We combine it with itself (Bitcoin's special rule for odd levels in
+ // the tree) to produce a higher level one.
+ if (pbranch && matchh) {
+ pbranch->push_back(h);
+ }
+ CHash256().Write(h.begin(), 32).Write(h.begin(), 32).Finalize(h.begin());
+ // Increment count to the value it would have if two entries at this
+ // level had existed.
+ count += (((uint32_t)1) << level);
+ level++;
+ // And propagate the result upwards accordingly.
+ while (!(count & (((uint32_t)1) << level))) {
+ if (pbranch) {
+ if (matchh) {
+ pbranch->push_back(inner[level]);
+ } else if (matchlevel == level) {
+ pbranch->push_back(h);
+ matchh = true;
+ }
+ }
+ CHash256().Write(inner[level].begin(), 32).Write(h.begin(), 32).Finalize(h.begin());
+ level++;
+ }
+ }
+ // Return result.
+ if (pmutated) *pmutated = mutated;
+ if (proot) *proot = h;
+}
+
+uint256 ComputeMerkleRoot(const std::vector<uint256>& leaves, bool* mutated) {
+ uint256 hash;
+ MerkleComputation(leaves, &hash, mutated, -1, NULL);
+ return hash;
+}
+
+std::vector<uint256> ComputeMerkleBranch(const std::vector<uint256>& leaves, uint32_t position) {
+ std::vector<uint256> ret;
+ MerkleComputation(leaves, NULL, NULL, position, &ret);
+ return ret;
+}
+
+uint256 ComputeMerkleRootFromBranch(const uint256& leaf, const std::vector<uint256>& vMerkleBranch, uint32_t nIndex) {
+ uint256 hash = leaf;
+ for (std::vector<uint256>::const_iterator it = vMerkleBranch.begin(); it != vMerkleBranch.end(); ++it) {
+ if (nIndex & 1) {
+ hash = Hash(BEGIN(*it), END(*it), BEGIN(hash), END(hash));
+ } else {
+ hash = Hash(BEGIN(hash), END(hash), BEGIN(*it), END(*it));
+ }
+ nIndex >>= 1;
+ }
+ return hash;
+}
+
+uint256 BlockMerkleRoot(const CBlock& block, bool* mutated)
+{
+ std::vector<uint256> leaves;
+ leaves.resize(block.vtx.size());
+ for (size_t s = 0; s < block.vtx.size(); s++) {
+ leaves[s] = block.vtx[s].GetHash();
+ }
+ return ComputeMerkleRoot(leaves, mutated);
+}
+
+std::vector<uint256> BlockMerkleBranch(const CBlock& block, uint32_t position)
+{
+ std::vector<uint256> leaves;
+ leaves.resize(block.vtx.size());
+ for (size_t s = 0; s < block.vtx.size(); s++) {
+ leaves[s] = block.vtx[s].GetHash();
+ }
+ return ComputeMerkleBranch(leaves, position);
+}
View
@@ -0,0 +1,32 @@
+// Copyright (c) 2015 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#ifndef BITCOIN_MERKLE
+#define BITCOIN_MERKLE
+
+#include <stdint.h>
+#include <vector>
+
+#include "primitives/transaction.h"
+#include "primitives/block.h"
+#include "uint256.h"
+
+uint256 ComputeMerkleRoot(const std::vector<uint256>& leaves, bool* mutated = NULL);
+std::vector<uint256> ComputeMerkleBranch(const std::vector<uint256>& leaves, uint32_t position);
+uint256 ComputeMerkleRootFromBranch(const uint256& leaf, const std::vector<uint256>& branch, uint32_t position);
+
+/*
+ * Compute the Merkle root of the transactions in a block.
+ * *mutated is set to true if a duplicated subtree was found.
+ */
+uint256 BlockMerkleRoot(const CBlock& block, bool* mutated = NULL);
+
+/*
+ * Compute the Merkle branch for the tree of transactions in a block, for a
+ * given position.
+ * This can be verified using ComputeMerkleRootFromBranch.
+ */
+std::vector<uint256> BlockMerkleBranch(const CBlock& block, uint32_t position);
+
+#endif
View
@@ -12,6 +12,7 @@
#include "checkpoints.h"
#include "checkqueue.h"
#include "consensus/consensus.h"
+#include "consensus/merkle.h"
#include "consensus/validation.h"
#include "hash.h"
#include "init.h"
@@ -2876,7 +2877,7 @@ bool CheckBlock(const CBlock& block, CValidationState& state, bool fCheckPOW, bo
// Check the merkle root.
if (fCheckMerkleRoot) {
bool mutated;
- uint256 hashMerkleRoot2 = block.ComputeMerkleRoot(&mutated);
+ uint256 hashMerkleRoot2 = BlockMerkleRoot(block, &mutated);
if (block.hashMerkleRoot != hashMerkleRoot2)
return state.DoS(100, error("CheckBlock(): hashMerkleRoot mismatch"),
REJECT_INVALID, "bad-txnmrklroot", true);
View
@@ -10,6 +10,7 @@
#include "chainparams.h"
#include "coins.h"
#include "consensus/consensus.h"
+#include "consensus/merkle.h"
#include "consensus/validation.h"
#include "hash.h"
#include "main.h"
@@ -373,7 +374,7 @@ void IncrementExtraNonce(CBlock* pblock, const CBlockIndex* pindexPrev, unsigned
assert(txCoinbase.vin[0].scriptSig.size() <= 100);
pblock->vtx[0] = txCoinbase;
- pblock->hashMerkleRoot = pblock->ComputeMerkleRoot();
+ pblock->hashMerkleRoot = BlockMerkleRoot(*pblock);
}
//////////////////////////////////////////////////////////////////////////////
View
@@ -15,69 +15,6 @@ uint256 CBlockHeader::GetHash() const
return SerializeHash(*this);
}
-uint256 CBlock::ComputeMerkleRoot(bool* fMutated) const
-{
- /* WARNING! If you're reading this because you're learning about crypto
- and/or designing a new system that will use merkle trees, keep in mind
- that the following merkle tree algorithm has a serious flaw related to
- duplicate txids, resulting in a vulnerability (CVE-2012-2459).
-
- The reason is that if the number of hashes in the list at a given time
- is odd, the last one is duplicated before computing the next level (which
- is unusual in Merkle trees). This results in certain sequences of
- transactions leading to the same merkle root. For example, these two
- trees:
-
- A A
- / \ / \
- B C B C
- / \ | / \ / \
- D E F D E F F
- / \ / \ / \ / \ / \ / \ / \
- 1 2 3 4 5 6 1 2 3 4 5 6 5 6
-
- for transaction lists [1,2,3,4,5,6] and [1,2,3,4,5,6,5,6] (where 5 and
- 6 are repeated) result in the same root hash A (because the hash of both
- of (F) and (F,F) is C).
-
- The vulnerability results from being able to send a block with such a
- transaction list, with the same merkle root, and the same block hash as
- the original without duplication, resulting in failed validation. If the
- receiving node proceeds to mark that block as permanently invalid
- however, it will fail to accept further unmodified (and thus potentially
- valid) versions of the same block. We defend against this by detecting
- the case where we would hash two identical hashes at the end of the list
- together, and treating that identically to the block having an invalid
- merkle root. Assuming no double-SHA256 collisions, this will detect all
- known ways of changing the transactions without affecting the merkle
- root.
- */
- std::vector<uint256> vMerkleTree;
- vMerkleTree.reserve(vtx.size() * 2 + 16); // Safe upper bound for the number of total nodes.
- for (std::vector<CTransaction>::const_iterator it(vtx.begin()); it != vtx.end(); ++it)
- vMerkleTree.push_back(it->GetHash());
- int j = 0;
- bool mutated = false;
- for (int nSize = vtx.size(); nSize > 1; nSize = (nSize + 1) / 2)
- {
- for (int i = 0; i < nSize; i += 2)
- {
- int i2 = std::min(i+1, nSize-1);
- if (i2 == i + 1 && i2 + 1 == nSize && vMerkleTree[j+i] == vMerkleTree[j+i2]) {
- // Two identical hashes at the end of the list at a particular level.
- mutated = true;
- }
- vMerkleTree.push_back(Hash(BEGIN(vMerkleTree[j+i]), END(vMerkleTree[j+i]),
- BEGIN(vMerkleTree[j+i2]), END(vMerkleTree[j+i2])));
- }
- j += nSize;
- }
- if (fMutated) {
- *fMutated = mutated;
- }
- return (vMerkleTree.empty() ? uint256() : vMerkleTree.back());
-}
-
std::string CBlock::ToString() const
{
std::stringstream s;
View
@@ -118,12 +118,6 @@ class CBlock : public CBlockHeader
return block;
}
- // Build the merkle tree for this block and return the merkle root.
- // If non-NULL, *mutated is set to whether mutation was detected in the merkle
- // tree (a duplication of transactions in the block leading to an identical
- // merkle root).
- uint256 ComputeMerkleRoot(bool* mutated = NULL) const;
-
std::string ToString() const;
};
View
@@ -72,5 +72,4 @@ BOOST_AUTO_TEST_CASE(test_combiner_all)
Test.disconnect(&ReturnTrue);
BOOST_CHECK(Test());
}
-
BOOST_AUTO_TEST_SUITE_END()
Oops, something went wrong.

0 comments on commit 61457c2

Please sign in to comment.