Auto merge of #3262 - str4d:2074-perf-1, r=<try>

Bitcoin 0.12 performance improvements

Cherry-picked from the following upstream PRs:

- bitcoin/bitcoin#6918
- bitcoin/bitcoin#6932

Part of #2074.

src/coins.cpp

@@ -358,6 +358,15 @@ CCoinsModifier CCoinsViewCache::ModifyCoins(const uint256 &txid) {
     return CCoinsModifier(*this, ret.first, cachedCoinUsage);
 }
 
+CCoinsModifier CCoinsViewCache::ModifyNewCoins(const uint256 &txid) {
+    assert(!hasModifier);
+    std::pair<CCoinsMap::iterator, bool> ret = cacheCoins.insert(std::make_pair(txid, CCoinsCacheEntry()));
+    ret.first->second.coins.Clear();
+    ret.first->second.flags = CCoinsCacheEntry::FRESH;
+    ret.first->second.flags |= CCoinsCacheEntry::DIRTY;
+    return CCoinsModifier(*this, ret.first, 0);
+}
+
 const CCoins* CCoinsViewCache::AccessCoins(const uint256 &txid) const {
     CCoinsMap::const_iterator it = FetchCoins(txid);
     if (it == cacheCoins.end()) {

src/coins.h

@@ -507,6 +507,16 @@ class CCoinsViewCache : public CCoinsViewBacked
      */
     CCoinsModifier ModifyCoins(const uint256 &txid);
 
+    /**
+     * Return a modifiable reference to a CCoins. Assumes that no entry with the given
+     * txid exists and creates a new one. This saves a database access in the case where
+     * the coins were to be wiped out by FromTx anyway. We rely on Zcash-derived block chains
+     * having no duplicate transactions, since BIP 30 and (except for the genesis block)
+     * BIP 34 have been enforced since launch. See the Zcash protocol specification, section
+     * "Bitcoin Improvement Proposals". Simultaneous modifications are not allowed.
+     */
+    CCoinsModifier ModifyNewCoins(const uint256 &txid);
+
     /**
      * Push the modifications applied to this cache to its base.
      * Failure to call this method before destruction will cause the changes to be forgotten.

src/init.cpp

@@ -27,6 +27,7 @@
 #include "net.h"
 #include "rpcserver.h"
 #include "script/standard.h"
+#include "script/sigcache.h"
 #include "scheduler.h"
 #include "txdb.h"
 #include "torcontrol.h"
@@ -471,7 +472,7 @@ std::string HelpMessage(HelpMessageMode mode)
     {
         strUsage += HelpMessageOpt("-limitfreerelay=<n>", strprintf("Continuously rate-limit free transactions to <n>*1000 bytes per minute (default: %u)", 15));
         strUsage += HelpMessageOpt("-relaypriority", strprintf("Require high priority for relaying free or low-fee transactions (default: %u)", 0));
-        strUsage += HelpMessageOpt("-maxsigcachesize=<n>", strprintf("Limit size of signature cache to <n> entries (default: %u)", 50000));
+        strUsage += HelpMessageOpt("-maxsigcachesize=<n>", strprintf("Limit size of signature cache to <n> MiB (default: %u)", DEFAULT_MAX_SIG_CACHE_SIZE));
     }
     strUsage += HelpMessageOpt("-minrelaytxfee=<amt>", strprintf(_("Fees (in %s/kB) smaller than this are considered zero fee for relaying (default: %s)"),
         CURRENCY_UNIT, FormatMoney(::minRelayTxFee.GetFeePerK())));

src/main.cpp

@@ -1903,7 +1903,7 @@ void UpdateCoins(const CTransaction& tx, CCoinsViewCache& inputs, CTxUndo &txund
     inputs.SetNullifiers(tx, true);
 
     // add outputs
-    inputs.ModifyCoins(tx.GetHash())->FromTx(tx, nHeight);
+    inputs.ModifyNewCoins(tx.GetHash())->FromTx(tx, nHeight);
 }
 
 void UpdateCoins(const CTransaction& tx, CCoinsViewCache& inputs, int nHeight)
@@ -2487,7 +2487,8 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin
             nFees += view.GetValueIn(tx)-tx.GetValueOut();
 
             std::vector<CScriptCheck> vChecks;
-            if (!ContextualCheckInputs(tx, state, view, fExpensiveChecks, flags, false, txdata[i], chainparams.GetConsensus(), consensusBranchId, nScriptCheckThreads ? &vChecks : NULL))
+            bool fCacheResults = fJustCheck; /* Don't cache results if we're actually connecting blocks (still consult the cache, though) */
+            if (!ContextualCheckInputs(tx, state, view, fExpensiveChecks, flags, fCacheResults, txdata[i], chainparams.GetConsensus(), consensusBranchId, nScriptCheckThreads ? &vChecks : NULL))
                 return false;
             control.Add(vChecks);
         }

src/script/sigcache.cpp

@@ -5,16 +5,29 @@
 
 #include "sigcache.h"
 
+#include "memusage.h"
 #include "pubkey.h"
 #include "random.h"
 #include "uint256.h"
 #include "util.h"
 
 #include <boost/thread.hpp>
-#include <boost/tuple/tuple_comparison.hpp>
+#include <boost/unordered_set.hpp>
 
 namespace {
 
+/**
+ * We're hashing a nonce into the entries themselves, so we don't need extra
+ * blinding in the set hash computation.
+ */
+class CSignatureCacheHasher
+{
+public:
+    size_t operator()(const uint256& key) const {
+        return key.GetCheapHash();
+    }
+};
+
 /**
  * Valid signature cache, to avoid doing expensive ECDSA signature checking
  * twice for every transaction (once when accepted into memory pool, and
@@ -23,52 +36,54 @@ namespace {
 class CSignatureCache
 {
 private:
-     //! sigdata_type is (signature hash, signature, public key):
-    typedef boost::tuple<uint256, std::vector<unsigned char>, CPubKey> sigdata_type;
-    std::set< sigdata_type> setValid;
+     //! Entries are SHA256(nonce || signature hash || public key || signature):
+    uint256 nonce;
+    typedef boost::unordered_set<uint256, CSignatureCacheHasher> map_type;
+    map_type setValid;
     boost::shared_mutex cs_sigcache;
 
+
 public:
+    CSignatureCache()
+    {
+        GetRandBytes(nonce.begin(), 32);
+    }
+
+    void
+    ComputeEntry(uint256& entry, const uint256 &hash, const std::vector<unsigned char>& vchSig, const CPubKey& pubkey)
+    {
+        CSHA256().Write(nonce.begin(), 32).Write(hash.begin(), 32).Write(&pubkey[0], pubkey.size()).Write(&vchSig[0], vchSig.size()).Finalize(entry.begin());
+    }
+
     bool
-    Get(const uint256 &hash, const std::vector<unsigned char>& vchSig, const CPubKey& pubKey)
+    Get(const uint256& entry)
     {
         boost::shared_lock<boost::shared_mutex> lock(cs_sigcache);
+        return setValid.count(entry);
+    }
 
-        sigdata_type k(hash, vchSig, pubKey);
-        std::set<sigdata_type>::iterator mi = setValid.find(k);
-        if (mi != setValid.end())
-            return true;
-        return false;
+    void Erase(const uint256& entry)
+    {
+        boost::unique_lock<boost::shared_mutex> lock(cs_sigcache);
+        setValid.erase(entry);
     }
 
-    void Set(const uint256 &hash, const std::vector<unsigned char>& vchSig, const CPubKey& pubKey)
+    void Set(const uint256& entry)
     {
-        // DoS prevention: limit cache size to less than 10MB
-        // (~200 bytes per cache entry times 50,000 entries)
-        // Since there can be no more than 20,000 signature operations per block
-        // 50,000 is a reasonable default.
-        int64_t nMaxCacheSize = GetArg("-maxsigcachesize", 50000);
+        size_t nMaxCacheSize = GetArg("-maxsigcachesize", DEFAULT_MAX_SIG_CACHE_SIZE) * ((size_t) 1 << 20);
         if (nMaxCacheSize <= 0) return;
 
         boost::unique_lock<boost::shared_mutex> lock(cs_sigcache);
-
-        while (static_cast<int64_t>(setValid.size()) > nMaxCacheSize)
+        while (memusage::DynamicUsage(setValid) > nMaxCacheSize)
         {
-            // Evict a random entry. Random because that helps
-            // foil would-be DoS attackers who might try to pre-generate
-            // and re-use a set of valid signatures just-slightly-greater
-            // than our cache size.
-            uint256 randomHash = GetRandHash();
-            std::vector<unsigned char> unused;
-            std::set<sigdata_type>::iterator it =
-                setValid.lower_bound(sigdata_type(randomHash, unused, unused));
-            if (it == setValid.end())
-                it = setValid.begin();
-            setValid.erase(*it);
+            map_type::size_type s = GetRand(setValid.bucket_count());
+            map_type::local_iterator it = setValid.begin(s);
+            if (it != setValid.end(s)) {
+                setValid.erase(*it);
+            }
         }
 
-        sigdata_type k(hash, vchSig, pubKey);
-        setValid.insert(k);
+        setValid.insert(entry);
     }
 };
 
@@ -78,13 +93,21 @@ bool CachingTransactionSignatureChecker::VerifySignature(const std::vector<unsig
 {
     static CSignatureCache signatureCache;
 
-    if (signatureCache.Get(sighash, vchSig, pubkey))
+    uint256 entry;
+    signatureCache.ComputeEntry(entry, sighash, vchSig, pubkey);
+
+    if (signatureCache.Get(entry)) {
+        if (!store) {
+            signatureCache.Erase(entry);
+        }
         return true;
+    }
 
     if (!TransactionSignatureChecker::VerifySignature(vchSig, pubkey, sighash))
         return false;
 
-    if (store)
-        signatureCache.Set(sighash, vchSig, pubkey);
+    if (store) {
+        signatureCache.Set(entry);
+    }
     return true;
 }

src/script/sigcache.h

@@ -10,6 +10,10 @@
 
 #include <vector>
 
+// DoS prevention: limit cache size to less than 40MB (over 500000
+// entries on 64-bit systems).
+static const unsigned int DEFAULT_MAX_SIG_CACHE_SIZE = 40;
+
 class CPubKey;
 
 class CachingTransactionSignatureChecker : public TransactionSignatureChecker

src/test/coins_tests.cpp

@@ -152,10 +152,13 @@ class CCoinsViewTest : public CCoinsView
                     CNullifiersMap& mapSaplingNullifiers)
     {
         for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end(); ) {
-            map_[it->first] = it->second.coins;
-            if (it->second.coins.IsPruned() && insecure_rand() % 3 == 0) {
-                // Randomly delete empty entries on write.
-                map_.erase(it->first);
+            if (it->second.flags & CCoinsCacheEntry::DIRTY) {
+                // Same optimization used in CCoinsViewDB is to only write dirty entries.
+                map_[it->first] = it->second.coins;
+                if (it->second.coins.IsPruned() && insecure_rand() % 3 == 0) {
+                    // Randomly delete empty entries on write.
+                    map_.erase(it->first);
+                }
             }
             mapCoins.erase(it++);
         }
@@ -185,10 +188,10 @@ class CCoinsViewTest : public CCoinsView
         BatchWriteNullifiers(mapSproutNullifiers, mapSproutNullifiers_);
         BatchWriteNullifiers(mapSaplingNullifiers, mapSaplingNullifiers_);
 
-        mapCoins.clear();
         mapSproutAnchors.clear();
         mapSaplingAnchors.clear();
-        hashBestBlock_ = hashBlock;
+        if (!hashBlock.IsNull())
+            hashBestBlock_ = hashBlock;
         hashBestSproutAnchor_ = hashSproutAnchor;
         hashBestSaplingAnchor_ = hashSaplingAnchor;
         return true;
@@ -878,6 +881,106 @@ BOOST_AUTO_TEST_CASE(coins_coinbase_spends)
     }
 }
 
+// This test is similar to the previous test
+// except the emphasis is on testing the functionality of UpdateCoins
+// random txs are created and UpdateCoins is used to update the cache stack
+BOOST_AUTO_TEST_CASE(updatecoins_simulation_test)
+{
+    // A simple map to track what we expect the cache stack to represent.
+    std::map<uint256, CCoins> result;
+
+    // The cache stack.
+    CCoinsViewTest base; // A CCoinsViewTest at the bottom.
+    std::vector<CCoinsViewCacheTest*> stack; // A stack of CCoinsViewCaches on top.
+    stack.push_back(new CCoinsViewCacheTest(&base)); // Start with one cache.
+
+    // Track the txids we've used and whether they have been spent or not
+    std::map<uint256, CAmount> coinbaseids;
+    std::set<uint256> alltxids;
+
+    for (unsigned int i = 0; i < NUM_SIMULATION_ITERATIONS; i++) {
+        {
+            CMutableTransaction tx;
+            tx.vin.resize(1);
+            tx.vout.resize(1);
+            tx.vout[0].nValue = i; //Keep txs unique
+            unsigned int height = insecure_rand();
+
+            // 1/10 times create a coinbase
+            if (insecure_rand() % 10 == 0 || coinbaseids.size() < 10) {
+                coinbaseids[tx.GetHash()] = tx.vout[0].nValue;
+                assert(CTransaction(tx).IsCoinBase());
+            }
+            // 9/10 times create a regular tx
+            else {
+                uint256 prevouthash;
+                // equally likely to spend coinbase or non coinbase
+                std::set<uint256>::iterator txIt = alltxids.lower_bound(GetRandHash());
+                if (txIt == alltxids.end()) {
+                    txIt = alltxids.begin();
+                }
+                prevouthash = *txIt;
+
+                // Construct the tx to spend the coins of prevouthash
+                tx.vin[0].prevout.hash = prevouthash;
+                tx.vin[0].prevout.n = 0;
+
+                // Update the expected result of prevouthash to know these coins are spent
+                CCoins& oldcoins = result[prevouthash];
+                oldcoins.Clear();
+
+                alltxids.erase(prevouthash);
+                coinbaseids.erase(prevouthash);
+
+                assert(!CTransaction(tx).IsCoinBase());
+            }
+            // Track this tx to possibly spend later
+            alltxids.insert(tx.GetHash());
+
+            // Update the expected result to know about the new output coins
+            CCoins &coins = result[tx.GetHash()];
+            coins.FromTx(tx, height);
+
+            CValidationState dummy;
+            UpdateCoins(tx, *(stack.back()), height);
+        }
+
+        // Once every 1000 iterations and at the end, verify the full cache.
+        if (insecure_rand() % 1000 == 1 || i == NUM_SIMULATION_ITERATIONS - 1) {
+            for (std::map<uint256, CCoins>::iterator it = result.begin(); it != result.end(); it++) {
+                const CCoins* coins = stack.back()->AccessCoins(it->first);
+                if (coins) {
+                    BOOST_CHECK(*coins == it->second);
+                 } else {
+                    BOOST_CHECK(it->second.IsPruned());
+                 }
+            }
+        }
+
+        if (insecure_rand() % 100 == 0) {
+            // Every 100 iterations, change the cache stack.
+            if (stack.size() > 0 && insecure_rand() % 2 == 0) {
+                stack.back()->Flush();
+                delete stack.back();
+                stack.pop_back();
+            }
+            if (stack.size() == 0 || (stack.size() < 4 && insecure_rand() % 2)) {
+                CCoinsView* tip = &base;
+                if (stack.size() > 0) {
+                    tip = stack.back();
+                }
+                stack.push_back(new CCoinsViewCacheTest(tip));
+           }
+        }
+    }
+
+    // Clean up the stack.
+    while (stack.size() > 0) {
+        delete stack.back();
+        stack.pop_back();
+    }
+}
+
 BOOST_AUTO_TEST_CASE(ccoins_serialization)
 {
     // Good example