Auto merge of #2912 - str4d:2074-main-refactor-1, r=<try>

Bitcoin Core refactoring and cleanups 1

Cherry-picked from the following upstream PRs:

- bitcoin/bitcoin#5994
- bitcoin/bitcoin#6538
- bitcoin/bitcoin#6163
- bitcoin/bitcoin#6982
- bitcoin/bitcoin#6986
- bitcoin/bitcoin#7053
- bitcoin/bitcoin#7444
- bitcoin/bitcoin#7793
  - Excluding some comments in `txmempool.h` on code we haven't yet pulled in.
- bitcoin/bitcoin#7916
- bitcoin/bitcoin#7815

Additionally, the Equihash parameters are moved into the consensus parameters.

Part of #2074.

doc/developer-notes.md

@@ -71,6 +71,12 @@ To describe a member or variable use:
 int var; //!< Detailed description after the member
 ```
 
+or
+```cpp
+//! Description before the member
+int var;
+```
+
 Also OK:
 ```c++
 ///

qa/rpc-tests/mempool_nu_activation.py

@@ -5,7 +5,8 @@
 
 from test_framework.test_framework import BitcoinTestFramework
 from test_framework.util import assert_equal, initialize_chain_clean, \
-    start_node, connect_nodes, wait_and_assert_operationid_status
+    start_node, connect_nodes, wait_and_assert_operationid_status, \
+    get_coinbase_address
 
 from decimal import Decimal
 
@@ -40,7 +41,7 @@ def run_test(self):
         self.sync_all()
 
         # Shield some ZEC
-        node1_taddr = self.nodes[1].getnewaddress()
+        node1_taddr = get_coinbase_address(self.nodes[1], 97)
         node0_zaddr = self.nodes[0].z_getnewaddress()
         recipients = [{'address': node0_zaddr, 'amount': Decimal('10')}]
         myopid = self.nodes[1].z_sendmany(node1_taddr, recipients, 1, Decimal('0'))

qa/rpc-tests/mempool_tx_input_limit.py

@@ -6,7 +6,8 @@
 from test_framework.test_framework import BitcoinTestFramework
 from test_framework.authproxy import JSONRPCException
 from test_framework.util import assert_equal, initialize_chain_clean, \
-    start_node, connect_nodes, wait_and_assert_operationid_status
+    start_node, connect_nodes, wait_and_assert_operationid_status, \
+    get_coinbase_address
 
 import time
 from decimal import Decimal
@@ -49,7 +50,7 @@ def run_test(self):
         node0_zaddr = self.nodes[0].z_getnewaddress()
 
         # Send three inputs from node 0 taddr to zaddr to get out of coinbase
-        node0_taddr = self.nodes[0].getnewaddress();
+        node0_taddr = get_coinbase_address(self.nodes[0], 3)
         recipients = []
         recipients.append({"address":node0_zaddr, "amount":Decimal('30.0')-Decimal('0.0001')}) # utxo amount less fee
         myopid = self.nodes[0].z_sendmany(node0_taddr, recipients)

qa/rpc-tests/paymentdisclosure.py

@@ -6,7 +6,8 @@
 from test_framework.test_framework import BitcoinTestFramework
 from test_framework.authproxy import JSONRPCException
 from test_framework.util import assert_equal, initialize_chain_clean, \
-    start_node, connect_nodes_bi, wait_and_assert_operationid_status
+    start_node, connect_nodes_bi, wait_and_assert_operationid_status, \
+    get_coinbase_address
 
 from decimal import Decimal
 
@@ -46,7 +47,7 @@ def run_test (self):
         assert_equal(self.nodes[1].getbalance(), 10)
         assert_equal(self.nodes[2].getbalance(), 30)
 
-        mytaddr = self.nodes[0].getnewaddress()
+        mytaddr = get_coinbase_address(self.nodes[0], 4)
         myzaddr = self.nodes[0].z_getnewaddress()
 
         # Check that Node 2 has payment disclosure disabled.

qa/rpc-tests/test_framework/util.py

@@ -402,3 +402,10 @@ def wait_and_assert_operationid_status(node, myopid, in_status='success', in_err
         assert(in_errormsg is not None)
         assert_equal(in_errormsg in errormsg, True)
     return txid
+
+def get_coinbase_address(node, expected_utxos):
+    addrs = [utxo['address'] for utxo in node.listunspent() if utxo['generated']]
+    assert(len(set(addrs)) > 0)
+    addrs = [a for a in set(addrs) if addrs.count(a) == expected_utxos]
+    assert(len(addrs) > 0)
+    return addrs[0]

qa/rpc-tests/wallet_1941.py

@@ -8,7 +8,8 @@
 from test_framework.test_framework import BitcoinTestFramework
 from test_framework.util import assert_equal, initialize_chain_clean, \
     initialize_datadir, start_nodes, start_node, connect_nodes_bi, \
-    bitcoind_processes, wait_and_assert_operationid_status
+    bitcoind_processes, wait_and_assert_operationid_status, \
+    get_coinbase_address
 
 from decimal import Decimal
 
@@ -46,7 +47,7 @@ def run_test (self):
         self.nodes[0].setmocktime(starttime)
         self.nodes[0].generate(101)
 
-        mytaddr = self.nodes[0].getnewaddress()     # where coins were mined
+        mytaddr = get_coinbase_address(self.nodes[0], 1)
         myzaddr = self.nodes[0].z_getnewaddress()
 
         # Send 10 coins to our zaddr.

qa/rpc-tests/wallet_anchorfork.py

@@ -6,7 +6,7 @@
 from test_framework.test_framework import BitcoinTestFramework
 from test_framework.util import assert_equal, initialize_chain_clean, \
     start_nodes, stop_nodes, connect_nodes_bi, \
-    wait_and_assert_operationid_status, wait_bitcoinds
+    wait_and_assert_operationid_status, wait_bitcoinds, get_coinbase_address
 from decimal import Decimal
 
 class WalletAnchorForkTest (BitcoinTestFramework):
@@ -43,7 +43,7 @@ def run_test (self):
         # At this point in time, commitment tree is the empty root
 
         # Node 0 creates a joinsplit transaction
-        mytaddr0 = self.nodes[0].getnewaddress()
+        mytaddr0 = get_coinbase_address(self.nodes[0], 4)
         myzaddr0 = self.nodes[0].z_getnewaddress()
         recipients = []
         recipients.append({"address":myzaddr0, "amount": Decimal('10.0') - Decimal('0.0001')})

qa/rpc-tests/wallet_nullifiers.py

@@ -6,7 +6,7 @@
 
 from test_framework.test_framework import BitcoinTestFramework
 from test_framework.util import assert_equal, start_node, \
-    start_nodes, connect_nodes_bi, bitcoind_processes
+    start_nodes, connect_nodes_bi, bitcoind_processes, get_coinbase_address
 
 import time
 from decimal import Decimal
@@ -22,7 +22,8 @@ def run_test (self):
         myzaddr0 = self.nodes[0].z_getnewaddress()
 
         # send node 0 taddr to zaddr to get out of coinbase
-        mytaddr = self.nodes[0].getnewaddress();
+        # Tests using the default cached chain have one address per coinbase output
+        mytaddr = get_coinbase_address(self.nodes[0], 1)
         recipients = []
         recipients.append({"address":myzaddr0, "amount":Decimal('10.0')-Decimal('0.0001')}) # utxo amount less fee
         myopid = self.nodes[0].z_sendmany(mytaddr, recipients)

qa/rpc-tests/wallet_overwintertx.py

@@ -5,7 +5,8 @@
 
 from test_framework.test_framework import BitcoinTestFramework
 from test_framework.util import assert_equal, initialize_chain_clean, \
-    start_nodes, connect_nodes_bi, wait_and_assert_operationid_status
+    start_nodes, connect_nodes_bi, wait_and_assert_operationid_status, \
+    get_coinbase_address
 
 from decimal import Decimal
 
@@ -31,7 +32,7 @@ def run_test (self):
         self.sync_all()
         # Node 0 has reward from blocks 1 to 98 which are spendable.
 
-        taddr0 = self.nodes[0].getnewaddress()
+        taddr0 = get_coinbase_address(self.nodes[0], 98)
         taddr1 = self.nodes[1].getnewaddress()
         taddr2 = self.nodes[2].getnewaddress()
         zaddr2 = self.nodes[2].z_getnewaddress()

qa/rpc-tests/wallet_protectcoinbase.py

@@ -8,7 +8,8 @@
 from test_framework.authproxy import JSONRPCException
 from test_framework.mininode import COIN
 from test_framework.util import assert_equal, initialize_chain_clean, \
-    start_nodes, connect_nodes_bi, wait_and_assert_operationid_status
+    start_nodes, connect_nodes_bi, wait_and_assert_operationid_status, \
+    get_coinbase_address
 
 import sys
 import time
@@ -75,7 +76,7 @@ def run_test (self):
         assert_equal("Coinbase funds can only be sent to a zaddr" in errorString, True)
 
         # Prepare to send taddr->zaddr
-        mytaddr = self.nodes[0].getnewaddress()
+        mytaddr = get_coinbase_address(self.nodes[0], 4)
         myzaddr = self.nodes[0].z_getnewaddress()
 
         # Node 3 will test that watch only address utxos are not selected

qa/rpc-tests/wallet_shieldcoinbase.py

@@ -7,7 +7,7 @@
 from test_framework.authproxy import JSONRPCException
 from test_framework.util import assert_equal, initialize_chain_clean, \
     start_node, connect_nodes_bi, sync_blocks, sync_mempools, \
-    wait_and_assert_operationid_status
+    wait_and_assert_operationid_status, get_coinbase_address
 
 from decimal import Decimal
 
@@ -34,17 +34,13 @@ def run_test (self):
         print "Mining blocks..."
 
         self.nodes[0].generate(1)
-        do_not_shield_taddr = self.nodes[0].getnewaddress()
-
         self.nodes[0].generate(4)
         walletinfo = self.nodes[0].getwalletinfo()
         assert_equal(walletinfo['immature_balance'], 50)
         assert_equal(walletinfo['balance'], 0)
         self.sync_all()
         self.nodes[2].generate(1)
-        self.nodes[2].getnewaddress()
         self.nodes[2].generate(1)
-        self.nodes[2].getnewaddress()
         self.nodes[2].generate(1)
         self.sync_all()
         self.nodes[1].generate(101)
@@ -53,8 +49,10 @@ def run_test (self):
         assert_equal(self.nodes[1].getbalance(), 10)
         assert_equal(self.nodes[2].getbalance(), 30)
 
+        do_not_shield_taddr = get_coinbase_address(self.nodes[0], 1)
+
         # Prepare to send taddr->zaddr
-        mytaddr = self.nodes[0].getnewaddress()
+        mytaddr = get_coinbase_address(self.nodes[0], 4)
         myzaddr = self.nodes[0].z_getnewaddress()
 
         # Shielding will fail when trying to spend from watch-only address
@@ -133,7 +131,7 @@ def run_test (self):
         self.sync_all()
         self.nodes[1].generate(100)
         self.sync_all()
-        mytaddr = self.nodes[0].getnewaddress()
+        mytaddr = get_coinbase_address(self.nodes[0], 800)
 
         # Shielding the 800 utxos will occur over two transactions, since max tx size is 100,000 bytes.
         # We don't verify shieldingValue as utxos are not selected in any specific order, so value can change on each test run.
@@ -166,7 +164,7 @@ def run_test (self):
 
         # Verify maximum number of utxos which node 2 can shield is limited by option -mempooltxinputlimit
         # This option is used when the limit parameter is set to 0.
-        mytaddr = self.nodes[2].getnewaddress()
+        mytaddr = get_coinbase_address(self.nodes[2], 20)
         result = self.nodes[2].z_shieldcoinbase(mytaddr, myzaddr, Decimal('0.0001'), 0)
         assert_equal(result["shieldingUTXOs"], Decimal('7'))
         assert_equal(result["remainingUTXOs"], Decimal('13'))
@@ -178,7 +176,7 @@ def run_test (self):
         # Verify maximum number of utxos which node 0 can shield is set by default limit parameter of 50
         self.nodes[0].generate(200)
         self.sync_all()
-        mytaddr = self.nodes[0].getnewaddress()
+        mytaddr = get_coinbase_address(self.nodes[0], 100)
         result = self.nodes[0].z_shieldcoinbase(mytaddr, myzaddr, Decimal('0.0001'))
         assert_equal(result["shieldingUTXOs"], Decimal('50'))
         assert_equal(result["remainingUTXOs"], Decimal('50'))

qa/rpc-tests/wallet_treestate.py

@@ -6,7 +6,8 @@
 
 from test_framework.test_framework import BitcoinTestFramework
 from test_framework.util import assert_equal, initialize_chain_clean, \
-    start_nodes, connect_nodes_bi, wait_and_assert_operationid_status
+    start_nodes, connect_nodes_bi, wait_and_assert_operationid_status, \
+    get_coinbase_address
 
 import time
 from decimal import Decimal
@@ -34,7 +35,7 @@ def run_test (self):
         self.nodes[1].generate(101)
         self.sync_all()
 
-        mytaddr = self.nodes[0].getnewaddress()     # where coins were mined
+        mytaddr = get_coinbase_address(self.nodes[0], 100)
         myzaddr = self.nodes[0].z_getnewaddress()
 
         # Spend coinbase utxos to create three notes of 9.99990000 each

src/amqp/amqppublishnotifier.cpp

@@ -3,6 +3,7 @@
 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 #include "amqppublishnotifier.h"
+#include "chainparams.h"
 #include "main.h"
 #include "util.h"
 
@@ -152,11 +153,12 @@ bool AMQPPublishRawBlockNotifier::NotifyBlock(const CBlockIndex *pindex)
 {
     LogPrint("amqp", "amqp: Publish rawblock %s\n", pindex->GetBlockHash().GetHex());
 
+    const Consensus::Params& consensusParams = Params().GetConsensus();
     CDataStream ss(SER_NETWORK, PROTOCOL_VERSION);
     {
         LOCK(cs_main);
         CBlock block;
-        if(!ReadBlockFromDisk(block, pindex)) {
+        if(!ReadBlockFromDisk(block, pindex, consensusParams)) {
             LogPrint("amqp", "amqp: Can't read block from disk");
             return false;
         }

src/chain.h

@@ -14,11 +14,63 @@
 
 #include <vector>
 
-#include <boost/foreach.hpp>
-
 static const int SPROUT_VALUE_VERSION = 1001400;
 static const int SAPLING_VALUE_VERSION = 1010100;
 
+class CBlockFileInfo
+{
+public:
+    unsigned int nBlocks;      //!< number of blocks stored in file
+    unsigned int nSize;        //!< number of used bytes of block file
+    unsigned int nUndoSize;    //!< number of used bytes in the undo file
+    unsigned int nHeightFirst; //!< lowest height of block in file
+    unsigned int nHeightLast;  //!< highest height of block in file
+    uint64_t nTimeFirst;       //!< earliest time of block in file
+    uint64_t nTimeLast;        //!< latest time of block in file
+
+    ADD_SERIALIZE_METHODS;
+
+    template <typename Stream, typename Operation>
+    inline void SerializationOp(Stream& s, Operation ser_action) {
+        READWRITE(VARINT(nBlocks));
+        READWRITE(VARINT(nSize));
+        READWRITE(VARINT(nUndoSize));
+        READWRITE(VARINT(nHeightFirst));
+        READWRITE(VARINT(nHeightLast));
+        READWRITE(VARINT(nTimeFirst));
+        READWRITE(VARINT(nTimeLast));
+    }
+
+     void SetNull() {
+         nBlocks = 0;
+         nSize = 0;
+         nUndoSize = 0;
+         nHeightFirst = 0;
+         nHeightLast = 0;
+         nTimeFirst = 0;
+         nTimeLast = 0;
+     }
+
+     CBlockFileInfo() {
+         SetNull();
+     }
+
+     std::string ToString() const;
+
+     /** update statistics (does not update nSize) */
+     void AddBlock(unsigned int nHeightIn, uint64_t nTimeIn) {
+         if (nBlocks==0 || nHeightFirst > nHeightIn)
+             nHeightFirst = nHeightIn;
+         if (nBlocks==0 || nTimeFirst > nTimeIn)
+             nTimeFirst = nTimeIn;
+         nBlocks++;
+         if (nHeightIn > nHeightLast)
+             nHeightLast = nHeightIn;
+         if (nTimeIn > nTimeLast)
+             nTimeLast = nTimeIn;
+     }
+};
+
 struct CDiskBlockPos
 {
     int nFile;