Merge remote-tracking branch 'upstream/master' into dandelion

configure.ac

@@ -2,7 +2,7 @@ dnl require autoconf 2.60 (AS_ECHO/AS_ECHO_N)
 AC_PREREQ([2.60])
 define(_CLIENT_VERSION_MAJOR, 4)
 define(_CLIENT_VERSION_MINOR, 7)
-define(_CLIENT_VERSION_REVISION, 1)
+define(_CLIENT_VERSION_REVISION, 3)
 define(_CLIENT_VERSION_BUILD, 0)
 define(_CLIENT_VERSION_IS_RELEASE, true)
 define(_CLIENT_BUILD_IS_TEST_RELEASE, false)

doc/Doxyfile

@@ -34,7 +34,7 @@ PROJECT_NAME           = NavCoin
 # This could be handy for archiving the generated documentation or
 # if some version control system is used.
 
-PROJECT_NUMBER         = 4.7.1
+PROJECT_NUMBER         = 4.7.3
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer

doc/release-notes/release-notes-4.7.2.md

@@ -0,0 +1,65 @@
+# NavCoin v4.7.2 Release Notes
+
+## Fix for Verify Chain
+
+<[Pull Request 634](https://github.com/navcoin/navcoin-core/pull/634)>
+<[Commit 049978e](https://github.com/navcoin/navcoin-core/commit/049978e246de960370f629fa38a6509fad0cf5c8)>
+
+This patch fixes [Issue 630](https://github.com/navcoin/navcoin-core/issues/630) and introduces several important changes.
+
+- New RPC command `getcfunddbstatehash` covered by functional test `cfunddb-statehash.py`
+- New state for payment requests `6` when those are paid.
+- The payment request parameter paidOnBlock is substituted by `stateChangedOnBlock` when state is `6`
+- New structure for storing the state history of CFundDB entries. Those are stored in a map associating blockhash and state, allowing to directly revert state transitions when reorganizations are seen.
+- `verifychain` now checks for the consistency of the CFundDB state hash when level 4 is specified.
+
+### IMPORTANT
+
+This set of changes will require older clients to reindex on launch, keeping the node offline for some hours at best. In order to reduce downtime, node operators can proceed as follows if needed:
+
+- Close node with old version.
+- `mkdir /tmp/reindexdata; cp -rf <data_folder>/blocks /tmp/reindexdata/; cp -rf <data_folder>/chainstate /tmp/reindexdata/`
+- Reopen node with old version. It will be again online
+- Launch in parallel a second instance of the node, this time using the new version with the parameters `-reindex -datadir=/tmp/reindexdata/`
+- Once the reindex finishes, close both nodes and copy back the reindexed data.
+- `rm -rf <data_folder>/blocks <data_folder>/chainstate; cp -rf /tmp/reindexdata/* <data_folder>`
+- Relaunch new version of the node.
+
+## CFundDB extra log and ensure read before modify
+
+<[Pull Request 622](https://github.com/navcoin/navcoin-core/pull/622)>
+<[Commit 37fa72e](https://github.com/navcoin/navcoin-core/commit/37fa72e386ad3daff58b92ed7dda1a9b0676a43b)>
+
+This PR adds extra log for all the modifications of the CFundDB and ensures entries are read in the memory cache before being modified.
+
+## Restart testnet
+
+<[Pull Request 628](https://github.com/navcoin/navcoin-core/pull/628)>
+<[Commit b8ed018](https://github.com/navcoin/navcoin-core/commit/b8ed0180a2deaf616c8e6b38aec42385f0a73879)>
+
+This pull request starts a new NavCoin testnet. If you're running a testnet node you will need to will need to wipe your testnet data directory and connect to the new testnet nodes. A list of some of the testnet nodes operated by NavCoin Core developers can be found on [Issue 626](https://github.com/navcoin/navcoin-core/issues/626). If you need testnet coins or want to be added to the list of nodes, please comment on the issue or join the #dev-testnet channel in [Discord](https://discord.gg/y4Vu9jw).
+
+## Full list of Merged PRs
+
+* [`Pull Request 652`](https://github.com/navcoin/navcoin-core/pull/652) [`Commit 0cde809`](https://github.com/navcoin/navcoin-core/commit/0cde80954fd2bdca1859c0e90dfcc3243dba6c61) Remove new version popup
+* [`Pull Request 651`](https://github.com/navcoin/navcoin-core/pull/651) [`Commit 95b524e`](https://github.com/navcoin/navcoin-core/commit/95b524eefbfb87190a9038397d6c6a2c8ea081c9) Update translations
+* [`Pull Request 648`](https://github.com/navcoin/navcoin-core/pull/648) [`Commit 09f0531`](https://github.com/navcoin/navcoin-core/commit/09f053152761aa254dae27a9da3e338e2e31e671) Update QT strings
+* [`Pull Request 647`](https://github.com/navcoin/navcoin-core/pull/647) [`Commit 0eafc3b`](https://github.com/navcoin/navcoin-core/commit/0eafc3b404503c985993a7069bc8cb160100911d) Reactivate CFund
+* [`Pull Request 641`](https://github.com/navcoin/navcoin-core/pull/641) [`Commit 494d4e2`](https://github.com/navcoin/navcoin-core/commit/494d4e2d598ad114c407d609d7b8141e4ab54f50) Add extra stats to getblock
+* [`Pull Request 634`](https://github.com/navcoin/navcoin-core/pull/634) [`Commit 049978e`](https://github.com/navcoin/navcoin-core/commit/049978e246de960370f629fa38a6509fad0cf5c8) Fix for verifychain
+* [`Pull Request 644`](https://github.com/navcoin/navcoin-core/pull/644) [`Commit 5d59483`](https://github.com/navcoin/navcoin-core/commit/5d59483c50d985d3053299febe941dcfb447be46) Adds proposalHash to the RPC getpaymentrequest
+* [`Pull Request 643`](https://github.com/navcoin/navcoin-core/pull/643) [`Commit d501c89`](https://github.com/navcoin/navcoin-core/commit/d501c89ea412760de2074d8706fc1684d42d1445) CPaymentRequest fields incorrect in diff
+* [`Pull Request 638`](https://github.com/navcoin/navcoin-core/pull/638) [`Commit 8131b42`](https://github.com/navcoin/navcoin-core/commit/8131b4236054c5ee57e253ce75dc77a9992a6bed) Fixed freezing GUI on reindex
+* [`Pull Request 632`](https://github.com/navcoin/navcoin-core/pull/632) [`Commit 2ad3391`](https://github.com/navcoin/navcoin-core/commit/2ad3391e5195720af2d288f923081c24df023d99) Fix reference to chain tip
+* [`Pull Request 629`](https://github.com/navcoin/navcoin-core/pull/629) [`Commit 2a6c64f`](https://github.com/navcoin/navcoin-core/commit/2a6c64f87858d3452b9b830a0853e993379449d6) Seed nodes
+* [`Pull Request 637`](https://github.com/navcoin/navcoin-core/pull/637) [`Commit ad883cd`](https://github.com/navcoin/navcoin-core/commit/ad883cdfa1a1e478f6db30beb41511e97a37bb28) Set DEFAULT_SCRIPTCHECK_THREADS to auto
+* [`Pull Request 624`](https://github.com/navcoin/navcoin-core/pull/624) [`Commit 7058550`](https://github.com/navcoin/navcoin-core/commit/7058550da5fe3a2f6ad5493fab763cea285f1a05) Updates to make the wallet.py and stakeimmaturebalance.py test more reliable
+* [`Pull Request 636`](https://github.com/navcoin/navcoin-core/pull/636) [`Commit e6f3c23`](https://github.com/navcoin/navcoin-core/commit/e6f3c23f41d5a3a4228f95bd9d67c6acff81f1a3) Only count stakes in main chain
+* [`Pull Request 633`](https://github.com/navcoin/navcoin-core/pull/633) [`Commit bfe9071`](https://github.com/navcoin/navcoin-core/commit/bfe90717910cfaa49562efdb14259deb6b208dac) Add second dns seeder
+* [`Pull Request 622`](https://github.com/navcoin/navcoin-core/pull/622) [`Commit 37fa72e`](https://github.com/navcoin/navcoin-core/commit/37fa72e386ad3daff58b92ed7dda1a9b0676a43b) CFundDB extra log and ensure read before modify
+* [`Pull Request 628`](https://github.com/navcoin/navcoin-core/pull/628) [`Commit b8ed018`](https://github.com/navcoin/navcoin-core/commit/b8ed0180a2deaf616c8e6b38aec42385f0a73879) Restart testnet
+* [`Pull Request 623`](https://github.com/navcoin/navcoin-core/pull/623) [`Commit 09ea936`](https://github.com/navcoin/navcoin-core/commit/09ea936f9bd1bb6557d541344345889252d6aef9) Disable CFund client functionality
+* [`Pull Request 609`](https://github.com/navcoin/navcoin-core/pull/609) [`Commit 1e73c05`](https://github.com/navcoin/navcoin-core/commit/1e73c05b17bc812057a866b414d66573211ad755) Added clearer error messages for the nRequest amount validation
+
+For additional information about new features, check [https://navcoin.org/en/notices/](https://navcoin.org/en/notices/) 
+

doc/release-notes/release-notes-4.7.3.md

@@ -0,0 +1,34 @@
+# NavCoin v4.7.3 Release Notes
+
+## Anti Header Spam v2
+
+<[Pull Request 656](https://github.com/navcoin/navcoin-core/pull/656)>
+<[Commit 5f11875](https://github.com/navcoin/navcoin-core/commit/5f118753a1900241e9cf8ea38281e4fe75cfeae8)>
+
+Introduces a new anti header spam system which improves the previous implementation and addresses the art-of-bug reports.
+
+Features:
+
+- Every time a header or block is received from another peer, its hash is added to a `points` list associated with the peer. 
+- Peers are discerned by their ip address, this means peers sharing ip address will also share the same `points` list. This can be changed with `-headerspamfilterignoreport` (default: `true`).
+- Before proceeding with the block or headers validation, the `points` list will be cleared removing all the hashes of blocks whose scripts have already been correctly validated.
+- The peer is banned if the size of the `points` list is greater than `MAX_HEADERS_RESULTS*2` once cleared of already validated blocks.
+- The maximum allowed size of the `points` list can be changed using the `-headerspamfiltermaxsize` parameter.
+- The log category `headerspam` has been added, which prints to the log the current size of a peers `points` list.
+- When `-debug=bench` is specified, execution time for the `updateState` function is logged.
+
+#### Considerations
+
+- The maximum size of the `points` list by default is 4,000. With a block time of 30 seconds, NavCoin sees an average of 2,880 blocks per day. A maximum value of 4000 is roughly one and a half times more than the count of blocks a peer needs to be behind the chain tip to be in Initial Block Download mode. When on IBD, the header spam filter is turned off. This ensures that normal synchronisation is not affected by this filter.
+- An attacker would be able to exhaust 32 bytes from the hash inserted in the `points` list + 181 bytes from the `CBlockIndex` inserted in `mapBlockIndex` for every invalid header/block before being banned. The `points` list is cleared when the attacker is banned, but those headers are not removed from `mapBlockIndex` or the hard disk in the current implementation. The size of CBlockIndex has been measured with:
+```c++
+    CBlockIndex* pindex = new CBlockIndex();
+    CDataStream ssPeers(SER_DISK, CLIENT_VERSION);
+    ss << CDiskBlockIndex(pindex);
+    std::vector<unsigned char>vch(ss.begin(), ss.end());
+    std::cout << to_string(vch.size()) << std::endl;
+```
+- The default maximum value means that a single malicious peer with a unique IP can exhaust at max `3,999*213=831 kilobytes` without being banned or `4,000*181=707 kilobytes` being banned.
+
+For additional information about new features, check [https://navcoin.org/en/notices/](https://navcoin.org/en/notices/) 
+

doc/release-process.md

@@ -35,6 +35,9 @@ E.g: `v2.5.0-rc.1`, `v2.5.0-rc.2`, `v2.5.0-rc.3`
 
 Please note any type of network security or stability issues will be prioritized and might not have any applied stand-down period.
 
+#### Feature Lockdown
+
+During the release candidate review period, no new pull requests should be merged into master except ones designed explicity to fix any issues found during testing of the release candidate. This ensures no new issues are inadvertently introduced into new release candidate or final release versions which could force the complete restart of the testing and review process.
 
 #### Release Candidates Preparation
 
@@ -200,10 +203,47 @@ people without access to `navcoin.org` to download the binary distribution.
 Also put it into the `optional_magnetlink:` slot in the YAML file for
 navcoin.org (see below for navcoin.org update instructions).
 
-- Update navcoin.org version
+### Prepare the Release Tag
+
+Once the release candidate is approved and promoted to a final release, a new release tag should be created with the final release notes and binaries attached.
+
+### Update Services
+
+Before publicly announcing the release the Bootstrap & NavPay servers should be updated to the new version.
+
+### Update The NavCoin Website
+
+Update the version number and download links on all translations of the Wallets page;
+
+https://github.com/navcoin/navcoin-org/tree/master/content/wallets
+
+Also create the notice for the release; 
+
+https://github.com/navcoin/navcoin-org/tree/master/content/notices
+
+The notice can be written manually by duplicating and modifying an existing notice, or through the admin section of the website. The admin section relies on your GitHub login having push access to the repo, so you will need to ensure you have the correct repository rights if you want to create it that way.
+
+The hero image for the release notice is usually generated with the navoin canva template to ensure it fits the social sharing spec and aligns wtih the brand guidelies.
+
+### Publicly Announce the release
+
+To ensure resonable due diligence is done to inform the communtiy of new software releases the final release should be announced on all possible NavCoin platforms;
+
+[Reddit](https://reddit.com/r/navcoin), [Twitter](https://twitter.com/navcoin), [Facebook](https://facebook.com/navcoin), [Telegram](https://t.me/navcoin), [Discord](https://discord.gg/y4Vu9jw), [BitcoinTalk](https://bitcointalk.org/index.php?topic=679791.new#new), [Medium](https://medium.com/nav-coin/), Blockfolio Signal & MailChimp.
+
+
+### Notify Exchanges, Commercial Nodes
+
+All exchanges should be notified of the update;
+
+https://navcoin.org/en/buy-navcoin
+
+Most of the exchanges have their contact emails are consolidated into a mailing list inside the admin@navcoin email account which can be set to the BCC to ensure they receive the update. For ones which are not part of the mailing list their is usually a support form on their site which needs to be filled out.
+
+Additional to exhcanges, the following wallets & services should be notified;
 
-- Announce the release:
+NavExplorer, NavPool, CryptoId, Coinomi, Magnum Wallet, AtomicDex & CoinPayments.
 
--- Communicate with enough time to economic actors of the Network (exchanges):
+### Celebrate with the Community
 
-- Celebrate
+Take the time to celebrate the success with everyone who helped to make the release happen! Every release is another step towards a new future where we've claimed back our financial freedom.

qa/pull-tester/rpc-tests.py

@@ -190,7 +190,8 @@
     'sendtoaddress.py',
     'stakeimmaturebalance.py',
     'rpc-help.py',
-    'createrawscriptaddress.py'
+    'createrawscriptaddress.py',
+    'cfunddb-statehash.py'
 ]
 #if ENABLE_ZMQ:
 #    testScripts.append('zmq_test.py')

qa/rpc-tests/cfund-fork-reorg-preq.py

@@ -17,7 +17,9 @@ def __init__(self):
         self.num_nodes = 2
 
     def setup_network(self, split=False):
-        self.nodes = self.setup_nodes()
+        self.nodes = []
+        self.nodes.append(start_node(0, self.options.tmpdir, ["-debug=dao"]))
+        self.nodes.append(start_node(1, self.options.tmpdir, ["-debug=dao"]))
         connect_nodes_bi(self.nodes, 0, 1)
         self.is_network_split = False
 
@@ -83,7 +85,7 @@ def run_test(self):
         best_block = self.nodes[1].getbestblockhash()
 
         self.stop_node(0)
-        self.nodes[0] = start_node(0, self.options.tmpdir, [])
+        self.nodes[0] = start_node(0, self.options.tmpdir, ["-debug=dao"])
 
         # Reconnect the nodes
         connect_nodes_bi(self.nodes, 0, 1)
@@ -100,8 +102,8 @@ def run_test(self):
         slow_gen(self.nodes[0], self.nodes[0].cfundstats()["votingPeriod"]["ending"] - self.nodes[0].cfundstats()["votingPeriod"]["current"])
         sync_blocks(self.nodes)
 
-        assert_equal(self.nodes[0].getpaymentrequest(paymentHash0)["status"], "accepted")
-        assert_equal(self.nodes[0].getblock(self.nodes[0].getpaymentrequest(paymentHash0)["paidOnBlock"]), self.nodes[1].getblock(self.nodes[1].getpaymentrequest(paymentHash0)["paidOnBlock"]))
+        assert_equal(self.nodes[0].getpaymentrequest(paymentHash0)["status"], "paid")
+        assert_equal(self.nodes[0].getblock(self.nodes[0].getpaymentrequest(paymentHash0)["stateChangedOnBlock"]), self.nodes[1].getblock(self.nodes[1].getpaymentrequest(paymentHash0)["stateChangedOnBlock"]))
         assert_equal(self.nodes[0].getbestblockhash(), self.nodes[1].getbestblockhash())
         assert_equal(self.nodes[0].getblock(self.nodes[0].getpaymentrequest(paymentHash0)["blockHash"]), self.nodes[1].getblock(self.nodes[1].getpaymentrequest(paymentHash0)["blockHash"]))
         assert_equal(self.nodes[0].getpaymentrequest(paymentHash0), self.nodes[1].getpaymentrequest(paymentHash0))
@@ -117,10 +119,10 @@ def create_raw_paymentrequest(self, amount, address, proposal_hash, description)
             [],
             {"6ac1": 1},
             json.dumps({
-              "v": 2, 
-              "h": proposal_hash, 
-              "n": amount, 
-              "s": signature, 
+              "v": 2,
+              "h": proposal_hash,
+              "n": amount,
+              "s": signature,
               "i": description,
               })
         )

qa/rpc-tests/cfund-fork-reorg-proposal.py

@@ -19,7 +19,9 @@ def __init__(self):
         self.num_nodes = 2
 
     def setup_network(self, split=False):
-        self.nodes = self.setup_nodes()
+        self.nodes = []
+        self.nodes.append(start_node(0, self.options.tmpdir, ["-debug=dao"]))
+        self.nodes.append(start_node(1, self.options.tmpdir, ["-debug=dao"]))
         connect_nodes_bi(self.nodes, 0, 1)
         self.is_network_split = False
 
@@ -59,19 +61,19 @@ def run_test(self):
         slow_gen(self.nodes[1], 1)
 
         # Assert that both hashes for payment request are identical
-        assert(proposalHash0 == proposalHash1)    
+        assert(proposalHash0 == proposalHash1)
 
         # Assert that both payment requests have been included in different block hashes
         assert(self.nodes[0].getproposal(proposalHash0)["blockHash"] != self.nodes[1].getproposal(proposalHash1)["blockHash"])
-    
+
         # Make the node vote yes
         self.nodes[1].proposalvote(proposalHash0, "yes")
 
         # End cycle 0
         slow_gen(self.nodes[1], 1)
         end_cycle(self.nodes[1])
 
-        # End cycle 1        
+        # End cycle 1
         slow_gen(self.nodes[1], 1)
         end_cycle(self.nodes[1])
 
@@ -96,7 +98,7 @@ def run_test(self):
         assert_equal(self.nodes[0].getblock(self.nodes[0].getproposal(proposalHash0)["blockHash"]), self.nodes[1].getblock(self.nodes[1].getproposal(proposalHash0)["blockHash"]))
         assert_equal(self.nodes[0].getproposal(proposalHash0), self.nodes[1].getproposal(proposalHash0))
 
-        # End cycle 2        
+        # End cycle 2
         slow_gen(self.nodes[1], 1)
         end_cycle(self.nodes[1])
         sync_blocks(self.nodes)
@@ -132,14 +134,14 @@ def run_test(self):
 
         # Verify both nodes accpeted the payment
 
-        assert_equal(self.nodes[0].getpaymentrequest(preqHash)["status"], "accepted")
-        assert_equal(self.nodes[0].getblock(self.nodes[0].getpaymentrequest(preqHash)["paidOnBlock"]), self.nodes[1].getblock(self.nodes[1].getpaymentrequest(preqHash)["paidOnBlock"]))
+        assert_equal(self.nodes[0].getpaymentrequest(preqHash)["status"], "paid")
+        assert_equal(self.nodes[0].getblock(self.nodes[0].getpaymentrequest(preqHash)["stateChangedOnBlock"]), self.nodes[1].getblock(self.nodes[1].getpaymentrequest(preqHash)["stateChangedOnBlock"]))
         assert_equal(self.nodes[0].getbestblockhash(), self.nodes[1].getbestblockhash())
         assert_equal(self.nodes[0].getblock(self.nodes[0].getpaymentrequest(preqHash)["blockHash"]), self.nodes[1].getblock(self.nodes[1].getpaymentrequest(preqHash)["blockHash"]))
         assert_equal(self.nodes[0].getpaymentrequest(preqHash), self.nodes[1].getpaymentrequest(preqHash))
 
         # Verify the payment was actually received
-        paidBlock = self.nodes[0].getblock(self.nodes[0].getpaymentrequest(preqHash)["paidOnBlock"])
+        paidBlock = self.nodes[0].getblock(self.nodes[0].getpaymentrequest(preqHash)["stateChangedOnBlock"])
         unspent = self.nodes[0].listunspent(0, 80)
 
         assert_equal(unspent[0]['address'], paymentAddress)