diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0766d89f55273..f5d63517b134d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -37,8 +37,8 @@ fixes or code moves with actual code changes. Commit messages should be verbose by default consisting of a short subject line (50 chars max), a blank line and detailed explanatory text as separate -paragraph(s); unless the title alone is self-explanatory (like "Corrected typo -in init.cpp") then a single title line is sufficient. Commit messages should be +paragraph(s), unless the title alone is self-explanatory (like "Corrected typo +in init.cpp") in which case a single title line is sufficient. Commit messages should be helpful to people reading your code in the future, so explain the reasoning for your decisions. Further explanation [here](http://chris.beams.io/posts/git-commit/). @@ -225,6 +225,37 @@ discussed extensively on the mailing list and IRC, be accompanied by a widely discussed BIP and have a generally widely perceived technical consensus of being a worthwhile change based on the judgement of the maintainers. +### Finding Reviewers + +As most reviewers are themselves developers with their own projects, the review +process can be quite lengthy, and some amount of patience is required. If you find +that you've been waiting for a pull request to be given attention for several +months, there may be a number of reasons for this, some of which you can do something +about: + + - It may be because of a feature freeze due to an upcoming release. During this time, + only bug fixes are taken into consideration. If your pull request is a new feature, + it will not be prioritized until the release is over. Wait for release. + - It may be because the changes you are suggesting do not appeal to people. Rather than + nits and critique, which require effort and means they care enough to spend time on your + contribution, thundering silence is a good sign of widespread (mild) dislike of a given change + (because people don't assume *others* won't actually like the proposal). Don't take + that personally, though! Instead, take another critical look at what you are suggesting + and see if it: changes too much, is too broad, doesn't adhere to the + [developer notes](doc/developer-notes.md), is dangerous or insecure, is messily written, etc. + Identify and address any of the issues you find. Then ask e.g. on IRC if someone could give + their opinion on the concept itself. + - It may be because your code is too complex for all but a few people. And those people + may not have realized your pull request even exists. A great way to find people who + are qualified and care about the code you are touching is the + [Git Blame feature](https://help.github.com/articles/tracing-changes-in-a-file/). Simply + find the person touching the code you are touching before you and see if you can find + them and give them a nudge. Don't be incessant about the nudging though. + - Finally, if all else fails, ask on IRC or elsewhere for someone to give your pull request + a look. If you think you've been waiting an unreasonably long amount of time (month+) for + no particular reason (few lines changed, etc), this is totally fine. Try to return the favor + when someone else is asking for feedback on their code, and universe balances out. + Release Policy -------------- diff --git a/Makefile.am b/Makefile.am index ad358fe68be11..eb42bc6db6d47 100644 --- a/Makefile.am +++ b/Makefile.am @@ -76,9 +76,6 @@ $(BITCOIN_WIN_INSTALLER): all-recursive echo error: could not build $@ @echo built $@ -$(if $(findstring src/,$(MAKECMDGOALS)),$(MAKECMDGOALS), none): FORCE - $(MAKE) -C src $(patsubst src/%,%,$@) - $(OSX_APP)/Contents/PkgInfo: $(MKDIR_P) $(@D) @echo "APPL????" > $@ diff --git a/configure.ac b/configure.ac index 0e16330b59dca..e51e9cf0be126 100644 --- a/configure.ac +++ b/configure.ac @@ -676,6 +676,10 @@ AX_BOOST_PROGRAM_OPTIONS AX_BOOST_THREAD AX_BOOST_CHRONO +dnl Boost 1.56 through 1.62 allow using std::atomic instead of its own atomic +dnl counter implementations. In 1.63 and later the std::atomic approach is default. +m4_pattern_allow(DBOOST_AC_USE_STD_ATOMIC) dnl otherwise it's treated like a macro +BOOST_CPPFLAGS="-DBOOST_SP_USE_STD_ATOMIC -DBOOST_AC_USE_STD_ATOMIC $BOOST_CPPFLAGS" if test x$use_reduce_exports = xyes; then AC_MSG_CHECKING([for working boost reduced exports]) diff --git a/doc/build-unix.md b/doc/build-unix.md index 31a88a1b18c64..b7eae2a630ad1 100644 --- a/doc/build-unix.md +++ b/doc/build-unix.md @@ -321,8 +321,10 @@ Clang is installed by default as `cc` compiler, this makes it easier to get started than on [OpenBSD](build-openbsd.md). Installing dependencies: pkg install autoconf automake libtool pkgconf - pkg install boost-libs openssl libevent2 + pkg install boost-libs openssl libevent + pkg install gmake +You need to use GNU make (`gmake`) instead of `make`. (`libressl` instead of `openssl` will also work) For the wallet (optional): @@ -338,7 +340,7 @@ Then build using: ./autogen.sh ./configure --with-incompatible-bdb BDB_CFLAGS="-I/usr/local/include/db5" BDB_LIBS="-L/usr/local/lib -ldb_cxx-5" - make + gmake *Note on debugging*: The version of `gdb` installed by default is [ancient and considered harmful](https://wiki.freebsd.org/GdbRetirement). It is not suitable for debugging a multi-threaded C++ program, not even for getting backtraces. Please install the package `gdb` and diff --git a/src/Makefile.am b/src/Makefile.am index b04b047c151be..c549b636ac9a8 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -474,6 +474,14 @@ DISTCLEANFILES = obj/build.h EXTRA_DIST = $(CTAES_DIST) + +config/bitcoin-config.h: config/stamp-h1 + @$(MAKE) -C $(top_builddir) $(subdir)/$(@) +config/stamp-h1: $(top_srcdir)/$(subdir)/config/bitcoin-config.h.in $(top_builddir)/config.status + $(AM_V_at)$(MAKE) -C $(top_builddir) $(subdir)/$(@) +$(top_srcdir)/$(subdir)/config/bitcoin-config.h.in: $(am__configure_deps) + $(AM_V_at)$(MAKE) -C $(top_srcdir) $(subdir)/config/bitcoin-config.h.in + clean-local: -$(MAKE) -C secp256k1 clean -$(MAKE) -C univalue clean diff --git a/src/checkqueue.h b/src/checkqueue.h index ea12df66dd0e5..63c104c02a114 100644 --- a/src/checkqueue.h +++ b/src/checkqueue.h @@ -5,6 +5,8 @@ #ifndef BITCOIN_CHECKQUEUE_H #define BITCOIN_CHECKQUEUE_H +#include "sync.h" + #include #include diff --git a/src/coins.h b/src/coins.h index 735c1adac5545..4c3d982ced047 100644 --- a/src/coins.h +++ b/src/coins.h @@ -6,6 +6,7 @@ #ifndef BITCOIN_COINS_H #define BITCOIN_COINS_H +#include "primitives/transaction.h" #include "compressor.h" #include "core_memusage.h" #include "hash.h" diff --git a/src/init.cpp b/src/init.cpp index 27762ff6bce6a..fa36deaa29331 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -25,6 +25,7 @@ #include "netbase.h" #include "net.h" #include "net_processing.h" +#include "policy/fees.h" #include "policy/policy.h" #include "rpc/server.h" #include "rpc/register.h" @@ -215,7 +216,7 @@ void Shutdown() fs::path est_path = GetDataDir() / FEE_ESTIMATES_FILENAME; CAutoFile est_fileout(fsbridge::fopen(est_path, "wb"), SER_DISK, CLIENT_VERSION); if (!est_fileout.IsNull()) - mempool.WriteFeeEstimates(est_fileout); + ::feeEstimator.Write(est_fileout); else LogPrintf("%s: Failed to write fee estimates to %s\n", __func__, est_path.string()); fFeeEstimatesInitialized = false; @@ -1556,7 +1557,7 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) CAutoFile est_filein(fsbridge::fopen(est_path, "rb"), SER_DISK, CLIENT_VERSION); // Allowed to fail as this file IS missing on first startup. if (!est_filein.IsNull()) - mempool.ReadFeeEstimates(est_filein); + ::feeEstimator.Read(est_filein); fFeeEstimatesInitialized = true; // ********************************************************* Step 8: load wallet diff --git a/src/net.cpp b/src/net.cpp index ceb7ed0c10328..dd3a7aa4f8f6b 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -1585,6 +1585,9 @@ void CConnman::ThreadDNSAddressSeed() LogPrintf("Loading addresses from DNS seeds (could take a while)\n"); BOOST_FOREACH(const CDNSSeedData &seed, vSeeds) { + if (interruptNet) { + return; + } if (HaveNameProxy()) { AddOneShot(seed.host); } else { @@ -1602,6 +1605,9 @@ void CConnman::ThreadDNSAddressSeed() found++; } } + if (interruptNet) { + return; + } // TODO: The seed name resolve may fail, yielding an IP of [::], which results in // addrman assigning the same source to results from different seeds. // This should switch to a hard-coded stable dummy IP for each seed name, so that the diff --git a/src/net.h b/src/net.h index e44941a111402..41561c51b8d75 100644 --- a/src/net.h +++ b/src/net.h @@ -11,7 +11,6 @@ #include "amount.h" #include "bloom.h" #include "compat.h" -#include "fs.h" #include "hash.h" #include "limitedmap.h" #include "netaddress.h" @@ -36,7 +35,6 @@ #include #include -class CAddrMan; class CScheduler; class CNode; diff --git a/src/policy/fees.cpp b/src/policy/fees.cpp index 38e07dc3450fa..f3f7f8378e568 100644 --- a/src/policy/fees.cpp +++ b/src/policy/fees.cpp @@ -7,14 +7,121 @@ #include "policy/policy.h" #include "amount.h" +#include "clientversion.h" #include "primitives/transaction.h" #include "random.h" #include "streams.h" #include "txmempool.h" #include "util.h" -void TxConfirmStats::Initialize(std::vector& defaultBuckets, - unsigned int maxConfirms, double _decay) +/** + * We will instantiate an instance of this class to track transactions that were + * included in a block. We will lump transactions into a bucket according to their + * approximate feerate and then track how long it took for those txs to be included in a block + * + * The tracking of unconfirmed (mempool) transactions is completely independent of the + * historical tracking of transactions that have been confirmed in a block. + */ +class TxConfirmStats +{ +private: + //Define the buckets we will group transactions into + std::vector buckets; // The upper-bound of the range for the bucket (inclusive) + std::map bucketMap; // Map of bucket upper-bound to index into all vectors by bucket + + // For each bucket X: + // Count the total # of txs in each bucket + // Track the historical moving average of this total over blocks + std::vector txCtAvg; + // and calculate the total for the current block to update the moving average + std::vector curBlockTxCt; + + // Count the total # of txs confirmed within Y blocks in each bucket + // Track the historical moving average of theses totals over blocks + std::vector > confAvg; // confAvg[Y][X] + // and calculate the totals for the current block to update the moving averages + std::vector > curBlockConf; // curBlockConf[Y][X] + + // Sum the total feerate of all tx's in each bucket + // Track the historical moving average of this total over blocks + std::vector avg; + // and calculate the total for the current block to update the moving average + std::vector curBlockVal; + + // Combine the conf counts with tx counts to calculate the confirmation % for each Y,X + // Combine the total value with the tx counts to calculate the avg feerate per bucket + + double decay; + + // Mempool counts of outstanding transactions + // For each bucket X, track the number of transactions in the mempool + // that are unconfirmed for each possible confirmation value Y + std::vector > unconfTxs; //unconfTxs[Y][X] + // transactions still unconfirmed after MAX_CONFIRMS for each bucket + std::vector oldUnconfTxs; + +public: + /** + * Create new TxConfirmStats. This is called by BlockPolicyEstimator's + * constructor with default values. + * @param defaultBuckets contains the upper limits for the bucket boundaries + * @param maxConfirms max number of confirms to track + * @param decay how much to decay the historical moving average per block + */ + TxConfirmStats(const std::vector& defaultBuckets, unsigned int maxConfirms, double decay); + + /** Clear the state of the curBlock variables to start counting for the new block */ + void ClearCurrent(unsigned int nBlockHeight); + + /** + * Record a new transaction data point in the current block stats + * @param blocksToConfirm the number of blocks it took this transaction to confirm + * @param val the feerate of the transaction + * @warning blocksToConfirm is 1-based and has to be >= 1 + */ + void Record(int blocksToConfirm, double val); + + /** Record a new transaction entering the mempool*/ + unsigned int NewTx(unsigned int nBlockHeight, double val); + + /** Remove a transaction from mempool tracking stats*/ + void removeTx(unsigned int entryHeight, unsigned int nBestSeenHeight, + unsigned int bucketIndex); + + /** Update our estimates by decaying our historical moving average and updating + with the data gathered from the current block */ + void UpdateMovingAverages(); + + /** + * Calculate a feerate estimate. Find the lowest value bucket (or range of buckets + * to make sure we have enough data points) whose transactions still have sufficient likelihood + * of being confirmed within the target number of confirmations + * @param confTarget target number of confirmations + * @param sufficientTxVal required average number of transactions per block in a bucket range + * @param minSuccess the success probability we require + * @param requireGreater return the lowest feerate such that all higher values pass minSuccess OR + * return the highest feerate such that all lower values fail minSuccess + * @param nBlockHeight the current block height + */ + double EstimateMedianVal(int confTarget, double sufficientTxVal, + double minSuccess, bool requireGreater, unsigned int nBlockHeight) const; + + /** Return the max number of confirms we're tracking */ + unsigned int GetMaxConfirms() const { return confAvg.size(); } + + /** Write state of estimation data to a file*/ + void Write(CAutoFile& fileout) const; + + /** + * Read saved state of estimation data from a file and replace all internal data structures and + * variables with this state. + */ + void Read(CAutoFile& filein); +}; + + +TxConfirmStats::TxConfirmStats(const std::vector& defaultBuckets, + unsigned int maxConfirms, double _decay) { decay = _decay; for (unsigned int i = 0; i < defaultBuckets.size(); i++) { @@ -77,7 +184,7 @@ void TxConfirmStats::UpdateMovingAverages() // returns -1 on error conditions double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal, double successBreakPoint, bool requireGreater, - unsigned int nBlockHeight) + unsigned int nBlockHeight) const { // Counters for a bucket (or range of buckets) double nConf = 0; // Number of tx's confirmed within the confTarget @@ -173,7 +280,7 @@ double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal, return median; } -void TxConfirmStats::Write(CAutoFile& fileout) +void TxConfirmStats::Write(CAutoFile& fileout) const { fileout << decay; fileout << buckets; @@ -290,9 +397,10 @@ void TxConfirmStats::removeTx(unsigned int entryHeight, unsigned int nBestSeenHe // of no harm to try to remove them again. bool CBlockPolicyEstimator::removeTx(uint256 hash) { + LOCK(cs_feeEstimator); std::map::iterator pos = mapMemPoolTxs.find(hash); if (pos != mapMemPoolTxs.end()) { - feeStats.removeTx(pos->second.blockHeight, nBestSeenHeight, pos->second.bucketIndex); + feeStats->removeTx(pos->second.blockHeight, nBestSeenHeight, pos->second.bucketIndex); mapMemPoolTxs.erase(hash); return true; } else { @@ -310,11 +418,17 @@ CBlockPolicyEstimator::CBlockPolicyEstimator() vfeelist.push_back(bucketBoundary); } vfeelist.push_back(INF_FEERATE); - feeStats.Initialize(vfeelist, MAX_BLOCK_CONFIRMS, DEFAULT_DECAY); + feeStats = new TxConfirmStats(vfeelist, MAX_BLOCK_CONFIRMS, DEFAULT_DECAY); +} + +CBlockPolicyEstimator::~CBlockPolicyEstimator() +{ + delete feeStats; } void CBlockPolicyEstimator::processTransaction(const CTxMemPoolEntry& entry, bool validFeeEstimate) { + LOCK(cs_feeEstimator); unsigned int txHeight = entry.GetHeight(); uint256 hash = entry.GetTx().GetHash(); if (mapMemPoolTxs.count(hash)) { @@ -343,7 +457,7 @@ void CBlockPolicyEstimator::processTransaction(const CTxMemPoolEntry& entry, boo CFeeRate feeRate(entry.GetFee(), entry.GetTxSize()); mapMemPoolTxs[hash].blockHeight = txHeight; - mapMemPoolTxs[hash].bucketIndex = feeStats.NewTx(txHeight, (double)feeRate.GetFeePerK()); + mapMemPoolTxs[hash].bucketIndex = feeStats->NewTx(txHeight, (double)feeRate.GetFeePerK()); } bool CBlockPolicyEstimator::processBlockTx(unsigned int nBlockHeight, const CTxMemPoolEntry* entry) @@ -367,13 +481,14 @@ bool CBlockPolicyEstimator::processBlockTx(unsigned int nBlockHeight, const CTxM // Feerates are stored and reported as BTC-per-kb: CFeeRate feeRate(entry->GetFee(), entry->GetTxSize()); - feeStats.Record(blocksToConfirm, (double)feeRate.GetFeePerK()); + feeStats->Record(blocksToConfirm, (double)feeRate.GetFeePerK()); return true; } void CBlockPolicyEstimator::processBlock(unsigned int nBlockHeight, std::vector& entries) { + LOCK(cs_feeEstimator); if (nBlockHeight <= nBestSeenHeight) { // Ignore side chains and re-orgs; assuming they are random // they don't affect the estimate. @@ -389,7 +504,7 @@ void CBlockPolicyEstimator::processBlock(unsigned int nBlockHeight, nBestSeenHeight = nBlockHeight; // Clear the current block state and update unconfirmed circular buffer - feeStats.ClearCurrent(nBlockHeight); + feeStats->ClearCurrent(nBlockHeight); unsigned int countedTxs = 0; // Repopulate the current block states @@ -399,7 +514,7 @@ void CBlockPolicyEstimator::processBlock(unsigned int nBlockHeight, } // Update all exponential averages with the current block state - feeStats.UpdateMovingAverages(); + feeStats->UpdateMovingAverages(); LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy after updating estimates for %u of %u txs in block, since last block %u of %u tracked, new mempool map size %u\n", countedTxs, entries.size(), trackedTxs, trackedTxs + untrackedTxs, mapMemPoolTxs.size()); @@ -408,14 +523,15 @@ void CBlockPolicyEstimator::processBlock(unsigned int nBlockHeight, untrackedTxs = 0; } -CFeeRate CBlockPolicyEstimator::estimateFee(int confTarget) +CFeeRate CBlockPolicyEstimator::estimateFee(int confTarget) const { + LOCK(cs_feeEstimator); // Return failure if trying to analyze a target we're not tracking // It's not possible to get reasonable estimates for confTarget of 1 - if (confTarget <= 1 || (unsigned int)confTarget > feeStats.GetMaxConfirms()) + if (confTarget <= 1 || (unsigned int)confTarget > feeStats->GetMaxConfirms()) return CFeeRate(0); - double median = feeStats.EstimateMedianVal(confTarget, SUFFICIENT_FEETXS, MIN_SUCCESS_PCT, true, nBestSeenHeight); + double median = feeStats->EstimateMedianVal(confTarget, SUFFICIENT_FEETXS, MIN_SUCCESS_PCT, true, nBestSeenHeight); if (median < 0) return CFeeRate(0); @@ -423,22 +539,28 @@ CFeeRate CBlockPolicyEstimator::estimateFee(int confTarget) return CFeeRate(median); } -CFeeRate CBlockPolicyEstimator::estimateSmartFee(int confTarget, int *answerFoundAtTarget, const CTxMemPool& pool) +CFeeRate CBlockPolicyEstimator::estimateSmartFee(int confTarget, int *answerFoundAtTarget, const CTxMemPool& pool) const { if (answerFoundAtTarget) *answerFoundAtTarget = confTarget; - // Return failure if trying to analyze a target we're not tracking - if (confTarget <= 0 || (unsigned int)confTarget > feeStats.GetMaxConfirms()) - return CFeeRate(0); - - // It's not possible to get reasonable estimates for confTarget of 1 - if (confTarget == 1) - confTarget = 2; double median = -1; - while (median < 0 && (unsigned int)confTarget <= feeStats.GetMaxConfirms()) { - median = feeStats.EstimateMedianVal(confTarget++, SUFFICIENT_FEETXS, MIN_SUCCESS_PCT, true, nBestSeenHeight); - } + + { + LOCK(cs_feeEstimator); + + // Return failure if trying to analyze a target we're not tracking + if (confTarget <= 0 || (unsigned int)confTarget > feeStats->GetMaxConfirms()) + return CFeeRate(0); + + // It's not possible to get reasonable estimates for confTarget of 1 + if (confTarget == 1) + confTarget = 2; + + while (median < 0 && (unsigned int)confTarget <= feeStats->GetMaxConfirms()) { + median = feeStats->EstimateMedianVal(confTarget++, SUFFICIENT_FEETXS, MIN_SUCCESS_PCT, true, nBestSeenHeight); + } + } // Must unlock cs_feeEstimator before taking mempool locks if (answerFoundAtTarget) *answerFoundAtTarget = confTarget - 1; @@ -454,19 +576,40 @@ CFeeRate CBlockPolicyEstimator::estimateSmartFee(int confTarget, int *answerFoun return CFeeRate(median); } -void CBlockPolicyEstimator::Write(CAutoFile& fileout) +bool CBlockPolicyEstimator::Write(CAutoFile& fileout) const { - fileout << nBestSeenHeight; - feeStats.Write(fileout); + try { + LOCK(cs_feeEstimator); + fileout << 139900; // version required to read: 0.13.99 or later + fileout << CLIENT_VERSION; // version that wrote the file + fileout << nBestSeenHeight; + feeStats->Write(fileout); + } + catch (const std::exception&) { + LogPrintf("CBlockPolicyEstimator::Write(): unable to read policy estimator data (non-fatal)\n"); + return false; + } + return true; } -void CBlockPolicyEstimator::Read(CAutoFile& filein, int nFileVersion) +bool CBlockPolicyEstimator::Read(CAutoFile& filein) { - int nFileBestSeenHeight; - filein >> nFileBestSeenHeight; - feeStats.Read(filein); - nBestSeenHeight = nFileBestSeenHeight; - // if nVersionThatWrote < 139900 then another TxConfirmStats (for priority) follows but can be ignored. + try { + LOCK(cs_feeEstimator); + int nVersionRequired, nVersionThatWrote, nFileBestSeenHeight; + filein >> nVersionRequired >> nVersionThatWrote; + if (nVersionRequired > CLIENT_VERSION) + return error("CBlockPolicyEstimator::Read(): up-version (%d) fee estimate file", nVersionRequired); + filein >> nFileBestSeenHeight; + feeStats->Read(filein); + nBestSeenHeight = nFileBestSeenHeight; + // if nVersionThatWrote < 139900 then another TxConfirmStats (for priority) follows but can be ignored. + } + catch (const std::exception&) { + LogPrintf("CBlockPolicyEstimator::Read(): unable to read policy estimator data (non-fatal)\n"); + return false; + } + return true; } FeeFilterRounder::FeeFilterRounder(const CFeeRate& minIncrementalFee) diff --git a/src/policy/fees.h b/src/policy/fees.h index dd01c90c45b74..34f07c7270751 100644 --- a/src/policy/fees.h +++ b/src/policy/fees.h @@ -8,6 +8,7 @@ #include "amount.h" #include "uint256.h" #include "random.h" +#include "sync.h" #include #include @@ -17,6 +18,7 @@ class CAutoFile; class CFeeRate; class CTxMemPoolEntry; class CTxMemPool; +class TxConfirmStats; /** \class CBlockPolicyEstimator * The BlockPolicyEstimator is used for estimating the feerate needed @@ -59,113 +61,6 @@ class CTxMemPool; * they've been outstanding. */ -/** - * We will instantiate an instance of this class to track transactions that were - * included in a block. We will lump transactions into a bucket according to their - * approximate feerate and then track how long it took for those txs to be included in a block - * - * The tracking of unconfirmed (mempool) transactions is completely independent of the - * historical tracking of transactions that have been confirmed in a block. - */ -class TxConfirmStats -{ -private: - //Define the buckets we will group transactions into - std::vector buckets; // The upper-bound of the range for the bucket (inclusive) - std::map bucketMap; // Map of bucket upper-bound to index into all vectors by bucket - - // For each bucket X: - // Count the total # of txs in each bucket - // Track the historical moving average of this total over blocks - std::vector txCtAvg; - // and calculate the total for the current block to update the moving average - std::vector curBlockTxCt; - - // Count the total # of txs confirmed within Y blocks in each bucket - // Track the historical moving average of theses totals over blocks - std::vector > confAvg; // confAvg[Y][X] - // and calculate the totals for the current block to update the moving averages - std::vector > curBlockConf; // curBlockConf[Y][X] - - // Sum the total feerate of all tx's in each bucket - // Track the historical moving average of this total over blocks - std::vector avg; - // and calculate the total for the current block to update the moving average - std::vector curBlockVal; - - // Combine the conf counts with tx counts to calculate the confirmation % for each Y,X - // Combine the total value with the tx counts to calculate the avg feerate per bucket - - double decay; - - // Mempool counts of outstanding transactions - // For each bucket X, track the number of transactions in the mempool - // that are unconfirmed for each possible confirmation value Y - std::vector > unconfTxs; //unconfTxs[Y][X] - // transactions still unconfirmed after MAX_CONFIRMS for each bucket - std::vector oldUnconfTxs; - -public: - /** - * Initialize the data structures. This is called by BlockPolicyEstimator's - * constructor with default values. - * @param defaultBuckets contains the upper limits for the bucket boundaries - * @param maxConfirms max number of confirms to track - * @param decay how much to decay the historical moving average per block - */ - void Initialize(std::vector& defaultBuckets, unsigned int maxConfirms, double decay); - - /** Clear the state of the curBlock variables to start counting for the new block */ - void ClearCurrent(unsigned int nBlockHeight); - - /** - * Record a new transaction data point in the current block stats - * @param blocksToConfirm the number of blocks it took this transaction to confirm - * @param val the feerate of the transaction - * @warning blocksToConfirm is 1-based and has to be >= 1 - */ - void Record(int blocksToConfirm, double val); - - /** Record a new transaction entering the mempool*/ - unsigned int NewTx(unsigned int nBlockHeight, double val); - - /** Remove a transaction from mempool tracking stats*/ - void removeTx(unsigned int entryHeight, unsigned int nBestSeenHeight, - unsigned int bucketIndex); - - /** Update our estimates by decaying our historical moving average and updating - with the data gathered from the current block */ - void UpdateMovingAverages(); - - /** - * Calculate a feerate estimate. Find the lowest value bucket (or range of buckets - * to make sure we have enough data points) whose transactions still have sufficient likelihood - * of being confirmed within the target number of confirmations - * @param confTarget target number of confirmations - * @param sufficientTxVal required average number of transactions per block in a bucket range - * @param minSuccess the success probability we require - * @param requireGreater return the lowest feerate such that all higher values pass minSuccess OR - * return the highest feerate such that all lower values fail minSuccess - * @param nBlockHeight the current block height - */ - double EstimateMedianVal(int confTarget, double sufficientTxVal, - double minSuccess, bool requireGreater, unsigned int nBlockHeight); - - /** Return the max number of confirms we're tracking */ - unsigned int GetMaxConfirms() { return confAvg.size(); } - - /** Write state of estimation data to a file*/ - void Write(CAutoFile& fileout); - - /** - * Read saved state of estimation data from a file and replace all internal data structures and - * variables with this state. - */ - void Read(CAutoFile& filein); -}; - - - /** Track confirm delays up to 25 blocks, can't estimate beyond that */ static const unsigned int MAX_BLOCK_CONFIRMS = 25; @@ -204,14 +99,12 @@ class CBlockPolicyEstimator public: /** Create new BlockPolicyEstimator and initialize stats tracking classes with default values */ CBlockPolicyEstimator(); + ~CBlockPolicyEstimator(); /** Process all the transactions that have been included in a block */ void processBlock(unsigned int nBlockHeight, std::vector& entries); - /** Process a transaction confirmed in a block*/ - bool processBlockTx(unsigned int nBlockHeight, const CTxMemPoolEntry* entry); - /** Process a transaction accepted to the mempool*/ void processTransaction(const CTxMemPoolEntry& entry, bool validFeeEstimate); @@ -219,19 +112,19 @@ class CBlockPolicyEstimator bool removeTx(uint256 hash); /** Return a feerate estimate */ - CFeeRate estimateFee(int confTarget); + CFeeRate estimateFee(int confTarget) const; /** Estimate feerate needed to get be included in a block within * confTarget blocks. If no answer can be given at confTarget, return an * estimate at the lowest target where one can be given. */ - CFeeRate estimateSmartFee(int confTarget, int *answerFoundAtTarget, const CTxMemPool& pool); + CFeeRate estimateSmartFee(int confTarget, int *answerFoundAtTarget, const CTxMemPool& pool) const; /** Write estimation data to a file */ - void Write(CAutoFile& fileout); + bool Write(CAutoFile& fileout) const; /** Read estimation data from a file */ - void Read(CAutoFile& filein, int nFileVersion); + bool Read(CAutoFile& filein); private: CFeeRate minTrackedFee; //!< Passed to constructor to avoid dependency on main @@ -247,10 +140,16 @@ class CBlockPolicyEstimator std::map mapMemPoolTxs; /** Classes to track historical data on transaction confirmations */ - TxConfirmStats feeStats; + TxConfirmStats* feeStats; unsigned int trackedTxs; unsigned int untrackedTxs; + + mutable CCriticalSection cs_feeEstimator; + + /** Process a transaction confirmed in a block*/ + bool processBlockTx(unsigned int nBlockHeight, const CTxMemPoolEntry* entry); + }; class FeeFilterRounder diff --git a/src/policy/policy.cpp b/src/policy/policy.cpp index 12caf194f55fe..bf1bd48b2b7cd 100644 --- a/src/policy/policy.cpp +++ b/src/policy/policy.cpp @@ -8,6 +8,7 @@ #include "policy/policy.h" #include "validation.h" +#include "coins.h" #include "tinyformat.h" #include "util.h" #include "utilstrencodings.h" diff --git a/src/qt/bitcoingui.cpp b/src/qt/bitcoingui.cpp index 54a86876feceb..343027b32830f 100644 --- a/src/qt/bitcoingui.cpp +++ b/src/qt/bitcoingui.cpp @@ -478,6 +478,7 @@ void BitcoinGUI::setClientModel(ClientModel *_clientModel) connect(_clientModel, SIGNAL(numConnectionsChanged(int)), this, SLOT(setNumConnections(int))); connect(_clientModel, SIGNAL(networkActiveChanged(bool)), this, SLOT(setNetworkActive(bool))); + modalOverlay->setKnownBestHeight(_clientModel->getHeaderTipHeight(), QDateTime::fromTime_t(_clientModel->getHeaderTipTime())); setNumBlocks(_clientModel->getNumBlocks(), _clientModel->getLastBlockDate(), _clientModel->getVerificationProgress(NULL), false); connect(_clientModel, SIGNAL(numBlocksChanged(int,QDateTime,double,bool)), this, SLOT(setNumBlocks(int,QDateTime,double,bool))); @@ -505,8 +506,6 @@ void BitcoinGUI::setClientModel(ClientModel *_clientModel) // initialize the disable state of the tray icon with the current value in the model. setTrayIconVisible(optionsModel->getHideTrayIcon()); } - - modalOverlay->setKnownBestHeight(clientModel->getHeaderTipHeight(), QDateTime::fromTime_t(clientModel->getHeaderTipTime())); } else { // Disable possibility to show main window via action toggleHideAction->setEnabled(false); diff --git a/src/qt/clientmodel.cpp b/src/qt/clientmodel.cpp index bb10e49422068..de00eacdb98f8 100644 --- a/src/qt/clientmodel.cpp +++ b/src/qt/clientmodel.cpp @@ -9,6 +9,7 @@ #include "guiutil.h" #include "peertablemodel.h" +#include "chain.h" #include "chainparams.h" #include "checkpoints.h" #include "clientversion.h" @@ -36,6 +37,8 @@ ClientModel::ClientModel(OptionsModel *_optionsModel, QObject *parent) : banTableModel(0), pollTimer(0) { + cachedBestHeaderHeight = -1; + cachedBestHeaderTime = -1; peerTableModel = new PeerTableModel(this); banTableModel = new BanTableModel(this); pollTimer = new QTimer(this); @@ -74,18 +77,28 @@ int ClientModel::getNumBlocks() const int ClientModel::getHeaderTipHeight() const { - LOCK(cs_main); - if (!pindexBestHeader) - return 0; - return pindexBestHeader->nHeight; + if (cachedBestHeaderHeight == -1) { + // make sure we initially populate the cache via a cs_main lock + // otherwise we need to wait for a tip update + LOCK(cs_main); + if (pindexBestHeader) { + cachedBestHeaderHeight = pindexBestHeader->nHeight; + cachedBestHeaderTime = pindexBestHeader->GetBlockTime(); + } + } + return cachedBestHeaderHeight; } int64_t ClientModel::getHeaderTipTime() const { - LOCK(cs_main); - if (!pindexBestHeader) - return 0; - return pindexBestHeader->GetBlockTime(); + if (cachedBestHeaderTime == -1) { + LOCK(cs_main); + if (pindexBestHeader) { + cachedBestHeaderHeight = pindexBestHeader->nHeight; + cachedBestHeaderTime = pindexBestHeader->GetBlockTime(); + } + } + return cachedBestHeaderTime; } quint64 ClientModel::getTotalBytesRecv() const @@ -283,6 +296,11 @@ static void BlockTipChanged(ClientModel *clientmodel, bool initialSync, const CB int64_t& nLastUpdateNotification = fHeader ? nLastHeaderTipUpdateNotification : nLastBlockTipUpdateNotification; + if (fHeader) { + // cache best headers time and height to reduce future cs_main locks + clientmodel->cachedBestHeaderHeight = pIndex->nHeight; + clientmodel->cachedBestHeaderTime = pIndex->GetBlockTime(); + } // if we are in-sync, update the UI regardless of last update time if (!initialSync || now - nLastUpdateNotification > MODEL_UPDATE_DELAY) { //pass a async signal to the UI thread diff --git a/src/qt/clientmodel.h b/src/qt/clientmodel.h index 2c10e633b84fd..4c92e2144e321 100644 --- a/src/qt/clientmodel.h +++ b/src/qt/clientmodel.h @@ -8,6 +8,8 @@ #include #include +#include + class AddressTableModel; class BanTableModel; class OptionsModel; @@ -81,6 +83,10 @@ class ClientModel : public QObject QString formatClientStartupTime() const; QString dataDir() const; + // caches for the best header + mutable std::atomic cachedBestHeaderHeight; + mutable std::atomic cachedBestHeaderTime; + private: OptionsModel *optionsModel; PeerTableModel *peerTableModel; diff --git a/src/qt/coincontroldialog.cpp b/src/qt/coincontroldialog.cpp index 1d19c65753d3d..38ad6e9aab6c4 100644 --- a/src/qt/coincontroldialog.cpp +++ b/src/qt/coincontroldialog.cpp @@ -15,6 +15,7 @@ #include "wallet/coincontrol.h" #include "init.h" +#include "policy/fees.h" #include "policy/policy.h" #include "validation.h" // For mempool #include "wallet/wallet.h" @@ -512,7 +513,7 @@ void CoinControlDialog::updateLabels(WalletModel *model, QDialog* dialog) nBytes -= 34; // Fee - nPayFee = CWallet::GetMinimumFee(nBytes, nTxConfirmTarget, mempool); + nPayFee = CWallet::GetMinimumFee(nBytes, nTxConfirmTarget, ::mempool, ::feeEstimator); if (nPayFee > 0 && coinControl->nMinimumTotalFee > nPayFee) nPayFee = coinControl->nMinimumTotalFee; @@ -592,7 +593,7 @@ void CoinControlDialog::updateLabels(WalletModel *model, QDialog* dialog) if (payTxFee.GetFeePerK() > 0) dFeeVary = (double)std::max(CWallet::GetRequiredFee(1000), payTxFee.GetFeePerK()) / 1000; else { - dFeeVary = (double)std::max(CWallet::GetRequiredFee(1000), mempool.estimateSmartFee(nTxConfirmTarget).GetFeePerK()) / 1000; + dFeeVary = (double)std::max(CWallet::GetRequiredFee(1000), ::feeEstimator.estimateSmartFee(nTxConfirmTarget, NULL, ::mempool).GetFeePerK()) / 1000; } QString toolTip4 = tr("Can vary +/- %1 satoshi(s) per input.").arg(dFeeVary); diff --git a/src/qt/sendcoinsdialog.cpp b/src/qt/sendcoinsdialog.cpp index ba6819f391e36..14f9f86f7bc3b 100644 --- a/src/qt/sendcoinsdialog.cpp +++ b/src/qt/sendcoinsdialog.cpp @@ -21,6 +21,7 @@ #include "validation.h" // mempool and minRelayTxFee #include "ui_interface.h" #include "txmempool.h" +#include "policy/fees.h" #include "wallet/wallet.h" #include @@ -114,7 +115,6 @@ SendCoinsDialog::SendCoinsDialog(const PlatformStyle *_platformStyle, QWidget *p ui->groupCustomFee->button((int)std::max(0, std::min(1, settings.value("nCustomFeeRadio").toInt())))->setChecked(true); ui->customFee->setValue(settings.value("nTransactionFee").toLongLong()); ui->checkBoxMinimumFee->setChecked(settings.value("fPayOnlyMinFee").toBool()); - ui->optInRBF->setCheckState(model->getDefaultWalletRbf() ? Qt::Checked : Qt::Unchecked); minimizeFeeSection(settings.value("fFeeSectionMinimized").toBool()); } @@ -175,6 +175,9 @@ void SendCoinsDialog::setModel(WalletModel *_model) updateSmartFeeLabel(); updateGlobalFeeVariables(); + // set default rbf checkbox state + ui->optInRBF->setCheckState(model->getDefaultWalletRbf() ? Qt::Checked : Qt::Unchecked); + // set the smartfee-sliders default value (wallets default conf.target or last stored value) QSettings settings; if (settings.value("nSmartFeeSliderPosition").toInt() == 0) @@ -660,7 +663,7 @@ void SendCoinsDialog::updateSmartFeeLabel() int nBlocksToConfirm = ui->sliderSmartFee->maximum() - ui->sliderSmartFee->value() + 2; int estimateFoundAtBlocks = nBlocksToConfirm; - CFeeRate feeRate = mempool.estimateSmartFee(nBlocksToConfirm, &estimateFoundAtBlocks); + CFeeRate feeRate = ::feeEstimator.estimateSmartFee(nBlocksToConfirm, &estimateFoundAtBlocks, ::mempool); if (feeRate <= CFeeRate(0)) // not enough data => minfee { ui->labelSmartFee->setText(BitcoinUnits::formatWithUnit(model->getOptionsModel()->getDisplayUnit(), diff --git a/src/qt/transactionrecord.cpp b/src/qt/transactionrecord.cpp index a9d9b6887ec3b..4bb260aa588ae 100644 --- a/src/qt/transactionrecord.cpp +++ b/src/qt/transactionrecord.cpp @@ -18,14 +18,8 @@ */ bool TransactionRecord::showTransaction(const CWalletTx &wtx) { - if (wtx.IsCoinBase()) - { - // Ensures we show generated coins / mined transactions at depth 1 - if (!wtx.IsInMainChain()) - { - return false; - } - } + // There are currently no cases where we hide transactions, but + // we may want to use this in the future for things like RBF. return true; } diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index 898765f125a55..cc19088d1a3c1 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -13,6 +13,7 @@ #include "transactiontablemodel.h" #include "base58.h" +#include "chain.h" #include "keystore.h" #include "validation.h" #include "net.h" // for g_connman diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index 662542cdf7fd5..a72e69ad3fb0c 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -116,6 +116,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "bumpfee", 1, "options" }, { "logging", 0, "include" }, { "logging", 1, "exclude" }, + { "disconnectnode", 1, "nodeid" }, { "name_scan", 1, "count" }, { "name_filter", 1, "maxage" }, { "name_filter", 2, "from" }, diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index 457a1ededf807..a6bdbb3699d69 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -15,6 +15,7 @@ #include "validation.h" #include "miner.h" #include "net.h" +#include "policy/fees.h" #include "pow.h" #include "rpc/blockchain.h" #include "rpc/server.h" @@ -822,7 +823,7 @@ UniValue estimatefee(const JSONRPCRequest& request) if (nBlocks < 1) nBlocks = 1; - CFeeRate feeRate = mempool.estimateFee(nBlocks); + CFeeRate feeRate = ::feeEstimator.estimateFee(nBlocks); if (feeRate == CFeeRate(0)) return -1.0; @@ -860,7 +861,7 @@ UniValue estimatesmartfee(const JSONRPCRequest& request) UniValue result(UniValue::VOBJ); int answerFound; - CFeeRate feeRate = mempool.estimateSmartFee(nBlocks, &answerFound); + CFeeRate feeRate = ::feeEstimator.estimateSmartFee(nBlocks, &answerFound, ::mempool); result.push_back(Pair("feerate", feeRate == CFeeRate(0) ? -1.0 : ValueFromAmount(feeRate.GetFeePerK()))); result.push_back(Pair("blocks", answerFound)); return result; diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index c6725546cc105..a6409af745eb2 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -4,6 +4,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include "base58.h" +#include "chain.h" #include "clientversion.h" #include "init.h" #include "validation.h" diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index 393f0b74a667b..92e6ede3e8711 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -234,23 +234,43 @@ UniValue addnode(const JSONRPCRequest& request) UniValue disconnectnode(const JSONRPCRequest& request) { - if (request.fHelp || request.params.size() != 1) + if (request.fHelp || request.params.size() == 0 || request.params.size() >= 3) throw std::runtime_error( - "disconnectnode \"address\" \n" - "\nImmediately disconnects from the specified node.\n" + "disconnectnode \"[address]\" [nodeid]\n" + "\nImmediately disconnects from the specified peer node.\n" + "\nStrictly one out of 'address' and 'nodeid' can be provided to identify the node.\n" + "\nTo disconnect by nodeid, either set 'address' to the empty string, or call using the named 'nodeid' argument only.\n" "\nArguments:\n" - "1. \"address\" (string, required) The IP address/port of the node\n" + "1. \"address\" (string, optional) The IP address/port of the node\n" + "2. \"nodeid\" (number, optional) The node ID (see getpeerinfo for node IDs)\n" "\nExamples:\n" + HelpExampleCli("disconnectnode", "\"192.168.0.6:8333\"") + + HelpExampleCli("disconnectnode", "\"\" 1") + HelpExampleRpc("disconnectnode", "\"192.168.0.6:8333\"") + + HelpExampleRpc("disconnectnode", "\"\", 1") ); if(!g_connman) throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); - bool ret = g_connman->DisconnectNode(request.params[0].get_str()); - if (!ret) + bool success; + const UniValue &address_arg = request.params[0]; + const UniValue &id_arg = request.params.size() < 2 ? NullUniValue : request.params[1]; + + if (!address_arg.isNull() && id_arg.isNull()) { + /* handle disconnect-by-address */ + success = g_connman->DisconnectNode(address_arg.get_str()); + } else if (!id_arg.isNull() && (address_arg.isNull() || (address_arg.isStr() && address_arg.get_str().empty()))) { + /* handle disconnect-by-id */ + NodeId nodeid = (NodeId) id_arg.get_int64(); + success = g_connman->DisconnectNode(nodeid); + } else { + throw JSONRPCError(RPC_INVALID_PARAMS, "Only one of address and nodeid should be provided."); + } + + if (!success) { throw JSONRPCError(RPC_CLIENT_NODE_NOT_CONNECTED, "Node not found in connected nodes"); + } return NullUniValue; } @@ -607,7 +627,7 @@ static const CRPCCommand commands[] = { "network", "ping", &ping, true, {} }, { "network", "getpeerinfo", &getpeerinfo, true, {} }, { "network", "addnode", &addnode, true, {"node","command"} }, - { "network", "disconnectnode", &disconnectnode, true, {"address"} }, + { "network", "disconnectnode", &disconnectnode, true, {"address", "nodeid"} }, { "network", "getaddednodeinfo", &getaddednodeinfo, true, {"node"} }, { "network", "getnettotals", &getnettotals, true, {} }, { "network", "getnetworkinfo", &getnetworkinfo, true, {} }, diff --git a/src/serialize.h b/src/serialize.h index e4d72d2348f17..e82ddf2c5a7bb 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -336,11 +336,18 @@ I ReadVarInt(Stream& is) I n = 0; while(true) { unsigned char chData = ser_readdata8(is); + if (n > (std::numeric_limits::max() >> 7)) { + throw std::ios_base::failure("ReadVarInt(): size too large"); + } n = (n << 7) | (chData & 0x7F); - if (chData & 0x80) + if (chData & 0x80) { + if (n == std::numeric_limits::max()) { + throw std::ios_base::failure("ReadVarInt(): size too large"); + } n++; - else + } else { return n; + } } } diff --git a/src/streams.h b/src/streams.h index 1387b9cf54a62..8dc5a19ead4b5 100644 --- a/src/streams.h +++ b/src/streams.h @@ -248,7 +248,8 @@ class CDataStream void insert(iterator it, std::vector::const_iterator first, std::vector::const_iterator last) { - assert(last - first >= 0); + if (last == first) return; + assert(last - first > 0); if (it == vch.begin() + nReadPos && (unsigned int)(last - first) <= nReadPos) { // special case for inserting at the front when there's room @@ -261,7 +262,8 @@ class CDataStream void insert(iterator it, const char* first, const char* last) { - assert(last - first >= 0); + if (last == first) return; + assert(last - first > 0); if (it == vch.begin() + nReadPos && (unsigned int)(last - first) <= nReadPos) { // special case for inserting at the front when there's room @@ -339,6 +341,8 @@ class CDataStream void read(char* pch, size_t nSize) { + if (nSize == 0) return; + // Read from the beginning of the buffer unsigned int nReadPosNext = nReadPos + nSize; if (nReadPosNext >= vch.size()) diff --git a/src/test/policyestimator_tests.cpp b/src/test/policyestimator_tests.cpp index bc2f49ef3faaa..ed6782ea34564 100644 --- a/src/test/policyestimator_tests.cpp +++ b/src/test/policyestimator_tests.cpp @@ -16,7 +16,8 @@ BOOST_FIXTURE_TEST_SUITE(policyestimator_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) { - CTxMemPool mpool; + CBlockPolicyEstimator feeEst; + CTxMemPool mpool(&feeEst); TestMemPoolEntryHelper entry; CAmount basefee(2000); CAmount deltaFee(100); @@ -78,16 +79,16 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) // At this point we should need to combine 5 buckets to get enough data points // So estimateFee(1,2,3) should fail and estimateFee(4) should return somewhere around // 8*baserate. estimateFee(4) %'s are 100,100,100,100,90 = average 98% - BOOST_CHECK(mpool.estimateFee(1) == CFeeRate(0)); - BOOST_CHECK(mpool.estimateFee(2) == CFeeRate(0)); - BOOST_CHECK(mpool.estimateFee(3) == CFeeRate(0)); - BOOST_CHECK(mpool.estimateFee(4).GetFeePerK() < 8*baseRate.GetFeePerK() + deltaFee); - BOOST_CHECK(mpool.estimateFee(4).GetFeePerK() > 8*baseRate.GetFeePerK() - deltaFee); + BOOST_CHECK(feeEst.estimateFee(1) == CFeeRate(0)); + BOOST_CHECK(feeEst.estimateFee(2) == CFeeRate(0)); + BOOST_CHECK(feeEst.estimateFee(3) == CFeeRate(0)); + BOOST_CHECK(feeEst.estimateFee(4).GetFeePerK() < 8*baseRate.GetFeePerK() + deltaFee); + BOOST_CHECK(feeEst.estimateFee(4).GetFeePerK() > 8*baseRate.GetFeePerK() - deltaFee); int answerFound; - BOOST_CHECK(mpool.estimateSmartFee(1, &answerFound) == mpool.estimateFee(4) && answerFound == 4); - BOOST_CHECK(mpool.estimateSmartFee(3, &answerFound) == mpool.estimateFee(4) && answerFound == 4); - BOOST_CHECK(mpool.estimateSmartFee(4, &answerFound) == mpool.estimateFee(4) && answerFound == 4); - BOOST_CHECK(mpool.estimateSmartFee(8, &answerFound) == mpool.estimateFee(8) && answerFound == 8); + BOOST_CHECK(feeEst.estimateSmartFee(1, &answerFound, mpool) == feeEst.estimateFee(4) && answerFound == 4); + BOOST_CHECK(feeEst.estimateSmartFee(3, &answerFound, mpool) == feeEst.estimateFee(4) && answerFound == 4); + BOOST_CHECK(feeEst.estimateSmartFee(4, &answerFound, mpool) == feeEst.estimateFee(4) && answerFound == 4); + BOOST_CHECK(feeEst.estimateSmartFee(8, &answerFound, mpool) == feeEst.estimateFee(8) && answerFound == 8); } } @@ -99,7 +100,7 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) // Second highest feerate has 100% chance of being included by 2 blocks, // so estimateFee(2) should return 9*baseRate etc... for (int i = 1; i < 10;i++) { - origFeeEst.push_back(mpool.estimateFee(i).GetFeePerK()); + origFeeEst.push_back(feeEst.estimateFee(i).GetFeePerK()); if (i > 2) { // Fee estimates should be monotonically decreasing BOOST_CHECK(origFeeEst[i-1] <= origFeeEst[i-2]); } @@ -118,10 +119,10 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) while (blocknum < 250) mpool.removeForBlock(block, ++blocknum); - BOOST_CHECK(mpool.estimateFee(1) == CFeeRate(0)); + BOOST_CHECK(feeEst.estimateFee(1) == CFeeRate(0)); for (int i = 2; i < 10;i++) { - BOOST_CHECK(mpool.estimateFee(i).GetFeePerK() < origFeeEst[i-1] + deltaFee); - BOOST_CHECK(mpool.estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee); + BOOST_CHECK(feeEst.estimateFee(i).GetFeePerK() < origFeeEst[i-1] + deltaFee); + BOOST_CHECK(feeEst.estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee); } @@ -141,8 +142,8 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) int answerFound; for (int i = 1; i < 10;i++) { - BOOST_CHECK(mpool.estimateFee(i) == CFeeRate(0) || mpool.estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee); - BOOST_CHECK(mpool.estimateSmartFee(i, &answerFound).GetFeePerK() > origFeeEst[answerFound-1] - deltaFee); + BOOST_CHECK(feeEst.estimateFee(i) == CFeeRate(0) || feeEst.estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee); + BOOST_CHECK(feeEst.estimateSmartFee(i, &answerFound, mpool).GetFeePerK() > origFeeEst[answerFound-1] - deltaFee); } // Mine all those transactions @@ -157,9 +158,9 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) } mpool.removeForBlock(block, 265); block.clear(); - BOOST_CHECK(mpool.estimateFee(1) == CFeeRate(0)); + BOOST_CHECK(feeEst.estimateFee(1) == CFeeRate(0)); for (int i = 2; i < 10;i++) { - BOOST_CHECK(mpool.estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee); + BOOST_CHECK(feeEst.estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee); } // Mine 200 more blocks where everything is mined every block @@ -179,9 +180,9 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) mpool.removeForBlock(block, ++blocknum); block.clear(); } - BOOST_CHECK(mpool.estimateFee(1) == CFeeRate(0)); + BOOST_CHECK(feeEst.estimateFee(1) == CFeeRate(0)); for (int i = 2; i < 10; i++) { - BOOST_CHECK(mpool.estimateFee(i).GetFeePerK() < origFeeEst[i-1] - deltaFee); + BOOST_CHECK(feeEst.estimateFee(i).GetFeePerK() < origFeeEst[i-1] - deltaFee); } // Test that if the mempool is limited, estimateSmartFee won't return a value below the mempool min fee @@ -190,8 +191,8 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) mpool.TrimToSize(1); BOOST_CHECK(mpool.GetMinFee(1).GetFeePerK() > feeV[5]); for (int i = 1; i < 10; i++) { - BOOST_CHECK(mpool.estimateSmartFee(i).GetFeePerK() >= mpool.estimateFee(i).GetFeePerK()); - BOOST_CHECK(mpool.estimateSmartFee(i).GetFeePerK() >= mpool.GetMinFee(1).GetFeePerK()); + BOOST_CHECK(feeEst.estimateSmartFee(i, NULL, mpool).GetFeePerK() >= feeEst.estimateFee(i).GetFeePerK()); + BOOST_CHECK(feeEst.estimateSmartFee(i, NULL, mpool).GetFeePerK() >= mpool.GetMinFee(1).GetFeePerK()); } } diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 639c779b360b5..ec193e377ee4c 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -5,7 +5,6 @@ #include "txmempool.h" -#include "clientversion.h" #include "consensus/consensus.h" #include "consensus/validation.h" #include "validation.h" @@ -17,7 +16,6 @@ #include "util.h" #include "utilmoneystr.h" #include "utiltime.h" -#include "version.h" CTxMemPoolEntry::CTxMemPoolEntry(const CTransactionRef& _tx, const CAmount& _nFee, int64_t _nTime, unsigned int _entryHeight, @@ -350,8 +348,8 @@ void CTxMemPoolEntry::UpdateAncestorState(int64_t modifySize, CAmount modifyFee, assert(int(nSigOpCostWithAncestors) >= 0); } -CTxMemPool::CTxMemPool() : - nTransactionsUpdated(0), +CTxMemPool::CTxMemPool(CBlockPolicyEstimator* estimator) : + nTransactionsUpdated(0), minerPolicyEstimator(estimator), names(*this) { _clear(); //lock free clear @@ -360,13 +358,6 @@ CTxMemPool::CTxMemPool() : // accepting transactions becomes O(N^2) where N is the number // of transactions in the pool nCheckFrequency = 0; - - minerPolicyEstimator = new CBlockPolicyEstimator(); -} - -CTxMemPool::~CTxMemPool() -{ - delete minerPolicyEstimator; } void CTxMemPool::pruneSpent(const uint256 &hashTx, CCoins &coins) @@ -445,7 +436,7 @@ bool CTxMemPool::addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry, nTransactionsUpdated++; totalTxSize += entry.GetTxSize(); - minerPolicyEstimator->processTransaction(entry, validFeeEstimate); + if (minerPolicyEstimator) {minerPolicyEstimator->processTransaction(entry, validFeeEstimate);} names.addUnchecked (hash, entry); vTxHashes.emplace_back(tx.GetWitnessHash(), newit); @@ -478,7 +469,7 @@ void CTxMemPool::removeUnchecked(txiter it, MemPoolRemovalReason reason) mapLinks.erase(it); mapTx.erase(it); nTransactionsUpdated++; - minerPolicyEstimator->removeTx(hash); + if (minerPolicyEstimator) {minerPolicyEstimator->removeTx(hash);} } // Calculates descendants of entry that are not already in setDescendants, and adds to @@ -615,7 +606,7 @@ void CTxMemPool::removeForBlock(const std::vector& vtx, unsigne entries.push_back(&*i); } // Before the txs in the new block have been removed from the mempool, update policy estimates - minerPolicyEstimator->processBlock(nBlockHeight, entries); + if (minerPolicyEstimator) {minerPolicyEstimator->processBlock(nBlockHeight, entries);} for (const auto& tx : vtx) { txiter it = mapTx.find(tx->GetHash()); @@ -882,51 +873,6 @@ TxMempoolInfo CTxMemPool::info(const uint256& hash) const return GetInfo(i); } -CFeeRate CTxMemPool::estimateFee(int nBlocks) const -{ - LOCK(cs); - return minerPolicyEstimator->estimateFee(nBlocks); -} -CFeeRate CTxMemPool::estimateSmartFee(int nBlocks, int *answerFoundAtBlocks) const -{ - LOCK(cs); - return minerPolicyEstimator->estimateSmartFee(nBlocks, answerFoundAtBlocks, *this); -} - -bool -CTxMemPool::WriteFeeEstimates(CAutoFile& fileout) const -{ - try { - LOCK(cs); - fileout << 139900; // version required to read: 0.13.99 or later - fileout << CLIENT_VERSION; // version that wrote the file - minerPolicyEstimator->Write(fileout); - } - catch (const std::exception&) { - LogPrintf("CTxMemPool::WriteFeeEstimates(): unable to write policy estimator data (non-fatal)\n"); - return false; - } - return true; -} - -bool -CTxMemPool::ReadFeeEstimates(CAutoFile& filein) -{ - try { - int nVersionRequired, nVersionThatWrote; - filein >> nVersionRequired >> nVersionThatWrote; - if (nVersionRequired > CLIENT_VERSION) - return error("CTxMemPool::ReadFeeEstimates(): up-version (%d) fee estimate file", nVersionRequired); - LOCK(cs); - minerPolicyEstimator->Read(filein, nVersionThatWrote); - } - catch (const std::exception&) { - LogPrintf("CTxMemPool::ReadFeeEstimates(): unable to read policy estimator data (non-fatal)\n"); - return false; - } - return true; -} - void CTxMemPool::PrioritiseTransaction(const uint256& hash, const CAmount& nFeeDelta) { { diff --git a/src/txmempool.h b/src/txmempool.h index 2d0f8f1fb0510..99fbaa083edbf 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -531,8 +531,7 @@ class CTxMemPool /** Create a new CTxMemPool. */ - CTxMemPool(); - ~CTxMemPool(); + CTxMemPool(CBlockPolicyEstimator* estimator = nullptr); /** * If sanity-checking is turned on, check makes sure the pool is @@ -701,19 +700,6 @@ class CTxMemPool TxMempoolInfo info(const uint256& hash) const; std::vector infoAll() const; - /** Estimate fee rate needed to get into the next nBlocks - * If no answer can be given at nBlocks, return an estimate - * at the lowest number of blocks where one can be given - */ - CFeeRate estimateSmartFee(int nBlocks, int *answerFoundAtBlocks = NULL) const; - - /** Estimate fee rate needed to get into the next nBlocks */ - CFeeRate estimateFee(int nBlocks) const; - - /** Write/Read estimates to disk */ - bool WriteFeeEstimates(CAutoFile& fileout) const; - bool ReadFeeEstimates(CAutoFile& filein); - size_t DynamicMemoryUsage() const; boost::signals2::signal NotifyEntryAdded; diff --git a/src/validation.cpp b/src/validation.cpp index 84fa1ec80fdf5..2b2ded72fe604 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -7,6 +7,7 @@ #include "arith_uint256.h" #include "auxpow.h" +#include "chain.h" #include "chainparams.h" #include "checkpoints.h" #include "checkqueue.h" @@ -81,7 +82,8 @@ uint256 hashAssumeValid; CFeeRate minRelayTxFee = CFeeRate(DEFAULT_MIN_RELAY_TX_FEE); CAmount maxTxFee = DEFAULT_TRANSACTION_MAXFEE; -CTxMemPool mempool; +CBlockPolicyEstimator feeEstimator; +CTxMemPool mempool(&feeEstimator); static void CheckBlockIndex(const Consensus::Params& consensusParams); @@ -1623,7 +1625,11 @@ bool ApplyTxInUndo(const CTxInUndo& undo, CCoinsViewCache& view, const COutPoint return fClean; } -bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockIndex* pindex, CCoinsViewCache& view, std::set& unexpiredNames, bool* pfClean) +/** Undo the effects of this block (with given index) on the UTXO set represented by coins. + * In case pfClean is provided, operation will try to be tolerant about errors, and *pfClean + * will be true if no problems were found. Otherwise, the return value will be false in case + * of problems. Note that in any case, coins may be modified. */ +static bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockIndex* pindex, CCoinsViewCache& view, std::set& unexpiredNames, bool* pfClean = NULL) { assert(pindex->GetBlockHash() == view.GetBestBlock()); @@ -1797,9 +1803,13 @@ static int64_t nTimeIndex = 0; static int64_t nTimeCallbacks = 0; static int64_t nTimeTotal = 0; -bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pindex, CCoinsViewCache& view, +/** Apply the effects of this block (with given index) on the UTXO set represented by coins. + * Validity checks that depend on the UTXO set are also done; ConnectBlock() + * can fail if those validity checks fail (among other reasons). */ +static bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pindex, + CCoinsViewCache& view, std::set& expiredNames, - const CChainParams& chainparams, bool fJustCheck) + const CChainParams& chainparams, bool fJustCheck = false) { AssertLockHeld(cs_main); assert(pindex); @@ -2043,12 +2053,6 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin int64_t nTime5 = GetTimeMicros(); nTimeIndex += nTime5 - nTime4; LogPrint(BCLog::BENCH, " - Index writing: %.2fms [%.2fs]\n", 0.001 * (nTime5 - nTime4), nTimeIndex * 0.000001); - // Watch for changes to the previous coinbase transaction. - static uint256 hashPrevBestCoinBase; - GetMainSignals().UpdatedTransaction(hashPrevBestCoinBase); - hashPrevBestCoinBase = block.vtx[0]->GetHash(); - - int64_t nTime6 = GetTimeMicros(); nTimeCallbacks += nTime6 - nTime5; LogPrint(BCLog::BENCH, " - Callbacks: %.2fms [%.2fs]\n", 0.001 * (nTime6 - nTime5), nTimeCallbacks * 0.000001); @@ -2825,7 +2829,7 @@ CBlockIndex* AddToBlockIndex(const CBlockHeader& block) } /** Mark a block as having its data received and checked (up to BLOCK_VALID_TRANSACTIONS). */ -bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBlockIndex *pindexNew, const CDiskBlockPos& pos) +static bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBlockIndex *pindexNew, const CDiskBlockPos& pos, const Consensus::Params& consensusParams) { pindexNew->nTx = block.vtx.size(); pindexNew->nChainTx = 0; @@ -2833,7 +2837,7 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl pindexNew->nDataPos = pos.nPos; pindexNew->nUndoPos = 0; pindexNew->nStatus |= BLOCK_HAVE_DATA; - if (IsWitnessEnabled(pindexNew->pprev, Params().GetConsensus())) { + if (IsWitnessEnabled(pindexNew->pprev, consensusParams)) { pindexNew->nStatus |= BLOCK_OPT_WITNESS; } pindexNew->RaiseValidity(BLOCK_VALID_TRANSACTIONS); @@ -3398,7 +3402,7 @@ static bool AcceptBlock(const std::shared_ptr& pblock, CValidation if (dbp == NULL) if (!WriteBlockToDisk(block, blockPos, chainparams.MessageStart())) AbortNode(state, "Failed to write block"); - if (!ReceivedBlockTransactions(block, state, pindex, blockPos)) + if (!ReceivedBlockTransactions(block, state, pindex, blockPos, chainparams.GetConsensus())) return error("AcceptBlock(): ReceivedBlockTransactions failed"); } catch (const std::runtime_error& e) { return AbortNode(state, std::string("System error: ") + e.what()); @@ -4037,7 +4041,7 @@ bool InitBlockIndex(const CChainParams& chainparams) if (!WriteBlockToDisk(block, blockPos, chainparams.MessageStart())) return error("LoadBlockIndex(): writing genesis block to disk failed"); CBlockIndex *pindex = AddToBlockIndex(block); - if (!ReceivedBlockTransactions(block, state, pindex, blockPos)) + if (!ReceivedBlockTransactions(block, state, pindex, blockPos, chainparams.GetConsensus())) return error("LoadBlockIndex(): genesis block not accepted"); // Force a chainstate write so that when we VerifyDB in a moment, it doesn't check stale data return FlushStateToDisk(state, FLUSH_STATE_ALWAYS); diff --git a/src/validation.h b/src/validation.h index 195a6f5b27dfc..bd3fde2665366 100644 --- a/src/validation.h +++ b/src/validation.h @@ -11,7 +11,6 @@ #endif #include "amount.h" -#include "chain.h" #include "coins.h" #include "fs.h" #include "protocol.h" // For CMessageHeader::MessageStartChars @@ -39,6 +38,7 @@ class CChainParams; class CInv; class CConnman; class CScriptCheck; +class CBlockPolicyEstimator; class CTxInUndo; class CTxMemPool; class CValidationInterface; @@ -169,6 +169,7 @@ struct BlockHasher extern CScript COINBASE_FLAGS; extern CCriticalSection cs_main; +extern CBlockPolicyEstimator feeEstimator; extern CTxMemPool mempool; typedef boost::unordered_map BlockMap; extern BlockMap mapBlockIndex; @@ -505,19 +506,6 @@ bool CheckBlock(const CBlock& block, CValidationState& state, const Consensus::P bool ContextualCheckBlockHeader(const CBlockHeader& block, CValidationState& state, const Consensus::Params& consensusParams, const CBlockIndex* pindexPrev, int64_t nAdjustedTime); bool ContextualCheckBlock(const CBlock& block, CValidationState& state, const Consensus::Params& consensusParams, const CBlockIndex* pindexPrev); -/** Apply the effects of this block (with given index) on the UTXO set represented by coins. - * Validity checks that depend on the UTXO set are also done; ConnectBlock() - * can fail if those validity checks fail (among other reasons). */ -bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pindex, CCoinsViewCache& coins, - std::set& expiredNames, - const CChainParams& chainparams, bool fJustCheck = false); - -/** Undo the effects of this block (with given index) on the UTXO set represented by coins. - * In case pfClean is provided, operation will try to be tolerant about errors, and *pfClean - * will be true if no problems were found. Otherwise, the return value will be false in case - * of problems. Note that in any case, coins may be modified. */ -bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockIndex* pindex, CCoinsViewCache& coins, std::set& unexpiredNames, bool* pfClean = NULL); - /** Check a block is completely valid from start to finish (only works on top of our current best block, with cs_main held) */ bool TestBlockValidity(CValidationState& state, const CChainParams& chainparams, const CBlock& block, CBlockIndex* pindexPrev, bool fCheckPOW = true, bool fCheckMerkleRoot = true); diff --git a/src/validationinterface.cpp b/src/validationinterface.cpp index a1979b4173b38..f8f74ad6f264e 100644 --- a/src/validationinterface.cpp +++ b/src/validationinterface.cpp @@ -17,7 +17,6 @@ void RegisterValidationInterface(CValidationInterface* pwalletIn) { g_signals.TransactionAddedToMempool.connect(boost::bind(&CValidationInterface::TransactionAddedToMempool, pwalletIn, _1)); g_signals.BlockConnected.connect(boost::bind(&CValidationInterface::BlockConnected, pwalletIn, _1, _2, _3, _4)); g_signals.BlockDisconnected.connect(boost::bind(&CValidationInterface::BlockDisconnected, pwalletIn, _1, _2, _3)); - g_signals.UpdatedTransaction.connect(boost::bind(&CValidationInterface::UpdatedTransaction, pwalletIn, _1)); g_signals.SetBestChain.connect(boost::bind(&CValidationInterface::SetBestChain, pwalletIn, _1)); g_signals.Inventory.connect(boost::bind(&CValidationInterface::Inventory, pwalletIn, _1)); g_signals.Broadcast.connect(boost::bind(&CValidationInterface::ResendWalletTransactions, pwalletIn, _1, _2)); @@ -32,7 +31,6 @@ void UnregisterValidationInterface(CValidationInterface* pwalletIn) { g_signals.Broadcast.disconnect(boost::bind(&CValidationInterface::ResendWalletTransactions, pwalletIn, _1, _2)); g_signals.Inventory.disconnect(boost::bind(&CValidationInterface::Inventory, pwalletIn, _1)); g_signals.SetBestChain.disconnect(boost::bind(&CValidationInterface::SetBestChain, pwalletIn, _1)); - g_signals.UpdatedTransaction.disconnect(boost::bind(&CValidationInterface::UpdatedTransaction, pwalletIn, _1)); g_signals.TransactionAddedToMempool.disconnect(boost::bind(&CValidationInterface::TransactionAddedToMempool, pwalletIn, _1)); g_signals.BlockConnected.disconnect(boost::bind(&CValidationInterface::BlockConnected, pwalletIn, _1, _2, _3, _4)); g_signals.BlockDisconnected.disconnect(boost::bind(&CValidationInterface::BlockDisconnected, pwalletIn, _1, _2, _3)); @@ -46,7 +44,6 @@ void UnregisterAllValidationInterfaces() { g_signals.Broadcast.disconnect_all_slots(); g_signals.Inventory.disconnect_all_slots(); g_signals.SetBestChain.disconnect_all_slots(); - g_signals.UpdatedTransaction.disconnect_all_slots(); g_signals.TransactionAddedToMempool.disconnect_all_slots(); g_signals.BlockConnected.disconnect_all_slots(); g_signals.BlockDisconnected.disconnect_all_slots(); diff --git a/src/validationinterface.h b/src/validationinterface.h index 40384c8bbc42a..aa4a10375f55c 100644 --- a/src/validationinterface.h +++ b/src/validationinterface.h @@ -37,7 +37,6 @@ class CValidationInterface { virtual void BlockConnected(const std::shared_ptr &block, const CBlockIndex *pindex, const std::vector &txnConflicted, const std::vector &vNameConflicts) {} virtual void BlockDisconnected(const std::shared_ptr &block, const CBlockIndex *pindexDelete, const std::vector &vNameConflicts) {} virtual void SetBestChain(const CBlockLocator &locator) {} - virtual void UpdatedTransaction(const uint256 &hash) {} virtual void Inventory(const uint256 &hash) {} virtual void ResendWalletTransactions(int64_t nBestBlockTime, CConnman* connman) {} virtual void BlockChecked(const CBlock&, const CValidationState&) {} @@ -61,8 +60,6 @@ struct CMainSignals { boost::signals2::signal &, const CBlockIndex *pindex, const std::vector &, const std::vector &vNameConflicts)> BlockConnected; /** Notifies listeners of a block being disconnected */ boost::signals2::signal &, const CBlockIndex *pindexDelete, const std::vector &vNameConflicts)> BlockDisconnected; - /** Notifies listeners of an updated transaction without new data (for now: a coinbase potentially becoming visible). */ - boost::signals2::signal UpdatedTransaction; /** Notifies listeners of a new active block chain. */ boost::signals2::signal SetBestChain; /** Notifies listeners about an inventory item being seen on the network. */ diff --git a/src/wallet/feebumper.cpp b/src/wallet/feebumper.cpp index 6b030935f37b9..82e59740655e6 100644 --- a/src/wallet/feebumper.cpp +++ b/src/wallet/feebumper.cpp @@ -5,6 +5,7 @@ #include "consensus/validation.h" #include "wallet/feebumper.h" #include "wallet/wallet.h" +#include "policy/fees.h" #include "policy/policy.h" #include "policy/rbf.h" #include "validation.h" //for mempool access @@ -159,11 +160,11 @@ CFeeBumper::CFeeBumper(const CWallet *pWallet, const uint256 txidIn, int newConf } else { // if user specified a confirm target then don't consider any global payTxFee if (specifiedConfirmTarget) { - nNewFee = CWallet::GetMinimumFee(maxNewTxSize, newConfirmTarget, mempool, CAmount(0)); + nNewFee = CWallet::GetMinimumFee(maxNewTxSize, newConfirmTarget, mempool, ::feeEstimator, CAmount(0)); } // otherwise use the regular wallet logic to select payTxFee or default confirm target else { - nNewFee = CWallet::GetMinimumFee(maxNewTxSize, newConfirmTarget, mempool); + nNewFee = CWallet::GetMinimumFee(maxNewTxSize, newConfirmTarget, mempool, ::feeEstimator); } nNewFeeRate = CFeeRate(nNewFee, maxNewTxSize); diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index 773679b678326..0d9fa895a10c8 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -95,6 +95,8 @@ UniValue importprivkey(const JSONRPCRequest& request) + HelpExampleCli("importprivkey", "\"mykey\"") + "\nImport using a label and without rescan\n" + HelpExampleCli("importprivkey", "\"mykey\" \"testing\" false") + + "\nImport using default blank label and without rescan\n" + + HelpExampleCli("importprivkey", "\"mykey\" \"\" false") + "\nAs a JSON-RPC call\n" + HelpExampleRpc("importprivkey", "\"mykey\", \"testing\", false") ); @@ -154,6 +156,31 @@ UniValue importprivkey(const JSONRPCRequest& request) return NullUniValue; } +UniValue abortrescan(const JSONRPCRequest& request) +{ + CWallet* const pwallet = GetWalletForJSONRPCRequest(request); + if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { + return NullUniValue; + } + + if (request.fHelp || request.params.size() > 0) + throw std::runtime_error( + "abortrescan\n" + "\nStops current wallet rescan triggered e.g. by an importprivkey call.\n" + "\nExamples:\n" + "\nImport a private key\n" + + HelpExampleCli("importprivkey", "\"mykey\"") + + "\nAbort the running wallet rescan\n" + + HelpExampleCli("abortrescan", "") + + "\nAs a JSON-RPC call\n" + + HelpExampleRpc("abortrescan", "") + ); + + if (!pwallet->IsScanning() || pwallet->IsAbortingRescan()) return false; + pwallet->AbortRescan(); + return true; +} + void ImportAddress(CWallet*, const CBitcoinAddress& address, const std::string& strLabel); void ImportScript(CWallet* const pwallet, const CScript& script, const std::string& strLabel, bool isRedeemScript) { diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 78b68daa2fcc3..4ff3d76f0fc7f 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -9,8 +9,10 @@ #include "consensus/validation.h" #include "core_io.h" #include "init.h" +#include "wallet/coincontrol.h" #include "validation.h" #include "net.h" +#include "policy/fees.h" #include "policy/policy.h" #include "policy/rbf.h" #include "rpc/server.h" @@ -2941,6 +2943,7 @@ UniValue bumpfee(const JSONRPCRequest& request) return result; } +extern UniValue abortrescan(const JSONRPCRequest& request); // in rpcdump.cpp extern UniValue dumpprivkey(const JSONRPCRequest& request); // in rpcdump.cpp extern UniValue importprivkey(const JSONRPCRequest& request); extern UniValue importaddress(const JSONRPCRequest& request); @@ -2963,6 +2966,7 @@ static const CRPCCommand commands[] = { "rawtransactions", "fundrawtransaction", &fundrawtransaction, false, {"hexstring","options"} }, { "hidden", "resendwallettransactions", &resendwallettransactions, true, {} }, { "wallet", "abandontransaction", &abandontransaction, false, {"txid"} }, + { "wallet", "abortrescan", &abortrescan, false, {} }, { "wallet", "addmultisigaddress", &addmultisigaddress, true, {"nrequired","keys","account"} }, { "wallet", "addwitnessaddress", &addwitnessaddress, true, {"address"} }, { "wallet", "backupwallet", &backupwallet, true, {"destination"} }, diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index 03353619213c4..8eeba72a064b2 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -424,6 +424,17 @@ BOOST_FIXTURE_TEST_CASE(rescan, TestChain100Setup) BOOST_CHECK_EQUAL(response.write(), strprintf("[{\"success\":false,\"error\":{\"code\":-1,\"message\":\"Failed to rescan before time %d, transactions may be missing.\"}},{\"success\":true}]", newTip->GetBlockTimeMax())); ::pwalletMain = backup; } + + // Verify ScanForWalletTransactions does not return null when the scan is + // elided due to the nTimeFirstKey optimization. + { + CWallet wallet; + { + LOCK(wallet.cs_wallet); + wallet.UpdateTimeFirstKey(newTip->GetBlockTime() + 7200 + 1); + } + BOOST_CHECK_EQUAL(newTip, wallet.ScanForWalletTransactions(newTip)); + } } // Check that GetImmatureCredit() returns a newly calculated value instead of diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index baf935a6c1551..b908af7d7e3f3 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -16,6 +16,7 @@ #include "keystore.h" #include "validation.h" #include "net.h" +#include "policy/fees.h" #include "policy/policy.h" #include "policy/rbf.h" #include "primitives/block.h" @@ -1553,18 +1554,21 @@ void CWalletTx::GetAccountAmounts(const std::string& strAccount, CAmount& nRecei * exist in the wallet will be updated. * * Returns pointer to the first block in the last contiguous range that was - * successfully scanned. - * + * successfully scanned or elided (elided if pIndexStart points at a block + * before CWallet::nTimeFirstKey). Returns null if there is no such range, or + * the range doesn't include chainActive.Tip(). */ CBlockIndex* CWallet::ScanForWalletTransactions(CBlockIndex* pindexStart, bool fUpdate) { - CBlockIndex* ret = nullptr; int64_t nNow = GetTime(); const CChainParams& chainParams = Params(); CBlockIndex* pindex = pindexStart; + CBlockIndex* ret = pindexStart; { LOCK2(cs_main, cs_wallet); + fAbortRescan = false; + fScanningWallet = true; // no need to read and scan block, if block was created before // our wallet birthday (as adjusted for block time variability) @@ -1574,7 +1578,7 @@ CBlockIndex* CWallet::ScanForWalletTransactions(CBlockIndex* pindexStart, bool f ShowProgress(_("Rescanning..."), 0); // show rescan progress in GUI as dialog or on splashscreen, if -rescan on startup double dProgressStart = GuessVerificationProgress(chainParams.TxData(), pindex); double dProgressTip = GuessVerificationProgress(chainParams.TxData(), chainActive.Tip()); - while (pindex) + while (pindex && !fAbortRescan) { if (pindex->nHeight % 100 == 0 && dProgressTip - dProgressStart > 0.0) ShowProgress(_("Rescanning..."), std::max(1, std::min(99, (int)((GuessVerificationProgress(chainParams.TxData(), pindex) - dProgressStart) / (dProgressTip - dProgressStart) * 100)))); @@ -1596,7 +1600,12 @@ CBlockIndex* CWallet::ScanForWalletTransactions(CBlockIndex* pindexStart, bool f LogPrintf("Still rescanning. At block %d. Progress=%f\n", pindex->nHeight, GuessVerificationProgress(chainParams.TxData(), pindex)); } } + if (pindex && fAbortRescan) { + LogPrintf("Rescan aborted at block %d. Progress=%f\n", pindex->nHeight, GuessVerificationProgress(chainParams.TxData(), pindex)); + } ShowProgress(_("Rescanning..."), 100); // hide progress dialog in GUI + + fScanningWallet = false; } return ret; } @@ -2697,7 +2706,7 @@ bool CWallet::CreateTransaction(const std::vector& vecSend, if (coinControl && coinControl->nConfirmTarget > 0) currentConfirmationTarget = coinControl->nConfirmTarget; - CAmount nFeeNeeded = GetMinimumFee(nBytes, currentConfirmationTarget, mempool); + CAmount nFeeNeeded = GetMinimumFee(nBytes, currentConfirmationTarget, ::mempool, ::feeEstimator); if (coinControl && nFeeNeeded > 0 && coinControl->nMinimumTotalFee > nFeeNeeded) { nFeeNeeded = coinControl->nMinimumTotalFee; } @@ -2871,19 +2880,19 @@ CAmount CWallet::GetRequiredFee(unsigned int nTxBytes) return std::max(minTxFee.GetFee(nTxBytes), ::minRelayTxFee.GetFee(nTxBytes)); } -CAmount CWallet::GetMinimumFee(unsigned int nTxBytes, unsigned int nConfirmTarget, const CTxMemPool& pool) +CAmount CWallet::GetMinimumFee(unsigned int nTxBytes, unsigned int nConfirmTarget, const CTxMemPool& pool, const CBlockPolicyEstimator& estimator) { // payTxFee is the user-set global for desired feerate - return GetMinimumFee(nTxBytes, nConfirmTarget, pool, payTxFee.GetFee(nTxBytes)); + return GetMinimumFee(nTxBytes, nConfirmTarget, pool, estimator, payTxFee.GetFee(nTxBytes)); } -CAmount CWallet::GetMinimumFee(unsigned int nTxBytes, unsigned int nConfirmTarget, const CTxMemPool& pool, CAmount targetFee) +CAmount CWallet::GetMinimumFee(unsigned int nTxBytes, unsigned int nConfirmTarget, const CTxMemPool& pool, const CBlockPolicyEstimator& estimator, CAmount targetFee) { CAmount nFeeNeeded = targetFee; // User didn't set: use -txconfirmtarget to estimate... if (nFeeNeeded == 0) { int estimateFoundTarget = nConfirmTarget; - nFeeNeeded = pool.estimateSmartFee(nConfirmTarget, &estimateFoundTarget).GetFee(nTxBytes); + nFeeNeeded = estimator.estimateSmartFee(nConfirmTarget, &estimateFoundTarget, pool).GetFee(nTxBytes); // ... unless we don't have enough mempool data for estimatefee, then use fallbackFee if (nFeeNeeded == 0) nFeeNeeded = fallbackFee.GetFee(nTxBytes); @@ -3476,17 +3485,6 @@ void CWallet::GetAllReserveKeys(std::set& setAddress) const } } -void CWallet::UpdatedTransaction(const uint256 &hashTx) -{ - { - LOCK(cs_wallet); - // Only notify UI if this transaction is in this wallet - std::map::const_iterator mi = mapWallet.find(hashTx); - if (mi != mapWallet.end()) - NotifyTransactionChanged(this, hashTx, CT_UPDATED); - } -} - void CWallet::GetScriptForMining(std::shared_ptr &script) { std::shared_ptr rKey = std::make_shared(this); @@ -3957,7 +3955,7 @@ bool CWallet::InitLoadWallet() std::string walletFile = GetArg("-wallet", DEFAULT_WALLET_DAT); - if (walletFile.find_first_of("/\\") != std::string::npos) { + if (boost::filesystem::path(walletFile).filename() != walletFile) { return InitError(_("-wallet parameter must only specify a filename (not a path)")); } else if (SanitizeString(walletFile, SAFE_CHARS_FILENAME) != walletFile) { return InitError(_("Invalid characters in -wallet filename")); diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index e610558d1fe9c..2374950b6b0c2 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -76,6 +76,7 @@ class CReserveKey; class CScript; class CScheduler; class CTxMemPool; +class CBlockPolicyEstimator; class CWalletTx; /** (client) version numbers for particular wallet features */ @@ -581,6 +582,8 @@ class CWallet : public CCryptoKeyStore, public CValidationInterface { private: static std::atomic fFlushScheduled; + std::atomic fAbortRescan; + std::atomic fScanningWallet; /** * Select a set of coins such that nValueRet >= nTargetValue and at least @@ -708,6 +711,8 @@ class CWallet : public CCryptoKeyStore, public CValidationInterface nTimeFirstKey = 0; fBroadcastTransactions = false; nRelockTime = 0; + fAbortRescan = false; + fScanningWallet = false; } std::map mapWallet; @@ -752,6 +757,13 @@ class CWallet : public CCryptoKeyStore, public CValidationInterface void UnlockAllCoins(); void ListLockedCoins(std::vector& vOutpts); + /* + * Rescan abort properties + */ + void AbortRescan() { fAbortRescan = true; } + bool IsAbortingRescan() { return fAbortRescan; } + bool IsScanning() { return fScanningWallet; } + /** * keystore implementation * Generate a new key @@ -864,12 +876,12 @@ class CWallet : public CCryptoKeyStore, public CValidationInterface * Estimate the minimum fee considering user set parameters * and the required fee */ - static CAmount GetMinimumFee(unsigned int nTxBytes, unsigned int nConfirmTarget, const CTxMemPool& pool); + static CAmount GetMinimumFee(unsigned int nTxBytes, unsigned int nConfirmTarget, const CTxMemPool& pool, const CBlockPolicyEstimator& estimator); /** * Estimate the minimum fee considering required fee and targetFee or if 0 * then fee estimation for nConfirmTarget */ - static CAmount GetMinimumFee(unsigned int nTxBytes, unsigned int nConfirmTarget, const CTxMemPool& pool, CAmount targetFee); + static CAmount GetMinimumFee(unsigned int nTxBytes, unsigned int nConfirmTarget, const CTxMemPool& pool, const CBlockPolicyEstimator& estimator, CAmount targetFee); /** * Return the minimum required fee taking into account the * floating relay fee and user set minimum transaction fee @@ -921,8 +933,6 @@ class CWallet : public CCryptoKeyStore, public CValidationInterface bool DelAddressBook(const CTxDestination& address); - void UpdatedTransaction(const uint256 &hashTx) override; - void Inventory(const uint256 &hash) override { { diff --git a/src/zmq/zmqpublishnotifier.cpp b/src/zmq/zmqpublishnotifier.cpp index b2963e9bded68..700c39f66ea82 100644 --- a/src/zmq/zmqpublishnotifier.cpp +++ b/src/zmq/zmqpublishnotifier.cpp @@ -2,6 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include "chain.h" #include "chainparams.h" #include "streams.h" #include "zmqpublishnotifier.h" diff --git a/test/README.md b/test/README.md index dec8db960d399..b40052b898a7c 100644 --- a/test/README.md +++ b/test/README.md @@ -25,6 +25,8 @@ The ZMQ functional test requires a python ZMQ library. To install it: Running tests locally ===================== +Build for your system first. Be sure to enable wallet, utils and daemon when you configure. Tests will not run otherwise. + Functional tests ---------------- diff --git a/test/functional/blockchain.py b/test/functional/blockchain.py index 596aed50ec9cf..de5e01c2e3836 100755 --- a/test/functional/blockchain.py +++ b/test/functional/blockchain.py @@ -6,6 +6,10 @@ Test the following RPCs: - gettxoutsetinfo + - getdifficulty + - getbestblockhash + - getblockhash + - getblockheader - verifychain Tests correspond to code in rpc/blockchain.cpp. @@ -40,6 +44,7 @@ def setup_network(self, split=False): def run_test(self): self._test_gettxoutsetinfo() self._test_getblockheader() + self._test_getdifficulty() self.nodes[0].verifychain(4, 0) def _test_gettxoutsetinfo(self): @@ -57,7 +62,8 @@ def _test_gettxoutsetinfo(self): def _test_getblockheader(self): node = self.nodes[0] - assert_raises_jsonrpc(-5, "Block not found", node.getblockheader, "nonsense") + assert_raises_jsonrpc(-5, "Block not found", + node.getblockheader, "nonsense") besthash = node.getbestblockhash() secondbesthash = node.getblockhash(199) @@ -79,5 +85,11 @@ def _test_getblockheader(self): assert isinstance(int(header['versionHex'], 16), int) assert isinstance(header['difficulty'], Decimal) + def _test_getdifficulty(self): + difficulty = self.nodes[0].getdifficulty() + # 1 hash in 2 should be valid, so difficulty should be 1/2**31 + # binary => decimal => binary math is why we do this check + assert abs(difficulty * 2**31 - 1) < 0.0001 + if __name__ == '__main__': BlockchainTest().main() diff --git a/test/functional/disconnect_ban.py b/test/functional/disconnect_ban.py new file mode 100755 index 0000000000000..3f451d49d23e2 --- /dev/null +++ b/test/functional/disconnect_ban.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +# Copyright (c) 2014-2016 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test node disconnect and ban behavior""" + +from test_framework.mininode import wait_until +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import (assert_equal, + assert_raises_jsonrpc, + connect_nodes_bi, + start_node, + stop_node, + ) + +class DisconnectBanTest(BitcoinTestFramework): + + def __init__(self): + super().__init__() + self.num_nodes = 2 + self.setup_clean_chain = False + + def setup_network(self): + self.nodes = self.setup_nodes() + connect_nodes_bi(self.nodes, 0, 1) + + def run_test(self): + self.log.info("Test setban and listbanned RPCs") + + self.log.info("setban: successfully ban single IP address") + assert_equal(len(self.nodes[1].getpeerinfo()), 2) # node1 should have 2 connections to node0 at this point + self.nodes[1].setban("127.0.0.1", "add") + wait_until(lambda: len(self.nodes[1].getpeerinfo()) == 0) + assert_equal(len(self.nodes[1].getpeerinfo()), 0) # all nodes must be disconnected at this point + assert_equal(len(self.nodes[1].listbanned()), 1) + + self.log.info("clearbanned: successfully clear ban list") + self.nodes[1].clearbanned() + assert_equal(len(self.nodes[1].listbanned()), 0) + self.nodes[1].setban("127.0.0.0/24", "add") + + self.log.info("setban: fail to ban an already banned subnet") + assert_equal(len(self.nodes[1].listbanned()), 1) + assert_raises_jsonrpc(-23, "IP/Subnet already banned", self.nodes[1].setban, "127.0.0.1", "add") + + self.log.info("setban: fail to ban an invalid subnet") + assert_raises_jsonrpc(-30, "Error: Invalid IP/Subnet", self.nodes[1].setban, "127.0.0.1/42", "add") + assert_equal(len(self.nodes[1].listbanned()), 1) # still only one banned ip because 127.0.0.1 is within the range of 127.0.0.0/24 + + self.log.info("setban remove: fail to unban a non-banned subnet") + assert_raises_jsonrpc(-30, "Error: Unban failed", self.nodes[1].setban, "127.0.0.1", "remove") + assert_equal(len(self.nodes[1].listbanned()), 1) + + self.log.info("setban remove: successfully unban subnet") + self.nodes[1].setban("127.0.0.0/24", "remove") + assert_equal(len(self.nodes[1].listbanned()), 0) + self.nodes[1].clearbanned() + assert_equal(len(self.nodes[1].listbanned()), 0) + + self.log.info("setban: test persistence across node restart") + self.nodes[1].setban("127.0.0.0/32", "add") + self.nodes[1].setban("127.0.0.0/24", "add") + self.nodes[1].setban("192.168.0.1", "add", 1) # ban for 1 seconds + self.nodes[1].setban("2001:4d48:ac57:400:cacf:e9ff:fe1d:9c63/19", "add", 1000) # ban for 1000 seconds + listBeforeShutdown = self.nodes[1].listbanned() + assert_equal("192.168.0.1/32", listBeforeShutdown[2]['address']) + wait_until(lambda: len(self.nodes[1].listbanned()) == 3) + + stop_node(self.nodes[1], 1) + + self.nodes[1] = start_node(1, self.options.tmpdir) + listAfterShutdown = self.nodes[1].listbanned() + assert_equal("127.0.0.0/24", listAfterShutdown[0]['address']) + assert_equal("127.0.0.0/32", listAfterShutdown[1]['address']) + assert_equal("/19" in listAfterShutdown[2]['address'], True) + + # Clear ban lists + self.nodes[1].clearbanned() + connect_nodes_bi(self.nodes, 0, 1) + + self.log.info("Test disconnectrnode RPCs") + + self.log.info("disconnectnode: fail to disconnect when calling with address and nodeid") + address1 = self.nodes[0].getpeerinfo()[0]['addr'] + node1 = self.nodes[0].getpeerinfo()[0]['addr'] + assert_raises_jsonrpc(-32602, "Only one of address and nodeid should be provided.", self.nodes[0].disconnectnode, address=address1, nodeid=node1) + + self.log.info("disconnectnode: fail to disconnect when calling with junk address") + assert_raises_jsonrpc(-29, "Node not found in connected nodes", self.nodes[0].disconnectnode, address="221B Baker Street") + + self.log.info("disconnectnode: successfully disconnect node by address") + address1 = self.nodes[0].getpeerinfo()[0]['addr'] + self.nodes[0].disconnectnode(address=address1) + wait_until(lambda: len(self.nodes[0].getpeerinfo()) == 1) + assert not [node for node in self.nodes[0].getpeerinfo() if node['addr'] == address1] + + self.log.info("disconnectnode: successfully reconnect node") + connect_nodes_bi(self.nodes, 0, 1) # reconnect the node + assert_equal(len(self.nodes[0].getpeerinfo()), 2) + assert [node for node in self.nodes[0].getpeerinfo() if node['addr'] == address1] + + self.log.info("disconnectnode: successfully disconnect node by node id") + id1 = self.nodes[0].getpeerinfo()[0]['id'] + self.nodes[0].disconnectnode(nodeid=id1) + wait_until(lambda: len(self.nodes[0].getpeerinfo()) == 1) + assert not [node for node in self.nodes[0].getpeerinfo() if node['id'] == id1] + +if __name__ == '__main__': + DisconnectBanTest().main() diff --git a/test/functional/nodehandling.py b/test/functional/nodehandling.py deleted file mode 100755 index a6b10a0d83d21..0000000000000 --- a/test/functional/nodehandling.py +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2014-2016 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test node handling.""" - -from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import * - -import urllib.parse - -class NodeHandlingTest (BitcoinTestFramework): - - def __init__(self): - super().__init__() - self.num_nodes = 4 - self.setup_clean_chain = False - - def run_test(self): - ########################### - # setban/listbanned tests # - ########################### - assert_equal(len(self.nodes[2].getpeerinfo()), 4) #we should have 4 nodes at this point - self.nodes[2].setban("127.0.0.1", "add") - time.sleep(3) #wait till the nodes are disconected - assert_equal(len(self.nodes[2].getpeerinfo()), 0) #all nodes must be disconnected at this point - assert_equal(len(self.nodes[2].listbanned()), 1) - self.nodes[2].clearbanned() - assert_equal(len(self.nodes[2].listbanned()), 0) - self.nodes[2].setban("127.0.0.0/24", "add") - assert_equal(len(self.nodes[2].listbanned()), 1) - # This will throw an exception because 127.0.0.1 is within range 127.0.0.0/24 - assert_raises_jsonrpc(-23, "IP/Subnet already banned", self.nodes[2].setban, "127.0.0.1", "add") - # This will throw an exception because 127.0.0.1/42 is not a real subnet - assert_raises_jsonrpc(-30, "Error: Invalid IP/Subnet", self.nodes[2].setban, "127.0.0.1/42", "add") - assert_equal(len(self.nodes[2].listbanned()), 1) #still only one banned ip because 127.0.0.1 is within the range of 127.0.0.0/24 - # This will throw an exception because 127.0.0.1 was not added above - assert_raises_jsonrpc(-30, "Error: Unban failed", self.nodes[2].setban, "127.0.0.1", "remove") - assert_equal(len(self.nodes[2].listbanned()), 1) - self.nodes[2].setban("127.0.0.0/24", "remove") - assert_equal(len(self.nodes[2].listbanned()), 0) - self.nodes[2].clearbanned() - assert_equal(len(self.nodes[2].listbanned()), 0) - - ##test persisted banlist - self.nodes[2].setban("127.0.0.0/32", "add") - self.nodes[2].setban("127.0.0.0/24", "add") - self.nodes[2].setban("192.168.0.1", "add", 1) #ban for 1 seconds - self.nodes[2].setban("2001:4d48:ac57:400:cacf:e9ff:fe1d:9c63/19", "add", 1000) #ban for 1000 seconds - listBeforeShutdown = self.nodes[2].listbanned() - assert_equal("192.168.0.1/32", listBeforeShutdown[2]['address']) #must be here - time.sleep(2) #make 100% sure we expired 192.168.0.1 node time - - #stop node - stop_node(self.nodes[2], 2) - - self.nodes[2] = start_node(2, self.options.tmpdir) - listAfterShutdown = self.nodes[2].listbanned() - assert_equal("127.0.0.0/24", listAfterShutdown[0]['address']) - assert_equal("127.0.0.0/32", listAfterShutdown[1]['address']) - assert_equal("/19" in listAfterShutdown[2]['address'], True) - - ########################### - # RPC disconnectnode test # - ########################### - url = urllib.parse.urlparse(self.nodes[1].url) - self.nodes[0].disconnectnode(url.hostname+":"+str(p2p_port(1))) - time.sleep(2) #disconnecting a node needs a little bit of time - for node in self.nodes[0].getpeerinfo(): - assert(node['addr'] != url.hostname+":"+str(p2p_port(1))) - - connect_nodes_bi(self.nodes,0,1) #reconnect the node - found = False - for node in self.nodes[0].getpeerinfo(): - if node['addr'] == url.hostname+":"+str(p2p_port(1)): - found = True - assert(found) - -if __name__ == '__main__': - NodeHandlingTest ().main () diff --git a/test/functional/p2p-segwit.py b/test/functional/p2p-segwit.py index cd7b788eb42c6..3dd78c8b7d246 100755 --- a/test/functional/p2p-segwit.py +++ b/test/functional/p2p-segwit.py @@ -8,7 +8,7 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * from test_framework.script import * -from test_framework.blocktools import create_block, create_coinbase, add_witness_commitment, WITNESS_COMMITMENT_HEADER +from test_framework.blocktools import create_block, create_coinbase, add_witness_commitment, get_witness_script, WITNESS_COMMITMENT_HEADER from test_framework.key import CECKey, CPubKey import time import random @@ -1721,15 +1721,10 @@ def test_getblocktemplate_before_lockin(self): assert('default_witness_commitment' in gbt_results) witness_commitment = gbt_results['default_witness_commitment'] - # TODO: this duplicates some code from blocktools.py, would be nice - # to refactor. # Check that default_witness_commitment is present. - block = CBlock() - witness_root = block.get_merkle_root([ser_uint256(0), ser_uint256(txid)]) - check_commitment = uint256_from_str(hash256(ser_uint256(witness_root)+ser_uint256(0))) - from test_framework.blocktools import WITNESS_COMMITMENT_HEADER - output_data = WITNESS_COMMITMENT_HEADER + ser_uint256(check_commitment) - script = CScript([OP_RETURN, output_data]) + witness_root = CBlock.get_merkle_root([ser_uint256(0), + ser_uint256(txid)]) + script = get_witness_script(witness_root, 0) assert_equal(witness_commitment, bytes_to_hex_str(script)) # undo mocktime diff --git a/test/functional/test_framework/blocktools.py b/test/functional/test_framework/blocktools.py index 2c9a0857dfa8c..5dcf516dc6e98 100644 --- a/test/functional/test_framework/blocktools.py +++ b/test/functional/test_framework/blocktools.py @@ -25,6 +25,13 @@ def create_block(hashprev, coinbase, nTime=None): # From BIP141 WITNESS_COMMITMENT_HEADER = b"\xaa\x21\xa9\xed" + +def get_witness_script(witness_root, witness_nonce): + witness_commitment = uint256_from_str(hash256(ser_uint256(witness_root)+ser_uint256(witness_nonce))) + output_data = WITNESS_COMMITMENT_HEADER + ser_uint256(witness_commitment) + return CScript([OP_RETURN, output_data]) + + # According to BIP141, blocks with witness rules active must commit to the # hash of all in-block transactions including witness. def add_witness_commitment(block, nonce=0): @@ -32,14 +39,12 @@ def add_witness_commitment(block, nonce=0): # transactions, with witnesses. witness_nonce = nonce witness_root = block.calc_witness_merkle_root() - witness_commitment = uint256_from_str(hash256(ser_uint256(witness_root)+ser_uint256(witness_nonce))) # witness_nonce should go to coinbase witness. block.vtx[0].wit.vtxinwit = [CTxInWitness()] block.vtx[0].wit.vtxinwit[0].scriptWitness.stack = [ser_uint256(witness_nonce)] # witness commitment is the last OP_RETURN output in coinbase - output_data = WITNESS_COMMITMENT_HEADER + ser_uint256(witness_commitment) - block.vtx[0].vout.append(CTxOut(0, CScript([OP_RETURN, output_data]))) + block.vtx[0].vout.append(CTxOut(0, get_witness_script(witness_root, witness_nonce))) block.vtx[0].rehash() block.hashMerkleRoot = block.calc_merkle_root() block.rehash() diff --git a/test/functional/test_framework/mininode.py b/test/functional/test_framework/mininode.py index 83e2f3f8f8dc2..5c84ac918a7bd 100755 --- a/test/functional/test_framework/mininode.py +++ b/test/functional/test_framework/mininode.py @@ -661,7 +661,8 @@ def serialize(self, with_witness=False): return r # Calculate the merkle root given a vector of transaction hashes - def get_merkle_root(self, hashes): + @classmethod + def get_merkle_root(cls, hashes): while len(hashes) > 1: newhashes = [] for i in range(0, len(hashes), 2): diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index aad2f6e5778f5..1cd9883890f53 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -27,9 +27,17 @@ # Formatting. Default colors to empty strings. BOLD, BLUE, RED, GREY = ("", ""), ("", ""), ("", ""), ("", "") -TICK = "✓ " -CROSS = "✖ " -CIRCLE = "○ " +try: + # Make sure python thinks it can write unicode to its stdout + "\u2713".encode("utf_8").decode(sys.stdout.encoding) + TICK = "✓ " + CROSS = "✖ " + CIRCLE = "○ " +except UnicodeDecodeError: + TICK = "P " + CROSS = "x " + CIRCLE = "o " + if os.name == 'posix': # primitive formatting on supported # terminal via ANSI escape sequences: @@ -84,7 +92,7 @@ 'multi_rpc.py', 'proxy_test.py', 'signrawtransactions.py', - 'nodehandling.py', + 'disconnect_ban.py', 'decodescript.py', 'blockchain.py', 'disablewallet.py', @@ -170,7 +178,8 @@ 'nulldummy.py', ] -ALL_SCRIPTS = BASE_SCRIPTS + ZMQ_SCRIPTS + EXTENDED_SCRIPTS +# Place EXTENDED_SCRIPTS first since it has the 3 longest running tests +ALL_SCRIPTS = EXTENDED_SCRIPTS + BASE_SCRIPTS + ZMQ_SCRIPTS NON_SCRIPTS = [ # These are python files that live in the functional tests directory, but are not test scripts. @@ -193,6 +202,7 @@ def main(): parser.add_argument('--force', '-f', action='store_true', help='run tests even on platforms where they are disabled by default (e.g. windows).') parser.add_argument('--help', '-h', '-?', action='store_true', help='print help text and exit') parser.add_argument('--jobs', '-j', type=int, default=4, help='how many test scripts to run in parallel. Default=4.') + parser.add_argument('--keepcache', '-k', action='store_true', help='the default behavior is to flush the cache directory on startup. --keepcache retains the cache from the previous testrun.') parser.add_argument('--quiet', '-q', action='store_true', help='only print results summary and failure logs') parser.add_argument('--nozmq', action='store_true', help='do not run the zmq tests') args, unknown_args = parser.parse_known_args() @@ -247,10 +257,9 @@ def main(): if enable_zmq: test_list += ZMQ_SCRIPTS if args.extended: - test_list += EXTENDED_SCRIPTS - # TODO: BASE_SCRIPTS and EXTENDED_SCRIPTS are sorted by runtime - # (for parallel running efficiency). This combined list will is no - # longer sorted. + # place the EXTENDED_SCRIPTS first since the three longest ones + # are there and the list is shorter + test_list = EXTENDED_SCRIPTS + test_list # Remove the test cases that the user has explicitly asked to exclude. if args.exclude: @@ -271,9 +280,23 @@ def main(): check_script_list(config["environment"]["SRCDIR"]) + if not args.keepcache: + shutil.rmtree("%s/test/cache" % config["environment"]["BUILDDIR"], ignore_errors=True) + run_tests(test_list, config["environment"]["SRCDIR"], config["environment"]["BUILDDIR"], config["environment"]["EXEEXT"], args.jobs, args.coverage, passon_args) def run_tests(test_list, src_dir, build_dir, exeext, jobs=1, enable_coverage=False, args=[]): + # Warn if bitcoind is already running (unix only) + try: + if subprocess.check_output(["pidof", "bitcoind"]) is not None: + print("%sWARNING!%s There is already a bitcoind process running on this system. Tests may fail unexpectedly due to resource contention!" % (BOLD[1], BOLD[0])) + except (OSError, subprocess.SubprocessError): + pass + + # Warn if there is a cache directory + cache_dir = "%s/test/cache" % build_dir + if os.path.isdir(cache_dir): + print("%sWARNING!%s There is a cache directory here: %s. If tests fail unexpectedly, try deleting the cache directory." % (BOLD[1], BOLD[0], cache_dir)) #Set env vars if "BITCOIND" not in os.environ: @@ -282,7 +305,7 @@ def run_tests(test_list, src_dir, build_dir, exeext, jobs=1, enable_coverage=Fal tests_dir = src_dir + '/test/functional/' flags = ["--srcdir={}/src".format(build_dir)] + args - flags.append("--cachedir=%s/test/cache" % build_dir) + flags.append("--cachedir=%s" % cache_dir) if enable_coverage: coverage = RPCCoverage() @@ -437,9 +460,10 @@ def check_script_list(src_dir): python_files = set([t for t in os.listdir(script_dir) if t[-3:] == ".py"]) missed_tests = list(python_files - set(map(lambda x: x.split()[0], ALL_SCRIPTS + NON_SCRIPTS + SKIPPED))) if len(missed_tests) != 0: - print("The following scripts are not being run:" + str(missed_tests)) - print("Check the test lists in test_runner.py") - sys.exit(1) + print("%sWARNING!%s The following scripts are not being run: %s. Check the test lists in test_runner.py." % (BOLD[1], BOLD[0], str(missed_tests))) + if os.getenv('TRAVIS') == 'true': + # On travis this warning is an error to prevent merging incomplete commits into master + sys.exit(1) class RPCCoverage(object): """