Merge branch 'bitcoin' into auxpow

Updated regtest chain parameters and tests for the Segwit changes in
Namecoin after  upstream fab17e8.

Removed BIP34Hash from chain parameters as that is never used at all
in Namecoin (only the height is).

Replaced BEGIN/END macros in auxpow code.  At the same time, removed all
other code explicitly dealing with endian-ness in favour of methods
encoding data independently from the system's byte order.

Note that the Qt unit test fails with this commit, but it also fails
upstream in Bitcoin in the same way.  That is likely due to an upstream
bug in bitcoin/bitcoin#11625.

build_msvc/libbitcoin_server/libbitcoin_server.vcxproj.in

@@ -154,12 +154,6 @@
     </Link>
   </ItemDefinitionGroup>
   <ItemGroup>
-    <ClCompile Include="..\..\src\rpc\net.cpp">
-      <ObjectFileName Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">$(IntDir)\netrpc.obj</ObjectFileName>
-      <ObjectFileName Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">$(IntDir)\netrpc.obj</ObjectFileName>
-      <ObjectFileName Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">$(IntDir)\netrpc.obj</ObjectFileName>
-      <ObjectFileName Condition="'$(Configuration)|$(Platform)'=='Release|x64'">$(IntDir)\netrpc.obj</ObjectFileName>
-    </ClCompile>
 @SOURCE_FILES@
   </ItemGroup>
   <ItemGroup>

configure.ac

@@ -295,7 +295,14 @@ if test x$use_sanitizers != x; then
   AX_CHECK_LINK_FLAG(
     [[-fsanitize=$use_sanitizers]],
     [[SANITIZER_LDFLAGS=-fsanitize=$use_sanitizers]],
-    [AC_MSG_ERROR([linker did not accept requested flags, you are missing required libraries])])
+    [AC_MSG_ERROR([linker did not accept requested flags, you are missing required libraries])],
+    [],
+    [AC_LANG_PROGRAM([[
+    #include <cstdint>
+    #include <cstddef>
+    extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { return 0; }
+    __attribute__((weak)) // allow for libFuzzer linking
+    ]],[[]])])
 fi
 
 ERROR_CXXFLAGS=
@@ -1094,7 +1101,7 @@ if test x$use_pkgconfig = xyes; then
       if test x$use_qr != xno; then
         BITCOIN_QT_CHECK([PKG_CHECK_MODULES([QR], [libqrencode], [have_qrencode=yes], [have_qrencode=no])])
       fi
-      if test x$build_bitcoin_cli$build_bitcoin_tx$build_bitcoind$bitcoin_enable_qt$use_tests != xnonononono; then
+      if test x$build_bitcoin_cli$build_bitcoind$bitcoin_enable_qt$use_tests != xnononono; then
         PKG_CHECK_MODULES([EVENT], [libevent],, [AC_MSG_ERROR(libevent not found.)])
         if test x$TARGET_OS != xwindows; then
           PKG_CHECK_MODULES([EVENT_PTHREADS], [libevent_pthreads],, [AC_MSG_ERROR(libevent_pthreads not found.)])
@@ -1119,7 +1126,7 @@ else
   AC_CHECK_HEADER([openssl/ssl.h],, AC_MSG_ERROR(libssl headers missing),)
   AC_CHECK_LIB([ssl],         [main],SSL_LIBS=-lssl, AC_MSG_ERROR(libssl missing))
 
-  if test x$build_bitcoin_cli$build_bitcoin_tx$build_bitcoind$bitcoin_enable_qt$use_tests != xnonononono; then
+  if test x$build_bitcoin_cli$build_bitcoind$bitcoin_enable_qt$use_tests != xnononono; then
     AC_CHECK_HEADER([event2/event.h],, AC_MSG_ERROR(libevent headers missing),)
     AC_CHECK_LIB([event],[main],EVENT_LIBS=-levent,AC_MSG_ERROR(libevent missing))
     if test x$TARGET_OS != xwindows; then

doc/README.md

@@ -5,7 +5,7 @@ Setup
 ---------------------
 Bitcoin Core is the original Bitcoin client and it builds the backbone of the network. It downloads and, by default, stores the entire history of Bitcoin transactions, which requires a few hundred gigabytes of disk space. Depending on the speed of your computer and network connection, the synchronization process can take anywhere from a few hours to a day or more.
 
-To download Bitcoin Core, visit [bitcoincore.org](https://bitcoincore.org/en/releases/).
+To download Bitcoin Core, visit [bitcoincore.org](https://bitcoincore.org/en/download/).
 
 Running
 ---------------------
@@ -41,9 +41,10 @@ The following are developer notes on how to build Bitcoin Core on your native pl
 - [macOS Build Notes](build-osx.md)
 - [Unix Build Notes](build-unix.md)
 - [Windows Build Notes](build-windows.md)
+- [FreeBSD Build Notes](build-freebsd.md)
 - [OpenBSD Build Notes](build-openbsd.md)
 - [NetBSD Build Notes](build-netbsd.md)
-- [Gitian Building Guide](gitian-building.md)
+- [Gitian Building Guide (External Link)](https://github.com/bitcoin-core/docs/blob/master/gitian-building.md)
 
 Development
 ---------------------

doc/fuzzing.md

@@ -3,10 +3,11 @@ Fuzz-testing Bitcoin Core
 
 A special test harness `test_bitcoin_fuzzy` is provided to provide an easy
 entry point for fuzzers and the like. In this document we'll describe how to
-use it with AFL.
+use it with AFL and libFuzzer.
 
-Building AFL
--------------
+## AFL
+
+### Building AFL
 
 It is recommended to always use the latest version of afl:
 ```
@@ -17,8 +18,7 @@ make
 export AFLPATH=$PWD
 ```
 
-Instrumentation
-----------------
+### Instrumentation
 
 To build Bitcoin Core using AFL instrumentation (this assumes that the
 `AFLPATH` was set as above):
@@ -39,8 +39,7 @@ compiling using `afl-clang-fast`/`afl-clang-fast++` the resulting
 features "persistent mode" and "deferred forkserver" can be used. See
 https://github.com/mcarpenter/afl/tree/master/llvm_mode for details.
 
-Preparing fuzzing
-------------------
+### Preparing fuzzing
 
 AFL needs an input directory with examples, and an output directory where it
 will place examples that it found. These can be anywhere in the file system,
@@ -60,8 +59,7 @@ Example inputs are available from:
 
 Extract these (or other starting inputs) into the `inputs` directory before starting fuzzing.
 
-Fuzzing
---------
+### Fuzzing
 
 To start the actual fuzzing use:
 ```
@@ -70,3 +68,21 @@ $AFLPATH/afl-fuzz -i ${AFLIN} -o ${AFLOUT} -m52 -- test/test_bitcoin_fuzzy
 
 You may have to change a few kernel parameters to test optimally - `afl-fuzz`
 will print an error and suggestion if so.
+
+## libFuzzer
+
+A recent version of `clang`, the address sanitizer and libFuzzer is needed (all
+found in the `compiler-rt` runtime libraries package).
+
+To build the `test/test_bitcoin_fuzzy` executable run
+
+```
+./configure --disable-ccache --with-sanitizers=fuzzer,address CC=clang CXX=clang++
+make
+```
+
+The fuzzer needs some inputs to work on, but the inputs or seeds can be used
+interchangably between libFuzzer and AFL.
+
+See https://llvm.org/docs/LibFuzzer.html#running on how to run the libFuzzer
+instrumented executable.

doc/release-process.md

@@ -23,7 +23,7 @@ Before every minor and major release:
 Before every major release:
 
 * Update hardcoded [seeds](/contrib/seeds/README.md), see [this pull request](https://github.com/bitcoin/bitcoin/pull/7415) for an example.
-* Update [`BLOCK_CHAIN_SIZE`](/src/qt/intro.cpp) to the current size plus some overhead.
+* Update [`src/chainparams.cpp`](/src/chainparams.cpp) m_assumed_blockchain_size and m_assumed_chain_state_size with the current size plus some overhead.
 * Update `src/chainparams.cpp` chainTxData with statistics about the transaction count and rate. Use the output of the RPC `getchaintxstats`, see
   [this pull request](https://github.com/bitcoin/bitcoin/pull/12270) for an example. Reviewers can verify the results by running `getchaintxstats <window_block_count> <window_last_block_hash>` with the `window_block_count` and `window_last_block_hash` from your output.
 * Update version of `contrib/gitian-descriptors/*.yml`: usually one'd want to do this on master after branching off the release - but be sure to at least do it before a new major release

src/Makefile.qt.include

@@ -120,6 +120,7 @@ QT_MOC_CPP = \
   qt/moc_bantablemodel.cpp \
   qt/moc_bitcoinaddressvalidator.cpp \
   qt/moc_bitcoinamountfield.cpp \
+  qt/moc_bitcoin.cpp \
   qt/moc_bitcoingui.cpp \
   qt/moc_bitcoinunits.cpp \
   qt/moc_clientmodel.cpp \
@@ -166,7 +167,6 @@ BITCOIN_MM = \
   qt/macos_appnap.mm
 
 QT_MOC = \
-  qt/bitcoin.moc \
   qt/bitcoinamountfield.moc \
   qt/intro.moc \
   qt/overviewpage.moc \
@@ -194,6 +194,7 @@ BITCOIN_QT_H = \
   qt/bantablemodel.h \
   qt/bitcoinaddressvalidator.h \
   qt/bitcoinamountfield.h \
+  qt/bitcoin.h \
   qt/bitcoingui.h \
   qt/bitcoinunits.h \
   qt/clientmodel.h \
@@ -302,6 +303,7 @@ RES_ICONS = \
 
 BITCOIN_QT_BASE_CPP = \
   qt/bantablemodel.cpp \
+  qt/bitcoin.cpp \
   qt/bitcoinaddressvalidator.cpp \
   qt/bitcoinamountfield.cpp \
   qt/bitcoingui.cpp \
@@ -382,6 +384,9 @@ qt_libbitcoinqt_a_OBJCXXFLAGS = $(AM_OBJCXXFLAGS) $(QT_PIE_FLAGS)
 
 qt_libbitcoinqt_a_SOURCES = $(BITCOIN_QT_CPP) $(BITCOIN_QT_H) $(QT_FORMS_UI) \
   $(QT_QRC) $(QT_QRC_LOCALE) $(QT_TS) $(PROTOBUF_PROTO) $(RES_ICONS) $(RES_IMAGES) $(RES_MOVIES)
+if TARGET_DARWIN
+  qt_libbitcoinqt_a_SOURCES += $(BITCOIN_MM)
+endif
 
 nodist_qt_libbitcoinqt_a_SOURCES = $(QT_MOC_CPP) $(QT_MOC) $(PROTOBUF_CC) \
   $(PROTOBUF_H) $(QT_QRC_CPP) $(QT_QRC_LOCALE_CPP)
@@ -404,10 +409,7 @@ qt_bitcoin_qt_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BITCOIN_QT_INCLUDE
   $(QT_INCLUDES) $(PROTOBUF_CFLAGS) $(QR_CFLAGS)
 qt_bitcoin_qt_CXXFLAGS = $(AM_CXXFLAGS) $(QT_PIE_FLAGS)
 
-qt_bitcoin_qt_SOURCES = qt/bitcoin.cpp
-if TARGET_DARWIN
-  qt_bitcoin_qt_SOURCES += $(BITCOIN_MM)
-endif
+qt_bitcoin_qt_SOURCES = qt/main.cpp
 if TARGET_WINDOWS
   qt_bitcoin_qt_SOURCES += $(BITCOIN_RC)
 endif

src/Makefile.qttest.include

@@ -6,6 +6,7 @@ bin_PROGRAMS += qt/test/test_bitcoin-qt
 TESTS += qt/test/test_bitcoin-qt
 
 TEST_QT_MOC_CPP = \
+  qt/test/moc_apptests.cpp \
   qt/test/moc_compattests.cpp \
   qt/test/moc_rpcnestedtests.cpp \
   qt/test/moc_uritests.cpp
@@ -22,6 +23,7 @@ endif # ENABLE_WALLET
 
 TEST_QT_H = \
   qt/test/addressbooktests.h \
+  qt/test/apptests.h \
   qt/test/compattests.h \
   qt/test/rpcnestedtests.h \
   qt/test/uritests.h \
@@ -40,6 +42,7 @@ qt_test_test_bitcoin_qt_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BITCOIN_
   $(QT_INCLUDES) $(QT_TEST_INCLUDES) $(PROTOBUF_CFLAGS)
 
 qt_test_test_bitcoin_qt_SOURCES = \
+  qt/test/apptests.cpp \
   qt/test/compattests.cpp \
   qt/test/rpcnestedtests.cpp \
   qt/test/test_main.cpp \

src/addrman.h

@@ -23,33 +23,31 @@
  */
 class CAddrInfo : public CAddress
 {
-
-
 public:
     //! last try whatsoever by us (memory only)
-    int64_t nLastTry;
+    int64_t nLastTry{0};
 
     //! last counted attempt (memory only)
-    int64_t nLastCountAttempt;
+    int64_t nLastCountAttempt{0};
 
 private:
     //! where knowledge about this address first came from
     CNetAddr source;
 
     //! last successful connection by us
-    int64_t nLastSuccess;
+    int64_t nLastSuccess{0};
 
     //! connection attempts since last successful attempt
-    int nAttempts;
+    int nAttempts{0};
 
     //! reference count in new sets (memory only)
-    int nRefCount;
+    int nRefCount{0};
 
     //! in tried set? (memory only)
-    bool fInTried;
+    bool fInTried{false};
 
     //! position in vRandom
-    int nRandomPos;
+    int nRandomPos{-1};
 
     friend class CAddrMan;
 
@@ -65,25 +63,12 @@ class CAddrInfo : public CAddress
         READWRITE(nAttempts);
     }
 
-    void Init()
-    {
-        nLastSuccess = 0;
-        nLastTry = 0;
-        nLastCountAttempt = 0;
-        nAttempts = 0;
-        nRefCount = 0;
-        fInTried = false;
-        nRandomPos = -1;
-    }
-
     CAddrInfo(const CAddress &addrIn, const CNetAddr &addrSource) : CAddress(addrIn), source(addrSource)
     {
-        Init();
     }
 
     CAddrInfo() : CAddress(), source()
     {
-        Init();
     }
 
     //! Calculate in which "tried" bucket this entry belongs
@@ -106,7 +91,6 @@ class CAddrInfo : public CAddress
 
     //! Calculate the relative chance this entry should be given when selecting nodes to connect to
     double GetChance(int64_t nNow = GetAdjustedTime()) const;
-
 };
 
 /** Stochastic address manager

src/auxpow.cpp

@@ -1,13 +1,12 @@
 // Copyright (c) 2009-2010 Satoshi Nakamoto
 // Copyright (c) 2011 Vince Durham
 // Copyright (c) 2009-2014 The Bitcoin developers
-// Copyright (c) 2014-2017 Daniel Kraft
+// Copyright (c) 2014-2019 Daniel Kraft
 // Distributed under the MIT/X11 software license, see the accompanying
 // file license.txt or http://www.opensource.org/licenses/mit-license.php.
 
 #include <auxpow.h>
 
-#include <compat/endian.h>
 #include <consensus/consensus.h>
 #include <consensus/merkle.h>
 #include <hash.h>
@@ -19,6 +18,26 @@
 #include <algorithm>
 #include <cassert>
 
+namespace
+{
+
+/**
+ * Decodes a 32-bit little endian integer from raw bytes.
+ */
+uint32_t
+DecodeLE32 (const unsigned char* bytes)
+{
+  uint32_t res = 0;
+  for (int i = 0; i < 4; ++i)
+    {
+      res <<= 8;
+      res |= bytes[3 - i];
+    }
+  return res;
+}
+
+} // anonymous namespace
+
 bool
 CAuxPow::check (const uint256& hashAuxBlock, int nChainId,
                 const Consensus::Params& params) const
@@ -49,8 +68,11 @@ CAuxPow::check (const uint256& hashAuxBlock, int nChainId,
     // Check that the same work is not submitted twice to our chain.
     //
 
+    const unsigned char* const mmHeaderBegin = pchMergedMiningHeader;
+    const unsigned char* const mmHeaderEnd
+        = mmHeaderBegin + sizeof (pchMergedMiningHeader);
     CScript::const_iterator pcHead =
-        std::search(script.begin(), script.end(), UBEGIN(pchMergedMiningHeader), UEND(pchMergedMiningHeader));
+        std::search(script.begin(), script.end(), mmHeaderBegin, mmHeaderEnd);
 
     CScript::const_iterator pc =
         std::search(script.begin(), script.end(), vchRootHash.begin(), vchRootHash.end());
@@ -62,7 +84,8 @@ CAuxPow::check (const uint256& hashAuxBlock, int nChainId,
     {
         // Enforce only one chain merkle root by checking that a single instance of the merged
         // mining header exists just before.
-        if (script.end() != std::search(pcHead + 1, script.end(), UBEGIN(pchMergedMiningHeader), UEND(pchMergedMiningHeader)))
+        if (script.end() != std::search(pcHead + 1, script.end(),
+                                        mmHeaderBegin, mmHeaderEnd))
             return error("Multiple merged mining headers in coinbase");
         if (pcHead + sizeof(pchMergedMiningHeader) != pc)
             return error("Merged mining header is not just before chain merkle root");
@@ -83,16 +106,12 @@ CAuxPow::check (const uint256& hashAuxBlock, int nChainId,
     if (script.end() - pc < 8)
         return error("Aux POW missing chain merkle tree size and nonce in parent coinbase");
 
-    uint32_t nSize;
-    memcpy(&nSize, &pc[0], 4);
-    nSize = le32toh (nSize);
+    const uint32_t nSize = DecodeLE32 (&pc[0]);
     const unsigned merkleHeight = vChainMerkleBranch.size ();
     if (nSize != (1u << merkleHeight))
         return error("Aux POW merkle branch size does not match parent coinbase");
 
-    uint32_t nNonce;
-    memcpy(&nNonce, &pc[4], 4);
-    nNonce = le32toh (nNonce);
+    const uint32_t nNonce = DecodeLE32 (&pc[4]);
     if (nChainIndex != getExpectedIndex (nNonce, nChainId, merkleHeight))
         return error("Aux POW wrong index");
 
@@ -136,9 +155,9 @@ CAuxPow::CheckMerkleBranch (uint256 hash,
        it != vMerkleBranch.end (); ++it)
   {
     if (nIndex & 1)
-      hash = Hash (BEGIN (*it), END (*it), BEGIN (hash), END (hash));
+      hash = Hash (it->begin (), it->end (), hash.begin (), hash.end ());
     else
-      hash = Hash (BEGIN (hash), END (hash), BEGIN (*it), END (*it));
+      hash = Hash (hash.begin (), hash.end (), it->begin (), it->end ());
     nIndex >>= 1;
   }
   return hash;