Merge Bitcoin 0.17.1 into Unit-e, take 2

.travis.yml

@@ -1,32 +1,36 @@
 branches:
   only:
     - master
-sudo: required
 dist: trusty
 os: linux
 language: minimal
 cache:
+  ccache: true
   directories:
     - depends/built
     - depends/sdk-sources
     - $HOME/.ccache
+stages:
+  - lint
+  - test
 git:
   depth: false  # full clone for git subtree check, this works around issue bitcoin/bitcoin#12388
 env:
   global:
     - MAKEJOBS=-j3
     - RUN_TESTS=false
-    - CHECK_DOC=0
+    - RUN_BENCH=false  # Set to true for any one job that has debug enabled, to quickly check bench is not crashing or hitting assertions
+    - DOCKER_NAME_TAG=ubuntu:18.04
+    - LC_ALL=C.UTF-8
     - BOOST_TEST_RANDOM=1$TRAVIS_BUILD_ID
     - CCACHE_SIZE=100M
     - CCACHE_TEMPDIR=/tmp/.ccache-temp
     - CCACHE_COMPRESS=1
+    - CCACHE_DIR=$HOME/.ccache
     - BASE_OUTDIR=$TRAVIS_BUILD_DIR/out
     - SDK_URL=https://bitcoincore.org/depends-sources/sdks
     - WINEDEBUG=fixme-all
-stages:
-  - lint
-  - test
+    - DOCKER_PACKAGES="build-essential libtool autotools-dev automake pkg-config bsdmainutils curl git ca-certificates ccache"
 before_install:
   - set -o errexit; source .travis/test_03_before_install.sh
 install:
@@ -36,63 +40,38 @@ before_script:
 script:
   - set -o errexit; source .travis/test_06_script.sh
 after_script:
-  - echo $TRAVIS_COMMIT_RANGE
-  - echo $TRAVIS_COMMIT_LOG
+    - echo $TRAVIS_COMMIT_RANGE
+    - echo $TRAVIS_COMMIT_LOG
 jobs:
   include:
     - stage: lint
       env:
-      dist: trusty
-      os: linux
-      sudo: false
       cache: false
       language: python
-      python: '2.7'
-      addons:
-        apt:
-          packages:
-            - python3-flake8
-      before_install:
-        - set -o errexit; source .travis/lint_03_before_install.sh
+      python: '3.6'
       install:
         - set -o errexit; source .travis/lint_04_install.sh
       before_script:
         - set -o errexit; source .travis/lint_05_before_script.sh
       script:
         - set -o errexit; source .travis/lint_06_script.sh
-
     - stage: test
       name: "ARM"
       env: >-
         HOST=arm-linux-gnueabihf
-        PACKAGES="g++-arm-linux-gnueabihf python3-pip"
+        PACKAGES="g++-arm-linux-gnueabihf"
         DEP_OPTS="NO_USB=1"
-        CHECK_DOC=1
         GOAL="install"
         UNITE_CONFIG="--enable-glibc-back-compat --enable-reduce-exports"
-
     - stage: test
       name: "Win32 (unit tests)"
       env: >-
         HOST=i686-w64-mingw32
         DPKG_ADD_ARCH="i386"
-        PACKAGES="python3 nsis g++-mingw-w64-i686 wine1.6"
+        PACKAGES="python3 nsis g++-mingw-w64-i686 wine-binfmt wine32"
         RUN_TESTS=true
         GOAL="install"
         UNITE_CONFIG="--enable-reduce-exports --enable-usbdevice"
-
-    - stage: test
-      name: "System libs (unit and functional tests)"
-      if: type = cron
-      env: >-
-        HOST=x86_64-unknown-linux-gnu
-        PACKAGES="python3-zmq libssl-dev libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-program-options-dev libboost-test-dev libboost-thread-dev libdb5.1++-dev libminiupnpc-dev libzmq3-dev libprotobuf-dev protobuf-compiler libqrencode-dev xvfb libhidapi-dev"
-        NO_DEPENDS=1
-        NEED_XVFB=1
-        RUN_TESTS=true
-        GOAL="install"
-        UNITE_CONFIG="--enable-zmq --with-incompatible-bdb --enable-glibc-back-compat --enable-reduce-exports CPPFLAGS=-DDEBUG_LOCKORDER --enable-usbdevice"
-
     - stage: test
       name: "32-bit + dash (unit and functional tests)"
       env: >-
@@ -102,29 +81,16 @@ jobs:
         RUN_TESTS=true
         GOAL="install"
         UNITE_CONFIG="--enable-zmq --enable-glibc-back-compat --enable-reduce-exports LDFLAGS=-static-libstdc++ --enable-usbdevice"
-        USE_SHELL="/bin/dash"
-
-    - stage: test
-      name: "Win64 (unit tests)"
-      if: type = cron
-      env: >-
-        HOST=x86_64-w64-mingw32
-        DPKG_ADD_ARCH="i386"
-        PACKAGES="python3 nsis g++-mingw-w64-x86-64 wine1.6"
-        RUN_TESTS=true
-        GOAL="install"
-        UNITE_CONFIG="--enable-reduce-exports --enable-usbdevice"
-
+        CONFIG_SHELL="/bin/dash"
     - stage: test
       name: "x86_64 Linux (unit and functional tests)"
       env: >-
         HOST=x86_64-unknown-linux-gnu
-        PACKAGES="python3-zmq protobuf-compiler libdbus-1-dev libharfbuzz-dev libudev-dev"
-        DEP_OPTS="NO_UPNP=1 DEBUG=1 ALLOW_HOST_PACKAGES=1"
+        PACKAGES="python3-zmq libssl1.0-dev libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libudev-dev libhidapi-dev libusb-1.0-0-dev"
+        NO_DEPENDS=1
         RUN_TESTS=true
         GOAL="install"
-        UNITE_CONFIG="--enable-zmq --enable-glibc-back-compat --enable-reduce-exports CPPFLAGS=-DDEBUG_LOCKORDER --enable-usbdevice"
-
+        UNITE_CONFIG="--enable-zmq --with-incompatible-bdb --enable-glibc-back-compat --enable-reduce-exports CPPFLAGS=-DDEBUG_LOCKORDER --enable-usbdevice"
     - stage: test
       name: "x86_64 Linux, No wallet (unit tests)"
       env: >-
@@ -134,12 +100,11 @@ jobs:
         RUN_TESTS=true
         GOAL="install"
         UNITE_CONFIG="--enable-glibc-back-compat --enable-reduce-exports"
-
     - stage: test
       name: "Cross-Mac"
       env: >-
-        HOST=x86_64-apple-darwin11
-        PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python-dev"
-        UNITE_CONFIG="--enable-reduce-exports --enable-werror --enable-usbdevice"
+        HOST=x86_64-apple-darwin14
+        PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python-dev python3-setuptools-git"
         OSX_SDK=10.11
-        GOAL="install"
+        GOAL="all deploy"
+        UNITE_CONFIG="--enable-reduce-exports --enable-werror --enable-usbdevice"

.travis/lint_03_before_install.sh

@@ -1 +1,3 @@
 #!/usr/bin/env bash
+
+export LC_ALL=C

.travis/lint_04_install.sh

@@ -1 +1,5 @@
 #!/usr/bin/env bash
+
+export LC_ALL=C
+
+travis_retry pip install flake8==3.5.0

.travis/lint_05_before_script.sh

@@ -5,4 +5,3 @@
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 export LC_ALL=C
-

.travis/lint_06_script.sh

@@ -4,11 +4,16 @@
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
-./contrib/devtools/git-subtree-check.sh src/crypto/ctaes
-./contrib/devtools/git-subtree-check.sh src/secp256k1
-./contrib/devtools/git-subtree-check.sh src/univalue
-./contrib/devtools/git-subtree-check.sh src/leveldb
-./contrib/devtools/check-doc.py
-./contrib/devtools/check-rpc-mappings.py .
-./contrib/devtools/lint-all.sh
+export LC_ALL=C
 
+if [ "$TRAVIS_EVENT_TYPE" = "pull_request" ]; then
+  test/lint/commit-script-check.sh $TRAVIS_COMMIT_RANGE
+fi
+
+test/lint/git-subtree-check.sh src/crypto/ctaes || true
+test/lint/git-subtree-check.sh src/secp256k1 || true
+test/lint/git-subtree-check.sh src/univalue || true
+test/lint/git-subtree-check.sh src/leveldb || true
+test/lint/check-doc.py
+test/lint/check-rpc-mappings.py .
+test/lint/lint-all.sh || true

.travis/test_03_before_install.sh

@@ -4,7 +4,19 @@
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
+export LC_ALL=C
 
 PATH=$(echo $PATH | tr ':' "\n" | sed '/\/opt\/python/d' | tr "\n" ":" | sed "s|::|:|g")
 export PATH
 
+BEGIN_FOLD () {
+  echo ""
+  CURRENT_FOLD_NAME=$1
+  echo "travis_fold:start:${CURRENT_FOLD_NAME}"
+}
+
+END_FOLD () {
+  RET=$?
+  echo "travis_fold:end:${CURRENT_FOLD_NAME}"
+  return $RET
+}

.travis/test_04_install.sh

@@ -4,15 +4,22 @@
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
+export LC_ALL=C
 
-if [ -n "$DPKG_ADD_ARCH" ]; then
-  sudo dpkg --add-architecture "$DPKG_ADD_ARCH"
+travis_retry docker pull $DOCKER_NAME_TAG
+env | grep -E '^(CCACHE_|WINEDEBUG|LC_ALL|BOOST_TEST_RANDOM|CONFIG_SHELL)' | tee /tmp/env
+if [[ $HOST = *-mingw32 ]]; then
+  DOCKER_ADMIN="--cap-add SYS_ADMIN"
 fi
+DOCKER_ID=$(docker run $DOCKER_ADMIN -idt --mount type=bind,src=$TRAVIS_BUILD_DIR,dst=$TRAVIS_BUILD_DIR --mount type=bind,src=$CCACHE_DIR,dst=$CCACHE_DIR -w $TRAVIS_BUILD_DIR --env-file /tmp/env $DOCKER_NAME_TAG)
 
-if [ -n "$PACKAGES" ]; then
-  travis_retry sudo apt-get update
-fi
+DOCKER_EXEC () {
+  docker exec $DOCKER_ID bash -c "cd $PWD && $*";
+}
 
-if [ -n "$PACKAGES" ]; then
-  travis_retry sudo apt-get install --no-install-recommends --no-upgrade -qq $PACKAGES
+if [ -n "$DPKG_ADD_ARCH" ]; then
+  DOCKER_EXEC dpkg --add-architecture "$DPKG_ADD_ARCH"
 fi
+
+travis_retry DOCKER_EXEC apt-get update
+travis_retry DOCKER_EXEC apt-get install --no-install-recommends --no-upgrade -qq $PACKAGES $DOCKER_PACKAGES

.travis/test_05_before_script.sh

@@ -4,9 +4,9 @@
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
+export LC_ALL=C
 
-unset CC
-unset CXX
+DOCKER_EXEC echo \> \$HOME/.unite  # Make sure default datadir does not exist and is never read by creating a dummy file
 
 mkdir -p depends/SDKs depends/sdk-sources
 
@@ -18,12 +18,10 @@ if [ -n "$OSX_SDK" -a -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then
   tar -C depends/SDKs -xf depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz
 fi
 
-if [ -z "$NO_DEPENDS" ]; then
-  make $MAKEJOBS -C depends HOST=$HOST $DEP_OPTS
+if [[ $HOST = *-mingw32 ]]; then
+  DOCKER_EXEC update-alternatives --set $HOST-g++ \$\(which $HOST-g++-posix\)
 fi
 
-# Start xvfb if needed, as documented at https://docs.travis-ci.com/user/gui-and-headless-browsers/#Using-xvfb-to-Run-Tests-That-Require-a-GUI
-if [ "$NEED_XVFB" = 1 ]; then
-  export DISPLAY=:99.0; /sbin/start-stop-daemon --start --pidfile /tmp/custom_xvfb_99.pid --make-pidfile --background --exec /usr/bin/Xvfb -- :99 -ac
+if [ -z "$NO_DEPENDS" ]; then
+  DOCKER_EXEC CONFIG_SHELL= make $MAKEJOBS -C depends HOST=$HOST $DEP_OPTS
 fi
-

.travis/test_06_script.sh

@@ -4,6 +4,7 @@
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
+export LC_ALL=C
 
 TRAVIS_COMMIT_LOG=$(git log --format=fuller -1)
 export TRAVIS_COMMIT_LOG
@@ -16,35 +17,55 @@ OUTDIR=$BASE_OUTDIR/$TRAVIS_PULL_REQUEST/$TRAVIS_JOB_NUMBER-$HOST
 UNITE_CONFIG_ALL="--disable-dependency-tracking --prefix=$TRAVIS_BUILD_DIR/depends/$HOST --bindir=$OUTDIR/bin --libdir=$OUTDIR/lib"
 
 if [ -z "$NO_DEPENDS" ]; then
-  depends/$HOST/native/bin/ccache --max-size=$CCACHE_SIZE
+  DOCKER_EXEC ccache --max-size=$CCACHE_SIZE
 fi
 
-test -n "$USE_SHELL" && eval '"$USE_SHELL" -c "./autogen.sh"' || ./autogen.sh
+BEGIN_FOLD autogen
+if [ -n "$CONFIG_SHELL" ] ; then
+  DOCKER_EXEC "$CONFIG_SHELL" -c "./autogen.sh"
+else
+  DOCKER_EXEC ./autogen.sh
+fi
+END_FOLD
 
 mkdir build && cd build
 
-../configure --cache-file=config.cache $UNITE_CONFIG_ALL $UNITE_CONFIG || ( cat config.log && false)
+BEGIN_FOLD configure
+DOCKER_EXEC ../configure --cache-file=config.cache $UNITE_CONFIG_ALL $UNITE_CONFIG || ( cat config.log && false)
+END_FOLD
 
-make distdir VERSION=$HOST
+BEGIN_FOLD distdir
+DOCKER_EXEC make distdir VERSION=$HOST
+END_FOLD
 
 cd unit-e-$HOST
 
-./configure --cache-file=../config.cache $UNITE_CONFIG_ALL $UNITE_CONFIG || ( cat config.log && false)
-
-make $MAKEJOBS $GOAL || ( echo "Build failure. Verbose build follows." && make $GOAL V=1 ; false )
+BEGIN_FOLD configure
+DOCKER_EXEC ./configure --cache-file=../config.cache $UNITE_CONFIG_ALL $UNITE_CONFIG || ( cat config.log && false)
+END_FOLD
 
-export LD_LIBRARY_PATH=$TRAVIS_BUILD_DIR/depends/$HOST/lib
+BEGIN_FOLD build
+DOCKER_EXEC make $MAKEJOBS $GOAL || ( echo "Build failure. Verbose build follows." && DOCKER_EXEC make $GOAL V=1 ; false )
+END_FOLD
 
 if [ "$RUN_TESTS" = "true" ]; then
-  while sleep 9m; do echo "=====[ $SECONDS seconds still running ]====="; done &
-  make $MAKEJOBS check VERBOSE=1
-  kill %1
+  BEGIN_FOLD unit-tests
+  DOCKER_EXEC LD_LIBRARY_PATH=$TRAVIS_BUILD_DIR/depends/$HOST/lib make $MAKEJOBS check VERBOSE=1
+  END_FOLD
+fi
+
+if [ "$RUN_BENCH" = "true" ]; then
+  BEGIN_FOLD bench
+  DOCKER_EXEC LD_LIBRARY_PATH=$TRAVIS_BUILD_DIR/depends/$HOST/lib $OUTDIR/bin/bench_unite -scaling=0.001
+  END_FOLD
 fi
 
 if [ "$TRAVIS_EVENT_TYPE" = "cron" ]; then
   extended="--extended --exclude feature_pruning,feature_dbcrash"
 fi
 
 if [ "$RUN_TESTS" = "true" ]; then
-  test/functional/test_runner.py --combinedlogslen=4000 --coverage --quiet ${extended}
+  BEGIN_FOLD functional-tests
+  DOCKER_EXEC test/functional/test_runner.py --combinedlogslen=4000 --coverage --quiet --failfast ${extended}
+  END_FOLD
 fi

Makefile.am

@@ -168,7 +168,8 @@ EXTRA_DIST += \
     test/util/data/txcreatescript4.json \
     test/util/data/txcreatesignv1.hex \
     test/util/data/txcreatesignv1.json \
-    test/util/data/txcreatesignv2.hex
+    test/util/data/txcreatesignv2.hex \
+    test/util/rpcauth-test.py
 
 CLEANFILES = $(OSX_DMG) $(UNITE_WIN_INSTALLER)
 
@@ -188,6 +189,10 @@ docs:
 	@echo "error: doxygen not found"
 endif
 
-clean-local:
+clean-docs:
+	rm -rf doc/doxygen
+
+clean-local: clean-docs
 	rm -rf coverage_percent.txt test_unite.coverage/ total.coverage/ test/tmp/ cache/ $(OSX_APP)
 	rm -rf test/functional/__pycache__ test/functional/test_framework/__pycache__ test/cache
+

README.md

@@ -37,7 +37,7 @@ features such as:
 
 We regularly merge upstream changes into the unit-e code base and also strive to
 contribute back changes which are relevant for upstream. The last upstream sync
-was done with the [0.16](https://github.com/bitcoin/bitcoin/tree/0.16) version,
+was done with the [0.17](https://github.com/bitcoin/bitcoin/tree/0.17) version,
 plus some changes cherry-picked from later development branches.
 
 The client is in a pre-testnet development phase. With the [launch of the

autogen.sh

@@ -3,6 +3,7 @@
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
+export LC_ALL=C
 set -e
 srcdir="$(dirname $0)"
 cd "$srcdir"