Upstream changes for the build process on travis

.travis.yml

@@ -1,73 +1,132 @@
-sudo: required
+# The test build matrix (stage: test) is constructed to test a wide range of
+# configurations, rather than a single pass/fail. This helps to catch build
+# failures and logic errors that present on platforms other than the ones the
+# author has tested.
+#
+# Some builders use the dependency-generator in `./depends`, rather than using
+# apt-get to install build dependencies. This guarantees that the tester is
+# using the same versions as Gitian, so the build results are nearly identical
+# to what would be found in a final release.
+#
+# In order to avoid rebuilding all dependencies for each build, the binaries
+# are cached and re-used when possible. Changes in the dependency-generator
+# will trigger cache-invalidation and rebuilds as necessary.
+#
+# These caches can be manually removed if necessary. This is one of the very
+# few manual operations that is possible with Travis, and it can be done by a
+# Bitcoin Core GitHub member via the Travis web interface [0].
+#
+# Travis CI uploads the cache after the script phase of the build [1].
+# However, the build is terminated without saving the chache if it takes over
+# 50 minutes [2]. Thus, if we spent too much time in early build stages, fail
+# with an error and save the cache.
+#
+# [0] https://travis-ci.org/bitcoin/bitcoin/caches
+# [1] https://docs.travis-ci.com/user/caching/#build-phases
+# [2] https://docs.travis-ci.com/user/customizing-the-build#build-timeouts
 
-#workaround for https://github.com/travis-ci/travis-ci/issues/5227
-addons:
-  hostname: bitcoin-tester
-
-dist: bionic
+dist: xenial
 os: linux
 language: minimal
 cache:
   ccache: true
   directories:
-  - depends/built
-  - depends/sdk-sources
-  - $HOME/.ccache
+    - depends/built
+    - depends/sdk-sources
+    - $HOME/.ccache
+stages:
+  - test
 env:
   global:
     - MAKEJOBS=-j3
-    - RUN_TESTS=false
-    - CHECK_DOC=0
+    - RUN_UNIT_TESTS=true
+    - RUN_FUNCTIONAL_TESTS=true
+    - DOCKER_NAME_TAG=ubuntu:18.04
     - BOOST_TEST_RANDOM=1$TRAVIS_BUILD_ID
     - CCACHE_SIZE=100M
     - CCACHE_TEMPDIR=/tmp/.ccache-temp
     - CCACHE_COMPRESS=1
     - CCACHE_DIR=$HOME/.ccache
     - BASE_OUTDIR=$TRAVIS_BUILD_DIR/out
     - SDK_URL=https://bitcoincore.org/depends-sources/sdks
-    - PYTHON_DEBUG=1
     - WINEDEBUG=fixme-all
-  matrix:
-# ARM
-    - HOST=arm-linux-gnueabihf PACKAGES="ccache g++-arm-linux-gnueabihf" DEP_OPTS="NO_QT=1" CHECK_DOC=1 GOAL="install" BITCOIN_CONFIG="--disable-shared --enable-glibc-back-compat --enable-reduce-exports CXXFLAGS=-Wno-psabi LDFLAGS=-static-libstdc++"
-# Win32
-    - HOST=i686-w64-mingw32 DPKG_ADD_ARCH="i386" DEP_OPTS="NO_QT=1" PACKAGES="ccache python3 nsis g++-mingw-w64-i686 wine1.6 bc openjdk-8-jre-headless" RUN_TESTS=false GOAL="install" BITCOIN_CONFIG="--enable-reduce-exports"
-# Win64
-    - HOST=x86_64-w64-mingw32 DPKG_ADD_ARCH="i386" DEP_OPTS="NO_QT=1" PACKAGES="ccache python3 nsis g++-mingw-w64-x86-64 wine1.6 bc openjdk-8-jre-headless" RUN_TESTS=false GOAL="install" BITCOIN_CONFIG="--enable-reduce-exports"
-# 32-bit + dash
-    - HOST=i686-pc-linux-gnu PACKAGES="ccache g++-multilib bc python3-zmq openjdk-8-jre-headless" DEP_OPTS="NO_QT=1" RUN_TESTS=false GOAL="install" BITCOIN_CONFIG="--enable-zmq --enable-glibc-back-compat --enable-reduce-exports LDFLAGS=-static-libstdc++" USE_SHELL="/bin/dash"
-# bitcoind
-    - HOST=x86_64-unknown-linux-gnu PACKAGES="ccache bc python3-zmq openjdk-8-jre-headless" DEP_OPTS="NO_QT=1 NO_UPNP=1 DEBUG=1" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--disable-shared --enable-zmq --enable-glibc-back-compat --enable-reduce-exports CPPFLAGS=-DDEBUG_LOCKORDER LDFLAGS=-static-libstdc++"
-# Cross-Mac
-    - HOST=x86_64-apple-darwin14 PACKAGES="ccache cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python-dev python3-setuptools" BITCOIN_CONFIG="--enable-reduce-exports" OSX_SDK=10.11 GOAL="deploy"
-
+    - DOCKER_PACKAGES="build-essential libtool autotools-dev automake pkg-config bsdmainutils curl git ca-certificates ccache python3 autoconf"
+    - CACHE_ERR_MSG="Error! Initial build successful, but not enough time remains to run later build stages and tests. Please manually re-run this job by using the travis restart button or asking a maintainer to restart. The next run should not time out because the build cache has been saved."
 before_install:
-    - export PATH=$(echo $PATH | tr ':' "\n" | sed '/\/opt\/python/d' | tr "\n" ":" | sed "s|::|:|g")
+  - set -o errexit; source .travis/test_03_before_install.sh
 install:
-    - if [ -n "$PPA" ]; then travis_retry sudo add-apt-repository "$PPA" -y; fi
-    - if [ -n "$DPKG_ADD_ARCH" ]; then sudo dpkg --add-architecture "$DPKG_ADD_ARCH" ; fi
-    - if [ -n "$PACKAGES" ]; then travis_retry sudo apt-get update; fi
-    - if [ -n "$PACKAGES" ]; then travis_retry sudo apt-get install --no-install-recommends --no-upgrade -qq $PACKAGES; fi
+  - set -o errexit; source .travis/test_04_install.sh
 before_script:
-    - unset CC; unset CXX
-    - if [ "$CHECK_DOC" = 1 ]; then contrib/devtools/check-doc.py; fi
-    - mkdir -p depends/SDKs depends/sdk-sources
-    - if [ -n "$OSX_SDK" -a ! -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then curl --location --fail $SDK_URL/MacOSX${OSX_SDK}.sdk.tar.gz -o depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz; fi
-    - if [ -n "$OSX_SDK" -a -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then tar -C depends/SDKs -xf depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz; fi
-    - make $MAKEJOBS -C depends HOST=$HOST $DEP_OPTS
+  - set -o errexit; source .travis/test_05_before_script.sh
 script:
-    - export TRAVIS_COMMIT_LOG=`git log --format=fuller -1`
-    - if [ -n "$USE_SHELL" ]; then export CONFIG_SHELL="$USE_SHELL"; fi
-    - OUTDIR=$BASE_OUTDIR/$TRAVIS_PULL_REQUEST/$TRAVIS_JOB_NUMBER-$HOST
-    - BITCOIN_CONFIG_ALL="--disable-dependency-tracking --prefix=$TRAVIS_BUILD_DIR/depends/$HOST --bindir=$OUTDIR/bin --libdir=$OUTDIR/lib"
-    - ccache --max-size=$CCACHE_SIZE
-    - test -n "$USE_SHELL" && eval '"$USE_SHELL" -c "./autogen.sh"' || ./autogen.sh
-    - mkdir build && cd build
-    - ../configure $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( cat config.log && false)
-    - make $MAKEJOBS $GOAL || ( echo "Build failure. Verbose build follows." && make $GOAL V=1 ; false )
-    - export LD_LIBRARY_PATH=$TRAVIS_BUILD_DIR/depends/$HOST/lib
-    - if [ "$RUN_TESTS" = "true" ]; then make $MAKEJOBS check VERBOSE=1; fi
-    - if [ "$RUN_TESTS" = "true" ]; then qa/pull-tester/rpc-tests.py -parallel=1 --coverage; fi
+  - export CONTINUE=1
+  - if [ $SECONDS -gt 1200 ]; then export CONTINUE=0; fi  # Likely the depends build took very long
+  - if [ $CONTINUE = "1" ]; then set -o errexit; source .travis/test_06_script_a.sh; else set +o errexit; echo "$CACHE_ERR_MSG"; false; fi
+  - if [ $SECONDS -gt 2000 ]; then export CONTINUE=0; fi  # Likely the build took very long; The tests take about 1000s, so we should abort if we have less than 50*60-1000=2000s left
+  - if [ $CONTINUE = "1" ]; then set -o errexit; source .travis/test_06_script_b.sh; else set +o errexit; echo "$CACHE_ERR_MSG"; false; fi
 after_script:
-    - echo $TRAVIS_COMMIT_RANGE
-    - echo $TRAVIS_COMMIT_LOG
+  - echo $TRAVIS_COMMIT_RANGE
+  - echo $TRAVIS_COMMIT_LOG
+jobs:
+  include:
+
+    - stage: test
+      name: 'ARM  [GOAL: install]  [no tests]'
+      env: >-
+        HOST=arm-linux-gnueabihf
+        PACKAGES="g++-arm-linux-gnueabihf"
+        RUN_UNIT_TESTS=false
+        RUN_FUNCTIONAL_TESTS=false
+        GOAL="install"
+        # -Wno-psabi is to disable ABI warnings: "note: parameter passing for argument of type ... changed in GCC 7.1"
+        # This could be removed once the ABI change warning does not show up by default
+        NAVCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports CXXFLAGS=-Wno-psabi"
+
+    - stage: test
+      name: 'Win32  [GOAL: install]  [no functional tests]'
+      env: >-
+        HOST=i686-w64-mingw32
+        DPKG_ADD_ARCH="i386"
+        PACKAGES="nsis g++-mingw-w64-i686 wine-binfmt wine32"
+        RUN_FUNCTIONAL_TESTS=false
+        GOAL="install"
+        NAVCOIN_CONFIG="--enable-reduce-exports --disable-gui-tests"
+
+    - stage: test
+      name: 'Win64  [GOAL: install]  [no functional tests]'
+      env: >-
+        HOST=x86_64-w64-mingw32
+        PACKAGES="nsis g++-mingw-w64-x86-64 wine-binfmt wine64"
+        RUN_FUNCTIONAL_TESTS=false
+        GOAL="install"
+        NAVCOIN_CONFIG="--enable-reduce-exports --disable-gui-tests"
+
+    - stage: test
+      name: 'i686 Linux  [GOAL: install] [dash shell]'
+      env: >-
+        HOST=i686-pc-linux-gnu
+        PACKAGES="g++-multilib python3-zmq"
+        DEP_OPTS="NO_QT=1"
+        GOAL="install"
+        NAVCOIN_CONFIG="--enable-zmq --enable-glibc-back-compat --enable-reduce-exports LDFLAGS=-static-libstdc++"
+        CONFIG_SHELL="/bin/dash"
+
+    - stage: test
+      name: 'x86_64 Linux  [GOAL: install]'
+      env: >-
+        HOST=x86_64-unknown-linux-gnu
+        PACKAGES="python3-zmq"
+        DEP_OPTS="NO_QT=1 NO_UPNP=1 DEBUG=1"
+        GOAL="install"
+        NAVCOIN_CONFIG="--enable-zmq --enable-glibc-back-compat --enable-reduce-exports CPPFLAGS=-DDEBUG_LOCKORDER"
+
+    - stage: test
+      name: 'macOS 10.10  [GOAL: deploy] [no tests]'
+      env: >-
+        HOST=x86_64-apple-darwin14
+        PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python3-dev python3-setuptools"
+        OSX_SDK=10.11
+        RUN_UNIT_TESTS=false
+        RUN_FUNCTIONAL_TESTS=false
+        GOAL="deploy"
+        NAVCOIN_CONFIG="--enable-reduce-exports --enable-werror"

.travis/README.md

@@ -0,0 +1,8 @@
+## travis build scripts
+
+The `.travis` directory contains scripts for each build step in each build stage.
+Currently the travis build defines one stage `test`. This stage has it's own
+[lifecycle](https://docs.travis-ci.com/user/customizing-the-build/#the-build-lifecycle).
+Every script in here is named and numbered according to which stage and lifecycle
+step it belongs to.
+

.travis/test_03_before_install.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2018 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+PATH=$(echo $PATH | tr ':' "\n" | sed '/\/opt\/python/d' | tr "\n" ":" | sed "s|::|:|g")
+# Add llvm-symbolizer directory to PATH. Needed to get symbolized stack traces from the sanitizers.
+PATH=$PATH:/usr/lib/llvm-6.0/bin/
+export PATH
+
+BEGIN_FOLD () {
+  echo ""
+  CURRENT_FOLD_NAME=$1
+  echo "travis_fold:start:${CURRENT_FOLD_NAME}"
+}
+
+END_FOLD () {
+  RET=$?
+  echo "travis_fold:end:${CURRENT_FOLD_NAME}"
+  if [ $RET != 0 ]; then
+    echo "${CURRENT_FOLD_NAME} failed with status code ${RET}"
+  fi
+}
+

.travis/test_04_install.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2018 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+travis_retry docker pull "$DOCKER_NAME_TAG"
+
+mkdir -p "${TRAVIS_BUILD_DIR}/sanitizer-output/"
+export ASAN_OPTIONS=""
+export LSAN_OPTIONS="suppressions=${TRAVIS_BUILD_DIR}/test/sanitizer_suppressions/lsan"
+export TSAN_OPTIONS="suppressions=${TRAVIS_BUILD_DIR}/test/sanitizer_suppressions/tsan:log_path=${TRAVIS_BUILD_DIR}/sanitizer-output/tsan"
+export UBSAN_OPTIONS="suppressions=${TRAVIS_BUILD_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1"
+env | grep -E '^(NAVCOIN_CONFIG|CCACHE_|WINEDEBUG|LC_ALL|BOOST_TEST_RANDOM|CONFIG_SHELL|(ASAN|LSAN|TSAN|UBSAN)_OPTIONS)' | tee /tmp/env
+if [[ $HOST = *-mingw32 ]]; then
+  DOCKER_ADMIN="--cap-add SYS_ADMIN"
+elif [[ $NAVCOIN_CONFIG = *--with-sanitizers=*address* ]]; then # If ran with (ASan + LSan), Docker needs access to ptrace (https://github.com/google/sanitizers/issues/764)
+  DOCKER_ADMIN="--cap-add SYS_PTRACE"
+fi
+DOCKER_ID=$(docker run $DOCKER_ADMIN -idt --mount type=bind,src=$TRAVIS_BUILD_DIR,dst=$TRAVIS_BUILD_DIR --mount type=bind,src=$CCACHE_DIR,dst=$CCACHE_DIR -w $TRAVIS_BUILD_DIR --env-file /tmp/env $DOCKER_NAME_TAG)
+
+DOCKER_EXEC () {
+  docker exec $DOCKER_ID bash -c "cd $PWD && $*"
+}
+
+if [ -n "$DPKG_ADD_ARCH" ]; then
+  DOCKER_EXEC dpkg --add-architecture "$DPKG_ADD_ARCH"
+fi
+
+INSTALL_COMMAND="apt-get install --no-install-recommends --no-upgrade -qq"
+
+travis_retry DOCKER_EXEC apt-get update
+travis_retry DOCKER_EXEC $INSTALL_COMMAND $PACKAGES $DOCKER_PACKAGES
+
+if [ "$RUN_FUNCTIONAL_TESTS" = "true" ]; then
+  BEGIN_FOLD local-ntp-server
+  travis_retry DOCKER_EXEC DEBIAN_FRONTEND=noninteractive $INSTALL_COMMAND ntp
+  travis_retry DOCKER_EXEC service ntp start
+  END_FOLD
+fi
+

.travis/test_05_before_script.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2018 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+DOCKER_EXEC echo \> \$HOME/.navcoin  # Make sure default datadir does not exist and is never read by creating a dummy file
+
+mkdir -p depends/SDKs depends/sdk-sources
+
+if [ -n "$OSX_SDK" -a ! -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then
+  curl --location --fail $SDK_URL/MacOSX${OSX_SDK}.sdk.tar.gz -o depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz
+fi
+if [ -n "$OSX_SDK" -a -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then
+  tar -C depends/SDKs -xf depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz
+fi
+if [[ $HOST = *-mingw32 ]]; then
+  DOCKER_EXEC update-alternatives --set $HOST-g++ \$\(which $HOST-g++-posix\)
+fi
+if [ -z "$NO_DEPENDS" ]; then
+  DOCKER_EXEC CONFIG_SHELL= make $MAKEJOBS -C depends HOST=$HOST $DEP_OPTS
+fi
+

.travis/test_06_script_a.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2018 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+TRAVIS_COMMIT_LOG=$(git log --format=fuller -1)
+export TRAVIS_COMMIT_LOG
+
+OUTDIR=$BASE_OUTDIR/$TRAVIS_PULL_REQUEST/$TRAVIS_JOB_NUMBER-$HOST
+NAVCOIN_CONFIG_ALL="--disable-dependency-tracking --prefix=$TRAVIS_BUILD_DIR/depends/$HOST --bindir=$OUTDIR/bin --libdir=$OUTDIR/lib"
+if [ -z "$NO_DEPENDS" ]; then
+  DOCKER_EXEC ccache --max-size=$CCACHE_SIZE
+fi
+
+BEGIN_FOLD autogen
+if [ -n "$CONFIG_SHELL" ]; then
+  DOCKER_EXEC "$CONFIG_SHELL" -c "./autogen.sh"
+else
+  DOCKER_EXEC ./autogen.sh
+fi
+END_FOLD
+
+mkdir build
+cd build || (echo "could not enter build directory"; exit 1)
+
+BEGIN_FOLD configure
+DOCKER_EXEC ../configure --cache-file=config.cache $NAVCOIN_CONFIG_ALL $NAVCOIN_CONFIG || ( cat config.log && false)
+END_FOLD
+
+set -o errtrace
+trap 'DOCKER_EXEC "cat ${TRAVIS_BUILD_DIR}/sanitizer-output/* 2> /dev/null"' ERR
+
+BEGIN_FOLD build
+DOCKER_EXEC make $MAKEJOBS $GOAL || ( echo "Build failure. Verbose build follows." && DOCKER_EXEC make $GOAL V=1 ; false )
+END_FOLD
+
+cd ${TRAVIS_BUILD_DIR} || (echo "could not enter travis build dir $TRAVIS_BUILD_DIR"; exit 1)

.travis/test_06_script_b.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2018 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+cd build || (echo "could not enter build directory"; exit 1)
+
+if [ "$RUN_UNIT_TESTS" = "true" ]; then
+  BEGIN_FOLD unit-tests
+  DOCKER_EXEC LD_LIBRARY_PATH=$TRAVIS_BUILD_DIR/depends/$HOST/lib make $MAKEJOBS check VERBOSE=1
+  END_FOLD
+fi
+
+if [ "$RUN_FUNCTIONAL_TESTS" = "true" ]; then
+  BEGIN_FOLD functional-tests
+  DOCKER_EXEC LOCAL_NTP=1 ./qa/pull-tester/rpc-tests.py -parallel=1 --coverage
+  END_FOLD
+fi

configure.ac

@@ -521,7 +521,7 @@ fi
 
 if test x$TARGET_OS != xwindows; then
   # All windows code is PIC, forcing it on just adds useless compile warnings
-  AX_CHECK_COMPILE_FLAG([-fPIC],[CFLAGS="$CFLAGS -fPIC"])
+  AX_CHECK_COMPILE_FLAG([-fPIC],[PIC_FLAGS="-fPIC"])
 fi
 
 if test x$use_hardening != xno; then

depends/packages/curl.mk

@@ -12,7 +12,7 @@ endef
 
 define $(package)_set_vars
   $(package)_config_env=CURL_CFLAG_EXTRAS="-DBUILDING_LIBCURL"
-  $(package)_config_opts=--with-ssl --enable-static --disable-shared
+  $(package)_config_opts=--with-ssl=$(host_prefix) --enable-static --disable-shared
   $(package)_config_opts_mingw32=--with-random=/dev/urandom
   $(package)_config_opts_x86_64_mingw32=--target=x86_64-w64-mingw32
   $(package)_config_opts_i686_mingw32=--target=i686-w64-mingw32