diff --git a/.appveyor.yml b/.appveyor.yml index 0c43e61592c4..ac17e2eeb6b6 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -33,23 +33,10 @@ before_build: - ps: Start-Process clcache-server - ps: fsutil behavior set disablelastaccess 0 # Enable Access time feature on Windows (for clcache) build_script: -- cmd: msbuild /p:TrackFileAccess=false /p:CLToolExe=clcache.exe build_msvc\bitcoin.sln /m /v:q /nologo +- cmd: msbuild /p:TrackFileAccess=false /p:CLToolExe=clcache.exe build_msvc\bitcoin.sln /m /v:n /nologo after_build: - ps: fsutil behavior set disablelastaccess 1 # Disable Access time feature on Windows (better performance) - ps: clcache -z -before_test: -- ps: ${conf_ini} = (Get-Content([IO.Path]::Combine(${env:APPVEYOR_BUILD_FOLDER}, "test", "config.ini.in"))) -- ps: ${conf_ini} = ${conf_ini}.Replace("@PACKAGE_NAME@", "Bitcoin Core") -- ps: ${conf_ini} = ${conf_ini}.Replace("@abs_top_srcdir@", ${env:APPVEYOR_BUILD_FOLDER}) -- ps: ${conf_ini} = ${conf_ini}.Replace("@abs_top_builddir@", ${env:APPVEYOR_BUILD_FOLDER}) -- ps: ${conf_ini} = ${conf_ini}.Replace("@EXEEXT@", ".exe") -- ps: ${conf_ini} = ${conf_ini}.Replace("@ENABLE_WALLET_TRUE@", "") -- ps: ${conf_ini} = ${conf_ini}.Replace("@BUILD_BITCOIN_CLI_TRUE@", "") -- ps: ${conf_ini} = ${conf_ini}.Replace("@BUILD_BITCOIND_TRUE@", "") -- ps: ${conf_ini} = ${conf_ini}.Replace("@ENABLE_ZMQ_TRUE@", "") -- ps: ${utf8} = New-Object System.Text.UTF8Encoding ${false} -- ps: '[IO.File]::WriteAllLines([IO.Path]::Combine(${env:APPVEYOR_BUILD_FOLDER}, "test", "config.ini"), ${conf_ini}, ${utf8})' -- ps: move "build_msvc\${env:PLATFORM}\${env:CONFIGURATION}\*.exe" src test_script: - cmd: src\test_bitcoin.exe -k stdout -e stdout 2> NUL - cmd: src\bench_bitcoin.exe -evals=1 -scaling=0 > NUL diff --git a/.cirrus.yml b/.cirrus.yml index ce17a223b18c..1e6e6937da4e 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -27,3 +27,27 @@ task: - gmake check ${MAKEJOBS} VERBOSE=1 functional_test_script: - ./test/functional/test_runner.py --jobs 9 --ci --extended --exclude feature_dbcrash --combinedlogslen=1000 --quiet --failfast +task: + name: "x86_64 Linux [GOAL: install] [bionic] [Using ./ci/ system]" + container: + image: ubuntu:18.04 + cpu: 8 + memory: 8G + timeout_in: 60m + env: + MAKEJOBS: "-j9" + RUN_CI_ON_HOST: "1" + CCACHE_SIZE: "200M" + CCACHE_DIR: "/tmp/ccache_dir" + ccache_cache: + folder: "/tmp/ccache_dir" + depends_built_cache: + folder: "/tmp/cirrus-ci-build/depends/built" + depends_sdk_cache: + folder: "/tmp/cirrus-ci-build/depends/sdk-sources" + install_script: + - apt-get update + - apt-get -y install git bash ccache + - ccache --max-size=${CCACHE_SIZE} + ci_script: + - ./ci/test_run_all.sh diff --git a/.gitignore b/.gitignore index 3ebf00ed521a..809e851ff111 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ *.tar.gz *.exe +*.pdb src/bitcoin src/bitcoind src/bitcoin-cli @@ -34,6 +35,7 @@ libtool src/config/bitcoin-config.h src/config/bitcoin-config.h.in src/config/stamp-h1 +src/obj share/setup.nsi share/qt/Info.plist @@ -76,7 +78,7 @@ src/qt/bitcoin-qt.includes # Only ignore unexpected patches *.patch -!depends/patches/*.patch +!depends/patches/**/*.patch #libtool object files *.lo @@ -85,10 +87,14 @@ src/qt/bitcoin-qt.includes # Compilation and Qt preprocessor part *.qm Makefile +!depends/Makefile bitcoin-qt Bitcoin-Qt.app background.tiff* +# Qt Creator +Makefile.am.user + # Unit-tests Makefile.test bitcoin-qt_test @@ -127,3 +133,7 @@ db4/ # clang-check *.plist + +osx_volname +dist/ +*.background.tiff diff --git a/.travis.yml b/.travis.yml index adf2140642b6..f515ab2b8775 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,7 +17,7 @@ # Bitcoin Core GitHub member via the Travis web interface [0]. # # Travis CI uploads the cache after the script phase of the build [1]. -# However, the build is terminated without saving the chache if it takes over +# However, the build is terminated without saving the cache if it takes over # 50 minutes [2]. Thus, if we spent too much time in early build stages, fail # with an error and save the cache. # @@ -37,38 +37,28 @@ cache: stages: - lint - test + - extended-lint env: global: - - MAKEJOBS=-j3 - - RUN_UNIT_TESTS=true - - RUN_FUNCTIONAL_TESTS=true - - RUN_FUZZ_TESTS=false - - DOCKER_NAME_TAG=ubuntu:18.04 - - BOOST_TEST_RANDOM=1$TRAVIS_BUILD_ID - - CCACHE_SIZE=100M - - CCACHE_TEMPDIR=/tmp/.ccache-temp - - CCACHE_COMPRESS=1 - - CCACHE_DIR=$HOME/.ccache - - BASE_OUTDIR=$TRAVIS_BUILD_DIR/out - - SDK_URL=https://bitcoincore.org/depends-sources/sdks - - WINEDEBUG=fixme-all - - DOCKER_PACKAGES="build-essential libtool autotools-dev automake pkg-config bsdmainutils curl git ca-certificates ccache" + - CI_RETRY_EXE="travis_retry" - CACHE_ERR_MSG="Error! Initial build successful, but not enough time remains to run later build stages and tests. Please manually re-run this job by using the travis restart button or asking a bitcoin maintainer to restart. The next run should not time out because the build cache has been saved." before_install: - - set -o errexit; source .travis/test_03_before_install.sh + - set -o errexit; source ./ci/test/00_setup_env.sh + - set -o errexit; source ./ci/test/03_before_install.sh install: - - set -o errexit; source .travis/test_04_install.sh + - set -o errexit; source ./ci/test/04_install.sh before_script: - - set -o errexit; source .travis/test_05_before_script.sh + - set -o errexit; source ./ci/test/05_before_script.sh script: - export CONTINUE=1 - if [ $SECONDS -gt 1200 ]; then export CONTINUE=0; fi # Likely the depends build took very long - - if [ $CONTINUE = "1" ]; then set -o errexit; source .travis/test_06_script_a.sh; else set +o errexit; echo "$CACHE_ERR_MSG"; false; fi + - if [ $TRAVIS_REPO_SLUG = "bitcoin/bitcoin" ]; then export CONTINUE=1; fi # Whitelisted repo (90 minutes build time) + - if [ $CONTINUE = "1" ]; then set -o errexit; source ./ci/test/06_script_a.sh; else set +o errexit; echo "$CACHE_ERR_MSG"; false; fi - if [ $SECONDS -gt 2000 ]; then export CONTINUE=0; fi # Likely the build took very long; The tests take about 1000s, so we should abort if we have less than 50*60-1000=2000s left - - if [ $CONTINUE = "1" ]; then set -o errexit; source .travis/test_06_script_b.sh; else set +o errexit; echo "$CACHE_ERR_MSG"; false; fi + - if [ $TRAVIS_REPO_SLUG = "bitcoin/bitcoin" ]; then export CONTINUE=1; fi # Whitelisted repo (90 minutes build time) + - if [ $CONTINUE = "1" ]; then set -o errexit; source ./ci/test/06_script_b.sh; else set +o errexit; echo "$CACHE_ERR_MSG"; false; fi after_script: - echo $TRAVIS_COMMIT_RANGE - - echo $TRAVIS_COMMIT_LOG jobs: include: @@ -79,110 +69,71 @@ jobs: language: python python: '3.5' # Oldest supported version according to doc/dependencies.md install: - - set -o errexit; source .travis/lint_04_install.sh + - set -o errexit; source ./ci/lint/04_install.sh before_script: - - set -o errexit; source .travis/lint_05_before_script.sh + - set -o errexit; source ./ci/lint/05_before_script.sh script: - - set -o errexit; source .travis/lint_06_script.sh + - set -o errexit; source ./ci/lint/06_script.sh + + - stage: extended-lint + name: 'extended lint [runtime >= 60 seconds]' + env: + cache: false + language: python + python: '3.5' + install: + - set -o errexit; source ./ci/extended_lint/04_install.sh + before_script: + - set -o errexit; source ./ci/lint/05_before_script.sh + script: + - set -o errexit; source ./ci/extended_lint/06_script.sh - stage: test name: 'ARM [GOAL: install] [no unit or functional tests]' env: >- - HOST=arm-linux-gnueabihf - PACKAGES="python3 g++-arm-linux-gnueabihf" - RUN_UNIT_TESTS=false - RUN_FUNCTIONAL_TESTS=false - GOAL="install" - # -Wno-psabi is to disable ABI warnings: "note: parameter passing for argument of type ... changed in GCC 7.1" - # This could be removed once the ABI change warning does not show up by default - BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports CXXFLAGS=-Wno-psabi" + FILE_ENV="./ci/test/00_setup_env_arm.sh" - stage: test name: 'Win64 [GOAL: deploy] [no gui or functional tests]' env: >- - HOST=x86_64-w64-mingw32 - PACKAGES="python3 nsis g++-mingw-w64-x86-64 wine-binfmt wine64" - RUN_FUNCTIONAL_TESTS=false - GOAL="deploy" - BITCOIN_CONFIG="--enable-reduce-exports --disable-gui-tests" + FILE_ENV="./ci/test/00_setup_env_win64.sh" - stage: test name: '32-bit + dash [GOAL: install] [GUI: no BIP70]' env: >- - HOST=i686-pc-linux-gnu - PACKAGES="g++-multilib python3-zmq" - GOAL="install" - BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --disable-bip70 --enable-glibc-back-compat --enable-reduce-exports LDFLAGS=-static-libstdc++" - CONFIG_SHELL="/bin/dash" + FILE_ENV="./ci/test/00_setup_env_i686.sh" - stage: test - name: 'x86_64 Linux [GOAL: install] [bionic] [uses qt5 dev package instead of depends Qt to speed up build and avoid timeout]' + name: 'x86_64 Linux [GOAL: install] [bionic] [uses qt5 dev package instead of depends Qt to speed up build and avoid timeout] [unsigned char]' env: >- - HOST=x86_64-unknown-linux-gnu - PACKAGES="python3-zmq qtbase5-dev qttools5-dev-tools protobuf-compiler libdbus-1-dev libharfbuzz-dev libprotobuf-dev" - DEP_OPTS="NO_QT=1 NO_UPNP=1 DEBUG=1 ALLOW_HOST_PACKAGES=1" - TEST_RUNNER_EXTRA="--coverage --extended --exclude feature_dbcrash" # Run extended tests so that coverage does not fail, but exclude the very slow dbcrash - GOAL="install" - BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --enable-glibc-back-compat --enable-reduce-exports --enable-debug CXXFLAGS=\"-g0 -O2\"" + FILE_ENV="./ci/test/00_setup_env_amd64_qt5.sh" - stage: test name: 'x86_64 Linux [GOAL: install] [trusty] [no functional tests, no depends, only system libs]' env: >- - HOST=x86_64-unknown-linux-gnu - DOCKER_NAME_TAG=ubuntu:14.04 - PACKAGES="python3-zmq qtbase5-dev qttools5-dev-tools libicu-dev libpng-dev libssl-dev libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev libdb5.1++-dev libminiupnpc-dev libzmq3-dev libprotobuf-dev protobuf-compiler libqrencode-dev" - NO_DEPENDS=1 - RUN_FUNCTIONAL_TESTS=false - GOAL="install" - BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=no" + FILE_ENV="./ci/test/00_setup_env_amd64_trusty.sh" - stage: test name: 'x86_64 Linux [GOAL: install] [xenial] [no depends, only system libs, sanitizers: thread (TSan), no wallet]' env: >- - HOST=x86_64-unknown-linux-gnu - DOCKER_NAME_TAG=ubuntu:16.04 - PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libssl-dev libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libprotobuf-dev protobuf-compiler libqrencode-dev" - NO_DEPENDS=1 - GOAL="install" - BITCOIN_CONFIG="--enable-zmq --disable-wallet --with-gui=qt5 CPPFLAGS=-DDEBUG_LOCKORDER --with-sanitizers=thread --disable-hardening --disable-asm CC=clang CXX=clang++" + FILE_ENV="./ci/test/00_setup_env_amd64_tsan.sh" - stage: test name: 'x86_64 Linux [GOAL: install] [bionic] [no depends, only system libs, sanitizers: address/leak (ASan + LSan) + undefined (UBSan) + integer]' env: >- - HOST=x86_64-unknown-linux-gnu - PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libssl1.0-dev libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libprotobuf-dev protobuf-compiler libqrencode-dev" - NO_DEPENDS=1 - GOAL="install" - BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=qt5 CPPFLAGS=-DDEBUG_LOCKORDER --with-sanitizers=address,integer,undefined CC=clang CXX=clang++" + FILE_ENV="./ci/test/00_setup_env_amd64_asan.sh" - stage: test name: 'x86_64 Linux [GOAL: install] [bionic] [no depends, only system libs, sanitizers: fuzzer,address]' env: >- - HOST=x86_64-unknown-linux-gnu - PACKAGES="clang llvm python3 libssl1.0-dev libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev" - NO_DEPENDS=1 - RUN_UNIT_TESTS=false - RUN_FUNCTIONAL_TESTS=false - RUN_FUZZ_TESTS=true - GOAL="install" - BITCOIN_CONFIG="--disable-wallet --disable-bench --with-utils=no --with-daemon=no --with-libs=no --with-gui=no --enable-fuzz --with-sanitizers=fuzzer,address CC=clang CXX=clang++" + FILE_ENV="./ci/test/00_setup_env_amd64_fuzz.sh" - stage: test name: 'x86_64 Linux [GOAL: install] [bionic] [no wallet]' env: >- - HOST=x86_64-unknown-linux-gnu - PACKAGES="python3-zmq" - DEP_OPTS="NO_WALLET=1" - GOAL="install" - BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports" + FILE_ENV="./ci/test/00_setup_env_amd64_nowallet.sh" - stage: test name: 'macOS 10.10 [GOAL: deploy] [no functional tests]' env: >- - HOST=x86_64-apple-darwin14 - PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python3-dev python3-setuptools" - OSX_SDK=10.11 - RUN_UNIT_TESTS=false - RUN_FUNCTIONAL_TESTS=false - GOAL="deploy" - BITCOIN_CONFIG="--enable-gui --enable-reduce-exports --enable-werror" + FILE_ENV="./ci/test/00_setup_env_mac.sh" diff --git a/.travis/README.md b/.travis/README.md deleted file mode 100644 index 21d1b9cc03ae..000000000000 --- a/.travis/README.md +++ /dev/null @@ -1,8 +0,0 @@ -## travis build scripts - -The `.travis` directory contains scripts for each build step in each build stage. -Currently the travis build defines two stages `lint` and `test`. Each stage has -it's own [lifecycle](https://docs.travis-ci.com/user/customizing-the-build/#the-build-lifecycle). -Every script in here is named and numbered according to which stage and lifecycle -step it belongs to. - diff --git a/.travis/test_04_install.sh b/.travis/test_04_install.sh deleted file mode 100755 index b589ee7a161d..000000000000 --- a/.travis/test_04_install.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C.UTF-8 - -travis_retry docker pull "$DOCKER_NAME_TAG" - -export DIR_FUZZ_IN=${TRAVIS_BUILD_DIR}/qa-assets -git clone https://github.com/bitcoin-core/qa-assets ${DIR_FUZZ_IN} -export DIR_FUZZ_IN=${DIR_FUZZ_IN}/fuzz_seed_corpus/ - -mkdir -p "${TRAVIS_BUILD_DIR}/sanitizer-output/" -export ASAN_OPTIONS="" -export LSAN_OPTIONS="suppressions=${TRAVIS_BUILD_DIR}/test/sanitizer_suppressions/lsan" -export TSAN_OPTIONS="suppressions=${TRAVIS_BUILD_DIR}/test/sanitizer_suppressions/tsan:log_path=${TRAVIS_BUILD_DIR}/sanitizer-output/tsan" -export UBSAN_OPTIONS="suppressions=${TRAVIS_BUILD_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1" -env | grep -E '^(BITCOIN_CONFIG|CCACHE_|WINEDEBUG|LC_ALL|BOOST_TEST_RANDOM|CONFIG_SHELL|(ASAN|LSAN|TSAN|UBSAN)_OPTIONS)' | tee /tmp/env -if [[ $HOST = *-mingw32 ]]; then - DOCKER_ADMIN="--cap-add SYS_ADMIN" -elif [[ $BITCOIN_CONFIG = *--with-sanitizers=*address* ]]; then # If ran with (ASan + LSan), Docker needs access to ptrace (https://github.com/google/sanitizers/issues/764) - DOCKER_ADMIN="--cap-add SYS_PTRACE" -fi -DOCKER_ID=$(docker run $DOCKER_ADMIN -idt --mount type=bind,src=$TRAVIS_BUILD_DIR,dst=$TRAVIS_BUILD_DIR --mount type=bind,src=$CCACHE_DIR,dst=$CCACHE_DIR -w $TRAVIS_BUILD_DIR --env-file /tmp/env $DOCKER_NAME_TAG) - -DOCKER_EXEC () { - docker exec $DOCKER_ID bash -c "cd $PWD && $*" -} - -travis_retry DOCKER_EXEC apt-get update -travis_retry DOCKER_EXEC apt-get install --no-install-recommends --no-upgrade -qq $PACKAGES $DOCKER_PACKAGES - diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a2456f5b8cc1..268259c82d99 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -80,21 +80,15 @@ The title of the pull request should be prefixed by the component or area that the pull request affects. Valid areas as: - *Consensus* for changes to consensus critical code - - *Docs* for changes to the documentation + - *Doc* for changes to the documentation - *Qt* for changes to bitcoin-qt + - *Log* Changes to log messages - *Mining* for changes to the mining code - *Net* or *P2P* for changes to the peer-to-peer network code + - *Refactor* for structural changes that do not change behavior - *RPC/REST/ZMQ* for changes to the RPC, REST or ZMQ APIs - *Scripts and tools* for changes to the scripts and tools - - *Tests* for changes to the bitcoin unit tests or QA tests - - *Trivial* should **only** be used for PRs that do not change generated - executable code. Notably, refactors (change of function arguments and code - reorganization) and changes in behavior should **not** be marked as trivial. - Examples of trivial PRs are changes to: - - comments - - whitespace - - variable names - - logging and messages + - *Test* for changes to the bitcoin unit tests or QA tests - *Utils and libraries* for changes to the utils and libraries - *Wallet* for changes to the wallet code @@ -103,10 +97,10 @@ Examples: Consensus: Add new opcode for BIP-XXXX OP_CHECKAWESOMESIG Net: Automatically create hidden service, listen on Tor Qt: Add feed bump button - Trivial: Fix typo in init.cpp + Log: Fix typo in log message Note that translations should not be submitted as pull requests, please see -[Translation Process](https://github.com/bitcoin/bitcoin/blob/master/doc/translation_process.md) +[Translation Process](https://github.com/bitcoin/bitcoin/blob/master/doc/translation_process.md) for more information on helping with translations. If a pull request is not to be considered for merging (yet), please @@ -322,7 +316,7 @@ The project leader is the release manager for each Bitcoin Core release. Copyright --------- -By contributing to this repository, you agree to license your work under the -MIT license unless specified otherwise in `contrib/debian/copyright` or at -the top of the file itself. Any work contributed where you are not the original +By contributing to this repository, you agree to license your work under the +MIT license unless specified otherwise in `contrib/debian/copyright` or at +the top of the file itself. Any work contributed where you are not the original author must contain its license header with the original author(s) and source. diff --git a/Makefile.am b/Makefile.am index 712f2a454de4..8b1e2a6b5b3c 100644 --- a/Makefile.am +++ b/Makefile.am @@ -310,4 +310,5 @@ clean-docs: clean-local: clean-docs rm -rf coverage_percent.txt test_bitcoin.coverage/ total.coverage/ test/tmp/ cache/ $(OSX_APP) rm -rf test/functional/__pycache__ test/functional/test_framework/__pycache__ test/cache share/rpcauth/__pycache__ + rm -rf osx_volname dist/ dpi36.background.tiff dpi72.background.tiff diff --git a/autogen.sh b/autogen.sh index 0c05626ccce5..2c434e9ef057 100755 --- a/autogen.sh +++ b/autogen.sh @@ -7,10 +7,10 @@ export LC_ALL=C set -e srcdir="$(dirname $0)" cd "$srcdir" -if [ -z ${LIBTOOLIZE} ] && GLIBTOOLIZE="`which glibtoolize 2>/dev/null`"; then +if [ -z ${LIBTOOLIZE} ] && GLIBTOOLIZE="$(command -v glibtoolize)"; then LIBTOOLIZE="${GLIBTOOLIZE}" export LIBTOOLIZE fi -which autoreconf >/dev/null || \ +command -v autoreconf >/dev/null || \ (echo "configuration failed, please install autoconf first" && exit 1) autoreconf --install --force --warnings=all diff --git a/build-aux/m4/bitcoin_qt.m4 b/build-aux/m4/bitcoin_qt.m4 index 1a7c5d5f7d39..675fb6d3fb63 100644 --- a/build-aux/m4/bitcoin_qt.m4 +++ b/build-aux/m4/bitcoin_qt.m4 @@ -268,7 +268,7 @@ AC_DEFUN([_BITCOIN_QT_CHECK_QT5],[ dnl Internal. Check if the included version of Qt is greater than Qt58. dnl Requires: INCLUDES must be populated as necessary. -dnl Output: bitcoin_cv_qt5=yes|no +dnl Output: bitcoin_cv_qt58=yes|no AC_DEFUN([_BITCOIN_QT_CHECK_QT58],[ AC_CACHE_CHECK(for > Qt 5.7, bitcoin_cv_qt58,[ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ @@ -355,7 +355,6 @@ AC_DEFUN([_BITCOIN_QT_FIND_STATIC_PLUGINS],[ PKG_CHECK_MODULES([QTFB], [Qt5FbSupport], [QT_LIBS="-lQt5FbSupport $QT_LIBS"]) fi if test "x$TARGET_OS" = xlinux; then - PKG_CHECK_MODULES([X11XCB], [x11-xcb], [QT_LIBS="$X11XCB_LIBS $QT_LIBS"]) PKG_CHECK_MODULES([QTXCBQPA], [Qt5XcbQpa], [QT_LIBS="$QTXCBQPA_LIBS $QT_LIBS"]) elif test "x$TARGET_OS" = xdarwin; then PKG_CHECK_MODULES([QTCLIPBOARD], [Qt5ClipboardSupport], [QT_LIBS="-lQt5ClipboardSupport $QT_LIBS"]) @@ -469,7 +468,6 @@ AC_DEFUN([_BITCOIN_QT_FIND_LIBS_WITHOUT_PKGCONFIG],[ ]) BITCOIN_QT_CHECK(AC_CHECK_LIB([z] ,[main],,AC_MSG_WARN([zlib not found. Assuming qt has it built-in]))) - BITCOIN_QT_CHECK(AC_SEARCH_LIBS([jpeg_create_decompress] ,[qtjpeg jpeg],,AC_MSG_WARN([libjpeg not found. Assuming qt has it built-in]))) if test x$bitcoin_cv_qt58 = xno; then BITCOIN_QT_CHECK(AC_SEARCH_LIBS([png_error] ,[qtpng png],,AC_MSG_WARN([libpng not found. Assuming qt has it built-in]))) BITCOIN_QT_CHECK(AC_SEARCH_LIBS([pcre16_exec], [qtpcre pcre16],,AC_MSG_WARN([libpcre16 not found. Assuming qt has it built-in]))) diff --git a/build_msvc/README.md b/build_msvc/README.md index d6543e6d6728..2e93979acaf5 100644 --- a/build_msvc/README.md +++ b/build_msvc/README.md @@ -44,7 +44,7 @@ The instructions below use `vcpkg` to install the dependencies. - Use Python to generate *.vcxproj from Makefile ``` - PS >python msvc-autogen.py + PS >py -3 msvc-autogen.py ``` - Build in Visual Studio. diff --git a/build_msvc/bench_bitcoin/bench_bitcoin.vcxproj b/build_msvc/bench_bitcoin/bench_bitcoin.vcxproj index 6ded6895cdfa..e64614c09d0b 100644 --- a/build_msvc/bench_bitcoin/bench_bitcoin.vcxproj +++ b/build_msvc/bench_bitcoin/bench_bitcoin.vcxproj @@ -20,9 +20,11 @@ + + @@ -68,9 +70,10 @@ - + + \ No newline at end of file diff --git a/build_msvc/bitcoin-cli/bitcoin-cli.vcxproj b/build_msvc/bitcoin-cli/bitcoin-cli.vcxproj index 72c55e47ccbb..e5e0e978f898 100644 --- a/build_msvc/bitcoin-cli/bitcoin-cli.vcxproj +++ b/build_msvc/bitcoin-cli/bitcoin-cli.vcxproj @@ -27,4 +27,5 @@ + diff --git a/build_msvc/bitcoin-tx/bitcoin-tx.vcxproj b/build_msvc/bitcoin-tx/bitcoin-tx.vcxproj index 6891a3e5ade0..4e9b4916a0fc 100644 --- a/build_msvc/bitcoin-tx/bitcoin-tx.vcxproj +++ b/build_msvc/bitcoin-tx/bitcoin-tx.vcxproj @@ -33,4 +33,5 @@ + diff --git a/build_msvc/bitcoin-wallet/bitcoin-wallet.vcxproj b/build_msvc/bitcoin-wallet/bitcoin-wallet.vcxproj index 996676baf90b..40c5db552288 100644 --- a/build_msvc/bitcoin-wallet/bitcoin-wallet.vcxproj +++ b/build_msvc/bitcoin-wallet/bitcoin-wallet.vcxproj @@ -51,4 +51,5 @@ + diff --git a/build_msvc/bitcoin_config.h b/build_msvc/bitcoin_config.h index ab13f73539aa..66cc1208a14f 100644 --- a/build_msvc/bitcoin_config.h +++ b/build_msvc/bitcoin_config.h @@ -421,4 +421,14 @@ /* Define for large files, on AIX-style hosts. */ /* #undef _LARGE_FILES */ +/* Windows Universal Platform constraints */ +#if !defined(WINAPI_FAMILY) || (WINAPI_FAMILY == WINAPI_FAMILY_DESKTOP_APP) +/* Either a desktop application without API restrictions, or and older system + before these macros were defined. */ + +/* ::wsystem is available */ +#define HAVE_SYSTEM 1 + +#endif // !WINAPI_FAMILY || WINAPI_FAMILY_DESKTOP_APP + #endif //BITCOIN_BITCOIN_CONFIG_H diff --git a/build_msvc/bitcoind/bitcoind.vcxproj b/build_msvc/bitcoind/bitcoind.vcxproj index 2c5dde301585..ae24cb100e68 100644 --- a/build_msvc/bitcoind/bitcoind.vcxproj +++ b/build_msvc/bitcoind/bitcoind.vcxproj @@ -45,4 +45,31 @@ + + + ..\..\test\config.ini.in + ..\..\test\config.ini + + + + + + + + + + + + + + diff --git a/build_msvc/common.init.vcxproj b/build_msvc/common.init.vcxproj index 8b6217e2b001..0d186b5af287 100644 --- a/build_msvc/common.init.vcxproj +++ b/build_msvc/common.init.vcxproj @@ -7,7 +7,6 @@ x86-windows-static x64-windows-static - $(Registry:HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Microsoft SDKs\Windows\v10.0@ProductVersion) $(Registry:HKEY_LOCAL_MACHINE\SOFTWARE\Wow6432Node\Microsoft\Microsoft SDKs\Windows\v10.0@ProductVersion) @@ -115,4 +114,5 @@ crypt32.lib;Iphlpapi.lib;ws2_32.lib;Shlwapi.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies) + diff --git a/build_msvc/common.vcxproj b/build_msvc/common.vcxproj new file mode 100644 index 000000000000..4bbcc3767f63 --- /dev/null +++ b/build_msvc/common.vcxproj @@ -0,0 +1,12 @@ + + +$(BuildDependsOn);CopyBuildArtifacts + + + + + + + + + diff --git a/build_msvc/libbitcoin_cli/libbitcoin_cli.vcxproj.in b/build_msvc/libbitcoin_cli/libbitcoin_cli.vcxproj.in index 5f6cf608af6a..620df72a2f76 100644 --- a/build_msvc/libbitcoin_cli/libbitcoin_cli.vcxproj.in +++ b/build_msvc/libbitcoin_cli/libbitcoin_cli.vcxproj.in @@ -12,4 +12,5 @@ + diff --git a/build_msvc/libbitcoin_common/libbitcoin_common.vcxproj.in b/build_msvc/libbitcoin_common/libbitcoin_common.vcxproj.in index 0bd9b3cd8910..b47d62b29587 100644 --- a/build_msvc/libbitcoin_common/libbitcoin_common.vcxproj.in +++ b/build_msvc/libbitcoin_common/libbitcoin_common.vcxproj.in @@ -12,4 +12,5 @@ + diff --git a/build_msvc/libbitcoin_crypto/libbitcoin_crypto.vcxproj.in b/build_msvc/libbitcoin_crypto/libbitcoin_crypto.vcxproj.in index abf1a7ac0267..32cb75bf871c 100644 --- a/build_msvc/libbitcoin_crypto/libbitcoin_crypto.vcxproj.in +++ b/build_msvc/libbitcoin_crypto/libbitcoin_crypto.vcxproj.in @@ -12,4 +12,5 @@ + diff --git a/build_msvc/libbitcoin_qt/libbitcoin_qt.vcxproj b/build_msvc/libbitcoin_qt/libbitcoin_qt.vcxproj index 73ba90aa8828..f21ba7a82bf7 100644 --- a/build_msvc/libbitcoin_qt/libbitcoin_qt.vcxproj +++ b/build_msvc/libbitcoin_qt/libbitcoin_qt.vcxproj @@ -406,9 +406,6 @@ true - - true - @@ -416,12 +413,6 @@ - - true - - - true - true @@ -455,9 +446,6 @@ true - - true - true @@ -473,9 +461,6 @@ true - - true - true @@ -497,9 +482,6 @@ true - - true - true @@ -518,9 +500,6 @@ true - - true - true @@ -530,15 +509,9 @@ true - - true - true - - true - true @@ -575,9 +548,6 @@ true - - true - true diff --git a/build_msvc/libbitcoin_server/libbitcoin_server.vcxproj.in b/build_msvc/libbitcoin_server/libbitcoin_server.vcxproj.in index 8e2ed394f8bd..58e90dbaeb43 100644 --- a/build_msvc/libbitcoin_server/libbitcoin_server.vcxproj.in +++ b/build_msvc/libbitcoin_server/libbitcoin_server.vcxproj.in @@ -15,4 +15,5 @@ + \ No newline at end of file diff --git a/build_msvc/libbitcoin_util/libbitcoin_util.vcxproj.in b/build_msvc/libbitcoin_util/libbitcoin_util.vcxproj.in index 71364d96b174..adf4fa03540f 100644 --- a/build_msvc/libbitcoin_util/libbitcoin_util.vcxproj.in +++ b/build_msvc/libbitcoin_util/libbitcoin_util.vcxproj.in @@ -12,4 +12,5 @@ + diff --git a/build_msvc/libbitcoin_wallet/libbitcoin_wallet.vcxproj.in b/build_msvc/libbitcoin_wallet/libbitcoin_wallet.vcxproj.in index b67f3904a489..9c8279c72aba 100644 --- a/build_msvc/libbitcoin_wallet/libbitcoin_wallet.vcxproj.in +++ b/build_msvc/libbitcoin_wallet/libbitcoin_wallet.vcxproj.in @@ -12,4 +12,5 @@ + diff --git a/build_msvc/libbitcoin_wallet_tool/libbitcoin_wallet_tool.vcxproj.in b/build_msvc/libbitcoin_wallet_tool/libbitcoin_wallet_tool.vcxproj.in index b8d0f4b44290..1a6b7b6b9267 100644 --- a/build_msvc/libbitcoin_wallet_tool/libbitcoin_wallet_tool.vcxproj.in +++ b/build_msvc/libbitcoin_wallet_tool/libbitcoin_wallet_tool.vcxproj.in @@ -12,4 +12,5 @@ + diff --git a/build_msvc/libbitcoin_zmq/libbitcoin_zmq.vcxproj.in b/build_msvc/libbitcoin_zmq/libbitcoin_zmq.vcxproj.in index 1b2c27e53ffa..e86eea81e656 100644 --- a/build_msvc/libbitcoin_zmq/libbitcoin_zmq.vcxproj.in +++ b/build_msvc/libbitcoin_zmq/libbitcoin_zmq.vcxproj.in @@ -12,4 +12,5 @@ + diff --git a/build_msvc/libbitcoinconsensus/libbitcoinconsensus.vcxproj b/build_msvc/libbitcoinconsensus/libbitcoinconsensus.vcxproj index 9c8931021b4c..4cb0bdc90218 100644 --- a/build_msvc/libbitcoinconsensus/libbitcoinconsensus.vcxproj +++ b/build_msvc/libbitcoinconsensus/libbitcoinconsensus.vcxproj @@ -33,4 +33,5 @@ + \ No newline at end of file diff --git a/build_msvc/libleveldb/libleveldb.vcxproj b/build_msvc/libleveldb/libleveldb.vcxproj index 4689bdc45e79..f855923c62da 100644 --- a/build_msvc/libleveldb/libleveldb.vcxproj +++ b/build_msvc/libleveldb/libleveldb.vcxproj @@ -59,4 +59,5 @@ + diff --git a/build_msvc/libsecp256k1/libsecp256k1.vcxproj b/build_msvc/libsecp256k1/libsecp256k1.vcxproj index 644f44350e74..99fb63fb0262 100644 --- a/build_msvc/libsecp256k1/libsecp256k1.vcxproj +++ b/build_msvc/libsecp256k1/libsecp256k1.vcxproj @@ -18,4 +18,5 @@ + \ No newline at end of file diff --git a/build_msvc/libunivalue/libunivalue.vcxproj b/build_msvc/libunivalue/libunivalue.vcxproj index 92a91157115b..0f13a5724135 100644 --- a/build_msvc/libunivalue/libunivalue.vcxproj +++ b/build_msvc/libunivalue/libunivalue.vcxproj @@ -15,4 +15,5 @@ + diff --git a/build_msvc/msbuild/tasks/replaceinfile.targets b/build_msvc/msbuild/tasks/replaceinfile.targets new file mode 100644 index 000000000000..2ccb8b30e053 --- /dev/null +++ b/build_msvc/msbuild/tasks/replaceinfile.targets @@ -0,0 +1,35 @@ + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/build_msvc/test_bitcoin/test_bitcoin.vcxproj b/build_msvc/test_bitcoin/test_bitcoin.vcxproj index 64bdb5f492a9..47e87b59e404 100644 --- a/build_msvc/test_bitcoin/test_bitcoin.vcxproj +++ b/build_msvc/test_bitcoin/test_bitcoin.vcxproj @@ -64,4 +64,5 @@ + diff --git a/build_msvc/testconsensus/testconsensus.vcxproj b/build_msvc/testconsensus/testconsensus.vcxproj index 9a6da3ecae76..776c40920abd 100644 --- a/build_msvc/testconsensus/testconsensus.vcxproj +++ b/build_msvc/testconsensus/testconsensus.vcxproj @@ -24,4 +24,5 @@ + diff --git a/ci/README.md b/ci/README.md new file mode 100644 index 000000000000..16c481158f33 --- /dev/null +++ b/ci/README.md @@ -0,0 +1,31 @@ +## ci scripts + +This directory contains scripts for each build step in each build stage. + +Currently three stages `lint`, `extended_lint` and `test` are defined. Each stage has its own lifecycle, similar to the +[Travis CI lifecycle](https://docs.travis-ci.com/user/job-lifecycle#the-job-lifecycle). Every script in here is named +and numbered according to which stage and lifecycle step it belongs to. + +### Running a stage locally + +To allow for a wide range of tested environments, but also ensure reproducibility to some extent, the test stage +requires `docker` to be installed. To install all requirements on Ubuntu, run + +``` +sudo apt install docker.io ccache bash git +``` + +To run the default test stage, + +``` +./ci/test_run_all.sh +``` + +To run the test stage with a specific configuration, + +``` +FILE_ENV="./ci/test/00_setup_env_arm.sh" ./ci/test_run_all.sh +``` + +Be aware that the tests will be build and run in-place, so please run at your own risk. +If the repository is not a fresh git clone, you might have to clean files from previous builds or test runs first. diff --git a/ci/extended_lint/04_install.sh b/ci/extended_lint/04_install.sh new file mode 100755 index 000000000000..123d874a84da --- /dev/null +++ b/ci/extended_lint/04_install.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C + +CPPCHECK_VERSION=1.86 +curl -s https://codeload.github.com/danmar/cppcheck/tar.gz/${CPPCHECK_VERSION} | tar -zxf - --directory /tmp/ +(cd /tmp/cppcheck-${CPPCHECK_VERSION}/ && make CFGDIR=/tmp/cppcheck-${CPPCHECK_VERSION}/cfg/ > /dev/null) +export PATH="$PATH:/tmp/cppcheck-${CPPCHECK_VERSION}/" diff --git a/ci/extended_lint/06_script.sh b/ci/extended_lint/06_script.sh new file mode 100755 index 000000000000..e8228c9c4d91 --- /dev/null +++ b/ci/extended_lint/06_script.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C + +test/lint/extended-lint-all.sh diff --git a/.travis/lint_04_install.sh b/ci/lint/04_install.sh similarity index 78% rename from .travis/lint_04_install.sh rename to ci/lint/04_install.sh index 62174620f286..20bff368a583 100755 --- a/.travis/lint_04_install.sh +++ b/ci/lint/04_install.sh @@ -6,9 +6,9 @@ export LC_ALL=C -travis_retry pip install codespell==1.15.0 -travis_retry pip install flake8==3.5.0 -travis_retry pip install vulture==0.29 +travis_retry pip3 install codespell==1.15.0 +travis_retry pip3 install flake8==3.5.0 +travis_retry pip3 install vulture==0.29 SHELLCHECK_VERSION=v0.6.0 curl -s "https://storage.googleapis.com/shellcheck/shellcheck-${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" | tar --xz -xf - --directory /tmp/ diff --git a/.travis/lint_05_before_script.sh b/ci/lint/05_before_script.sh similarity index 100% rename from .travis/lint_05_before_script.sh rename to ci/lint/05_before_script.sh diff --git a/.travis/lint_06_script.sh b/ci/lint/06_script.sh similarity index 90% rename from .travis/lint_06_script.sh rename to ci/lint/06_script.sh index eeebc80ec0b8..c7dea599dcd7 100755 --- a/.travis/lint_06_script.sh +++ b/ci/lint/06_script.sh @@ -18,7 +18,7 @@ test/lint/check-doc.py test/lint/check-rpc-mappings.py . test/lint/lint-all.sh -if [ "$TRAVIS_REPO_SLUG" = "bitcoin/bitcoin" -a "$TRAVIS_EVENT_TYPE" = "cron" ]; then +if [ "$TRAVIS_REPO_SLUG" = "bitcoin/bitcoin" ] && [ "$TRAVIS_EVENT_TYPE" = "cron" ]; then git log --merges --before="2 days ago" -1 --format='%H' > ./contrib/verify-commits/trusted-sha512-root-commit travis_retry gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys $( $max_sleep ? $max_sleep : t)}"` + fi +} + +__log_out() { + echo "$1" 1>&2 +} + +# Paramters: max_tries min_sleep max_sleep constant_sleep fail_script EXECUTION_COMMAND +retry() +{ + local max_tries="$1"; shift + local min_sleep="$1"; shift + local max_sleep="$1"; shift + local constant_sleep="$1"; shift + local fail_script="$1"; shift + if [ -n "$VERBOSE" ]; then + __log_out "Retry Parameters: max_tries=$max_tries min_sleep=$min_sleep max_sleep=$max_sleep constant_sleep=$constant_sleep" + if [ -n "$fail_script" ]; then __log_out "Fail script: $fail_script"; fi + __log_out "" + __log_out "Execution Command: $*" + __log_out "" + fi + + local attempts=0 + local return_code=1 + + + while [[ $return_code -ne 0 && $attempts -le $max_tries ]]; do + if [ $attempts -gt 0 ]; then + __sleep_amount + __log_out "Before retry #$attempts: sleeping $sleep_time seconds" + sleep $sleep_time + fi + + P="$1" + for param in "${@:2}"; do P="$P '$param'"; done + #TODO: replace single quotes in each arg with '"'"' ? + export RETRY_ATTEMPT=$attempts + bash -c "$P" + return_code=$? + #__log_out "Process returned $return_code on attempt $attempts" + if [ $return_code -eq 127 ]; then + # command not found + exit $return_code + elif [ $return_code -ne 0 ]; then + attempts=$[$attempts +1] + fi + done + + if [ $attempts -gt $max_tries ]; then + if [ -n "$fail_script" ]; then + __log_out "Retries exhausted, running fail script" + eval $fail_script + else + __log_out "Retries exhausted" + fi + fi + + exit $return_code +} + +# If we're being sourced, don't worry about such things +if [ "$BASH_SOURCE" == "$0" ]; then + # Prints the help text + help() + { + local retry=$(basename $0) + cat < /dev/null + if [[ $? -ne 4 ]]; then + echo "I’m sorry, 'getopt --test' failed in this environment. Please load GNU getopt." + exit 1 + fi + + OPTIONS=vt:s:m:x:f: + LONGOPTIONS=verbose,tries:,sleep:,min:,max:,fail: + + PARSED=$($GETOPT_BIN --options="$OPTIONS" --longoptions="$LONGOPTIONS" --name "$0" -- "$@") + if [[ $? -ne 0 ]]; then + # e.g. $? == 1 + # then getopt has complained about wrong arguments to stdout + exit 2 + fi + # read getopt’s output this way to handle the quoting right: + eval set -- "$PARSED" + + max_tries=10 + min_sleep=0.3 + max_sleep=60.0 + constant_sleep= + fail_script= + + # now enjoy the options in order and nicely split until we see -- + while true; do + case "$1" in + -v|--verbose) + VERBOSE=true + shift + ;; + -t|--tries) + max_tries="$2" + shift 2 + ;; + -s|--sleep) + constant_sleep="$2" + shift 2 + ;; + -m|--min) + min_sleep="$2" + shift 2 + ;; + -x|--max) + max_sleep="$2" + shift 2 + ;; + -f|--fail) + fail_script="$2" + shift 2 + ;; + --) + shift + break + ;; + *) + echo "Programming error" + exit 3 + ;; + esac + done + + retry "$max_tries" "$min_sleep" "$max_sleep" "$constant_sleep" "$fail_script" "$@" + +fi diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh new file mode 100755 index 000000000000..09b37f8240cd --- /dev/null +++ b/ci/test/00_setup_env.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +echo "Setting default values in env" + +BASE_ROOT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../../ >/dev/null 2>&1 && pwd ) +export BASE_ROOT_DIR + +export MAKEJOBS=${MAKEJOBS:--j3} +export HOST=${HOST:-x86_64-unknown-linux-gnu} +export RUN_UNIT_TESTS=${RUN_UNIT_TESTS:-true} +export RUN_FUNCTIONAL_TESTS=${RUN_FUNCTIONAL_TESTS:-true} +export RUN_FUZZ_TESTS=${RUN_FUZZ_TESTS:-false} +export DOCKER_NAME_TAG=${DOCKER_NAME_TAG:-ubuntu:18.04} +export BOOST_TEST_RANDOM=${BOOST_TEST_RANDOM:-1$TRAVIS_BUILD_ID} +export CCACHE_SIZE=${CCACHE_SIZE:-100M} +export CCACHE_TEMPDIR=${CCACHE_TEMPDIR:-/tmp/.ccache-temp} +export CCACHE_COMPRESS=${CCACHE_COMPRESS:-1} +export CCACHE_DIR=${CCACHE_DIR:-$HOME/.ccache} +export BASE_BUILD_DIR=${BASE_BUILD_DIR:-${TRAVIS_BUILD_DIR:-$BASE_ROOT_DIR}} +export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_BUILD_DIR/out/$HOST} +export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks} +export WINEDEBUG=${WINEDEBUG:-fixme-all} +export DOCKER_PACKAGES=${DOCKER_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3} +export GOAL=${GOAL:-install} +export DIR_QA_ASSETS=${DIR_QA_ASSETS:-${BASE_BUILD_DIR}/qa-assets} +export PATH=${BASE_ROOT_DIR}/ci/retry:$PATH +export CI_RETRY_EXE=${CI_RETRY_EXE:retry} + +echo "Setting specific values in env" +if [ -n "${FILE_ENV}" ]; then + set -o errexit; + # shellcheck disable=SC1090 + source "${FILE_ENV}" +fi diff --git a/ci/test/00_setup_env_amd64_asan.sh b/ci/test/00_setup_env_amd64_asan.sh new file mode 100644 index 000000000000..9d20b6a72b3b --- /dev/null +++ b/ci/test/00_setup_env_amd64_asan.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +export HOST=x86_64-unknown-linux-gnu +export PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libssl1.0-dev libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libprotobuf-dev protobuf-compiler libqrencode-dev" +export NO_DEPENDS=1 +export GOAL="install" +export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=qt5 CPPFLAGS=-DDEBUG_LOCKORDER --with-sanitizers=address,integer,undefined CC=clang CXX=clang++" diff --git a/ci/test/00_setup_env_amd64_fuzz.sh b/ci/test/00_setup_env_amd64_fuzz.sh new file mode 100644 index 000000000000..edcb65af286b --- /dev/null +++ b/ci/test/00_setup_env_amd64_fuzz.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +export HOST=x86_64-unknown-linux-gnu +export PACKAGES="clang llvm python3 libssl1.0-dev libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev" +export NO_DEPENDS=1 +export RUN_UNIT_TESTS=false +export RUN_FUNCTIONAL_TESTS=false +export RUN_FUZZ_TESTS=true +export GOAL="install" +export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,address CC=clang CXX=clang++" diff --git a/ci/test/00_setup_env_amd64_nowallet.sh b/ci/test/00_setup_env_amd64_nowallet.sh new file mode 100644 index 000000000000..d5a2ba311192 --- /dev/null +++ b/ci/test/00_setup_env_amd64_nowallet.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +export HOST=x86_64-unknown-linux-gnu +export PACKAGES="python3-zmq" +export DEP_OPTS="NO_WALLET=1" +export GOAL="install" +export BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports" diff --git a/ci/test/00_setup_env_amd64_qt5.sh b/ci/test/00_setup_env_amd64_qt5.sh new file mode 100644 index 000000000000..77b1531be478 --- /dev/null +++ b/ci/test/00_setup_env_amd64_qt5.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +export HOST=x86_64-unknown-linux-gnu +export PACKAGES="python3-zmq qtbase5-dev qttools5-dev-tools protobuf-compiler libdbus-1-dev libharfbuzz-dev libprotobuf-dev" +export DEP_OPTS="NO_QT=1 NO_UPNP=1 DEBUG=1 ALLOW_HOST_PACKAGES=1" +export TEST_RUNNER_EXTRA="--coverage --extended --exclude feature_dbcrash" # Run extended tests so that coverage does not fail, but exclude the very slow dbcrash +export GOAL="install" +export BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --enable-glibc-back-compat --enable-reduce-exports --enable-debug CFLAGS=\"-g0 -O2 -funsigned-char\" CXXFLAGS=\"-g0 -O2 -funsigned-char\"" diff --git a/ci/test/00_setup_env_amd64_trusty.sh b/ci/test/00_setup_env_amd64_trusty.sh new file mode 100644 index 000000000000..cc0f1196e78a --- /dev/null +++ b/ci/test/00_setup_env_amd64_trusty.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +export HOST=x86_64-unknown-linux-gnu +export DOCKER_NAME_TAG=ubuntu:14.04 +export PACKAGES="python3-zmq qtbase5-dev qttools5-dev-tools libicu-dev libpng-dev libssl-dev libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev libdb5.1++-dev libzmq3-dev libprotobuf-dev protobuf-compiler libqrencode-dev" +export NO_DEPENDS=1 +export RUN_FUNCTIONAL_TESTS=false +export GOAL="install" +export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=no" diff --git a/ci/test/00_setup_env_amd64_tsan.sh b/ci/test/00_setup_env_amd64_tsan.sh new file mode 100644 index 000000000000..c127e284bdfe --- /dev/null +++ b/ci/test/00_setup_env_amd64_tsan.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +export HOST=x86_64-unknown-linux-gnu +export DOCKER_NAME_TAG=ubuntu:16.04 +export PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libssl-dev libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libprotobuf-dev protobuf-compiler libqrencode-dev" +export NO_DEPENDS=1 +export GOAL="install" +export BITCOIN_CONFIG="--enable-zmq --disable-wallet --with-gui=qt5 CPPFLAGS=-DDEBUG_LOCKORDER --with-sanitizers=thread --disable-hardening --disable-asm CC=clang CXX=clang++" diff --git a/ci/test/00_setup_env_arm.sh b/ci/test/00_setup_env_arm.sh new file mode 100644 index 000000000000..ac7ace8c3b63 --- /dev/null +++ b/ci/test/00_setup_env_arm.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +export HOST=arm-linux-gnueabihf +export PACKAGES="python3 g++-arm-linux-gnueabihf" +export RUN_UNIT_TESTS=false +export RUN_FUNCTIONAL_TESTS=false +export GOAL="install" +# -Wno-psabi is to disable ABI warnings: "note: parameter passing for argument of type ... changed in GCC 7.1" +# This could be removed once the ABI change warning does not show up by default +export BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports CXXFLAGS=-Wno-psabi" diff --git a/ci/test/00_setup_env_i686.sh b/ci/test/00_setup_env_i686.sh new file mode 100644 index 000000000000..768e2ac558dc --- /dev/null +++ b/ci/test/00_setup_env_i686.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +export HOST=i686-pc-linux-gnu +export PACKAGES="g++-multilib python3-zmq" +export GOAL="install" +export BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --disable-bip70 --enable-glibc-back-compat --enable-reduce-exports LDFLAGS=-static-libstdc++" +export CONFIG_SHELL="/bin/dash" diff --git a/ci/test/00_setup_env_mac.sh b/ci/test/00_setup_env_mac.sh new file mode 100644 index 000000000000..f384ba92636c --- /dev/null +++ b/ci/test/00_setup_env_mac.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +export HOST=x86_64-apple-darwin14 +export PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python3-dev python3-setuptools" +export OSX_SDK=10.11 +export RUN_UNIT_TESTS=false +export RUN_FUNCTIONAL_TESTS=false +export GOAL="deploy" +export BITCOIN_CONFIG="--enable-gui --enable-reduce-exports --enable-werror" diff --git a/ci/test/00_setup_env_win64.sh b/ci/test/00_setup_env_win64.sh new file mode 100644 index 000000000000..1e04c4287af1 --- /dev/null +++ b/ci/test/00_setup_env_win64.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +export HOST=x86_64-w64-mingw32 +export PACKAGES="python3 nsis g++-mingw-w64-x86-64 wine-binfmt wine64" +export RUN_FUNCTIONAL_TESTS=false +export GOAL="deploy" +export BITCOIN_CONFIG="--enable-reduce-exports --disable-gui-tests" diff --git a/.travis/test_03_before_install.sh b/ci/test/03_before_install.sh similarity index 87% rename from .travis/test_03_before_install.sh rename to ci/test/03_before_install.sh index 3c9fcf3f983a..5086114ba16e 100755 --- a/.travis/test_03_before_install.sh +++ b/ci/test/03_before_install.sh @@ -6,7 +6,6 @@ export LC_ALL=C.UTF-8 -PATH=$(echo $PATH | tr ':' "\n" | sed '/\/opt\/python/d' | tr "\n" ":" | sed "s|::|:|g") # Add llvm-symbolizer directory to PATH. Needed to get symbolized stack traces from the sanitizers. PATH=$PATH:/usr/lib/llvm-6.0/bin/ export PATH diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh new file mode 100755 index 000000000000..3535746e83f1 --- /dev/null +++ b/ci/test/04_install.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2018 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +ccache echo "Creating ccache dir if it didn't already exist" + +if [ ! -d ${DIR_QA_ASSETS} ]; then + git clone https://github.com/bitcoin-core/qa-assets ${DIR_QA_ASSETS} +fi +export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/ + +mkdir -p "${BASE_BUILD_DIR}/sanitizer-output/" +export ASAN_OPTIONS="" +export LSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/lsan" +export TSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/tsan:log_path=${BASE_BUILD_DIR}/sanitizer-output/tsan" +export UBSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1" +env | grep -E '^(BITCOIN_CONFIG|CCACHE_|WINEDEBUG|LC_ALL|BOOST_TEST_RANDOM|CONFIG_SHELL|(ASAN|LSAN|TSAN|UBSAN)_OPTIONS)' | tee /tmp/env +if [[ $HOST = *-mingw32 ]]; then + DOCKER_ADMIN="--cap-add SYS_ADMIN" +elif [[ $BITCOIN_CONFIG = *--with-sanitizers=*address* ]]; then # If ran with (ASan + LSan), Docker needs access to ptrace (https://github.com/google/sanitizers/issues/764) + DOCKER_ADMIN="--cap-add SYS_PTRACE" +fi + +if [ -z "$RUN_CI_ON_HOST" ]; then + echo "Creating $DOCKER_NAME_TAG container to run in" + ${CI_RETRY_EXE} docker pull "$DOCKER_NAME_TAG" + + DOCKER_ID=$(docker run $DOCKER_ADMIN -idt --mount type=bind,src=$BASE_BUILD_DIR,dst=$BASE_BUILD_DIR --mount type=bind,src=$CCACHE_DIR,dst=$CCACHE_DIR -w $BASE_BUILD_DIR --env-file /tmp/env $DOCKER_NAME_TAG) + + DOCKER_EXEC () { + docker exec $DOCKER_ID bash -c "cd $PWD && $*" + } +else + echo "Running on host system without docker wrapper" + DOCKER_EXEC () { + bash -c "cd $PWD && $*" + } +fi + +DOCKER_EXEC free -m -h +DOCKER_EXEC echo "Number of CPUs \(nproc\): $(nproc)" + +${CI_RETRY_EXE} DOCKER_EXEC apt-get update +${CI_RETRY_EXE} DOCKER_EXEC apt-get install --no-install-recommends --no-upgrade -qq $PACKAGES $DOCKER_PACKAGES + diff --git a/.travis/test_05_before_script.sh b/ci/test/05_before_script.sh similarity index 82% rename from .travis/test_05_before_script.sh rename to ci/test/05_before_script.sh index 70caee979c14..516d3fc0428b 100755 --- a/.travis/test_05_before_script.sh +++ b/ci/test/05_before_script.sh @@ -10,10 +10,10 @@ DOCKER_EXEC echo \> \$HOME/.bitcoin # Make sure default datadir does not exist mkdir -p depends/SDKs depends/sdk-sources -if [ -n "$OSX_SDK" -a ! -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then +if [ -n "$OSX_SDK" ] && [ ! -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then curl --location --fail $SDK_URL/MacOSX${OSX_SDK}.sdk.tar.gz -o depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz fi -if [ -n "$OSX_SDK" -a -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then +if [ -n "$OSX_SDK" ] && [ -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then tar -C depends/SDKs -xf depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz fi if [[ $HOST = *-mingw32 ]]; then @@ -22,4 +22,3 @@ fi if [ -z "$NO_DEPENDS" ]; then DOCKER_EXEC CONFIG_SHELL= make $MAKEJOBS -C depends HOST=$HOST $DEP_OPTS fi - diff --git a/.travis/test_06_script_a.sh b/ci/test/06_script_a.sh similarity index 70% rename from .travis/test_06_script_a.sh rename to ci/test/06_script_a.sh index 8cc593f93652..eb6ade7919f8 100755 --- a/.travis/test_06_script_a.sh +++ b/ci/test/06_script_a.sh @@ -6,11 +6,7 @@ export LC_ALL=C.UTF-8 -TRAVIS_COMMIT_LOG=$(git log --format=fuller -1) -export TRAVIS_COMMIT_LOG - -OUTDIR=$BASE_OUTDIR/$TRAVIS_PULL_REQUEST/$TRAVIS_JOB_NUMBER-$HOST -BITCOIN_CONFIG_ALL="--disable-dependency-tracking --prefix=$TRAVIS_BUILD_DIR/depends/$HOST --bindir=$OUTDIR/bin --libdir=$OUTDIR/lib" +BITCOIN_CONFIG_ALL="--disable-dependency-tracking --prefix=$BASE_BUILD_DIR/depends/$HOST --bindir=$BASE_OUTDIR/bin --libdir=$BASE_OUTDIR/lib" if [ -z "$NO_DEPENDS" ]; then DOCKER_EXEC ccache --max-size=$CCACHE_SIZE fi @@ -23,7 +19,7 @@ else fi END_FOLD -mkdir build +mkdir -p build cd build || (echo "could not enter build directory"; exit 1) BEGIN_FOLD configure @@ -41,10 +37,10 @@ DOCKER_EXEC ./configure --cache-file=../config.cache $BITCOIN_CONFIG_ALL $BITCOI END_FOLD set -o errtrace -trap 'DOCKER_EXEC "cat ${TRAVIS_BUILD_DIR}/sanitizer-output/* 2> /dev/null"' ERR +trap 'DOCKER_EXEC "cat ${BASE_BUILD_DIR}/sanitizer-output/* 2> /dev/null"' ERR BEGIN_FOLD build DOCKER_EXEC make $MAKEJOBS $GOAL || ( echo "Build failure. Verbose build follows." && DOCKER_EXEC make $GOAL V=1 ; false ) END_FOLD -cd ${TRAVIS_BUILD_DIR} || (echo "could not enter travis build dir $TRAVIS_BUILD_DIR"; exit 1) +cd ${BASE_BUILD_DIR} || (echo "could not enter travis build dir $BASE_BUILD_DIR"; exit 1) diff --git a/.travis/test_06_script_b.sh b/ci/test/06_script_b.sh similarity index 67% rename from .travis/test_06_script_b.sh rename to ci/test/06_script_b.sh index e40055a6ee13..d179ce81c0a1 100755 --- a/.travis/test_06_script_b.sh +++ b/ci/test/06_script_b.sh @@ -10,13 +10,13 @@ cd "build/bitcoin-$HOST" || (echo "could not enter distdir build/bitcoin-$HOST"; if [ "$RUN_UNIT_TESTS" = "true" ]; then BEGIN_FOLD unit-tests - DOCKER_EXEC LD_LIBRARY_PATH=$TRAVIS_BUILD_DIR/depends/$HOST/lib make $MAKEJOBS check VERBOSE=1 + DOCKER_EXEC LD_LIBRARY_PATH=$BASE_BUILD_DIR/depends/$HOST/lib make $MAKEJOBS check VERBOSE=1 END_FOLD fi if [ "$RUN_FUNCTIONAL_TESTS" = "true" ]; then BEGIN_FOLD functional-tests - DOCKER_EXEC test/functional/test_runner.py --ci --combinedlogslen=4000 ${TEST_RUNNER_EXTRA} --quiet --failfast + DOCKER_EXEC test/functional/test_runner.py --ci --ansi --combinedlogslen=4000 ${TEST_RUNNER_EXTRA} --quiet --failfast END_FOLD fi @@ -26,4 +26,4 @@ if [ "$RUN_FUZZ_TESTS" = "true" ]; then END_FOLD fi -cd ${TRAVIS_BUILD_DIR} || (echo "could not enter travis build dir $TRAVIS_BUILD_DIR"; exit 1) +cd ${BASE_BUILD_DIR} || (echo "could not enter travis build dir $BASE_BUILD_DIR"; exit 1) diff --git a/ci/test_run_all.sh b/ci/test_run_all.sh new file mode 100755 index 000000000000..a1d4bd19524d --- /dev/null +++ b/ci/test_run_all.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +set -o errexit; source ./ci/test/00_setup_env.sh +set -o errexit; source ./ci/test/03_before_install.sh +set -o errexit; source ./ci/test/04_install.sh +set -o errexit; source ./ci/test/05_before_script.sh +set -o errexit; source ./ci/test/06_script_a.sh +set -o errexit; source ./ci/test/06_script_b.sh diff --git a/configure.ac b/configure.ac index 5e1a5e14a132..adbb1c5a4c9a 100644 --- a/configure.ac +++ b/configure.ac @@ -1,8 +1,8 @@ dnl require autoconf 2.60 (AS_ECHO/AS_ECHO_N) AC_PREREQ([2.60]) define(_CLIENT_VERSION_MAJOR, 0) -define(_CLIENT_VERSION_MINOR, 18) -define(_CLIENT_VERSION_REVISION, 99) +define(_CLIENT_VERSION_MINOR, 1) +define(_CLIENT_VERSION_REVISION, 1) define(_CLIENT_VERSION_BUILD, 0) define(_CLIENT_VERSION_RC, 0) define(_CLIENT_VERSION_IS_RELEASE, false) @@ -133,7 +133,7 @@ AC_ARG_ENABLE(gui-tests, AC_ARG_WITH([rapidcheck], [AS_HELP_STRING([--with-rapidcheck], - [enable RapidCheck property based tests (default is yes if librapidcheck is found)])], + [enable RapidCheck property-based tests (default is yes if librapidcheck is found)])], [use_rapidcheck=$withval], [use_rapidcheck=auto]) @@ -148,7 +148,8 @@ AC_ARG_ENABLE([extended-functional-tests], [use_extended_functional_tests=no]) AC_ARG_ENABLE([fuzz], - AS_HELP_STRING([--enable-fuzz],[enable building of fuzz targets (default no)]), + AS_HELP_STRING([--enable-fuzz], + [enable building of fuzz targets (default no). enabling this will disable all other targets]), [enable_fuzz=$enableval], [enable_fuzz=no]) @@ -238,7 +239,7 @@ AM_CONDITIONAL(ENABLE_MAN, test "$enable_man" != no) # Enable debug AC_ARG_ENABLE([debug], [AS_HELP_STRING([--enable-debug], - [use debug compiler flags and macros (default is no)])], + [use compiler flags and macros suited for debugging (default is no)])], [enable_debug=$enableval], [enable_debug=no]) @@ -270,12 +271,9 @@ if test "x$enable_debug" = xyes; then if test "x$CXXFLAGS_overridden" = xno; then CXXFLAGS="" fi - # Prefer -Og, fall back to -O0 if that is unavailable. - AX_CHECK_COMPILE_FLAG( - [-Og], - [[DEBUG_CXXFLAGS="$DEBUG_CXXFLAGS -Og"]], - [AX_CHECK_COMPILE_FLAG([-O0],[[DEBUG_CXXFLAGS="$DEBUG_CXXFLAGS -O0"]],,[[$CXXFLAG_WERROR]])], - [[$CXXFLAG_WERROR]]) + + # Disable all optimizations + AX_CHECK_COMPILE_FLAG([-O0], [[DEBUG_CXXFLAGS="$DEBUG_CXXFLAGS -O0"]],,[[$CXXFLAG_WERROR]]) # Prefer -g3, fall back to -g if that is unavailable. AX_CHECK_COMPILE_FLAG( @@ -322,6 +320,7 @@ if test "x$enable_werror" = "xyes"; then AC_MSG_ERROR("enable-werror set but -Werror is not usable") fi AX_CHECK_COMPILE_FLAG([-Werror=vla],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=vla"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Werror=switch],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=switch"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Werror=thread-safety-analysis],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=thread-safety-analysis"],,[[$CXXFLAG_WERROR]]) fi @@ -330,6 +329,7 @@ if test "x$CXXFLAGS_overridden" = "xno"; then AX_CHECK_COMPILE_FLAG([-Wextra],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wextra"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wformat],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wformat"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wvla],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wvla"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Wswitch],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wswitch"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wformat-security],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wformat-security"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wthread-safety-analysis],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wthread-safety-analysis"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wrange-loop-analysis],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wrange-loop-analysis"],,[[$CXXFLAG_WERROR]]) @@ -925,6 +925,29 @@ if test x$use_reduce_exports = xyes; then [AC_MSG_ERROR([Cannot set default symbol visibility. Use --disable-reduce-exports.])]) fi +AC_MSG_CHECKING([for std::system]) +AC_LINK_IFELSE( + [ AC_LANG_PROGRAM( + [[ #include ]], + [[ int nErr = std::system(""); ]] + )], + [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_STD__SYSTEM, 1, Define to 1 if you have the `std::system' function.)], + [ AC_MSG_RESULT(no) ] +) + +AC_MSG_CHECKING([for ::_wsystem]) +AC_LINK_IFELSE( + [ AC_LANG_PROGRAM( + [[ ]], + [[ int nErr = ::_wsystem(""); ]] + )], + [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_WSYSTEM, 1, Define to 1 if you have the `::wsystem' function.)], + [ AC_MSG_RESULT(no) ] +) + +# Define to 1 if std::system or ::wsystem (Windows) is available +AC_DEFINE([HAVE_SYSTEM], [HAVE_STD__SYSTEM || HAVE_WSYSTEM], [std::system or ::wsystem]) + LEVELDB_CPPFLAGS= LIBLEVELDB= LIBMEMENV= @@ -933,6 +956,29 @@ AC_SUBST(LEVELDB_CPPFLAGS) AC_SUBST(LIBLEVELDB) AC_SUBST(LIBMEMENV) +dnl enable-fuzz should disable all other targets +if test "x$enable_fuzz" = "xyes"; then + AC_MSG_WARN(enable-fuzz will disable all other targets) + build_bitcoin_utils=no + build_bitcoin_cli=no + build_bitcoin_tx=no + build_bitcoin_wallet=no + build_bitcoind=no + build_bitcoin_libs=no + bitcoin_enable_qt=no + bitcoin_enable_qt_test=no + bitcoin_enable_qt_dbus=no + enable_wallet=no + use_bench=no + use_upnp=no + use_zmq=no +else + BITCOIN_QT_INIT + + dnl sets $bitcoin_enable_qt, $bitcoin_enable_qt_test, $bitcoin_enable_qt_dbus + BITCOIN_QT_CONFIGURE([$use_pkgconfig]) +fi + if test x$enable_wallet != xno; then dnl Check for libdb_cxx only if wallet enabled BITCOIN_FIND_BDB48 @@ -945,12 +991,27 @@ if test x$use_upnp != xno; then [AC_CHECK_LIB([miniupnpc], [upnpDiscover], [MINIUPNPC_LIBS=-lminiupnpc], [have_miniupnpc=no])], [have_miniupnpc=no] ) +dnl The minimum supported miniUPnPc API version is set to 10. This keeps compatibility +dnl with Ubuntu 16.04 LTS and Debian 8 libminiupnpc-dev packages. +if test x$have_miniupnpc != xno; then + AC_MSG_CHECKING([whether miniUPnPc API version is supported]) + AC_PREPROC_IFELSE([AC_LANG_PROGRAM([[ + @%:@include + ]], [[ + #if MINIUPNPC_API_VERSION >= 10 + // Everything is okay + #else + # error miniUPnPc API version is too old + #endif + ]])],[ + AC_MSG_RESULT(yes) + ],[ + AC_MSG_RESULT(no) + AC_MSG_WARN([miniUPnPc API version < 10 is unsupported, disabling UPnP support.]) + have_miniupnpc=no + ]) +fi fi - -BITCOIN_QT_INIT - -dnl sets $bitcoin_enable_qt, $bitcoin_enable_qt_test, $bitcoin_enable_qt_dbus -BITCOIN_QT_CONFIGURE([$use_pkgconfig]) if test x$build_bitcoin_wallet$build_bitcoin_cli$build_bitcoin_tx$build_bitcoind$bitcoin_enable_qt$use_tests$use_bench = xnonononononono; then use_boost=no @@ -1214,7 +1275,7 @@ AC_CHECK_DECLS([EVP_MD_CTX_new],,,[AC_INCLUDES_DEFAULT ]) CXXFLAGS="${save_CXXFLAGS}" -dnl RapidCheck Property Based Testing +dnl RapidCheck property-based testing enable_property_tests=no if test "x$use_rapidcheck" = xauto; then @@ -1343,9 +1404,10 @@ dnl enable upnp support AC_MSG_CHECKING([whether to build with support for UPnP]) if test x$have_miniupnpc = xno; then if test x$use_upnp = xyes; then - AC_MSG_ERROR("UPnP requested but cannot be built. use --without-miniupnpc") + AC_MSG_ERROR("UPnP requested but cannot be built. Use --without-miniupnpc.") fi AC_MSG_RESULT(no) + use_upnp=no else if test x$use_upnp != xno; then AC_MSG_RESULT(yes) @@ -1560,7 +1622,7 @@ if test x$need_bundled_univalue = xyes; then AC_CONFIG_SUBDIRS([src/univalue]) fi -ac_configure_args="${ac_configure_args} --disable-shared --with-pic --with-bignum=no --enable-module-recovery --disable-jni" +ac_configure_args="${ac_configure_args} --disable-shared --with-pic --enable-benchmark=no --with-bignum=no --enable-module-recovery --enable-module-schnorrsig --enable-experimental --disable-jni" AC_CONFIG_SUBDIRS([src/secp256k1]) AC_OUTPUT @@ -1595,6 +1657,7 @@ fi echo " with zmq = $use_zmq" echo " with test = $use_tests" if test x$use_tests != xno; then + echo " with prop = $enable_property_tests" echo " with fuzz = $enable_fuzz" fi echo " with bench = $use_bench" diff --git a/contrib/README.md b/contrib/README.md index 8915919766b1..e9e72f6686f8 100644 --- a/contrib/README.md +++ b/contrib/README.md @@ -3,10 +3,10 @@ Repository Tools ### [Developer tools](/contrib/devtools) ### Specific tools for developers working on this repository. -Contains the script `github-merge.py` for merging GitHub pull requests securely and signing them using GPG. +Additional tools, including the `github-merge.py` script, are available in the [maintainer-tools](https://github.com/bitcoin-core/bitcoin-maintainer-tools) repository. ### [Verify-Commits](/contrib/verify-commits) ### -Tool to verify that every merge commit was signed by a developer using the above `github-merge.py` script. +Tool to verify that every merge commit was signed by a developer using the `github-merge.py` script. ### [Linearize](/contrib/linearize) ### Construct a linear, no-fork, best version of the blockchain. diff --git a/contrib/debian/copyright b/contrib/debian/copyright index 2d5b0188d25c..0eccbacb96a5 100644 --- a/contrib/debian/copyright +++ b/contrib/debian/copyright @@ -26,21 +26,14 @@ License: GNU-All-permissive-License Files: src/qt/res/icons/add.png src/qt/res/icons/address-book.png src/qt/res/icons/chevron.png - src/qt/res/icons/configure.png - src/qt/res/icons/debugwindow.png src/qt/res/icons/edit.png src/qt/res/icons/editcopy.png src/qt/res/icons/editpaste.png src/qt/res/icons/export.png src/qt/res/icons/eye.png - src/qt/res/icons/filesave.png src/qt/res/icons/history.png - src/qt/res/icons/info.png - src/qt/res/icons/key.png src/qt/res/icons/lock_*.png - src/qt/res/icons/open.png src/qt/res/icons/overview.png - src/qt/res/icons/quit.png src/qt/res/icons/receive.png src/qt/res/icons/remove.png src/qt/res/icons/send.png @@ -60,7 +53,7 @@ Files: src/qt/res/icons/connect*.png Copyright: Marco Falke Luke Dashjr License: Expat -Comment: Inspired by Stephan Hutchings Typicons +Comment: Inspired by Stephen Hutchings' Typicons Files: src/qt/res/icons/tx_mined.png src/qt/res/src/mine.svg @@ -72,21 +65,17 @@ Files: src/qt/res/icons/tx_mined.png src/qt/res/src/hd_enabled.svg Copyright: Jonas Schnelli License: Expat -Comment: Files: src/qt/res/icons/clock*.png src/qt/res/icons/eye_*.png src/qt/res/icons/tx_in*.png - src/qt/res/icons/verify.png src/qt/res/src/clock_*.svg src/qt/res/src/tx_*.svg - src/qt/res/src/verify.svg -Copyright: Stephan Hutching, Jonas Schnelli +Copyright: Stephen Hutchings, Jonas Schnelli License: Expat -Comment: Modifications of Stephan Hutchings Typicons +Comment: Modifications of Stephen Hutchings' Typicons -Files: src/qt/res/icons/about.png - src/qt/res/icons/bitcoin.* +Files: src/qt/res/icons/bitcoin.* share/pixmaps/bitcoin* src/qt/res/src/bitcoin.svg Copyright: Bitboy, Jonas Schnelli diff --git a/contrib/devtools/README.md b/contrib/devtools/README.md index 4994d7f0a528..3d1024c7a513 100644 --- a/contrib/devtools/README.md +++ b/contrib/devtools/README.md @@ -89,66 +89,6 @@ example: BUILDDIR=$PWD/build contrib/devtools/gen-manpages.sh ``` -github-merge.py -=============== - -A small script to automate merging pull-requests securely and sign them with GPG. - -For example: - - ./github-merge.py 3077 - -(in any git repository) will help you merge pull request #3077 for the -bitcoin/bitcoin repository. - -What it does: -* Fetch master and the pull request. -* Locally construct a merge commit. -* Show the diff that merge results in. -* Ask you to verify the resulting source tree (so you can do a make -check or whatever). -* Ask you whether to GPG sign the merge commit. -* Ask you whether to push the result upstream. - -This means that there are no potential race conditions (where a -pullreq gets updated while you're reviewing it, but before you click -merge), and when using GPG signatures, that even a compromised GitHub -couldn't mess with the sources. - -Setup ---------- -Configuring the github-merge tool for the bitcoin repository is done in the following way: - - git config githubmerge.repository bitcoin/bitcoin - git config githubmerge.testcmd "make -j4 check" (adapt to whatever you want to use for testing) - git config --global user.signingkey mykeyid - -Authentication (optional) --------------------------- - -The API request limit for unauthenticated requests is quite low, but the -limit for authenticated requests is much higher. If you start running -into rate limiting errors it can be useful to set an authentication token -so that the script can authenticate requests. - -- First, go to [Personal access tokens](https://github.com/settings/tokens). -- Click 'Generate new token'. -- Fill in an arbitrary token description. No further privileges are needed. -- Click the `Generate token` button at the bottom of the form. -- Copy the generated token (should be a hexadecimal string) - -Then do: - - git config --global user.ghtoken "pasted token" - -Create and verify timestamps of merge commits ---------------------------------------------- -To create or verify timestamps on the merge commits, install the OpenTimestamps -client via `pip3 install opentimestamps-client`. Then, download the gpg wrapper -`ots-git-gpg-wrapper.sh` and set it as git's `gpg.program`. See -[the ots git integration documentation](https://github.com/opentimestamps/opentimestamps-client/blob/master/doc/git-integration.md#usage) -for further details. - optimize-pngs.py ================ diff --git a/contrib/devtools/copyright_header.py b/contrib/devtools/copyright_header.py index f2987f226054..fc01e570aa89 100755 --- a/contrib/devtools/copyright_header.py +++ b/contrib/devtools/copyright_header.py @@ -34,7 +34,7 @@ "src/univalue/", ] -INCLUDE = ['*.h', '*.cpp', '*.cc', '*.c', '*.py'] +INCLUDE = ['*.h', '*.cpp', '*.cc', '*.c', '*.mm', '*.py'] INCLUDE_COMPILED = re.compile('|'.join([fnmatch.translate(m) for m in INCLUDE])) def applies_to_file(filename): @@ -90,14 +90,12 @@ def compile_copyright_regex(copyright_style, year_style, name): EXPECTED_HOLDER_NAMES = [ "Satoshi Nakamoto\n", "The Bitcoin Core developers\n", - "Bitcoin Core Developers\n", "BitPay Inc\.\n", "University of Illinois at Urbana-Champaign\.\n", "Pieter Wuille\n", "Wladimir J. van der Laan\n", "Jeff Garzik\n", "Jan-Klaas Kollhof\n", - "Sam Rushing\n", "ArtForz -- public domain half-a-node\n", "Intel Corporation", "The Zcash developers", diff --git a/contrib/devtools/gen-manpages.sh b/contrib/devtools/gen-manpages.sh index 63b9847100a0..dbdb62287759 100755 --- a/contrib/devtools/gen-manpages.sh +++ b/contrib/devtools/gen-manpages.sh @@ -16,7 +16,7 @@ BITCOINQT=${BITCOINQT:-$BINDIR/qt/bitcoin-qt} [ ! -x $BITCOIND ] && echo "$BITCOIND not found or not executable." && exit 1 # The autodetected version git tag can screw up manpage output a little bit -BTCVER=($($BITCOINCLI --version | head -n1 | awk -F'[ -]' '{ print $6, $7 }')) +read -r -a BTCVER <<< "$($BITCOINCLI --version | head -n1 | awk -F'[ -]' '{ print $6, $7 }')" # Create a footer file with copyright content. # This gets autodetected fine for bitcoind if --version-string is not set, diff --git a/contrib/devtools/github-merge.py b/contrib/devtools/github-merge.py deleted file mode 100755 index 03179cec2475..000000000000 --- a/contrib/devtools/github-merge.py +++ /dev/null @@ -1,384 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2016-2017 Bitcoin Core Developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -# This script will locally construct a merge commit for a pull request on a -# github repository, inspect it, sign it and optionally push it. - -# The following temporary branches are created/overwritten and deleted: -# * pull/$PULL/base (the current master we're merging onto) -# * pull/$PULL/head (the current state of the remote pull request) -# * pull/$PULL/merge (github's merge) -# * pull/$PULL/local-merge (our merge) - -# In case of a clean merge that is accepted by the user, the local branch with -# name $BRANCH is overwritten with the merged result, and optionally pushed. -import os -from sys import stdin,stdout,stderr -import argparse -import hashlib -import subprocess -import sys -import json -import codecs -from urllib.request import Request, urlopen -from urllib.error import HTTPError - -# External tools (can be overridden using environment) -GIT = os.getenv('GIT','git') -BASH = os.getenv('BASH','bash') - -# OS specific configuration for terminal attributes -ATTR_RESET = '' -ATTR_PR = '' -COMMIT_FORMAT = '%H %s (%an)%d' -if os.name == 'posix': # if posix, assume we can use basic terminal escapes - ATTR_RESET = '\033[0m' - ATTR_PR = '\033[1;36m' - COMMIT_FORMAT = '%C(bold blue)%H%Creset %s %C(cyan)(%an)%Creset%C(green)%d%Creset' - -def git_config_get(option, default=None): - ''' - Get named configuration option from git repository. - ''' - try: - return subprocess.check_output([GIT,'config','--get',option]).rstrip().decode('utf-8') - except subprocess.CalledProcessError: - return default - -def get_response(req_url, ghtoken): - req = Request(req_url) - if ghtoken is not None: - req.add_header('Authorization', 'token ' + ghtoken) - return urlopen(req) - -def retrieve_json(req_url, ghtoken, use_pagination=False): - ''' - Retrieve json from github. - Return None if an error happens. - ''' - try: - reader = codecs.getreader('utf-8') - if not use_pagination: - return json.load(reader(get_response(req_url, ghtoken))) - - obj = [] - page_num = 1 - while True: - req_url_page = '{}?page={}'.format(req_url, page_num) - result = get_response(req_url_page, ghtoken) - obj.extend(json.load(reader(result))) - - link = result.headers.get('link', None) - if link is not None: - link_next = [l for l in link.split(',') if 'rel="next"' in l] - if len(link_next) > 0: - page_num = int(link_next[0][link_next[0].find("page=")+5:link_next[0].find(">")]) - continue - break - return obj - except HTTPError as e: - error_message = e.read() - print('Warning: unable to retrieve pull information from github: %s' % e) - print('Detailed error: %s' % error_message) - return None - except Exception as e: - print('Warning: unable to retrieve pull information from github: %s' % e) - return None - -def retrieve_pr_info(repo,pull,ghtoken): - req_url = "https://api.github.com/repos/"+repo+"/pulls/"+pull - return retrieve_json(req_url,ghtoken) - -def retrieve_pr_comments(repo,pull,ghtoken): - req_url = "https://api.github.com/repos/"+repo+"/issues/"+pull+"/comments" - return retrieve_json(req_url,ghtoken,use_pagination=True) - -def retrieve_pr_reviews(repo,pull,ghtoken): - req_url = "https://api.github.com/repos/"+repo+"/pulls/"+pull+"/reviews" - return retrieve_json(req_url,ghtoken,use_pagination=True) - -def ask_prompt(text): - print(text,end=" ",file=stderr) - stderr.flush() - reply = stdin.readline().rstrip() - print("",file=stderr) - return reply - -def get_symlink_files(): - files = sorted(subprocess.check_output([GIT, 'ls-tree', '--full-tree', '-r', 'HEAD']).splitlines()) - ret = [] - for f in files: - if (int(f.decode('utf-8').split(" ")[0], 8) & 0o170000) == 0o120000: - ret.append(f.decode('utf-8').split("\t")[1]) - return ret - -def tree_sha512sum(commit='HEAD'): - # request metadata for entire tree, recursively - files = [] - blob_by_name = {} - for line in subprocess.check_output([GIT, 'ls-tree', '--full-tree', '-r', commit]).splitlines(): - name_sep = line.index(b'\t') - metadata = line[:name_sep].split() # perms, 'blob', blobid - assert(metadata[1] == b'blob') - name = line[name_sep+1:] - files.append(name) - blob_by_name[name] = metadata[2] - - files.sort() - # open connection to git-cat-file in batch mode to request data for all blobs - # this is much faster than launching it per file - p = subprocess.Popen([GIT, 'cat-file', '--batch'], stdout=subprocess.PIPE, stdin=subprocess.PIPE) - overall = hashlib.sha512() - for f in files: - blob = blob_by_name[f] - # request blob - p.stdin.write(blob + b'\n') - p.stdin.flush() - # read header: blob, "blob", size - reply = p.stdout.readline().split() - assert(reply[0] == blob and reply[1] == b'blob') - size = int(reply[2]) - # hash the blob data - intern = hashlib.sha512() - ptr = 0 - while ptr < size: - bs = min(65536, size - ptr) - piece = p.stdout.read(bs) - if len(piece) == bs: - intern.update(piece) - else: - raise IOError('Premature EOF reading git cat-file output') - ptr += bs - dig = intern.hexdigest() - assert(p.stdout.read(1) == b'\n') # ignore LF that follows blob data - # update overall hash with file hash - overall.update(dig.encode("utf-8")) - overall.update(" ".encode("utf-8")) - overall.update(f) - overall.update("\n".encode("utf-8")) - p.stdin.close() - if p.wait(): - raise IOError('Non-zero return value executing git cat-file') - return overall.hexdigest() - -def get_acks_from_comments(head_commit, comments): - assert len(head_commit) == 6 - ack_str ='\n\nACKs for commit {}:\n'.format(head_commit) - for c in comments: - review = [l for l in c['body'].split('\r\n') if 'ACK' in l and head_commit in l] - if review: - ack_str += ' {}:\n'.format(c['user']['login']) - ack_str += ' {}\n'.format(review[0]) - return ack_str - -def print_merge_details(pull, title, branch, base_branch, head_branch): - print('%s#%s%s %s %sinto %s%s' % (ATTR_RESET+ATTR_PR,pull,ATTR_RESET,title,ATTR_RESET+ATTR_PR,branch,ATTR_RESET)) - subprocess.check_call([GIT,'log','--graph','--topo-order','--pretty=format:'+COMMIT_FORMAT,base_branch+'..'+head_branch]) - -def parse_arguments(): - epilog = ''' - In addition, you can set the following git configuration variables: - githubmerge.repository (mandatory), - user.signingkey (mandatory), - user.ghtoken (default: none). - githubmerge.host (default: git@github.com), - githubmerge.branch (no default), - githubmerge.testcmd (default: none). - ''' - parser = argparse.ArgumentParser(description='Utility to merge, sign and push github pull requests', - epilog=epilog) - parser.add_argument('pull', metavar='PULL', type=int, nargs=1, - help='Pull request ID to merge') - parser.add_argument('branch', metavar='BRANCH', type=str, nargs='?', - default=None, help='Branch to merge against (default: githubmerge.branch setting, or base branch for pull, or \'master\')') - return parser.parse_args() - -def main(): - # Extract settings from git repo - repo = git_config_get('githubmerge.repository') - host = git_config_get('githubmerge.host','git@github.com') - opt_branch = git_config_get('githubmerge.branch',None) - testcmd = git_config_get('githubmerge.testcmd') - ghtoken = git_config_get('user.ghtoken') - signingkey = git_config_get('user.signingkey') - if repo is None: - print("ERROR: No repository configured. Use this command to set:", file=stderr) - print("git config githubmerge.repository /", file=stderr) - sys.exit(1) - if signingkey is None: - print("ERROR: No GPG signing key set. Set one using:",file=stderr) - print("git config --global user.signingkey ",file=stderr) - sys.exit(1) - - if host.startswith(('https:','http:')): - host_repo = host+"/"+repo+".git" - else: - host_repo = host+":"+repo - - # Extract settings from command line - args = parse_arguments() - pull = str(args.pull[0]) - - # Receive pull information from github - info = retrieve_pr_info(repo,pull,ghtoken) - if info is None: - sys.exit(1) - comments = retrieve_pr_comments(repo,pull,ghtoken) + retrieve_pr_reviews(repo,pull,ghtoken) - if comments is None: - sys.exit(1) - title = info['title'].strip() - body = info['body'].strip() - # precedence order for destination branch argument: - # - command line argument - # - githubmerge.branch setting - # - base branch for pull (as retrieved from github) - # - 'master' - branch = args.branch or opt_branch or info['base']['ref'] or 'master' - - # Initialize source branches - head_branch = 'pull/'+pull+'/head' - base_branch = 'pull/'+pull+'/base' - merge_branch = 'pull/'+pull+'/merge' - local_merge_branch = 'pull/'+pull+'/local-merge' - - devnull = open(os.devnull, 'w', encoding="utf8") - try: - subprocess.check_call([GIT,'checkout','-q',branch]) - except subprocess.CalledProcessError: - print("ERROR: Cannot check out branch %s." % (branch), file=stderr) - sys.exit(3) - try: - subprocess.check_call([GIT,'fetch','-q',host_repo,'+refs/pull/'+pull+'/*:refs/heads/pull/'+pull+'/*', - '+refs/heads/'+branch+':refs/heads/'+base_branch]) - except subprocess.CalledProcessError: - print("ERROR: Cannot find pull request #%s or branch %s on %s." % (pull,branch,host_repo), file=stderr) - sys.exit(3) - try: - subprocess.check_call([GIT,'log','-q','-1','refs/heads/'+head_branch], stdout=devnull, stderr=stdout) - except subprocess.CalledProcessError: - print("ERROR: Cannot find head of pull request #%s on %s." % (pull,host_repo), file=stderr) - sys.exit(3) - try: - subprocess.check_call([GIT,'log','-q','-1','refs/heads/'+merge_branch], stdout=devnull, stderr=stdout) - except subprocess.CalledProcessError: - print("ERROR: Cannot find merge of pull request #%s on %s." % (pull,host_repo), file=stderr) - sys.exit(3) - subprocess.check_call([GIT,'checkout','-q',base_branch]) - subprocess.call([GIT,'branch','-q','-D',local_merge_branch], stderr=devnull) - subprocess.check_call([GIT,'checkout','-q','-b',local_merge_branch]) - - try: - # Go up to the repository's root. - toplevel = subprocess.check_output([GIT,'rev-parse','--show-toplevel']).strip() - os.chdir(toplevel) - # Create unsigned merge commit. - if title: - firstline = 'Merge #%s: %s' % (pull,title) - else: - firstline = 'Merge #%s' % (pull,) - message = firstline + '\n\n' - message += subprocess.check_output([GIT,'log','--no-merges','--topo-order','--pretty=format:%H %s (%an)',base_branch+'..'+head_branch]).decode('utf-8') - message += '\n\nPull request description:\n\n ' + body.replace('\n', '\n ') + '\n' - message += get_acks_from_comments(head_commit=subprocess.check_output([GIT,'log','-1','--pretty=format:%H',head_branch]).decode('utf-8')[:6], comments=comments) - try: - subprocess.check_call([GIT,'merge','-q','--commit','--no-edit','--no-ff','--no-gpg-sign','-m',message.encode('utf-8'),head_branch]) - except subprocess.CalledProcessError: - print("ERROR: Cannot be merged cleanly.",file=stderr) - subprocess.check_call([GIT,'merge','--abort']) - sys.exit(4) - logmsg = subprocess.check_output([GIT,'log','--pretty=format:%s','-n','1']).decode('utf-8') - if logmsg.rstrip() != firstline.rstrip(): - print("ERROR: Creating merge failed (already merged?).",file=stderr) - sys.exit(4) - - symlink_files = get_symlink_files() - for f in symlink_files: - print("ERROR: File %s was a symlink" % f) - if len(symlink_files) > 0: - sys.exit(4) - - # Put tree SHA512 into the message - try: - first_sha512 = tree_sha512sum() - message += '\n\nTree-SHA512: ' + first_sha512 - except subprocess.CalledProcessError: - print("ERROR: Unable to compute tree hash") - sys.exit(4) - try: - subprocess.check_call([GIT,'commit','--amend','--no-gpg-sign','-m',message.encode('utf-8')]) - except subprocess.CalledProcessError: - print("ERROR: Cannot update message.", file=stderr) - sys.exit(4) - - print_merge_details(pull, title, branch, base_branch, head_branch) - print() - - # Run test command if configured. - if testcmd: - if subprocess.call(testcmd,shell=True): - print("ERROR: Running %s failed." % testcmd,file=stderr) - sys.exit(5) - - # Show the created merge. - diff = subprocess.check_output([GIT,'diff',merge_branch+'..'+local_merge_branch]) - subprocess.check_call([GIT,'diff',base_branch+'..'+local_merge_branch]) - if diff: - print("WARNING: merge differs from github!",file=stderr) - reply = ask_prompt("Type 'ignore' to continue.") - if reply.lower() == 'ignore': - print("Difference with github ignored.",file=stderr) - else: - sys.exit(6) - else: - # Verify the result manually. - print("Dropping you on a shell so you can try building/testing the merged source.",file=stderr) - print("Run 'git diff HEAD~' to show the changes being merged.",file=stderr) - print("Type 'exit' when done.",file=stderr) - if os.path.isfile('/etc/debian_version'): # Show pull number on Debian default prompt - os.putenv('debian_chroot',pull) - subprocess.call([BASH,'-i']) - - second_sha512 = tree_sha512sum() - if first_sha512 != second_sha512: - print("ERROR: Tree hash changed unexpectedly",file=stderr) - sys.exit(8) - - # Sign the merge commit. - print_merge_details(pull, title, branch, base_branch, head_branch) - while True: - reply = ask_prompt("Type 's' to sign off on the above merge, or 'x' to reject and exit.").lower() - if reply == 's': - try: - subprocess.check_call([GIT,'commit','-q','--gpg-sign','--amend','--no-edit']) - break - except subprocess.CalledProcessError: - print("Error while signing, asking again.",file=stderr) - elif reply == 'x': - print("Not signing off on merge, exiting.",file=stderr) - sys.exit(1) - - # Put the result in branch. - subprocess.check_call([GIT,'checkout','-q',branch]) - subprocess.check_call([GIT,'reset','-q','--hard',local_merge_branch]) - finally: - # Clean up temporary branches. - subprocess.call([GIT,'checkout','-q',branch]) - subprocess.call([GIT,'branch','-q','-D',head_branch],stderr=devnull) - subprocess.call([GIT,'branch','-q','-D',base_branch],stderr=devnull) - subprocess.call([GIT,'branch','-q','-D',merge_branch],stderr=devnull) - subprocess.call([GIT,'branch','-q','-D',local_merge_branch],stderr=devnull) - - # Push the result. - while True: - reply = ask_prompt("Type 'push' to push the result to %s, branch %s, or 'x' to exit without pushing." % (host_repo,branch)).lower() - if reply == 'push': - subprocess.check_call([GIT,'push',host_repo,'refs/heads/'+branch]) - break - elif reply == 'x': - sys.exit(1) - -if __name__ == '__main__': - main() - diff --git a/contrib/devtools/symbol-check.py b/contrib/devtools/symbol-check.py index 7729dd7257dd..dd35d862c924 100755 --- a/contrib/devtools/symbol-check.py +++ b/contrib/devtools/symbol-check.py @@ -67,8 +67,6 @@ 'ld-linux-armhf.so.3', # 32-bit ARM dynamic linker 'ld-linux-riscv64-lp64d.so.1', # 64-bit RISC-V dynamic linker # bitcoin-qt only -'libX11-xcb.so.1', # part of X11 -'libX11.so.6', # part of X11 'libxcb.so.1', # part of X11 'libfontconfig.so.1', # font support 'libfreetype.so.6', # font parsing diff --git a/contrib/gitian-keys/keys.txt b/contrib/gitian-keys/keys.txt index 33f0f7e5b05d..9222a40b175e 100644 --- a/contrib/gitian-keys/keys.txt +++ b/contrib/gitian-keys/keys.txt @@ -1,3 +1,4 @@ +9D3CC86A72F8494342EA5FD10A41BDC3F4FAFF1C Aaron Clauson (sipsorcery) 617C90010B3BD370B0AC7D424BB42E31C79111B8 Akira Takizawa E944AE667CF960B1004BC32FCA662BE18B877A60 Andreas Schildbach 152812300785C96444D3334D17565732E08E5E41 Andrew Chow diff --git a/contrib/guix/README.md b/contrib/guix/README.md new file mode 100644 index 000000000000..4dfa1729a599 --- /dev/null +++ b/contrib/guix/README.md @@ -0,0 +1,229 @@ +# Bootstrappable Bitcoin Core Builds + +This directory contains the files necessary to perform bootstrappable Bitcoin +Core builds. + +[Bootstrappability][b17e] furthers our binary security guarantees by allowing us +to _audit and reproduce_ our toolchain instead of blindly _trusting_ binary +downloads. + +We achieve bootstrappability by using Guix as a functional package manager. + +## Requirements + +Conservatively, a x86_64 machine with: + +- 2 or more logical cores +- 4GB of free disk space on the partition that /gnu/store will reside in +- 24GB of free disk space on the partition that the Bitcoin Core git repository + resides in + +> Note: these requirements are slightly less onerous than those of Gitian builds + +## Setup + +### Installing Guix + +If you're just testing this out, you can use the +[Dockerfile][fanquake/guix-docker] for convenience. It automatically speeds up +your builds by [using substitutes](#speeding-up-builds-with-substitute-servers). +If you don't want this behaviour, refer to the [next +section](#choosing-your-security-model). + +Otherwise, follow the [Guix installation guide][guix/bin-install]. + +> Note: For those who like to keep their filesystems clean, Guix is designed to +> be very standalone and _will not_ conflict with your system's package +> manager/existing setup. It _only_ touches `/var/guix`, `/gnu`, and +> `~/.config/guix`. + +### Choosing your security model + +Guix allows us to achieve better binary security by using our CPU time to build +everything from scratch. However, it doesn't sacrifice user choice in pursuit of +this: users can decide whether or not to bootstrap and to use substitutes. + +After installation, you may want to consider [adding substitute +servers](#speeding-up-builds-with-substitute-servers) to speed up your build if +that fits your security model (say, if you're just testing that this works). +This is skippable if you're using the [Dockerfile][fanquake/guix-docker]. + +If you prefer not to use any substitutes, make sure to set +`ADDITIONAL_GUIX_ENVIRONMENT_FLAGS` like the following snippet. The first build +will take a while, but the resulting packages will be cached for future builds. + +```sh +export ADDITIONAL_GUIX_ENVIRONMENT_FLAGS='--no-substitutes' +``` + +Likewise, to perform a bootstrapped build (takes even longer): + +```sh +export ADDITIONAL_GUIX_ENVIRONMENT_FLAGS='--bootstrap --no-substitutes' +``` + +### Using the right Guix + +Once Guix is installed, deploy our patched version into your current Guix +profile. The changes there are slowly being upstreamed. + +```sh +guix pull --url=https://github.com/dongcarl/guix.git \ + --commit=82c77e52b8b46e0a3aad2cb12307c2e30547deec \ + --max-jobs=4 # change accordingly +``` + +Make sure that you are using your current profile. (You are prompted to do this +at the end of the `guix pull`) + +```bash +export PATH="${HOME}/.config/guix/current/bin${PATH:+:}$PATH" +``` + +> Note: There is ongoing work to eliminate this entire section using Guix +> [inferiors][guix/inferiors] and [channels][guix/channels]. + +## Usage + +### As a Development Environment + +For a Bitcoin Core depends development environment, simply invoke + +```sh +guix environment --manifest=contrib/guix/manifest.scm +``` + +And you'll land back in your shell with all the build dependencies required for +a `depends` build injected into your environment. + +### As a Tool for Deterministic Builds + +From the top of a clean Bitcoin Core repository: + +```sh +./contrib/guix/guix-build.sh +``` + +After the build finishes successfully (check the status code please), compare +hashes: + +```sh +find output/ -type f -print0 | sort -z | xargs -r0 sha256sum +``` + +#### Recognized environment variables + +* _**HOSTS**_ + + Override the space-separated list of platform triples for which to perform a + bootstrappable build. _(defaults to "i686-linux-gnu x86\_64-linux-gnu + arm-linux-gnueabihf aarch64-linux-gnu riscv64-linux-gnu")_ + + > Windows and OS X platform triplet support are WIP. + +* _**SOURCES_PATH**_ + + Set the depends tree download cache for sources. This is passed through to the + depends tree. Setting this to the same directory across multiple builds of the + depends tree can eliminate unnecessary redownloading of package sources. + +* _**MAX_JOBS**_ + + Override the maximum number of jobs to run simultaneously, you might want to + do so on a memory-limited machine. This may be passed to `make` as in `make + --jobs="$MAX_JOBS"` or `xargs` as in `xargs -P"$MAX_JOBS"`. _(defaults to the + value of `nproc` outside the container)_ + +* _**SOURCE_DATE_EPOCH**_ + + Override the reference UNIX timestamp used for bit-for-bit reproducibility, + the variable name conforms to [standard][r12e/source-date-epoch]. _(defaults + to the output of `$(git log --format=%at -1)`)_ + +* _**V**_ + + If non-empty, will pass `V=1` to all `make` invocations, making `make` output + verbose. + +* _**ADDITIONAL_GUIX_ENVIRONMENT_FLAGS**_ + + Additional flags to be passed to `guix environment`. For a fully-bootstrapped + build, set this to `--bootstrap --no-substitutes` (refer to the [security + model section](#choosing-your-security-model) for more details). Note that a + fully-bootstrapped build will take quite a long time on the first run. + +## Tips and Tricks + +### Speeding up builds with substitute servers + +_This whole section is automatically done in the convenience +[Dockerfiles][fanquake/guix-docker]_ + +For those who are used to life in the fast _(and trustful)_ lane, you can use +[substitute servers][guix/substitutes] to enable binary downloads of packages. + +> For those who only want to use substitutes from the official Guix build farm +> and have authorized the build farm's signing key during Guix's installation, +> you don't need to do anything. + +#### Authorize the signing keys + +For the official Guix build farm at https://ci.guix.gnu.org, run as root: + +``` +guix archive --authorize < ~root/.config/guix/current/share/guix/ci.guix.gnu.org.pub +``` + +For dongcarl's substitute server at https://guix.carldong.io, run as root: + +```sh +wget -qO- 'https://guix.carldong.io/signing-key.pub' | guix archive --authorize +``` + +#### Use the substitute servers + +The official Guix build farm at https://ci.guix.gnu.org is automatically used +unless the `--no-substitutes` flag is supplied. + +This can be overridden for all `guix` invocations by passing the +`--substitute-urls` option to your invocation of `guix-daemon`. This can also be +overridden on a call-by-call basis by passing the same `--substitute-urls` +option to client tools such at `guix environment`. + +To use dongcarl's substitute server for Bitcoin Core builds after having +[authorized his signing key](#authorize-the-signing-keys): + +``` +export ADDITIONAL_GUIX_ENVIRONMENT_FLAGS='--substitute-urls="https://guix.carldong.io https://ci.guix.gnu.org"' +``` + +## FAQ + +### How can I trust the binary installation? + +As mentioned at the bottom of [this manual page][guix/bin-install]: + +> The binary installation tarballs can be (re)produced and verified simply by +> running the following command in the Guix source tree: +> +> make guix-binary.x86_64-linux.tar.xz + +### When will Guix be packaged in debian? + +Vagrant Cascadian has been making good progress on this +[here][debian/guix-package]. We have all the pieces needed to put up an APT +repository and will likely put one up soon. + +[b17e]: http://bootstrappable.org/ +[r12e/source-date-epoch]: https://reproducible-builds.org/docs/source-date-epoch/ + +[guix/install.sh]: https://git.savannah.gnu.org/cgit/guix.git/plain/etc/guix-install.sh +[guix/bin-install]: https://www.gnu.org/software/guix/manual/en/html_node/Binary-Installation.html +[guix/env-setup]: https://www.gnu.org/software/guix/manual/en/html_node/Build-Environment-Setup.html +[guix/substitutes]: https://www.gnu.org/software/guix/manual/en/html_node/Substitutes.html +[guix/substitute-server-auth]: https://www.gnu.org/software/guix/manual/en/html_node/Substitute-Server-Authorization.html +[guix/inferiors]: https://www.gnu.org/software/guix/manual/en/html_node/Inferiors.html +[guix/channels]: https://www.gnu.org/software/guix/manual/en/html_node/Channels.html + +[debian/guix-package]: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=850644 +[fanquake/guix-docker]: https://github.com/fanquake/core-review/tree/master/guix diff --git a/contrib/guix/guix-build.sh b/contrib/guix/guix-build.sh new file mode 100755 index 000000000000..f8ba8c7ed2ad --- /dev/null +++ b/contrib/guix/guix-build.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +export LC_ALL=C +set -e -o pipefail + +# Determine the maximum number of jobs to run simultaneously (overridable by +# environment) +MAX_JOBS="${MAX_JOBS:-$(nproc)}" + +# Download the depends sources now as we won't have internet access in the build +# container +make -C "${PWD}/depends" -j"$MAX_JOBS" download ${V:+V=1} ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} + +# Determine the reference time used for determinism (overridable by environment) +SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git log --format=%at -1)}" + +# Deterministically build Bitcoin Core for HOSTs (overriable by environment) +for host in ${HOSTS=i686-linux-gnu x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv64-linux-gnu}; do + + # Display proper warning when the user interrupts the build + trap 'echo "** INT received while building ${host}, you may want to clean up the relevant output and distsrc-* directories before rebuilding"' INT + + # Run the build script 'contrib/guix/libexec/build.sh' in the build + # container specified by 'contrib/guix/manifest.scm' + # shellcheck disable=SC2086 + guix environment --manifest="${PWD}/contrib/guix/manifest.scm" \ + --container \ + --pure \ + --no-cwd \ + --share="$PWD"=/bitcoin \ + ${SOURCES_PATH:+--share="$SOURCES_PATH"} \ + ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \ + -- env HOST="$host" \ + MAX_JOBS="$MAX_JOBS" \ + SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:?unable to determine value}" \ + ${V:+V=1} \ + ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} \ + bash -c "cd /bitcoin && bash contrib/guix/libexec/build.sh" + +done diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh new file mode 100644 index 000000000000..56b972a5cbbb --- /dev/null +++ b/contrib/guix/libexec/build.sh @@ -0,0 +1,206 @@ +#!/usr/bin/env bash +export LC_ALL=C +set -e -o pipefail + +# Check that environment variables assumed to be set by the environment are set +echo "Building for platform triple ${HOST:?not set} with reference timestamp ${SOURCE_DATE_EPOCH:?not set}..." +echo "At most ${MAX_JOBS:?not set} jobs will run at once..." + +##################### +# Environment Setup # +##################### + +# The depends folder also serves as a base-prefix for depends packages for +# $HOSTs after successfully building. +BASEPREFIX="${PWD}/depends" + +# Setup an output directory for our build +OUTDIR="${OUTDIR:-${PWD}/output}" +[ -e "$OUTDIR" ] || mkdir -p "$OUTDIR" + +# Setup the directory where our Bitcoin Core build for HOST will occur +DISTSRC="${DISTSRC:-${PWD}/distsrc-${HOST}}" +if [ -e "$DISTSRC" ]; then + echo "DISTSRC directory '${DISTSRC}' exists, probably because of previous builds... Aborting..." + exit 1 +else + mkdir -p "$DISTSRC" +fi + +# Given a package name and an output name, return the path of that output in our +# current guix environment +store_path() { + grep --extended-regexp "/[^-]{32}-${1}-cross-${HOST}-[^-]+${2:+-${2}}" "${GUIX_ENVIRONMENT}/manifest" \ + | head --lines=1 \ + | sed --expression='s|^[[:space:]]*"||' \ + --expression='s|"[[:space:]]*$||' +} + +# Determine output paths to use in CROSS_* environment variables +CROSS_GLIBC="$(store_path glibc)" +CROSS_GLIBC_STATIC="$(store_path glibc static)" +CROSS_KERNEL="$(store_path linux-libre-headers)" +CROSS_GCC="$(store_path gcc)" + +# Set environment variables to point Guix's cross-toolchain to the right +# includes/libs for $HOST +export CROSS_C_INCLUDE_PATH="${CROSS_GCC}/include:${CROSS_GLIBC}/include:${CROSS_KERNEL}/include" +export CROSS_CPLUS_INCLUDE_PATH="${CROSS_GCC}/include/c++:${CROSS_GLIBC}/include:${CROSS_KERNEL}/include" +export CROSS_LIBRARY_PATH="${CROSS_GLIBC}/lib:${CROSS_GLIBC_STATIC}/lib:${CROSS_GCC}/lib:${CROSS_GCC}/${HOST}/lib:${CROSS_KERNEL}/lib" + +# Disable Guix ld auto-rpath behavior +export GUIX_LD_WRAPPER_DISABLE_RPATH=yes + +# Make /usr/bin if it doesn't exist +[ -e /usr/bin ] || mkdir -p /usr/bin + +# Symlink file and env to a conventional path +[ -e /usr/bin/file ] || ln -s --no-dereference "$(command -v file)" /usr/bin/file +[ -e /usr/bin/env ] || ln -s --no-dereference "$(command -v env)" /usr/bin/env + +# Determine the correct value for -Wl,--dynamic-linker for the current $HOST +glibc_dynamic_linker=$( + case "$HOST" in + i686-linux-gnu) echo /lib/ld-linux.so.2 ;; + x86_64-linux-gnu) echo /lib64/ld-linux-x86-64.so.2 ;; + arm-linux-gnueabihf) echo /lib/ld-linux-armhf.so.3 ;; + aarch64-linux-gnu) echo /lib/ld-linux-aarch64.so.1 ;; + riscv64-linux-gnu) echo /lib/ld-linux-riscv64-lp64d.so.1 ;; + *) exit 1 ;; + esac +) + +# Environment variables for determinism +export QT_RCC_TEST=1 +export QT_RCC_SOURCE_DATE_OVERRIDE=1 +export TAR_OPTIONS="--owner=0 --group=0 --numeric-owner --mtime='@${SOURCE_DATE_EPOCH}' --sort=name" +export TZ="UTC" + +#################### +# Depends Building # +#################### + +# Build the depends tree, overriding variables that assume multilib gcc +make -C depends --jobs="$MAX_JOBS" HOST="$HOST" \ + ${V:+V=1} \ + ${SOURCES_PATH+SOURCES_PATH="$SOURCES_PATH"} \ + i686_linux_CC=i686-linux-gnu-gcc \ + i686_linux_CXX=i686-linux-gnu-g++ \ + i686_linux_AR=i686-linux-gnu-ar \ + i686_linux_RANLIB=i686-linux-gnu-ranlib \ + i686_linux_NM=i686-linux-gnu-nm \ + i686_linux_STRIP=i686-linux-gnu-strip \ + x86_64_linux_CC=x86_64-linux-gnu-gcc \ + x86_64_linux_CXX=x86_64-linux-gnu-g++ \ + x86_64_linux_AR=x86_64-linux-gnu-ar \ + x86_64_linux_RANLIB=x86_64-linux-gnu-ranlib \ + x86_64_linux_NM=x86_64-linux-gnu-nm \ + x86_64_linux_STRIP=x86_64-linux-gnu-strip \ + qt_config_opts_i686_linux='-platform linux-g++ -xplatform bitcoin-linux-g++' + + +########################### +# Source Tarball Building # +########################### + +# Create the source tarball and move it to "${OUTDIR}/src" if not already there +if [ -z "$(find "${OUTDIR}/src" -name 'bitcoin-*.tar.gz')" ]; then + ./autogen.sh + env CONFIG_SITE="${BASEPREFIX}/${HOST}/share/config.site" ./configure --prefix=/ + make dist GZIP_ENV='-9n' ${V:+V=1} + mkdir -p "${OUTDIR}/src" + mv "$(find "${PWD}" -name 'bitcoin-*.tar.gz')" "${OUTDIR}/src/" +fi + +# Determine the full path to our source tarball +SOURCEDIST="$(find "${OUTDIR}/src" -name 'bitcoin-*.tar.gz')" +# Determine our distribution name (e.g. bitcoin-0.18.0) +DISTNAME="$(basename "$SOURCEDIST" '.tar.gz')" + +########################### +# Binary Tarball Building # +########################### + +# Create a spec file to normalize ssp linking behaviour +spec_file="$(mktemp)" +cat << EOF > "$spec_file" +*link_ssp: +%{fstack-protector|fstack-protector-all|fstack-protector-strong|fstack-protector-explicit:} +EOF + +# Similar flags to Gitian +CONFIGFLAGS="--enable-glibc-back-compat --enable-reduce-exports --disable-bench --disable-gui-tests" +HOST_CFLAGS="-O2 -g -specs=${spec_file} -ffile-prefix-map=${PWD}=." +HOST_CXXFLAGS="-O2 -g -specs=${spec_file} -ffile-prefix-map=${PWD}=." +HOST_LDFLAGS="-Wl,--as-needed -Wl,--dynamic-linker=$glibc_dynamic_linker -static-libstdc++" + +# Make $HOST-specific native binaries from depends available in $PATH +export PATH="${BASEPREFIX}/${HOST}/native/bin:${PATH}" +( + cd "$DISTSRC" + + # Extract the source tarball + tar --strip-components=1 -xf "${SOURCEDIST}" + + # Configure this DISTSRC for $HOST + # shellcheck disable=SC2086 + env CONFIG_SITE="${BASEPREFIX}/${HOST}/share/config.site" \ + ./configure --prefix=/ \ + --disable-ccache \ + --disable-maintainer-mode \ + --disable-dependency-tracking \ + ${CONFIGFLAGS} \ + CFLAGS="${HOST_CFLAGS}" \ + CXXFLAGS="${HOST_CXXFLAGS}" \ + LDFLAGS="${HOST_LDFLAGS}" + + sed -i.old 's/-lstdc++ //g' config.status libtool src/univalue/config.status src/univalue/libtool + + # Build Bitcoin Core + make --jobs="$MAX_JOBS" ${V:+V=1} + + # Perform basic ELF security checks on a series of executables. + make -C src --jobs=1 check-security ${V:+V=1} + # Check that executables only contain allowed gcc, glibc and libstdc++ + # version symbols for Linux distro back-compatibility. + make -C src --jobs=1 check-symbols ${V:+V=1} + + # Setup the directory where our Bitcoin Core build for HOST will be + # installed. This directory will also later serve as the input for our + # binary tarballs. + INSTALLPATH="${PWD}/installed/${DISTNAME}" + mkdir -p "${INSTALLPATH}" + # Install built Bitcoin Core to $INSTALLPATH + make install DESTDIR="${INSTALLPATH}" ${V:+V=1} + ( + cd installed + + # Prune libtool and object archives + find . -name "lib*.la" -delete + find . -name "lib*.a" -delete + + # Prune pkg-config files + rm -r "${DISTNAME}/lib/pkgconfig" + + # Split binaries and libraries from their debug symbols + { + find "${DISTNAME}/bin" -type f -executable -print0 + find "${DISTNAME}/lib" -type f -print0 + } | xargs -0 -n1 -P"$MAX_JOBS" -I{} "${DISTSRC}/contrib/devtools/split-debug.sh" {} {} {}.dbg + + cp "${DISTSRC}/doc/README.md" "${DISTNAME}/" + + # Finally, deterministically produce {non-,}debug binary tarballs ready + # for release + find "${DISTNAME}" -not -name "*.dbg" -print0 \ + | sort --zero-terminated \ + | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ + | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}.tar.gz" && exit 1 ) + find "${DISTNAME}" -name "*.dbg" -print0 \ + | sort --zero-terminated \ + | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ + | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}-debug.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}-debug.tar.gz" && exit 1 ) + ) +) diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm new file mode 100644 index 000000000000..ca11d7a0f00a --- /dev/null +++ b/contrib/guix/manifest.scm @@ -0,0 +1,158 @@ +(use-modules (gnu) + (gnu packages) + (gnu packages autotools) + (gnu packages base) + (gnu packages bash) + (gnu packages check) + (gnu packages commencement) + (gnu packages compression) + (gnu packages cross-base) + (gnu packages file) + (gnu packages gawk) + (gnu packages gcc) + (gnu packages linux) + (gnu packages perl) + (gnu packages pkg-config) + (gnu packages python) + (gnu packages shells) + (guix build-system trivial) + (guix gexp) + (guix packages) + (guix profiles) + (guix utils)) + +(define (make-ssp-fixed-gcc xgcc) + "Given a XGCC package, return a modified package that uses the SSP function +from glibc instead of from libssp.so. Taken from: +http://www.linuxfromscratch.org/hlfs/view/development/chapter05/gcc-pass1.html" + (package + (inherit xgcc) + (arguments + (substitute-keyword-arguments (package-arguments xgcc) + ((#:make-flags flags) + `(cons "gcc_cv_libc_provides_ssp=yes" ,flags)))))) + +(define (make-gcc-rpath-link xgcc) + "Given a XGCC package, return a modified package that replace each instance of +-rpath in the default system spec that's inserted by Guix with -rpath-link" + (package + (inherit xgcc) + (arguments + (substitute-keyword-arguments (package-arguments xgcc) + ((#:phases phases) + `(modify-phases ,phases + (add-after 'pre-configure 'replace-rpath-with-rpath-link + (lambda _ + (substitute* (cons "gcc/config/rs6000/sysv4.h" + (find-files "gcc/config" + "^gnu-user.*\\.h$")) + (("-rpath=") "-rpath-link=")) + #t)))))))) + +(define (make-cross-toolchain target + base-gcc-for-libc + base-kernel-headers + base-libc + base-gcc) + "Create a cross-compilation toolchain package for TARGET" + (let* ((xbinutils (cross-binutils target)) + ;; 1. Build a cross-compiling gcc without targeting any libc, derived + ;; from BASE-GCC-FOR-LIBC + (xgcc-sans-libc (cross-gcc target + #:xgcc base-gcc-for-libc + #:xbinutils xbinutils)) + ;; 2. Build cross-compiled kernel headers with XGCC-SANS-LIBC, derived + ;; from BASE-KERNEL-HEADERS + (xkernel (cross-kernel-headers target + base-kernel-headers + xgcc-sans-libc + xbinutils)) + ;; 3. Build a cross-compiled libc with XGCC-SANS-LIBC and XKERNEL, + ;; derived from BASE-LIBC + (xlibc (cross-libc target + base-libc + xgcc-sans-libc + xbinutils + xkernel)) + ;; 4. Build a cross-compiling gcc targeting XLIBC, derived from + ;; BASE-GCC + (xgcc (cross-gcc target + #:xgcc base-gcc + #:xbinutils xbinutils + #:libc xlibc))) + ;; Define a meta-package that propagates the resulting XBINUTILS, XLIBC, and + ;; XGCC + (package + (name (string-append target "-toolchain")) + (version (package-version xgcc)) + (source #f) + (build-system trivial-build-system) + (arguments '(#:builder (begin (mkdir %output) #t))) + (propagated-inputs + `(("binutils" ,xbinutils) + ("libc" ,xlibc) + ("libc:static" ,xlibc "static") + ("gcc" ,xgcc))) + (synopsis (string-append "Complete GCC tool chain for " target)) + (description (string-append "This package provides a complete GCC tool +chain for " target " development.")) + (home-page (package-home-page xgcc)) + (license (package-license xgcc))))) + +(define* (make-bitcoin-cross-toolchain target + #:key + (base-gcc-for-libc gcc-5) + (base-kernel-headers linux-libre-headers-4.19) + (base-libc glibc-2.27) + (base-gcc (make-gcc-rpath-link + (make-ssp-fixed-gcc gcc-9)))) + "Convienience wrapper around MAKE-CROSS-TOOLCHAIN with default values +desirable for building Bitcoin Core release binaries." + (make-cross-toolchain target + base-gcc-for-libc + base-kernel-headers + base-libc + base-gcc)) + +(packages->manifest + (list ;; The Basics + bash-minimal + which + coreutils + util-linux + ;; File(system) inspection + file + grep + diffutils + findutils + ;; File transformation + patch + gawk + sed + ;; Compression and archiving + tar + bzip2 + gzip + xz + zlib + ;; Build tools + gnu-make + libtool + autoconf + automake + pkg-config + ;; Scripting + perl + python-3.7 + ;; Native gcc 9 toolchain targeting glibc 2.27 + (make-gcc-toolchain gcc-9 glibc-2.27) + ;; Cross gcc 9 toolchains targeting glibc 2.27 + (make-bitcoin-cross-toolchain "i686-linux-gnu") + (make-bitcoin-cross-toolchain "x86_64-linux-gnu") + (make-bitcoin-cross-toolchain "aarch64-linux-gnu") + (make-bitcoin-cross-toolchain "arm-linux-gnueabihf") + ;; The glibc 2.27 for riscv64 needs gcc 7 to successfully build (see: + ;; https://www.gnu.org/software/gcc/gcc-7/changes.html#riscv). The final + ;; toolchain is still a gcc 9 toolchain targeting glibc 2.27. + (make-bitcoin-cross-toolchain "riscv64-linux-gnu" + #:base-gcc-for-libc gcc-7))) diff --git a/contrib/install_db4.sh b/contrib/install_db4.sh index 47594a5b0a9b..4b848dda99ee 100755 --- a/contrib/install_db4.sh +++ b/contrib/install_db4.sh @@ -14,7 +14,7 @@ if [ -z "${1}" ]; then fi expand_path() { - echo "$(cd "${1}" && pwd -P)" + cd "${1}" && pwd -P } BDB_PREFIX="$(expand_path ${1})/db4"; shift; @@ -23,7 +23,7 @@ BDB_HASH='12edc0df75bf9abd7f82f821795bcee50f42cb2e5f76a6a281b85732798364ef' BDB_URL="https://download.oracle.com/berkeley-db/${BDB_VERSION}.tar.gz" check_exists() { - which "$1" >/dev/null 2>&1 + command -v "$1" >/dev/null } sha256_check() { @@ -95,7 +95,9 @@ make install echo echo "db4 build complete." echo +# shellcheck disable=SC2016 echo 'When compiling bitcoind, run `./configure` in the following way:' echo echo " export BDB_PREFIX='${BDB_PREFIX}'" +# shellcheck disable=SC2016 echo ' ./configure BDB_LIBS="-L${BDB_PREFIX}/lib -ldb_cxx-4.8" BDB_CFLAGS="-I${BDB_PREFIX}/include" ...' diff --git a/contrib/macdeploy/detached-sig-apply.sh b/contrib/macdeploy/detached-sig-apply.sh index f8503e4de8c5..af2b11fa0db0 100755 --- a/contrib/macdeploy/detached-sig-apply.sh +++ b/contrib/macdeploy/detached-sig-apply.sh @@ -36,13 +36,13 @@ if [ -z "${CODESIGN_ALLOCATE}" ]; then fi find ${TEMPDIR} -name "*.sign" | while read i; do - SIZE=`stat -c %s "${i}"` - TARGET_FILE="`echo "${i}" | sed 's/\.sign$//'`" + SIZE=$(stat -c %s "${i}") + TARGET_FILE="$(echo "${i}" | sed 's/\.sign$//')" echo "Allocating space for the signature of size ${SIZE} in ${TARGET_FILE}" ${CODESIGN_ALLOCATE} -i "${TARGET_FILE}" -a ${ARCH} ${SIZE} -o "${i}.tmp" - OFFSET=`${PAGESTUFF} "${i}.tmp" -p | tail -2 | grep offset | sed 's/[^0-9]*//g'` + OFFSET=$(${PAGESTUFF} "${i}.tmp" -p | tail -2 | grep offset | sed 's/[^0-9]*//g') if [ -z ${QUIET} ]; then echo "Attaching signature at offset ${OFFSET}" fi diff --git a/contrib/macdeploy/detached-sig-create.sh b/contrib/macdeploy/detached-sig-create.sh index 5281ebcc47dc..938bcd163869 100755 --- a/contrib/macdeploy/detached-sig-create.sh +++ b/contrib/macdeploy/detached-sig-create.sh @@ -14,7 +14,7 @@ TEMPLIST=${TEMPDIR}/signatures.txt OUT=signature-osx.tar.gz OUTROOT=osx -if [ ! -n "$1" ]; then +if [ -z "$1" ]; then echo "usage: $0 " echo "example: $0 -s MyIdentity" exit 1 @@ -26,20 +26,20 @@ mkdir -p ${TEMPDIR} ${CODESIGN} -f --file-list ${TEMPLIST} "$@" "${BUNDLE}" grep -v CodeResources < "${TEMPLIST}" | while read i; do - TARGETFILE="${BUNDLE}/`echo "${i}" | sed "s|.*${BUNDLE}/||"`" - SIZE=`pagestuff "$i" -p | tail -2 | grep size | sed 's/[^0-9]*//g'` - OFFSET=`pagestuff "$i" -p | tail -2 | grep offset | sed 's/[^0-9]*//g'` + TARGETFILE="${BUNDLE}/$(echo "${i}" | sed "s|.*${BUNDLE}/||")" + SIZE=$(pagestuff "$i" -p | tail -2 | grep size | sed 's/[^0-9]*//g') + OFFSET=$(pagestuff "$i" -p | tail -2 | grep offset | sed 's/[^0-9]*//g') SIGNFILE="${TEMPDIR}/${OUTROOT}/${TARGETFILE}.sign" - DIRNAME="`dirname "${SIGNFILE}"`" + DIRNAME="$(dirname "${SIGNFILE}")" mkdir -p "${DIRNAME}" echo "Adding detached signature for: ${TARGETFILE}. Size: ${SIZE}. Offset: ${OFFSET}" dd if="$i" of="${SIGNFILE}" bs=1 skip=${OFFSET} count=${SIZE} 2>/dev/null done grep CodeResources < "${TEMPLIST}" | while read i; do - TARGETFILE="${BUNDLE}/`echo "${i}" | sed "s|.*${BUNDLE}/||"`" + TARGETFILE="${BUNDLE}/$(echo "${i}" | sed "s|.*${BUNDLE}/||")" RESOURCE="${TEMPDIR}/${OUTROOT}/${TARGETFILE}" - DIRNAME="`dirname "${RESOURCE}"`" + DIRNAME="$(dirname "${RESOURCE}")" mkdir -p "${DIRNAME}" echo "Adding resource for: \"${TARGETFILE}\"" cp "${i}" "${RESOURCE}" diff --git a/contrib/qos/tc.sh b/contrib/qos/tc.sh index 5f9b87d9b23f..ccb0f4f8958e 100644 --- a/contrib/qos/tc.sh +++ b/contrib/qos/tc.sh @@ -33,7 +33,7 @@ tc class add dev ${IF} parent 1:1 classid 1:11 htb rate ${LIMIT} ceil ${LIMIT} p tc filter add dev ${IF} parent 1: protocol ip prio 1 handle 1 fw classid 1:10 tc filter add dev ${IF} parent 1: protocol ip prio 2 handle 2 fw classid 1:11 -if [ ! -z "${LOCALNET_V6}" ] ; then +if [ -n "${LOCALNET_V6}" ] ; then # v6 cannot have the same priority value as v4 tc filter add dev ${IF} parent 1: protocol ipv6 prio 3 handle 1 fw classid 1:10 tc filter add dev ${IF} parent 1: protocol ipv6 prio 4 handle 2 fw classid 1:11 @@ -56,7 +56,7 @@ fi iptables -t mangle -A OUTPUT -p tcp -m tcp --dport 8333 ! -d ${LOCALNET_V4} -j MARK --set-mark 0x2 iptables -t mangle -A OUTPUT -p tcp -m tcp --sport 8333 ! -d ${LOCALNET_V4} -j MARK --set-mark 0x2 -if [ ! -z "${LOCALNET_V6}" ] ; then +if [ -n "${LOCALNET_V6}" ] ; then ip6tables -t mangle -A OUTPUT -p tcp -m tcp --dport 8333 ! -d ${LOCALNET_V6} -j MARK --set-mark 0x4 ip6tables -t mangle -A OUTPUT -p tcp -m tcp --sport 8333 ! -d ${LOCALNET_V6} -j MARK --set-mark 0x4 fi diff --git a/contrib/verify-commits/gpg.sh b/contrib/verify-commits/gpg.sh index 7a10ba7d7d53..288e4ccacb86 100755 --- a/contrib/verify-commits/gpg.sh +++ b/contrib/verify-commits/gpg.sh @@ -21,7 +21,7 @@ else # they've created a collision for. Not the most likely attack, but preventing # it is pretty easy so we do so as a "belt-and-suspenders" measure. GPG_RES="" - for LINE in "$(gpg --version)"; do + for LINE in $(gpg --version); do case "$LINE" in "gpg (GnuPG) 1.4.1"*|"gpg (GnuPG) 2.0."*) echo "Please upgrade to at least gpg 2.1.10 to check for weak signatures" > /dev/stderr @@ -35,7 +35,7 @@ else done [ "$GPG_RES" = "" ] && GPG_RES="$(printf '%s\n' "$INPUT" | gpg --trust-model always --weak-digest sha1 "$@" 2>/dev/null)" fi -for LINE in $(echo "$GPG_RES"); do +for LINE in $GPG_RES; do case "$LINE" in "[GNUPG:] VALIDSIG "*) while read KEY; do diff --git a/contrib/verify-commits/verify-commits.py b/contrib/verify-commits/verify-commits.py index 255ce7509291..9ec8663fbae7 100755 --- a/contrib/verify-commits/verify-commits.py +++ b/contrib/verify-commits/verify-commits.py @@ -16,7 +16,7 @@ def tree_sha512sum(commit='HEAD'): """Calculate the Tree-sha512 for the commit. - This is copied from github-merge.py.""" + This is copied from github-merge.py. See https://github.com/bitcoin-core/bitcoin-maintainer-tools.""" # request metadata for entire tree, recursively files = [] diff --git a/contrib/verifybinaries/README.md b/contrib/verifybinaries/README.md index 3ffe0a2f2895..4209fdb3644e 100644 --- a/contrib/verifybinaries/README.md +++ b/contrib/verifybinaries/README.md @@ -6,7 +6,7 @@ Make sure you obtain the proper release signing key and verify the fingerprint w ```sh $ gpg --fingerprint "Bitcoin Core binary release signing key" -pub 4096R/36C2E964 2015-06-24 [expires: 2017-02-13] +pub 4096R/36C2E964 2015-06-24 [expires: YYYY-MM-DD] Key fingerprint = 01EA 5486 DE18 A882 D4C2 6845 90C8 019E 36C2 E964 uid Wladimir J. van der Laan (Bitcoin Core binary release signing key) ``` diff --git a/contrib/verifybinaries/verify.sh b/contrib/verifybinaries/verify.sh index fc7492ad3b79..bfe74aa4fa57 100755 --- a/contrib/verifybinaries/verify.sh +++ b/contrib/verifybinaries/verify.sh @@ -13,7 +13,7 @@ export LC_ALL=C function clean_up { - for file in $* + for file in "$@" do rm "$file" 2> /dev/null done @@ -82,22 +82,20 @@ else exit 2 fi -#first we fetch the file containing the signature -WGETOUT=$(wget -N "$HOST1$BASEDIR$SIGNATUREFILENAME" 2>&1) - -#and then see if wget completed successfully -if [ $? -ne 0 ]; then +if ! WGETOUT=$(wget -N "$HOST1$BASEDIR$SIGNATUREFILENAME" 2>&1); then echo "Error: couldn't fetch signature file. Have you specified the version number in the following format?" + # shellcheck disable=SC1087 echo "[$VERSIONPREFIX]-[$RCVERSIONSTRING[0-9]] (example: ${VERSIONPREFIX}0.10.4-${RCVERSIONSTRING}1)" echo "wget output:" + # shellcheck disable=SC2001 echo "$WGETOUT"|sed 's/^/\t/g' exit 2 fi -WGETOUT=$(wget -N -O "$SIGNATUREFILENAME.2" "$HOST2$BASEDIR$SIGNATUREFILENAME" 2>&1) -if [ $? -ne 0 ]; then +if ! WGETOUT=$(wget -N -O "$SIGNATUREFILENAME.2" "$HOST2$BASEDIR$SIGNATUREFILENAME" 2>&1); then echo "bitcoin.org failed to provide signature file, but bitcoincore.org did?" echo "wget output:" + # shellcheck disable=SC2001 echo "$WGETOUT"|sed 's/^/\t/g' clean_up $SIGNATUREFILENAME exit 3 @@ -128,6 +126,7 @@ if [ $RET -ne 0 ]; then fi echo "gpg output:" + # shellcheck disable=SC2001 echo "$GPGOUT"|sed 's/^/\t/g' clean_up $SIGNATUREFILENAME $SIGNATUREFILENAME.2 $TMPFILE exit "$RET" diff --git a/contrib/windeploy/detached-sig-create.sh b/contrib/windeploy/detached-sig-create.sh index 15f8108cf011..cc42422b23f4 100755 --- a/contrib/windeploy/detached-sig-create.sh +++ b/contrib/windeploy/detached-sig-create.sh @@ -8,7 +8,7 @@ if [ -z "$OSSLSIGNCODE" ]; then OSSLSIGNCODE=osslsigncode fi -if [ ! -n "$1" ]; then +if [ -z "$1" ]; then echo "usage: $0 " echo "example: $0 -key codesign.key" exit 1 @@ -23,7 +23,7 @@ TIMESERVER=http://timestamp.comodoca.com CERTFILE="win-codesign.cert" mkdir -p "${OUTSUBDIR}" -basename -a `ls -1 "${SRCDIR}"/*-unsigned.exe` | while read UNSIGNED; do +basename -a $(ls -1 "${SRCDIR}"/*-unsigned.exe) | while read UNSIGNED; do echo Signing "${UNSIGNED}" "${OSSLSIGNCODE}" sign -certs "${CERTFILE}" -t "${TIMESERVER}" -in "${SRCDIR}/${UNSIGNED}" -out "${WORKDIR}/${UNSIGNED}" "$@" "${OSSLSIGNCODE}" extract-signature -pem -in "${WORKDIR}/${UNSIGNED}" -out "${OUTSUBDIR}/${UNSIGNED}.pem" && rm "${WORKDIR}/${UNSIGNED}" diff --git a/depends/packages/dbus.mk b/depends/packages/dbus.mk deleted file mode 100644 index ad10b0fdd7fb..000000000000 --- a/depends/packages/dbus.mk +++ /dev/null @@ -1,27 +0,0 @@ -package=dbus -$(package)_version=1.10.18 -$(package)_download_path=https://dbus.freedesktop.org/releases/dbus -$(package)_file_name=$(package)-$($(package)_version).tar.gz -$(package)_sha256_hash=6049ddd5f3f3e2618f615f1faeda0a115104423a7996b7aa73e2f36e38cc514a -$(package)_dependencies=expat - -define $(package)_set_vars - $(package)_config_opts=--disable-tests --disable-doxygen-docs --disable-xml-docs --disable-shared --without-x -endef - -define $(package)_config_cmds - $($(package)_autoconf) -endef - -define $(package)_build_cmds - $(MAKE) -C dbus libdbus-1.la -endef - -define $(package)_stage_cmds - $(MAKE) -C dbus DESTDIR=$($(package)_staging_dir) install-libLTLIBRARIES install-dbusincludeHEADERS install-nodist_dbusarchincludeHEADERS && \ - $(MAKE) DESTDIR=$($(package)_staging_dir) install-pkgconfigDATA -endef - -define $(package)_postprocess_cmds - rm lib/*.la -endef diff --git a/depends/packages/expat.mk b/depends/packages/expat.mk index b811f84a380b..478438191504 100644 --- a/depends/packages/expat.mk +++ b/depends/packages/expat.mk @@ -1,11 +1,11 @@ package=expat -$(package)_version=2.2.6 -$(package)_download_path=https://github.com/libexpat/libexpat/releases/download/R_2_2_6/ +$(package)_version=2.2.7 +$(package)_download_path=https://github.com/libexpat/libexpat/releases/download/R_2_2_7/ $(package)_file_name=$(package)-$($(package)_version).tar.bz2 -$(package)_sha256_hash=17b43c2716d521369f82fc2dc70f359860e90fa440bea65b3b85f0b246ea81f2 +$(package)_sha256_hash=cbc9102f4a31a8dafd42d642e9a3aa31e79a0aedaa1f6efd2795ebc83174ec18 define $(package)_set_vars - $(package)_config_opts=--disable-shared --without-docbook + $(package)_config_opts=--disable-shared --without-docbook --without-tests --without-examples $(package)_config_opts_linux=--with-pic endef diff --git a/depends/packages/libX11.mk b/depends/packages/libX11.mk deleted file mode 100644 index f46bd9219e9b..000000000000 --- a/depends/packages/libX11.mk +++ /dev/null @@ -1,32 +0,0 @@ -package=libX11 -$(package)_version=1.6.2 -$(package)_download_path=https://xorg.freedesktop.org/releases/individual/lib/ -$(package)_file_name=$(package)-$($(package)_version).tar.bz2 -$(package)_sha256_hash=2aa027e837231d2eeea90f3a4afe19948a6eb4c8b2bec0241eba7dbc8106bd16 -$(package)_dependencies=libxcb xtrans xextproto xproto - -define $(package)_set_vars - # See libXext for --disable-malloc0returnsnull rationale. - $(package)_config_opts=--disable-xkb --disable-static --disable-malloc0returnsnull - $(package)_config_opts_linux=--with-pic -endef - -define $(package)_preprocess_cmds - cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub . -endef - -define $(package)_config_cmds - $($(package)_autoconf) -endef - -define $(package)_build_cmds - $(MAKE) -endef - -define $(package)_stage_cmds - $(MAKE) DESTDIR=$($(package)_staging_dir) install -endef - -define $(package)_postprocess_cmds - rm lib/*.la -endef diff --git a/depends/packages/libXext.mk b/depends/packages/libXext.mk deleted file mode 100644 index 77f32a340ee2..000000000000 --- a/depends/packages/libXext.mk +++ /dev/null @@ -1,53 +0,0 @@ -package=libXext -$(package)_version=1.3.3 -$(package)_download_path=https://xorg.freedesktop.org/releases/individual/lib/ -$(package)_file_name=$(package)-$($(package)_version).tar.bz2 -$(package)_sha256_hash=b518d4d332231f313371fdefac59e3776f4f0823bcb23cf7c7305bfb57b16e35 -$(package)_dependencies=xproto xextproto libX11 libXau - -define $(package)_set_vars - # A number of steps in the autoconfig process implicitly assume that the build - # system and the host system are the same. For example, library components - # want to build and run test programs to determine the behavior of certain - # host system elements. This is clearly impossible when crosscompiling. To - # work around these issues, the --enable-malloc0returnsnull (or - # --disable-malloc0returnsnull, depending on the host system) must be passed - # to configure. - # -- https://www.x.org/wiki/CrossCompilingXorg/ - # - # Concretely, between the releases of libXext 1.3.2 and 1.3.3, - # XORG_CHECK_MALLOC_ZERO from xorg-macros was changed to use the autoconf - # cache, expecting cross-compilation environments to seed this cache as there - # is no single correct value when cross compiling (think uclibc, musl, etc.). - # You can see the actual change in commit 72fdc868b56fe2b7bdc9a69872651baeca72 - # in the freedesktop/xorg-macros repo. - # - # As a result of this change, if we don't seed the cache and we don't use - # either --{en,dis}able-malloc0returnsnull, the AC_RUN_IFELSE block has no - # optional action-if-cross-compiling argument and configure prints an error - # message and exits as documented in the autoconf manual. Prior to this - # commit, the AC_RUN_IFELSE block had an action-if-cross-compiling argument - # which set the more pessimistic default value MALLOC_ZERO_RETURNS_NULL=yes. - # This is why the flag was not required prior to libXext 1.3.3. - $(package)_config_opts=--disable-static --disable-malloc0returnsnull -endef - -define $(package)_preprocess_cmds - cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub . -endef - -define $(package)_config_cmds - $($(package)_autoconf) -endef - -define $(package)_build_cmds - $(MAKE) -endef - -define $(package)_stage_cmds - $(MAKE) DESTDIR=$($(package)_staging_dir) install -endef - -define $(package)_postprocess_cmds - rm lib/*.la -endef diff --git a/depends/packages/libxcb.mk b/depends/packages/libxcb.mk index 9402951826de..90bc15f9f4bf 100644 --- a/depends/packages/libxcb.mk +++ b/depends/packages/libxcb.mk @@ -3,10 +3,22 @@ $(package)_version=1.10 $(package)_download_path=https://xcb.freedesktop.org/dist $(package)_file_name=$(package)-$($(package)_version).tar.bz2 $(package)_sha256_hash=98d9ab05b636dd088603b64229dd1ab2d2cc02ab807892e107d674f9c3f2d5b5 -$(package)_dependencies=xcb_proto libXau xproto +$(package)_dependencies=xcb_proto libXau define $(package)_set_vars $(package)_config_opts=--disable-static +# Because we pass -qt-xcb to Qt, it will compile in a set of xcb helper libraries and extensions, +# so we skip building all of the extensions here. +# More info is available from: https://doc.qt.io/qt-5.9/linux-requirements.html +$(package)_config_opts += --disable-composite --disable-damage --disable-dpms +$(package)_config_opts += --disable-dri2 --disable-dri3 --disable-glx +$(package)_config_opts += --disable-present --disable-randr --disable-record +$(package)_config_opts += --disable-render --disable-resource --disable-screensaver +$(package)_config_opts += --disable-shape --disable-shm --disable-sync +$(package)_config_opts += --disable-xevie --disable-xfixes --disable-xfree86-dri +$(package)_config_opts += --disable-xinerama --disable-xinput --disable-xkb +$(package)_config_opts += --disable-xprint --disable-selinux --disable-xtest +$(package)_config_opts += --disable-xv --disable-xvmc endef define $(package)_preprocess_cmds diff --git a/depends/packages/packages.mk b/depends/packages/packages.mk index 9a08e3089270..9edcd1eb38e5 100644 --- a/depends/packages/packages.mk +++ b/depends/packages/packages.mk @@ -3,7 +3,7 @@ packages:=boost openssl libevent qt_native_packages = native_protobuf qt_packages = qrencode protobuf zlib -qt_linux_packages:=qt expat dbus libxcb xcb_proto libXau xproto freetype fontconfig libX11 xextproto libXext xtrans +qt_linux_packages:=qt expat libxcb xcb_proto libXau xproto freetype fontconfig rapidcheck_packages = rapidcheck diff --git a/depends/packages/qt.mk b/depends/packages/qt.mk index 23cde9ee6d9e..2610c1e7484c 100644 --- a/depends/packages/qt.mk +++ b/depends/packages/qt.mk @@ -5,10 +5,10 @@ $(package)_suffix=opensource-src-$($(package)_version).tar.xz $(package)_file_name=qtbase-$($(package)_suffix) $(package)_sha256_hash=36dd9574f006eaa1e5af780e4b33d11fe39d09fd7c12f3b9d83294174bd28f00 $(package)_dependencies=openssl zlib -$(package)_linux_dependencies=freetype fontconfig libxcb libX11 xproto libXext +$(package)_linux_dependencies=freetype fontconfig libxcb $(package)_build_subdir=qtbase $(package)_qt_libs=corelib network widgets gui plugins testlib -$(package)_patches=fix_qt_pkgconfig.patch mac-qmake.conf fix_configure_mac.patch fix_no_printer.patch fix_rcc_determinism.patch fix_riscv64_arch.patch xkb-default.patch +$(package)_patches=fix_qt_pkgconfig.patch mac-qmake.conf fix_configure_mac.patch fix_no_printer.patch fix_rcc_determinism.patch fix_riscv64_arch.patch xkb-default.patch no-xlib.patch $(package)_qttranslations_file_name=qttranslations-$($(package)_suffix) $(package)_qttranslations_sha256_hash=b36da7d93c3ab6fca56b32053bb73bc619c8b192bb89b74e3bcde2705f1c2a14 @@ -35,9 +35,11 @@ $(package)_config_opts += -no-freetype $(package)_config_opts += -no-gif $(package)_config_opts += -no-glib $(package)_config_opts += -no-icu +$(package)_config_opts += -no-ico $(package)_config_opts += -no-iconv $(package)_config_opts += -no-kms $(package)_config_opts += -no-linuxfb +$(package)_config_opts += -no-libjpeg $(package)_config_opts += -no-libudev $(package)_config_opts += -no-mtdev $(package)_config_opts += -no-openvg @@ -63,26 +65,42 @@ $(package)_config_opts += -pch $(package)_config_opts += -pkg-config $(package)_config_opts += -prefix $(host_prefix) $(package)_config_opts += -qt-libpng -$(package)_config_opts += -qt-libjpeg $(package)_config_opts += -qt-pcre $(package)_config_opts += -qt-harfbuzz $(package)_config_opts += -system-zlib $(package)_config_opts += -static $(package)_config_opts += -silent $(package)_config_opts += -v +$(package)_config_opts += -no-feature-bearermanagement +$(package)_config_opts += -no-feature-colordialog +$(package)_config_opts += -no-feature-commandlineparser +$(package)_config_opts += -no-feature-concurrent $(package)_config_opts += -no-feature-dial +$(package)_config_opts += -no-feature-filesystemwatcher +$(package)_config_opts += -no-feature-fontcombobox $(package)_config_opts += -no-feature-ftp +$(package)_config_opts += -no-feature-image_heuristic_mask +$(package)_config_opts += -no-feature-keysequenceedit $(package)_config_opts += -no-feature-lcdnumber $(package)_config_opts += -no-feature-pdf -$(package)_config_opts += -no-feature-printer $(package)_config_opts += -no-feature-printdialog -$(package)_config_opts += -no-feature-concurrent +$(package)_config_opts += -no-feature-printer +$(package)_config_opts += -no-feature-printpreviewdialog +$(package)_config_opts += -no-feature-printpreviewwidget +$(package)_config_opts += -no-feature-regularexpression +$(package)_config_opts += -no-feature-sessionmanager $(package)_config_opts += -no-feature-sql $(package)_config_opts += -no-feature-statemachine $(package)_config_opts += -no-feature-syntaxhighlighter $(package)_config_opts += -no-feature-textbrowser $(package)_config_opts += -no-feature-textodfwriter +$(package)_config_opts += -no-feature-topleveldomain $(package)_config_opts += -no-feature-udpsocket +$(package)_config_opts += -no-feature-undocommand +$(package)_config_opts += -no-feature-undogroup +$(package)_config_opts += -no-feature-undostack +$(package)_config_opts += -no-feature-undoview +$(package)_config_opts += -no-feature-vnc $(package)_config_opts += -no-feature-wizard $(package)_config_opts += -no-feature-xml @@ -98,8 +116,9 @@ endif $(package)_config_opts_linux = -qt-xkbcommon-x11 $(package)_config_opts_linux += -qt-xcb +$(package)_config_opts_linux += -no-xcb-xlib +$(package)_config_opts_linux += -no-feature-xlib $(package)_config_opts_linux += -system-freetype -$(package)_config_opts_linux += -no-feature-sessionmanager $(package)_config_opts_linux += -fontconfig $(package)_config_opts_linux += -no-opengl $(package)_config_opts_arm_linux += -platform linux-g++ -xplatform bitcoin-linux-g++ @@ -156,11 +175,13 @@ define $(package)_preprocess_cmds echo "!host_build: QMAKE_CXXFLAGS += $($(package)_cxxflags) $($(package)_cppflags)" >> qtbase/mkspecs/common/gcc-base.conf && \ echo "!host_build: QMAKE_LFLAGS += $($(package)_ldflags)" >> qtbase/mkspecs/common/gcc-base.conf && \ patch -p1 -i $($(package)_patch_dir)/fix_riscv64_arch.patch &&\ + patch -p1 -i $($(package)_patch_dir)/no-xlib.patch &&\ echo "QMAKE_LINK_OBJECT_MAX = 10" >> qtbase/mkspecs/win32-g++/qmake.conf &&\ echo "QMAKE_LINK_OBJECT_SCRIPT = object_script" >> qtbase/mkspecs/win32-g++/qmake.conf &&\ sed -i.old "s|QMAKE_CFLAGS = |!host_build: QMAKE_CFLAGS = $($(package)_cflags) $($(package)_cppflags) |" qtbase/mkspecs/win32-g++/qmake.conf && \ sed -i.old "s|QMAKE_LFLAGS = |!host_build: QMAKE_LFLAGS = $($(package)_ldflags) |" qtbase/mkspecs/win32-g++/qmake.conf && \ - sed -i.old "s|QMAKE_CXXFLAGS = |!host_build: QMAKE_CXXFLAGS = $($(package)_cxxflags) $($(package)_cppflags) |" qtbase/mkspecs/win32-g++/qmake.conf + sed -i.old "s|QMAKE_CXXFLAGS = |!host_build: QMAKE_CXXFLAGS = $($(package)_cxxflags) $($(package)_cppflags) |" qtbase/mkspecs/win32-g++/qmake.conf && \ + sed -i.old "s/LIBRARY_PATH/(CROSS_)?\0/g" qtbase/mkspecs/features/toolchain.prf endef define $(package)_config_cmds diff --git a/depends/packages/rapidcheck.mk b/depends/packages/rapidcheck.mk index a35e091c8085..fa4fb3c78259 100644 --- a/depends/packages/rapidcheck.mk +++ b/depends/packages/rapidcheck.mk @@ -8,6 +8,10 @@ define $(package)_config_cmds cmake -DCMAKE_INSTALL_PREFIX=$($(package)_staging_dir)$(host_prefix) -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=true -DRC_INSTALL_ALL_EXTRAS=ON endef +define $(package)_preprocess_cmds + sed -i.old 's/ -Wall//' CMakeLists.txt +endef + define $(package)_build_cmds $(MAKE) rapidcheck endef diff --git a/depends/packages/xextproto.mk b/depends/packages/xextproto.mk deleted file mode 100644 index 157b76edf6fc..000000000000 --- a/depends/packages/xextproto.mk +++ /dev/null @@ -1,25 +0,0 @@ -package=xextproto -$(package)_version=7.3.0 -$(package)_download_path=https://xorg.freedesktop.org/releases/individual/proto -$(package)_file_name=$(package)-$($(package)_version).tar.bz2 -$(package)_sha256_hash=f3f4b23ac8db9c3a9e0d8edb591713f3d70ef9c3b175970dd8823dfc92aa5bb0 - -define $(package)_preprocess_cmds - cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub . -endef - -define $(package)_set_vars -$(package)_config_opts=--disable-shared -endef - -define $(package)_config_cmds - $($(package)_autoconf) -endef - -define $(package)_build_cmds - $(MAKE) -endef - -define $(package)_stage_cmds - $(MAKE) DESTDIR=$($(package)_staging_dir) install -endef diff --git a/depends/packages/xtrans.mk b/depends/packages/xtrans.mk deleted file mode 100644 index 1993ff8344cb..000000000000 --- a/depends/packages/xtrans.mk +++ /dev/null @@ -1,26 +0,0 @@ -package=xtrans -$(package)_version=1.3.4 -$(package)_download_path=https://xorg.freedesktop.org/releases/individual/lib/ -$(package)_file_name=$(package)-$($(package)_version).tar.bz2 -$(package)_sha256_hash=054d4ee3efd52508c753e9f7bc655ef185a29bd2850dd9e2fc2ccc33544f583a -$(package)_dependencies= - -define $(package)_set_vars -$(package)_config_opts_linux=--with-pic --disable-shared -endef - -define $(package)_preprocess_cmds - cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub . -endef - -define $(package)_config_cmds - $($(package)_autoconf) -endef - -define $(package)_build_cmds - $(MAKE) -endef - -define $(package)_stage_cmds - $(MAKE) DESTDIR=$($(package)_staging_dir) install -endef diff --git a/depends/patches/qt/no-xlib.patch b/depends/patches/qt/no-xlib.patch new file mode 100644 index 000000000000..fe82c2c73cb0 --- /dev/null +++ b/depends/patches/qt/no-xlib.patch @@ -0,0 +1,69 @@ +From 9563cef873ae82e06f60708d706d054717e801ce Mon Sep 17 00:00:00 2001 +From: Carl Dong +Date: Thu, 18 Jul 2019 17:22:05 -0400 +Subject: [PATCH] Wrap xlib related code blocks in #if's + +They are not necessary to compile QT. +--- + qtbase/src/plugins/platforms/xcb/qxcbcursor.cpp | 8 ++++++++ + 1 file changed, 8 insertions(+) + +diff --git a/qtbase/src/plugins/platforms/xcb/qxcbcursor.cpp b/qtbase/src/plugins/platforms/xcb/qxcbcursor.cpp +index 7c62c2e2b3..c05c6c0a07 100644 +--- a/qtbase/src/plugins/platforms/xcb/qxcbcursor.cpp ++++ b/qtbase/src/plugins/platforms/xcb/qxcbcursor.cpp +@@ -49,7 +49,9 @@ + #include + #include + #include ++#if QT_CONFIG(xcb_xlib) && QT_CONFIG(library) + #include ++#endif + #include + #include + +@@ -384,6 +386,7 @@ void QXcbCursor::changeCursor(QCursor *cursor, QWindow *widget) + w->setCursor(c, isBitmapCursor); + } + ++#if QT_CONFIG(xcb_xlib) && QT_CONFIG(library) + static int cursorIdForShape(int cshape) + { + int cursorId = 0; +@@ -437,6 +440,7 @@ static int cursorIdForShape(int cshape) + } + return cursorId; + } ++#endif + + xcb_cursor_t QXcbCursor::createNonStandardCursor(int cshape) + { +@@ -558,7 +562,9 @@ static xcb_cursor_t loadCursor(void *dpy, int cshape) + xcb_cursor_t QXcbCursor::createFontCursor(int cshape) + { + xcb_connection_t *conn = xcb_connection(); ++#if QT_CONFIG(xcb_xlib) && QT_CONFIG(library) + int cursorId = cursorIdForShape(cshape); ++#endif + xcb_cursor_t cursor = XCB_NONE; + + // Try Xcursor first +@@ -589,6 +595,7 @@ xcb_cursor_t QXcbCursor::createFontCursor(int cshape) + // Non-standard X11 cursors are created from bitmaps + cursor = createNonStandardCursor(cshape); + ++#if QT_CONFIG(xcb_xlib) && QT_CONFIG(library) + // Create a glpyh cursor if everything else failed + if (!cursor && cursorId) { + cursor = xcb_generate_id(conn); +@@ -596,6 +603,7 @@ xcb_cursor_t QXcbCursor::createFontCursor(int cshape) + cursorId, cursorId + 1, + 0xFFFF, 0xFFFF, 0xFFFF, 0, 0, 0); + } ++#endif + + if (cursor && cshape >= 0 && cshape < Qt::LastCursor && connection()->hasXFixes()) { + const char *name = cursorNames[cshape]; +-- +2.22.0 + diff --git a/doc/Doxyfile.in b/doc/Doxyfile.in index 58c65fb7e255..399d54eb85d5 100644 --- a/doc/Doxyfile.in +++ b/doc/Doxyfile.in @@ -453,7 +453,7 @@ EXTRACT_PACKAGE = NO # included in the documentation. # The default value is: NO. -EXTRACT_STATIC = NO +EXTRACT_STATIC = YES # If the EXTRACT_LOCAL_CLASSES tag is set to YES, classes (and structs) defined # locally in source files will be included in the documentation. If set to NO, diff --git a/doc/README.md b/doc/README.md index b4fa933c8ec4..d3017de2abaf 100644 --- a/doc/README.md +++ b/doc/README.md @@ -74,6 +74,7 @@ The Bitcoin repo's [root README](/README.md) contains relevant information on th - [bitcoin.conf Configuration File](bitcoin-conf.md) - [Files](files.md) - [Fuzz-testing](fuzzing.md) +- [Reduce Memory](reduce-memory.md) - [Reduce Traffic](reduce-traffic.md) - [Tor Support](tor.md) - [Init Scripts (systemd/upstart/openrc)](init.md) diff --git a/doc/benchmarking.md b/doc/benchmarking.md index 48f81cf6c18f..8e3d88ab7ae0 100644 --- a/doc/benchmarking.md +++ b/doc/benchmarking.md @@ -2,10 +2,17 @@ Benchmarking ============ Bitcoin Core has an internal benchmarking framework, with benchmarks -for cryptographic algorithms (e.g. SHA1, SHA256, SHA512, RIPEMD160), as well as the rolling bloom filter. +for cryptographic algorithms (e.g. SHA1, SHA256, SHA512, RIPEMD160, Poly1305, ChaCha20), rolling bloom filter, coins selection, +thread queue, wallet balance. Running --------------------- + +For benchmarks purposes you only need to compile `bitcoin_bench`. Beware of configuring without `--enable-debug` as this would impact +benchmarking by unlatching log printers and lock analysis. + + make -C src bench_bitcoin + After compiling bitcoin-core, the benchmarks can be run with: src/bench/bench_bitcoin @@ -13,43 +20,29 @@ After compiling bitcoin-core, the benchmarks can be run with: The output will look similar to: ``` # Benchmark, evals, iterations, total, min, max, median -Base58CheckEncode, 5, 320000, 120.772, 7.49351e-05, 7.59374e-05, 7.54759e-05 -Base58Decode, 5, 800000, 122.833, 3.0467e-05, 3.11732e-05, 3.06304e-05 -Base58Encode, 5, 470000, 137.094, 5.81061e-05, 5.85109e-05, 5.84462e-05 -BenchLockedPool, 5, 530, 34.2023, 0.0128247, 0.0129613, 0.0129026 -CCheckQueueSpeedPrevectorJob, 5, 1400, 26.1762, 0.00365048, 0.00388629, 0.00367108 -CCoinsCaching, 5, 170000, 48.1074, 5.60229e-05, 5.72316e-05, 5.66214e-05 -CoinSelection, 5, 650, 34.6426, 0.0105801, 0.0107699, 0.010664 -DeserializeAndCheckBlockTest, 5, 160, 39.2084, 0.0483662, 0.0494199, 0.0490138 -DeserializeBlockTest, 5, 130, 23.8129, 0.0357731, 0.0373763, 0.0365858 -FastRandom_1bit, 5, 440000000, 38.1609, 1.72974e-08, 1.73882e-08, 1.73478e-08 -FastRandom_32bit, 5, 110000000, 72.8237, 1.29992e-07, 1.37014e-07, 1.30115e-07 -MempoolEviction, 5, 41000, 89.8883, 0.000432748, 0.000446857, 0.000438483 -PrevectorClear, 5, 5600, 47.9229, 0.00169952, 0.0017455, 0.00170315 -PrevectorDestructor, 5, 5700, 44.5498, 0.0015561, 0.00156977, 0.00156469 -RIPEMD160, 5, 440, 135.988, 0.0615496, 0.062268, 0.0617779 -RollingBloom, 5, 1500000, 36.5109, 4.80961e-06, 4.97463e-06, 4.85811e-06 -SHA1, 5, 570, 51.808, 0.018065, 0.0182623, 0.0181865 -SHA256, 5, 340, 8.31841, 0.00483231, 0.00499803, 0.00485486 -SHA256_32b, 5, 4700000, 10.469, 4.43441e-07, 4.47611e-07, 4.45223e-07 -SHA512, 5, 330, 33.3408, 0.02017, 0.0202554, 0.0201921 -SipHash_32b, 5, 40000000, 38.7088, 1.91103e-07, 1.96998e-07, 1.93792e-07 -Sleep100ms, 5, 10, 5.01062, 0.100131, 0.100368, 0.100147 -Trig, 5, 12000000, 5.95494, 9.78115e-08, 1.04354e-07, 9.80682e-08 -VerifyScriptBench, 5, 6300, 9.02493, 0.000285566, 0.000288433, 0.000286175 +AssembleBlock, 5, 700, 1.79954, 0.000510913, 0.000517018, 0.000514497 +... ``` Help --------------------- -`-?` will print a list of options and exit: - src/bench/bench_bitcoin -? + src/bench/bench_bitcoin --help + +To print options like scaling factor or per-benchmark filter. Notes --------------------- More benchmarks are needed for, in no particular order: - Script Validation -- CCoinDBView caching - Coins database - Memory pool -- Wallet coin selection +- Cuckoo Cache +- P2P throughput + +Going Further +-------------------- + +To monitor Bitcoin Core performance more in depth (like reindex or IBD): https://github.com/chaincodelabs/bitcoinperf + +To generate Flame Graphs for Bitcoin Core: https://github.com/eklitzke/bitcoin/blob/flamegraphs/doc/flamegraphs.md diff --git a/doc/bips.md b/doc/bips.md index eb24ce6f66bb..3085fa424b03 100644 --- a/doc/bips.md +++ b/doc/bips.md @@ -12,8 +12,8 @@ BIPs that are implemented by Bitcoin Core (up-to-date up to **v0.18.0**): * [`BIP 31`](https://github.com/bitcoin/bips/blob/master/bip-0031.mediawiki): The 'pong' protocol message (and the protocol version bump to 60001) has been implemented since **v0.6.1** ([PR #1081](https://github.com/bitcoin/bitcoin/pull/1081)). * [`BIP 32`](https://github.com/bitcoin/bips/blob/master/bip-0032.mediawiki): Hierarchical Deterministic Wallets has been implemented since **v0.13.0** ([PR #8035](https://github.com/bitcoin/bitcoin/pull/8035)). * [`BIP 34`](https://github.com/bitcoin/bips/blob/master/bip-0034.mediawiki): The rule that requires blocks to contain their height (number) in the coinbase input, and the introduction of version 2 blocks has been implemented since **v0.7.0**. The rule took effect for version 2 blocks as of *block 224413* (March 5th 2013), and version 1 blocks are no longer allowed since *block 227931* (March 25th 2013) ([PR #1526](https://github.com/bitcoin/bitcoin/pull/1526)). -* [`BIP 35`](https://github.com/bitcoin/bips/blob/master/bip-0035.mediawiki): The 'mempool' protocol message (and the protocol version bump to 60002) has been implemented since **v0.7.0** ([PR #1641](https://github.com/bitcoin/bitcoin/pull/1641)). -* [`BIP 37`](https://github.com/bitcoin/bips/blob/master/bip-0037.mediawiki): The bloom filtering for transaction relaying, partial Merkle trees for blocks, and the protocol version bump to 70001 (enabling low-bandwidth SPV clients) has been implemented since **v0.8.0** ([PR #1795](https://github.com/bitcoin/bitcoin/pull/1795)). +* [`BIP 35`](https://github.com/bitcoin/bips/blob/master/bip-0035.mediawiki): The 'mempool' protocol message (and the protocol version bump to 60002) has been implemented since **v0.7.0** ([PR #1641](https://github.com/bitcoin/bitcoin/pull/1641)). As of **v0.13.0**, this is only available for `NODE_BLOOM` (BIP 111) peers. +* [`BIP 37`](https://github.com/bitcoin/bips/blob/master/bip-0037.mediawiki): The bloom filtering for transaction relaying, partial Merkle trees for blocks, and the protocol version bump to 70001 (enabling low-bandwidth SPV clients) has been implemented since **v0.8.0** ([PR #1795](https://github.com/bitcoin/bitcoin/pull/1795)). Disabled by default since **v0.19.0**, can be enabled by the `-peerbloomfilters` option. * [`BIP 42`](https://github.com/bitcoin/bips/blob/master/bip-0042.mediawiki): The bug that would have caused the subsidy schedule to resume after block 13440000 was fixed in **v0.9.2** ([PR #3842](https://github.com/bitcoin/bitcoin/pull/3842)). * [`BIP 61`](https://github.com/bitcoin/bips/blob/master/bip-0061.mediawiki): The 'reject' protocol message (and the protocol version bump to 70002) was added in **v0.9.0** ([PR #3185](https://github.com/bitcoin/bitcoin/pull/3185)). Starting **v0.17.0**, whether to send reject messages can be configured with the `-enablebip61` option, and support is deprecated as of **v0.18.0**. * [`BIP 65`](https://github.com/bitcoin/bips/blob/master/bip-0065.mediawiki): The CHECKLOCKTIMEVERIFY softfork was merged in **v0.12.0** ([PR #6351](https://github.com/bitcoin/bitcoin/pull/6351)), and backported to **v0.11.2** and **v0.10.4**. Mempool-only CLTV was added in [PR #6124](https://github.com/bitcoin/bitcoin/pull/6124). diff --git a/doc/bitcoin-conf.md b/doc/bitcoin-conf.md index 88ecb8fe6523..f8146b5d7592 100644 --- a/doc/bitcoin-conf.md +++ b/doc/bitcoin-conf.md @@ -30,6 +30,21 @@ Network specific options can be: - placed into sections with headers `[main]` (not `[mainnet]`), `[test]` (not `[testnet]`) or `[regtest]`; - prefixed with a chain name; e.g., `regtest.maxmempool=100`. +Network specific options take precedence over non-network specific options. +If multiple values for the same option are found with the same precedence, the +first one is generally chosen. + +This means that given the following configuration, `regtest.rpcport` is set to `3000`: + +``` +regtest=1 +rpcport=2000 +regtest.rpcport=3000 + +[regtest] +rpcport=4000 +``` + ## Configuration File Path The configuration file is not automatically created; you can create it using your favorite text editor. By default, the configuration file name is `bitcoin.conf` and it is located in the Bitcoin data directory, but both the Bitcoin data directory and the configuration file path may be changed using the `-datadir` and `-conf` command-line options. diff --git a/doc/build-osx.md b/doc/build-osx.md index d28a3d97aadb..65cfce6b1557 100644 --- a/doc/build-osx.md +++ b/doc/build-osx.md @@ -1,33 +1,37 @@ -macOS Build Instructions and Notes -==================================== +# macOS Build Instructions and Notes + The commands in this guide should be executed in a Terminal application. -The built-in one is located in `/Applications/Utilities/Terminal.app`. +The built-in one is located in +``` +/Applications/Utilities/Terminal.app +``` -Preparation ------------ +## Preparation Install the macOS command line tools: -`xcode-select --install` +```shell +xcode-select --install +``` When the popup appears, click `Install`. Then install [Homebrew](https://brew.sh). -Dependencies ----------------------- - - brew install automake berkeley-db4 libtool boost miniupnpc openssl pkg-config protobuf python qt libevent qrencode +## Dependencies +```shell +brew install automake berkeley-db4 libtool boost miniupnpc openssl pkg-config protobuf python qt libevent qrencode +``` See [dependencies.md](dependencies.md) for a complete overview. If you want to build the disk image with `make deploy` (.dmg / optional), you need RSVG: +```shell +brew install librsvg +``` - brew install librsvg - -Berkeley DB ------------ +## Berkeley DB It is recommended to use Berkeley DB 4.8. If you have to build it yourself, -you can use [the installation script included in contrib/](/contrib/install_db4.sh) +you can use [this](/contrib/install_db4.sh) script to install it like so: ```shell @@ -38,172 +42,167 @@ from the root of the repository. **Note**: You only need Berkeley DB if the wallet is enabled (see [*Disable-wallet mode*](/doc/build-osx.md#disable-wallet-mode)). -Build Bitcoin Core ------------------------- +## Build Bitcoin Core 1. Clone the Bitcoin Core source code: - - git clone https://github.com/bitcoin/bitcoin - cd bitcoin + ```shell + git clone https://github.com/bitcoin/bitcoin + cd bitcoin + ``` 2. Build Bitcoin Core: Configure and build the headless Bitcoin Core binaries as well as the GUI (if Qt is found). You can disable the GUI build by passing `--without-gui` to configure. - - ./autogen.sh - ./configure - make + ```shell + ./autogen.sh + ./configure + make + ``` 3. It is recommended to build and run the unit tests: - - make check - -4. You can also create a .dmg that contains the .app bundle (optional): - - make deploy - -Disable-wallet mode --------------------- -When the intention is to run only a P2P node without a wallet, Bitcoin Core may be compiled in -disable-wallet mode with: - - ./configure --disable-wallet + ```shell + make check + ``` + +4. You can also create a `.dmg` that contains the `.app` bundle (optional): + ```shell + make deploy + ``` + +## `disable-wallet` mode +When the intention is to run only a P2P node without a wallet, Bitcoin Core may be +compiled in `disable-wallet` mode with: +```shell +./configure --disable-wallet +``` In this case there is no dependency on Berkeley DB 4.8. Mining is also possible in disable-wallet mode using the `getblocktemplate` RPC call. -Running -------- - +## Running Bitcoin Core is now available at `./src/bitcoind` Before running, you may create an empty configuration file: +```shell +mkdir -p "/Users/${USER}/Library/Application Support/Bitcoin" - mkdir -p "/Users/${USER}/Library/Application Support/Bitcoin" - - touch "/Users/${USER}/Library/Application Support/Bitcoin/bitcoin.conf" +touch "/Users/${USER}/Library/Application Support/Bitcoin/bitcoin.conf" - chmod 600 "/Users/${USER}/Library/Application Support/Bitcoin/bitcoin.conf" +chmod 600 "/Users/${USER}/Library/Application Support/Bitcoin/bitcoin.conf" +``` -The first time you run bitcoind, it will start downloading the blockchain. This process could take many hours, or even days on slower than average systems. +The first time you run bitcoind, it will start downloading the blockchain. This process could +take many hours, or even days on slower than average systems. You can monitor the download process by looking at the debug.log file: +```shell +tail -f $HOME/Library/Application\ Support/Bitcoin/debug.log +``` - tail -f $HOME/Library/Application\ Support/Bitcoin/debug.log - -Other commands: -------- - - ./src/bitcoind -daemon # Starts the bitcoin daemon. - ./src/bitcoin-cli --help # Outputs a list of command-line options. - ./src/bitcoin-cli help # Outputs a list of RPC commands when the daemon is running. - -Notes ------ - -* Tested on OS X 10.10 Yosemite through macOS 10.13 High Sierra on 64-bit Intel processors only. - -* Building with downloaded Qt binaries is not officially supported. See the notes in [#7714](https://github.com/bitcoin/bitcoin/issues/7714) +## Other commands: +```shell +./src/bitcoind -daemon # Starts the bitcoin daemon. +./src/bitcoin-cli --help # Outputs a list of command-line options. +./src/bitcoin-cli help # Outputs a list of RPC commands when the daemon is running. +``` -Deterministic macOS DMG Notes ------------------------------ +## Notes +* Tested on OS X 10.10 Yosemite through macOS 10.14 Mojave on 64-bit Intel +processors only. +* Building with downloaded Qt binaries is not officially supported. See the notes in [#7714](https://github.com/bitcoin/bitcoin/issues/7714). -Working macOS DMGs are created in Linux by combining a recent clang, -the Apple binutils (ld, ar, etc) and DMG authoring tools. +## Deterministic macOS DMG Notes +Working macOS DMGs are created in Linux by combining a recent `clang`, the Apple +`binutils` (`ld`, `ar`, etc) and DMG authoring tools. -Apple uses clang extensively for development and has upstreamed the necessary -functionality so that a vanilla clang can take advantage. It supports the use -of -F, -target, -mmacosx-version-min, and --sysroot, which are all necessary -when building for macOS. +Apple uses `clang` extensively for development and has upstreamed the necessary +functionality so that a vanilla clang can take advantage. It supports the use of `-F`, +`-target`, `-mmacosx-version-min`, and `--sysroot`, which are all necessary when +building for macOS. -Apple's version of binutils (called cctools) contains lots of functionality -missing in the FSF's binutils. In addition to extra linker options for -frameworks and sysroots, several other tools are needed as well such as -install_name_tool, lipo, and nmedit. These do not build under linux, so they -have been patched to do so. The work here was used as a starting point: -[mingwandroid/toolchain4](https://github.com/mingwandroid/toolchain4). +Apple's version of `binutils` (called `cctools`) contains lots of functionality missing in the +FSF's `binutils`. In addition to extra linker options for frameworks and sysroots, several +other tools are needed as well such as `install_name_tool`, `lipo`, and `nmedit`. These +do not build under Linux, so they have been patched to do so. The work here was used as +a starting point: [mingwandroid/toolchain4](https://github.com/mingwandroid/toolchain4). -In order to build a working toolchain, the following source packages are needed -from Apple: cctools, dyld, and ld64. +In order to build a working toolchain, the following source packages are needed from +Apple: `cctools`, `dyld`, and `ld64`. -These tools inject timestamps by default, which produce non-deterministic -binaries. The ZERO_AR_DATE environment variable is used to disable that. +These tools inject timestamps by default, which produce non-deterministic binaries. The +`ZERO_AR_DATE` environment variable is used to disable that. -This version of cctools has been patched to use the current version of clang's -headers and its libLTO.so rather than those from llvmgcc, as it was -originally done in toolchain4. +This version of `cctools` has been patched to use the current version of `clang`'s headers +and its `libLTO.so` rather than those from `llvmgcc`, as it was originally done in `toolchain4`. -To complicate things further, all builds must target an Apple SDK. These SDKs -are free to download, but not redistributable. -To obtain it, register for a developer account, then download the [Xcode 7.3.1 dmg](https://developer.apple.com/devcenter/download.action?path=/Developer_Tools/Xcode_7.3.1/Xcode_7.3.1.dmg). +To complicate things further, all builds must target an Apple SDK. These SDKs are free to +download, but not redistributable. To obtain it, register for an Apple Developer Account, +then download the [Xcode 7.3.1 dmg](https://developer.apple.com/devcenter/download.action?path=/Developer_Tools/Xcode_7.3.1/Xcode_7.3.1.dmg). -This file is several gigabytes in size, but only a single directory inside is -needed: +This file is several gigabytes in size, but only a single directory inside is needed: ``` Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.11.sdk ``` -Unfortunately, the usual linux tools (7zip, hpmount, loopback mount) are incapable of opening this file. -To create a tarball suitable for Gitian input, there are two options: +Unfortunately, the usual Linux tools (7zip, hpmount, loopback mount) are incapable of +opening this file. To create a tarball suitable for Gitian input, there are two options: -Using macOS, you can mount the dmg, and then create it with: -``` - $ hdiutil attach Xcode_7.3.1.dmg - $ tar -C /Volumes/Xcode/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/ -czf MacOSX10.11.sdk.tar.gz MacOSX10.11.sdk +Using macOS, you can mount the DMG, and then create it with: +```shell +hdiutil attach Xcode_7.3.1.dmg +tar -C /Volumes/Xcode/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/ -czf MacOSX10.11.sdk.tar.gz MacOSX10.11.sdk ``` -Alternatively, you can use 7zip and SleuthKit to extract the files one by one. -The script contrib/macdeploy/extract-osx-sdk.sh automates this. First ensure -the dmg file is in the current directory, and then run the script. You may wish -to delete the intermediate 5.hfs file and MacOSX10.11.sdk (the directory) when -you've confirmed the extraction succeeded. +Alternatively, you can use 7zip and SleuthKit to extract the files one by one. The script +[`extract-osx-sdk.sh`](./../contrib/macdeploy/extract-osx-sdk.sh) automates this. First +ensure the DMG file is in the current directory, and then run the script. You may wish to +delete the `intermediate 5.hfs` file and `MacOSX10.11.sdk` (the directory) when you've +confirmed the extraction succeeded. -```bash +```shell apt-get install p7zip-full sleuthkit contrib/macdeploy/extract-osx-sdk.sh rm -rf 5.hfs MacOSX10.11.sdk ``` -The Gitian descriptors build 2 sets of files: Linux tools, then Apple binaries -which are created using these tools. The build process has been designed to -avoid including the SDK's files in Gitian's outputs. All interim tarballs are -fully deterministic and may be freely redistributed. +The Gitian descriptors build 2 sets of files: Linux tools, then Apple binaries which are +created using these tools. The build process has been designed to avoid including the +SDK's files in Gitian's outputs. All interim tarballs are fully deterministic and may be freely +redistributed. -genisoimage is used to create the initial DMG. It is not deterministic as-is, -so it has been patched. A system genisoimage will work fine, but it will not -be deterministic because the file-order will change between invocations. -The patch can be seen here: [theuni/osx-cross-depends](https://raw.githubusercontent.com/theuni/osx-cross-depends/master/patches/cdrtools/genisoimage.diff). -No effort was made to fix this cleanly, so it likely leaks memory badly. But -it's only used for a single invocation, so that's no real concern. +`genisoimage` is used to create the initial DMG. It is not deterministic as-is, so it has been +patched. A system `genisoimage` will work fine, but it will not be deterministic because +the file-order will change between invocations. The patch can be seen here: [theuni/osx-cross-depends](https://raw.githubusercontent.com/theuni/osx-cross-depends/master/patches/cdrtools/genisoimage.diff). +No effort was made to fix this cleanly, so it likely leaks memory badly. But it's only used for +a single invocation, so that's no real concern. -genisoimage cannot compress DMGs, so afterwards, the 'dmg' tool from the -libdmg-hfsplus project is used to compress it. There are several bugs in this -tool and its maintainer has seemingly abandoned the project. It has been forked -and is available (with fixes) here: [theuni/libdmg-hfsplus](https://github.com/theuni/libdmg-hfsplus). +`genisoimage` cannot compress DMGs, so afterwards, the DMG tool from the +`libdmg-hfsplus` project is used to compress it. There are several bugs in this tool and its +maintainer has seemingly abandoned the project. It has been forked and is available +(with fixes) here: [theuni/libdmg-hfsplus](https://github.com/theuni/libdmg-hfsplus). -The 'dmg' tool has the ability to create DMGs from scratch as well, but this -functionality is broken. Only the compression feature is currently used. -Ideally, the creation could be fixed and genisoimage would no longer be necessary. +The DMG tool has the ability to create DMGs from scratch as well, but this functionality is +broken. Only the compression feature is currently used. Ideally, the creation could be fixed +and `genisoimage` would no longer be necessary. Background images and other features can be added to DMG files by inserting a -.DS_Store before creation. This is generated by the script -contrib/macdeploy/custom_dsstore.py. - -As of OS X 10.9 Mavericks, using an Apple-blessed key to sign binaries is a -requirement in order to satisfy the new Gatekeeper requirements. Because this -private key cannot be shared, we'll have to be a bit creative in order for the -build process to remain somewhat deterministic. Here's how it works: - -- Builders use Gitian to create an unsigned release. This outputs an unsigned - dmg which users may choose to bless and run. It also outputs an unsigned app - structure in the form of a tarball, which also contains all of the tools - that have been previously (deterministically) built in order to create a - final dmg. -- The Apple keyholder uses this unsigned app to create a detached signature, - using the script that is also included there. Detached signatures are available from this [repository](https://github.com/bitcoin-core/bitcoin-detached-sigs). -- Builders feed the unsigned app + detached signature back into Gitian. It - uses the pre-built tools to recombine the pieces into a deterministic dmg. +`.DS_Store` before creation. This is generated by the script +`contrib/macdeploy/custom_dsstore.py`. + +As of OS X 10.9 Mavericks, using an Apple-blessed key to sign binaries is a requirement in +order to satisfy the new Gatekeeper requirements. Because this private key cannot be +shared, we'll have to be a bit creative in order for the build process to remain somewhat +deterministic. Here's how it works: + +- Builders use Gitian to create an unsigned release. This outputs an unsigned DMG which + users may choose to bless and run. It also outputs an unsigned app structure in the form + of a tarball, which also contains all of the tools that have been previously (deterministically) + built in order to create a final DMG. +- The Apple keyholder uses this unsigned app to create a detached signature, using the + script that is also included there. Detached signatures are available from this [repository](https://github.com/bitcoin-core/bitcoin-detached-sigs). +- Builders feed the unsigned app + detached signature back into Gitian. It uses the + pre-built tools to recombine the pieces into a deterministic DMG. diff --git a/doc/build-unix.md b/doc/build-unix.md index da65bc347a16..eb88aca050ab 100644 --- a/doc/build-unix.md +++ b/doc/build-unix.md @@ -61,6 +61,14 @@ tuned to conserve memory with additional CXXFLAGS: ./configure CXXFLAGS="--param ggc-min-expand=1 --param ggc-min-heapsize=32768" +Alternatively, or in addition, debugging information can be skipped for compilation. The default compile flags are +`-g -O2`, and can be changed with: + + ./configure CXXFLAGS="-O2" + +Finally, clang (often less resource hungry) can be used instead of gcc, which is used by default: + + ./configure CXX=clang++ CC=clang ## Linux Distribution Specific Instructions @@ -78,7 +86,7 @@ Now, you can either build from self-compiled [depends](/depends/README.md) or in BerkeleyDB is required for the wallet. -Ubuntu and Debian have their own libdb-dev and libdb++-dev packages, but these will install +Ubuntu and Debian have their own `libdb-dev` and `libdb++-dev` packages, but these will install BerkeleyDB 5.1 or later. This will break binary wallet compatibility with the distributed executables, which are based on BerkeleyDB 4.8. If you do not care about wallet compatibility, pass `--with-incompatible-bdb` to configure. @@ -88,7 +96,7 @@ Otherwise, you can build from self-compiled `depends` (see above). To build Bitcoin Core without wallet, see [*Disable-wallet mode*](/doc/build-unix.md#disable-wallet-mode) -Optional (see --with-miniupnpc and --enable-upnp-default): +Optional (see `--with-miniupnpc` and `--enable-upnp-default`): sudo apt-get install libminiupnpc-dev @@ -122,10 +130,14 @@ Build requirements: sudo dnf install gcc-c++ libtool make autoconf automake openssl-devel libevent-devel boost-devel libdb4-devel libdb4-cxx-devel python3 -Optional: +Optional (see `--with-miniupnpc` and `--enable-upnp-default`): sudo dnf install miniupnpc-devel +ZMQ dependencies (provides ZMQ API): + + sudo dnf install zeromq-devel + To build with Qt 5 you need the following: sudo dnf install qt5-qttools-devel qt5-qtbase-devel protobuf-devel diff --git a/doc/dependencies.md b/doc/dependencies.md index 4c0e8d25675f..0b23ca0a2d56 100644 --- a/doc/dependencies.md +++ b/doc/dependencies.md @@ -7,26 +7,24 @@ These are the dependencies currently used by Bitcoin Core. You can find instruct | --- | --- | --- | --- | --- | --- | | Berkeley DB | [4.8.30](https://www.oracle.com/technetwork/database/database-technologies/berkeleydb/downloads/index.html) | 4.8.x | No | | | | Boost | [1.70.0](https://www.boost.org/users/download/) | [1.47.0](https://github.com/bitcoin/bitcoin/pull/8920) | No | | | -| Clang | | [3.3+](https://llvm.org/releases/download.html) (C++11 support) | | | | -| D-Bus | [1.10.18](https://cgit.freedesktop.org/dbus/dbus/tree/NEWS?h=dbus-1.10) | | No | Yes | | -| Expat | [2.2.6](https://libexpat.github.io/) | | No | Yes | | +| Clang | | [3.3+](https://releases.llvm.org/download.html) (C++11 support) | | | | +| Expat | [2.2.7](https://libexpat.github.io/) | | No | Yes | | | fontconfig | [2.12.1](https://www.freedesktop.org/software/fontconfig/release/) | | No | Yes | | | FreeType | [2.7.1](https://download.savannah.gnu.org/releases/freetype) | | No | | | | GCC | | [4.8+](https://gcc.gnu.org/) (C++11 support) | | | | | HarfBuzz-NG | | | | | | | libevent | [2.1.8-stable](https://github.com/libevent/libevent/releases) | 2.0.22 | No | | | -| libjpeg | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk#L65) | -| libpng | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk#L64) | +| libpng | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) | | librsvg | | | | | | | MiniUPnPc | [2.0.20180203](http://miniupnp.free.fr/files) | | No | | | | OpenSSL | [1.0.1k](https://www.openssl.org/source) | | Yes | | | -| PCRE | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk#L66) | +| PCRE | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) | | protobuf | [2.6.1](https://github.com/google/protobuf/releases) | | No | | | | Python (tests) | | [3.5](https://www.python.org/downloads) | | | | | qrencode | [3.4.4](https://fukuchi.org/works/qrencode) | | No | | | | Qt | [5.9.7](https://download.qt.io/official_releases/qt/) | [5.5.1](https://github.com/bitcoin/bitcoin/issues/13478) | No | | | -| XCB | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk#L87) (Linux only) | -| xkbcommon | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk#L86) (Linux only) | +| XCB | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) (Linux only) | +| xkbcommon | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) (Linux only) | | ZeroMQ | [4.3.1](https://github.com/zeromq/libzmq/releases) | 4.0.0 | No | | | | zlib | [1.2.11](https://zlib.net/) | | | | No | diff --git a/doc/developer-notes.md b/doc/developer-notes.md index ecd720539e7b..d1114b0c73f6 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -62,7 +62,7 @@ tool to clean up patches automatically before submission. - Braces on the same line for everything else. - 4 space indentation (no tabs) for every block except namespaces. - No indentation for `public`/`protected`/`private` or for `namespace`. - - No extra spaces inside parenthesis; don't do ( this ) + - No extra spaces inside parenthesis; don't do ( this ). - No space after function names; one space after `if`, `for` and `while`. - If an `if` only has a single-statement `then`-clause, it can appear on the same line as the `if`, without braces. In every other case, @@ -72,12 +72,12 @@ tool to clean up patches automatically before submission. - **Symbol naming conventions**. These are preferred in new code, but are not required when doing so would need changes to significant pieces of existing code. - - Variable (including function arguments) and namespace names are all lowercase, and may use `_` to + - Variable (including function arguments) and namespace names are all lowercase and may use `_` to separate words (snake_case). - Class member variables have a `m_` prefix. - Global variables have a `g_` prefix. - Constant names are all uppercase, and use `_` to separate words. - - Class names, function names and method names are UpperCamelCase + - Class names, function names, and method names are UpperCamelCase (PascalCase). Do not prefix class names with `C`. - Test suite naming convention: The Boost test suite in file `src/test/foo_tests.cpp` should be named `foo_tests`. Test suite names @@ -134,6 +134,7 @@ Bitcoin Core uses [Doxygen](http://www.doxygen.nl/) to generate its official doc Use Doxygen-compatible comment blocks for functions, methods, and fields. For example, to describe a function use: + ```c++ /** * ... text ... @@ -143,11 +144,12 @@ For example, to describe a function use: */ bool function(int arg1, const char *arg2) ``` + A complete list of `@xxx` commands can be found at http://www.doxygen.nl/manual/commands.html. As Doxygen recognizes the comments by the delimiters (`/**` and `*/` in this case), you don't *need* to provide any commands for a comment to be valid; just a description text is fine. -To describe a class use the same construct above the class definition: +To describe a class, use the same construct above the class definition: ```c++ /** * Alerts are for notifying old versions if they become too obsolete and @@ -189,7 +191,7 @@ but the above styles are favored. Documentation can be generated with `make docs` and cleaned up with `make clean-docs`. The resulting files are located in `doc/doxygen/html`; open `index.html` to view the homepage. -Before running `make docs`, you will need to install dependencies `doxygen` and `dot`. For example, on MacOS via Homebrew: +Before running `make docs`, you will need to install dependencies `doxygen` and `dot`. For example, on macOS via Homebrew: ``` brew install doxygen --with-graphviz ``` @@ -231,7 +233,7 @@ that run in `-regtest` mode. Bitcoin Core is a multi-threaded application, and deadlocks or other multi-threading bugs can be very difficult to track down. The `--enable-debug` configure option adds `-DDEBUG_LOCKORDER` to the compiler flags. This inserts -run-time checks to keep track of which locks are held, and adds warnings to the +run-time checks to keep track of which locks are held and adds warnings to the debug.log file if inconsistencies are detected. ### Valgrind suppressions file @@ -299,7 +301,7 @@ $ perf record \ -p `pgrep bitcoind` -- sleep 60 ``` -You could then analyze the results by running +You could then analyze the results by running: ```sh perf report --stdio | c++filt | less @@ -364,7 +366,7 @@ Additional resources: Locking/mutex usage notes ------------------------- -The code is multi-threaded, and uses mutexes and the +The code is multi-threaded and uses mutexes and the `LOCK` and `TRY_LOCK` macros to protect data structures. Deadlocks due to inconsistent lock ordering (thread 1 locks `cs_main` and then @@ -375,7 +377,7 @@ reported in the debug.log file. Re-architecting the core code so there are better-defined interfaces between the various components is a goal, with any necessary locking -done by the components (e.g. see the self-contained `CBasicKeyStore` class +done by the components (e.g. see the self-contained `FillableSigningProvider` class and its `cs_KeyStore` lock for example). Threads @@ -389,7 +391,7 @@ Threads - ThreadDNSAddressSeed : Loads addresses of peers from the DNS. -- ThreadMapPort : Universal plug-and-play startup/shutdown +- ThreadMapPort : Universal plug-and-play startup/shutdown. - ThreadSocketHandler : Sends/Receives data from peers on port 8333. @@ -408,7 +410,7 @@ Threads Ignoring IDE/editor files -------------------------- -In closed-source environments in which everyone uses the same IDE it is common +In closed-source environments in which everyone uses the same IDE, it is common to add temporary files it produces to the project-wide `.gitignore` file. However, in open source software such as Bitcoin Core, where everyone uses @@ -446,19 +448,19 @@ pay attention to for reviewers of Bitcoin Core code. General Bitcoin Core ---------------------- -- New features should be exposed on RPC first, then can be made available in the GUI +- New features should be exposed on RPC first, then can be made available in the GUI. - *Rationale*: RPC allows for better automatic testing. The test suite for - the GUI is very limited + the GUI is very limited. -- Make sure pull requests pass Travis CI before merging +- Make sure pull requests pass Travis CI before merging. - *Rationale*: Makes sure that they pass thorough testing, and that the tester will keep passing - on the master branch. Otherwise all new pull requests will start failing the tests, resulting in - confusion and mayhem + on the master branch. Otherwise, all new pull requests will start failing the tests, resulting in + confusion and mayhem. - *Explanation*: If the test suite is to be updated for a change, this has to - be done first + be done first. Wallet ------- @@ -466,13 +468,13 @@ Wallet - Make sure that no crashes happen with run-time option `-disablewallet`. - *Rationale*: In RPC code that conditionally uses the wallet (such as - `validateaddress`) it is easy to forget that global pointer `pwalletMain` + `validateaddress`), it is easy to forget that global pointer `pwalletMain` can be nullptr. See `test/functional/disablewallet.py` for functional tests - exercising the API with `-disablewallet` + exercising the API with `-disablewallet`. -- Include `db_cxx.h` (BerkeleyDB header) only when `ENABLE_WALLET` is set +- Include `db_cxx.h` (BerkeleyDB header) only when `ENABLE_WALLET` is set. - - *Rationale*: Otherwise compilation of the disable-wallet build will fail in environments without BerkeleyDB + - *Rationale*: Otherwise compilation of the disable-wallet build will fail in environments without BerkeleyDB. General C++ ------------- @@ -483,26 +485,26 @@ Guidelines](https://isocpp.github.io/CppCoreGuidelines/). Common misconceptions are clarified in those sections: - Passing (non-)fundamental types in the [C++ Core - Guideline](https://isocpp.github.io/CppCoreGuidelines/CppCoreGuidelines#Rf-conventional) + Guideline](https://isocpp.github.io/CppCoreGuidelines/CppCoreGuidelines#Rf-conventional). -- Assertions should not have side-effects +- Assertions should not have side-effects. - *Rationale*: Even though the source code is set to refuse to compile with assertions disabled, having side-effects in assertions is unexpected and - makes the code harder to understand + makes the code harder to understand. -- If you use the `.h`, you must link the `.cpp` +- If you use the `.h`, you must link the `.cpp`. - *Rationale*: Include files define the interface for the code in implementation files. Including one but not linking the other is confusing. Please avoid that. Moving functions from - the `.h` to the `.cpp` should not result in build errors + the `.h` to the `.cpp` should not result in build errors. -- Use the RAII (Resource Acquisition Is Initialization) paradigm where possible. For example by using +- Use the RAII (Resource Acquisition Is Initialization) paradigm where possible. For example, by using `unique_ptr` for allocations in a function. - - *Rationale*: This avoids memory and resource leaks, and ensures exception safety + - *Rationale*: This avoids memory and resource leaks, and ensures exception safety. -- Use `MakeUnique()` to construct objects owned by `unique_ptr`s +- Use `MakeUnique()` to construct objects owned by `unique_ptr`s. - *Rationale*: `MakeUnique` is concise and ensures exception safety in complex expressions. `MakeUnique` is a temporary project local implementation of `std::make_unique` (C++14). @@ -510,27 +512,27 @@ Common misconceptions are clarified in those sections: C++ data structures -------------------- -- Never use the `std::map []` syntax when reading from a map, but instead use `.find()` +- Never use the `std::map []` syntax when reading from a map, but instead use `.find()`. - *Rationale*: `[]` does an insert (of the default element) if the item doesn't exist in the map yet. This has resulted in memory leaks in the past, as well as - race conditions (expecting read-read behavior). Using `[]` is fine for *writing* to a map + race conditions (expecting read-read behavior). Using `[]` is fine for *writing* to a map. - Do not compare an iterator from one data structure with an iterator of - another data structure (even if of the same type) + another data structure (even if of the same type). - *Rationale*: Behavior is undefined. In C++ parlor this means "may reformat - the universe", in practice this has resulted in at least one hard-to-debug crash bug + the universe", in practice this has resulted in at least one hard-to-debug crash bug. - Watch out for out-of-bounds vector access. `&vch[vch.size()]` is illegal, including `&vch[0]` for an empty vector. Use `vch.data()` and `vch.data() + vch.size()` instead. -- Vector bounds checking is only enabled in debug mode. Do not rely on it +- Vector bounds checking is only enabled in debug mode. Do not rely on it. - Initialize all non-static class members where they are defined. If this is skipped for a good reason (i.e., optimization on the critical - path), add an explicit comment about this + path), add an explicit comment about this. - *Rationale*: Ensure determinism by avoiding accidental use of uninitialized values. Also, static analyzers balk about this. @@ -554,12 +556,12 @@ class A `int8_t`. Do not use bare `char` unless it is to pass to a third-party API. This type can be signed or unsigned depending on the architecture, which can lead to interoperability problems or dangerous conditions such as - out-of-bounds array accesses + out-of-bounds array accesses. -- Prefer explicit constructions over implicit ones that rely on 'magical' C++ behavior +- Prefer explicit constructions over implicit ones that rely on 'magical' C++ behavior. - *Rationale*: Easier to understand what is happening, thus easier to spot mistakes, even for those - that are not language lawyers + that are not language lawyers. Strings and formatting ------------------------ @@ -567,17 +569,17 @@ Strings and formatting - Be careful of `LogPrint` versus `LogPrintf`. `LogPrint` takes a `category` argument, `LogPrintf` does not. - *Rationale*: Confusion of these can result in runtime exceptions due to - formatting mismatch, and it is easy to get wrong because of subtly similar naming + formatting mismatch, and it is easy to get wrong because of subtly similar naming. -- Use `std::string`, avoid C string manipulation functions +- Use `std::string`, avoid C string manipulation functions. - *Rationale*: C++ string handling is marginally safer, less scope for - buffer overflows and surprises with `\0` characters. Also some C string manipulations - tend to act differently depending on platform, or even the user locale + buffer overflows, and surprises with `\0` characters. Also, some C string manipulations + tend to act differently depending on platform, or even the user locale. -- Use `ParseInt32`, `ParseInt64`, `ParseUInt32`, `ParseUInt64`, `ParseDouble` from `utilstrencodings.h` for number parsing +- Use `ParseInt32`, `ParseInt64`, `ParseUInt32`, `ParseUInt64`, `ParseDouble` from `utilstrencodings.h` for number parsing. - - *Rationale*: These functions do overflow checking, and avoid pesky locale issues. + - *Rationale*: These functions do overflow checking and avoid pesky locale issues. - Avoid using locale dependent functions if possible. You can use the provided [`lint-locale-dependence.sh`](/test/lint/lint-locale-dependence.sh) @@ -607,14 +609,14 @@ Strings and formatting `wcstoll`, `wcstombs`, `wcstoul`, `wcstoull`, `wcstoumax`, `wcswidth`, `wcsxfrm`, `wctob`, `wctomb`, `wctrans`, `wctype`, `wcwidth`, `wprintf` -- For `strprintf`, `LogPrint`, `LogPrintf` formatting characters don't need size specifiers +- For `strprintf`, `LogPrint`, `LogPrintf` formatting characters don't need size specifiers. - - *Rationale*: Bitcoin Core uses tinyformat, which is type safe. Leave them out to avoid confusion + - *Rationale*: Bitcoin Core uses tinyformat, which is type safe. Leave them out to avoid confusion. Variable names -------------- -Although the shadowing warning (`-Wshadow`) is not enabled by default (it prevents issues rising +Although the shadowing warning (`-Wshadow`) is not enabled by default (it prevents issues arising from using a different variable with the same name), please name variables so that their names do not shadow variables defined in the source code. @@ -633,22 +635,20 @@ AddressBookPage::AddressBookPage(Mode _mode) ``` When using nested cycles, do not name the inner cycle variable the same as in -upper cycle etc. - +the upper cycle, etc. Threads and synchronization ---------------------------- - Build and run tests with `-DDEBUG_LOCKORDER` to verify that no potential deadlocks are introduced. As of 0.12, this is defined by default when - configuring with `--enable-debug` + configuring with `--enable-debug`. - When using `LOCK`/`TRY_LOCK` be aware that the lock exists in the context of the current scope, so surround the statement and the code that needs the lock - with braces + with braces. OK: - ```c++ { TRY_LOCK(cs_vNodes, lockNodes); @@ -657,7 +657,6 @@ Threads and synchronization ``` Wrong: - ```c++ TRY_LOCK(cs_vNodes, lockNodes); { @@ -679,13 +678,11 @@ Scripts `#!/usr/bin/env bash` searches the user's PATH to find the bash binary. OK: - ```bash #!/usr/bin/env bash ``` Wrong: - ```bash #!/bin/bash ``` @@ -694,9 +691,9 @@ Source code organization -------------------------- - Implementation code should go into the `.cpp` file and not the `.h`, unless necessary due to template usage or - when performance due to inlining is critical + when performance due to inlining is critical. - - *Rationale*: Shorter and simpler header files are easier to read, and reduce compile time + - *Rationale*: Shorter and simpler header files are easier to read and reduce compile time. - Use only the lowercase alphanumerics (`a-z0-9`), underscore (`_`) and hyphen (`-`) in source code filenames. @@ -714,7 +711,7 @@ Source code organization - Don't import anything into the global namespace (`using namespace ...`). Use fully specified types such as `std::string`. - - *Rationale*: Avoids symbol conflicts + - *Rationale*: Avoids symbol conflicts. - Terminate namespaces with a comment (`// namespace mynamespace`). The comment should be placed on the same line as the brace closing the namespace, e.g. @@ -729,7 +726,7 @@ namespace { } // namespace ``` - - *Rationale*: Avoids confusion about the namespace context + - *Rationale*: Avoids confusion about the namespace context. - Use `#include ` bracket syntax instead of `#include "primitives/transactions.h"` quote syntax. @@ -752,13 +749,13 @@ namespace { GUI ----- -- Do not display or manipulate dialogs in model code (classes `*Model`) +- Do not display or manipulate dialogs in model code (classes `*Model`). - *Rationale*: Model classes pass through events and data from the core, they should not interact with the user. That's where View classes come in. The converse also holds: try to not directly access core data structures from Views. -- Avoid adding slow or blocking code in the GUI thread. In particular do not +- Avoid adding slow or blocking code in the GUI thread. In particular, do not add new `interfaces::Node` and `interfaces::Wallet` method calls, even if they may be fast now, in case they are changed to lock or communicate across processes in the future. @@ -777,12 +774,12 @@ Subtrees Several parts of the repository are subtrees of software maintained elsewhere. Some of these are maintained by active developers of Bitcoin Core, in which case changes should probably go -directly upstream without being PRed directly against the project. They will be merged back in the next +directly upstream without being PRed directly against the project. They will be merged back in the next subtree merge. -Others are external projects without a tight relationship with our project. Changes to these should also -be sent upstream but bugfixes may also be prudent to PR against Bitcoin Core so that they can be integrated -quickly. Cosmetic changes should be purely taken upstream. +Others are external projects without a tight relationship with our project. Changes to these should also +be sent upstream, but bugfixes may also be prudent to PR against Bitcoin Core so that they can be integrated +quickly. Cosmetic changes should be purely taken upstream. There is a tool in `test/lint/git-subtree-check.sh` to check a subtree directory for consistency with its upstream repository. @@ -812,11 +809,11 @@ you must be aware of. ### File Descriptor Counts -In most configurations we use the default LevelDB value for `max_open_files`, +In most configurations, we use the default LevelDB value for `max_open_files`, which is 1000 at the time of this writing. If LevelDB actually uses this many -file descriptors it will cause problems with Bitcoin's `select()` loop, because +file descriptors, it will cause problems with Bitcoin's `select()` loop, because it may cause new sockets to be created where the fd value is >= 1024. For this -reason, on 64-bit Unix systems we rely on an internal LevelDB optimization that +reason, on 64-bit Unix systems, we rely on an internal LevelDB optimization that uses `mmap()` + `close()` to open table files without actually retaining references to the table file descriptors. If you are upgrading LevelDB, you must sanity check the changes to make sure that this assumption remains valid. @@ -841,14 +838,14 @@ details. It is possible for LevelDB changes to inadvertently change consensus compatibility between nodes. This happened in Bitcoin 0.8 (when LevelDB was -first introduced). When upgrading LevelDB you should review the upstream changes +first introduced). When upgrading LevelDB, you should review the upstream changes to check for issues affecting consensus compatibility. For example, if LevelDB had a bug that accidentally prevented a key from being returned in an edge case, and that bug was fixed upstream, the bug "fix" would -be an incompatible consensus change. In this situation the correct behavior +be an incompatible consensus change. In this situation, the correct behavior would be to revert the upstream fix before applying the updates to Bitcoin's -copy of LevelDB. In general you should be wary of any upstream changes affecting +copy of LevelDB. In general, you should be wary of any upstream changes affecting what data is returned from LevelDB queries. Scripted diffs @@ -867,7 +864,7 @@ To create a scripted-diff: - `-BEGIN VERIFY SCRIPT-` - `-END VERIFY SCRIPT-` -The scripted-diff is verified by the tool `test/lint/commit-script-check.sh`. The tool's default behavior when supplied +The scripted-diff is verified by the tool `test/lint/commit-script-check.sh`. The tool's default behavior, when supplied with a commit is to verify all scripted-diffs from the beginning of time up to said commit. Internally, the tool passes the first supplied argument to `git rev-list --reverse` to determine which commits to verify script-diffs for, ignoring commits that don't conform to the commit message format described above. @@ -900,23 +897,23 @@ RPC interface guidelines A few guidelines for introducing and reviewing new RPC interfaces: -- Method naming: use consecutive lower-case names such as `getrawtransaction` and `submitblock` +- Method naming: use consecutive lower-case names such as `getrawtransaction` and `submitblock`. - - *Rationale*: Consistency with existing interface. + - *Rationale*: Consistency with the existing interface. - Argument naming: use snake case `fee_delta` (and not, e.g. camel case `feeDelta`) - - *Rationale*: Consistency with existing interface. + - *Rationale*: Consistency with the existing interface. - Use the JSON parser for parsing, don't manually parse integers or strings from arguments unless absolutely necessary. - *Rationale*: Introduces hand-rolled string manipulation code at both the caller and callee sites, - which is error prone, and it is easy to get things such as escaping wrong. + which is error-prone, and it is easy to get things such as escaping wrong. JSON already supports nested data structures, no need to re-invent the wheel. - *Exception*: AmountFromValue can parse amounts as string. This was introduced because many JSON - parsers and formatters hard-code handling decimal numbers as floating point + parsers and formatters hard-code handling decimal numbers as floating-point values, resulting in potential loss of precision. This is unacceptable for monetary values. **Always** use `AmountFromValue` and `ValueFromAmount` when inputting or outputting monetary values. The only exceptions to this are @@ -925,7 +922,7 @@ A few guidelines for introducing and reviewing new RPC interfaces: - Missing arguments and 'null' should be treated the same: as default values. If there is no default value, both cases should fail in the same way. The easiest way to follow this - guideline is detect unspecified arguments with `params[x].isNull()` instead of + guideline is to detect unspecified arguments with `params[x].isNull()` instead of `params.size() <= x`. The former returns true if the argument is either null or missing, while the latter returns true if is missing, and false if it is null. @@ -959,7 +956,7 @@ A few guidelines for introducing and reviewing new RPC interfaces: from there. - A RPC method must either be a wallet method or a non-wallet method. Do not - introduce new methods that differ in behavior based on presence of a wallet. + introduce new methods that differ in behavior based on the presence of a wallet. - *Rationale*: as well as complicating the implementation and interfering with the introduction of multi-wallet, wallet and non-wallet code should be @@ -967,7 +964,7 @@ A few guidelines for introducing and reviewing new RPC interfaces: - Try to make the RPC response a JSON object. - - *Rationale*: If a RPC response is not a JSON object then it is harder to avoid API breakage if + - *Rationale*: If a RPC response is not a JSON object, then it is harder to avoid API breakage if new data in the response is needed. - Wallet RPCs call BlockUntilSyncedToCurrentChain to maintain consistency with diff --git a/doc/fuzzing.md b/doc/fuzzing.md index f9221dde5bd9..3dc6be8b860b 100644 --- a/doc/fuzzing.md +++ b/doc/fuzzing.md @@ -46,7 +46,7 @@ export AFLPATH=$PWD To build Bitcoin Core using AFL instrumentation (this assumes that the `AFLPATH` was set as above): ``` -./configure --disable-ccache --disable-shared --enable-tests --enable-fuzz --disable-wallet --disable-bench --with-utils=no --with-daemon=no --with-libs=no --with-gui=no CC=${AFLPATH}/afl-gcc CXX=${AFLPATH}/afl-g++ +./configure --disable-ccache --disable-shared --enable-tests --enable-fuzz CC=${AFLPATH}/afl-gcc CXX=${AFLPATH}/afl-g++ export AFL_HARDEN=1 cd src/ make @@ -83,7 +83,7 @@ found in the `compiler-rt` runtime libraries package). To build all fuzz targets with libFuzzer, run ``` -./configure --disable-ccache --disable-wallet --disable-bench --with-utils=no --with-daemon=no --with-libs=no --with-gui=no --enable-fuzz --with-sanitizers=fuzzer,address CC=clang CXX=clang++ +./configure --disable-ccache --enable-fuzz --with-sanitizers=fuzzer,address CC=clang CXX=clang++ make ``` diff --git a/doc/rapidcheck.md b/doc/rapidcheck.md new file mode 100644 index 000000000000..397a907f17c0 --- /dev/null +++ b/doc/rapidcheck.md @@ -0,0 +1,84 @@ +# RapidCheck property-based testing for Bitcoin Core + +## Concept + +Property-based testing is experimentally being added to Bitcoin Core with +[RapidCheck](https://github.com/emil-e/rapidcheck), a C++ framework for +property-based testing inspired by the Haskell library +[QuickCheck](https://hackage.haskell.org/package/QuickCheck). + +RapidCheck performs random testing of program properties. A specification of the +program is given in the form of properties which functions should satisfy, and +RapidCheck tests that the properties hold in a large number of randomly +generated cases. + +If an exception is found, RapidCheck tries to find the smallest case, for some +definition of smallest, for which the property is still false and displays it as +a counter-example. For example, if the input is an integer, RapidCheck tries to +find the smallest integer for which the property is false. + +## Running + +If RapidCheck is installed, Bitcoin Core will automatically run the +property-based tests with the unit tests during `make check`, unless the +`--without-rapidcheck` flag is passed when configuring. + +For more information, run `./configure --help` and see `--with-rapidcheck` under +Optional Packages. + +## Setup + +The following instructions have been tested with Linux Debian and macOS. + +1. Clone the RapidCheck source code and cd into the repository. + + ```shell + git clone https://github.com/emil-e/rapidcheck.git + cd rapidcheck + ``` + +2. Build RapidCheck (requires CMake to be installed). + + ```shell + cmake -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=true -DRC_ENABLE_BOOST_TEST=ON $(pwd) + make && make install + ``` + +3. Configure Bitcoin Core with RapidCheck. + + `cd` to the directory of your local bitcoin repository and run + `./configure`. In the output you should see: + + ```shell + checking rapidcheck.h usability... yes + checking rapidcheck.h presence... yes + checking for rapidcheck.h... yes + [...] + Options used to compile and link: + [...] + with test = yes + with prop = yes + ``` + +4. Build Bitcoin Core with RapidCheck. + + Now you can run `make` and should see the property-based tests compiled with + the unit tests: + + ```shell + Making all in src + [...] + CXX test/gen/test_bitcoin-crypto_gen.o + CXX test/test_bitcoin-key_properties.o + ``` + +5. Run the unit tests with `make check`. The property-based tests will be run + with the unit tests. + + ```shell + Running tests: crypto_tests from test/crypto_tests.cpp + [...] + Running tests: key_properties from test/key_properties.cpp + ``` + +That's it! You are now running property-based tests in Bitcoin Core. diff --git a/doc/reduce-memory.md b/doc/reduce-memory.md new file mode 100644 index 000000000000..8d8ccdfedc2c --- /dev/null +++ b/doc/reduce-memory.md @@ -0,0 +1,49 @@ +# Reduce Memory + +There are a few parameters that can be dialed down to reduce the memory usage of `bitcoind`. This can be useful on embedded systems or small VPSes. + +## In-memory caches + +The size of some in-memory caches can be reduced. As caches trade off memory usage for performance, reducing these will usually have a negative effect on performance. + +- `-dbcache=` - the UTXO database cache size, this defaults to `450`. The unit is MiB (1024). + - The minimum value for `-dbcache` is 4. + - A lower `-dbcache` makes initial sync time much longer. After the initial sync, the effect is less pronounced for most use-cases, unless fast validation of blocks is important, such as for mining. + +## Memory pool + +- In Bitcoin Core there is a memory pool limiter which can be configured with `-maxmempool=`, where `` is the size in MB (1000). The default value is `300`. + - The minimum value for `-maxmempool` is 5. + - A lower maximum mempool size means that transactions will be evicted sooner. This will affect any uses of `bitcoind` that process unconfirmed transactions. + +- To completely disable mempool functionality there is the option `-blocksonly`. This will make the client opt out of receiving (and thus relaying) transactions completely, except as part of blocks. + + - Do not use this when using the client to broadcast transactions as any transaction sent will stick out like a sore thumb, affecting privacy. When used with the wallet it should be combined with `-walletbroadcast=0` and `-spendzeroconfchange=0`. Another mechanism for broadcasting outgoing transactions (if any) should be used. + +- Since `0.14.0`, unused memory allocated to the mempool (default: 300MB) is shared with the UTXO cache, so when trying to reduce memory usage you should limit the mempool, with the `-maxmempool` command line argument. + +## Number of peers + +- `-maxconnections=` - the maximum number of connections, this defaults to `125`. Each active connection takes up some memory. Only significant if incoming + connections are enabled, otherwise the number of connections will never be more than `8`. + +## Thread configuration + +For each thread a thread stack needs to be allocated. By default on Linux, +threads take up 8MiB for the thread stack on a 64-bit system, and 4MiB in a +32-bit system. + +- `-par=` - the number of script verification threads, defaults to the number of cores in the system minus one. +- `-rpcthreads=` - the number of threads used for processing RPC requests, defaults to `4`. + +## Linux specific + +By default, since glibc `2.10`, the C library will create up to two heap arenas per core. This is known to cause excessive memory usage in some scenarios. To avoid this make a script that sets `MALLOC_ARENA_MAX` before starting bitcoind: + +```bash +#!/bin/bash +export MALLOC_ARENA_MAX=1 +bitcoind +``` + +The behavior was introduced to increase CPU locality of allocated memory and performance with concurrent allocation, so this setting could in theory reduce performance. However, in Bitcoin Core very little parallel allocation happens, so the impact is expected to be small or absent. diff --git a/doc/release-notes-0.18.1-16257.md b/doc/release-notes-0.18.1-16257.md new file mode 100644 index 000000000000..21867b7fb2f5 --- /dev/null +++ b/doc/release-notes-0.18.1-16257.md @@ -0,0 +1,6 @@ +Wallet changes +-------------- +When creating a transaction with a fee above `-maxtxfee` (default 0.1 BTC), +the RPC commands `walletcreatefundedpsbt` and `fundrawtransaction` will now fail +instead of rounding down the fee. Beware that the `feeRate` argument is specified +in BTC per kilobyte, not satoshi per byte. diff --git a/doc/release-notes-15427.md b/doc/release-notes-15427.md new file mode 100644 index 000000000000..25edfd440281 --- /dev/null +++ b/doc/release-notes-15427.md @@ -0,0 +1,9 @@ +Updated RPCs +------------ + +The `utxoupdatepsbt` RPC method has been updated to take a `descriptors` +argument. When provided, input and output scripts and keys will be filled in +when known, and P2SH-witness inputs will be filled in from the UTXO set when a +descriptor is provided that shows they're spending segwit outputs. + +See the RPC help text for full details. diff --git a/doc/release-notes-15993.md b/doc/release-notes-15993.md new file mode 100644 index 000000000000..493c7126ee9d --- /dev/null +++ b/doc/release-notes-15993.md @@ -0,0 +1,3 @@ +Build system changes +-------------------- +The minimum supported miniUPnPc API version is set to 10. This keeps compatibility with Ubuntu 16.04 LTS and Debian 8 `libminiupnpc-dev` packages. Please note, on Debian this package is still vulnerable to [CVE-2017-8798](https://security-tracker.debian.org/tracker/CVE-2017-8798) (in jessie only) and [CVE-2017-1000494](https://security-tracker.debian.org/tracker/CVE-2017-1000494) (both in jessie and in stretch). diff --git a/doc/release-notes-16060.md b/doc/release-notes-16060.md new file mode 100644 index 000000000000..7e150d10e752 --- /dev/null +++ b/doc/release-notes-16060.md @@ -0,0 +1,15 @@ +Low-level RPC changes +---------------------- + +- Soft fork reporting in the `getblockchaininfo` return object has been + updated. For full details, see the RPC help text. In summary: + - The `bip9_softforks` sub-object is no longer returned + - The `softforks` sub-object now returns an object keyed by soft fork name, + rather than an array + - Each softfork object in the `softforks` object contains a `type` value which + is either `buried` (for soft fork deployments where the activation height is + hard-coded into the client implementation), or `bip9` (for soft fork deployments + where activation is controlled by BIP 9 signaling). + +- `getblocktemplate` no longer returns a `rules` array containing `CSV` + and `segwit` (the BIP 9 deployments that are currently in active state). diff --git a/doc/release-notes-16383.md b/doc/release-notes-16383.md new file mode 100644 index 000000000000..801571516728 --- /dev/null +++ b/doc/release-notes-16383.md @@ -0,0 +1,8 @@ +RPC changes +----------- + +RPCs which have an `include_watchonly` argument or `includeWatching` +option now default to `true` for watch-only wallets. Affected RPCs +are: `getbalance`, `listreceivedbyaddress`, `listreceivedbylabel`, +`listtransactions`, `listsinceblock`, `gettransaction`, +`walletcreatefundedpsbt`, and `fundrawtransaction`. diff --git a/doc/release-notes-16394.md b/doc/release-notes-16394.md new file mode 100644 index 000000000000..f09cba4b6d55 --- /dev/null +++ b/doc/release-notes-16394.md @@ -0,0 +1,4 @@ +RPC changes +----------- +`createwallet` now returns a warning if an empty string is used as an encryption password, and does not encrypt the wallet, instead of raising an error. +This makes it easier to disable encryption but also specify other options when using the `bitcoin-cli` tool. diff --git a/doc/release-notes-16647.md b/doc/release-notes-16647.md new file mode 100644 index 000000000000..7e5539cb0d87 --- /dev/null +++ b/doc/release-notes-16647.md @@ -0,0 +1,3 @@ +RPC changes +----------- +`getmempoolentry` now provides a `weight` field containing the transaction weight as defined in BIP 141. diff --git a/doc/release-notes.md b/doc/release-notes.md index 9efb6cbabbda..e0673ae7eeda 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -19,8 +19,8 @@ Bitcoin Core version *version* is now available from: -This is a new major version release, including new features, various bugfixes -and performance improvements, as well as updated translations. +This release includes new features, various bug fixes and performance +improvements, as well as updated translations. Please report bugs using the issue tracker at GitHub: @@ -42,30 +42,19 @@ Upgrading directly from a version of Bitcoin Core that has reached its EOL is possible, but might take some time if the datadir needs to be migrated. Old wallet versions of Bitcoin Core are generally supported. -Downgrading warning -------------------- - -The chainstate database for this release is not compatible with previous -releases, so if you run 0.15 and then decide to switch back to any -older version, you will need to run the old release with the `-reindex-chainstate` -option to rebuild the chainstate data structures in the old format. - -If your node has pruning enabled, this will entail re-downloading and -processing the entire blockchain. - Compatibility ============== Bitcoin Core is supported and extensively tested on operating systems using -the Linux kernel, macOS 10.10+, and Windows 7 and newer. It is not recommended +the Linux kernel, macOS 10.10+, and Windows 7 and newer. It is not recommended to use Bitcoin Core on unsupported systems. Bitcoin Core should also work on most other Unix-like systems but is not -frequently tested on them. +as frequently tested on them. -From 0.17.0 onwards, macOS <10.10 is no longer supported. 0.17.0 is +From 0.17.0 onwards, macOS <10.10 is no longer supported. 0.17.0 is built using Qt 5.9.x, which doesn't support versions of macOS older than -10.10. Additionally, Bitcoin Core does not yet change appearance when +10.10. Additionally, Bitcoin Core does not yet change appearance when macOS "dark mode" is activated. In addition to previously-supported CPU platforms, this release's @@ -99,10 +88,30 @@ Low-level Changes section below. `-limitancestorcount`, `-limitdescendantcount` and `-walletrejectlongchains` command line arguments. +Deprecated or removed RPCs +-------------------------- + +- The `totalFee` option of the `bumpfee` RPC has been deprecated and will be + removed in 0.20. To continue using this option start with + `-deprecatedrpc=totalFee`. See the `bumpfee` RPC help text for more details. Low-level changes ================= +RPC +--- + + +Tests +----- + +- The regression test chain, that can be enabled by the `-regtest` command line + flag, now requires transactions to not violate standard policy by default. + Making the default the same as for mainnet, makes it easier to test mainnet + behavior on regtest. Be reminded that the testnet still allows non-standard + txs by default and that the policy can be locally adjusted with the + `-acceptnonstdtxn` command line flag for both test chains. + Configuration ------------ diff --git a/doc/release-notes/release-notes-0.18.1.md b/doc/release-notes/release-notes-0.18.1.md new file mode 100644 index 000000000000..483cc5075e71 --- /dev/null +++ b/doc/release-notes/release-notes-0.18.1.md @@ -0,0 +1,136 @@ +Bitcoin Core version 0.18.1 is now available from: + + + +This is a new minor version release, including new features, various bug +fixes and performance improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + + +To receive security and update notifications, please subscribe to: + + + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has +completely shut down (which might take a few minutes for older +versions), then run the installer (on Windows) or just copy over +`/Applications/Bitcoin-Qt` (on Mac) or `bitcoind`/`bitcoin-qt` (on +Linux). + +The first time you run version 0.15.0 or newer, your chainstate database +will be converted to a new format, which will take anywhere from a few +minutes to half an hour, depending on the speed of your machine. + +Note that the block database format also changed in version 0.8.0 and +there is no automatic upgrade code from before version 0.8 to version +0.15.0 or later. Upgrading directly from 0.7.x and earlier without +redownloading the blockchain is not supported. However, as usual, old +wallet versions are still supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems +using the Linux kernel, macOS 10.10+, and Windows 7 and newer. It is not +recommended to use Bitcoin Core on unsupported systems. + +Bitcoin Core should also work on most other Unix-like systems but is not +as frequently tested on them. + +From 0.17.0 onwards, macOS <10.10 is no longer supported. 0.17.0 is +built using Qt 5.9.x, which doesn't support versions of macOS older than +10.10. Additionally, Bitcoin Core does not yet change appearance when +macOS "dark mode" is activated. + +Known issues +============ + +Wallet GUI +---------- + +For advanced users who have both (1) enabled coin control features, and +(2) are using multiple wallets loaded at the same time: The coin control +input selection dialog can erroneously retain wrong-wallet state when +switching wallets using the dropdown menu. For now, it is recommended +not to use coin control features with multiple wallets loaded. + +0.18.1 change log +================= + +### P2P protocol and network code +- #15990 Add tests and documentation for blocksonly (MarcoFalke) +- #16021 Avoid logging transaction decode errors to stderr (MarcoFalke) +- #16405 fix: tor: Call `event_base_loopbreak` from the event's callback (promag) +- #16412 Make poll in InterruptibleRecv only filter for POLLIN events (tecnovert) + +### Wallet +- #15913 Add -ignorepartialspends to list of ignored wallet options (luke-jr) + +### RPC and other APIs +- #15991 Bugfix: fix pruneblockchain returned prune height (jonasschnelli) +- #15899 Document iswitness flag and fix bug in converttopsbt (MarcoFalke) +- #16026 Ensure that uncompressed public keys in a multisig always returns a legacy address (achow101) +- #14039 Disallow extended encoding for non-witness transactions (sipa) +- #16210 add 2nd arg to signrawtransactionwithkey examples (dooglus) +- #16250 signrawtransactionwithkey: report error when missing redeemScript/witnessScript (ajtowns) + +### GUI +- #16044 fix the bug of OPEN CONFIGURATION FILE on Mac (shannon1916) +- #15957 Show "No wallets available" in open menu instead of nothing (meshcollider) +- #16118 Enable open wallet menu on setWalletController (promag) +- #16135 Set progressDialog to nullptr (promag) +- #16231 Fix open wallet menu initialization order (promag) +- #16254 Set `AA_EnableHighDpiScaling` attribute early (hebasto) +- #16122 Enable console line edit on setClientModel (promag) +- #16348 Assert QMetaObject::invokeMethod result (promag) + +### Build system +- #15985 Add test for GCC bug 90348 (sipa) +- #15947 Install bitcoin-wallet manpage (domob1812) +- #15983 build with -fstack-reuse=none (MarcoFalke) + +### Tests and QA +- #15826 Pure python EC (sipa) +- #15893 Add test for superfluous witness record in deserialization (instagibbs) +- #14818 Bugfix: test/functional/rpc_psbt: Remove check for specific error message that depends on uncertain assumptions (luke-jr) +- #15831 Add test that addmultisigaddress fails for watchonly addresses (MarcoFalke) + +### Documentation +- #15890 Remove text about txes always relayed from -whitelist (harding) + +### Miscellaneous +- #16095 Catch by reference not value in wallettool (kristapsk) +- #16205 Replace fprintf with tfm::format (MarcoFalke) + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- Andrew Chow +- Anthony Towns +- Chris Moore +- Daniel Kraft +- David A. Harding +- fanquake +- Gregory Sanders +- Hennadii Stepanov +- John Newbery +- Jonas Schnelli +- João Barbosa +- Kristaps Kaupe +- Luke Dashjr +- MarcoFalke +- Michele Federici +- Pieter Wuille +- Samuel Dobson +- shannon1916 +- tecnovert +- Wladimir J. van der Laan + +As well as everyone that helped translating on [Transifex](https://www.transifex.com/projects/p/bitcoin/). diff --git a/doc/release-notes/release-notes-16152.md b/doc/release-notes/release-notes-16152.md new file mode 100644 index 000000000000..9c77cb9ae5e5 --- /dev/null +++ b/doc/release-notes/release-notes-16152.md @@ -0,0 +1,7 @@ +P2P Changes +----------- +- The default value for the -peerbloomfilters configuration option (and, thus, NODE_BLOOM support) has been changed to false. + This resolves well-known DoS vectors in Bitcoin Core, especially for nodes with spinning disks. It is not anticipated that + this will result in a significant lack of availability of NODE_BLOOM-enabled nodes in the coming years, however, clients + which rely on the availability of NODE_BLOOM-supporting nodes on the P2P network should consider the process of migrating + to a more modern (and less trustful and privacy-violating) alternative over the coming years. diff --git a/share/examples/bitcoin.conf b/share/examples/bitcoin.conf index b5475dc1c68d..709d8b002b75 100644 --- a/share/examples/bitcoin.conf +++ b/share/examples/bitcoin.conf @@ -70,8 +70,8 @@ # server=1 tells Bitcoin-Qt and bitcoind to accept JSON-RPC commands #server=0 -# Bind to given address to listen for JSON-RPC connections. Use [host]:port notation for IPv6. -# This option can be specified multiple times (default: bind to all interfaces) +# Bind to given address to listen for JSON-RPC connections. +# Refer to the manpage or bitcoind -help for further details. #rpcbind= # If no rpcpassword is set, rpc cookie auth is sought. The default `-rpccookiefile` name diff --git a/share/genbuild.sh b/share/genbuild.sh index 38c9ba176cf8..cd6214d5fab2 100755 --- a/share/genbuild.sh +++ b/share/genbuild.sh @@ -24,9 +24,9 @@ git_check_in_repo() { DESC="" SUFFIX="" -if [ "${BITCOIN_GENBUILD_NO_GIT}" != "1" -a -e "$(which git 2>/dev/null)" -a "$(git rev-parse --is-inside-work-tree 2>/dev/null)" = "true" ] && git_check_in_repo share/genbuild.sh; then +if [ "${BITCOIN_GENBUILD_NO_GIT}" != "1" ] && [ -e "$(command -v git)" ] && [ "$(git rev-parse --is-inside-work-tree 2>/dev/null)" = "true" ] && git_check_in_repo share/genbuild.sh; then # clean 'dirty' status of touched files that haven't been modified - git diff >/dev/null 2>/dev/null + git diff >/dev/null 2>/dev/null # if latest commit is tagged and not dirty, then override using the tag name RAWDESC=$(git describe --abbrev=0 2>/dev/null) diff --git a/share/qt/extract_strings_qt.py b/share/qt/extract_strings_qt.py index 76ef12eeece0..3cc0fa1b1dda 100755 --- a/share/qt/extract_strings_qt.py +++ b/share/qt/extract_strings_qt.py @@ -76,10 +76,7 @@ def parse_po(text): #endif """) f.write('static const char UNUSED *bitcoin_strings[] = {\n') -f.write('QT_TRANSLATE_NOOP("bitcoin-core", "%s"),\n' % (os.getenv('PACKAGE_NAME'),)) f.write('QT_TRANSLATE_NOOP("bitcoin-core", "%s"),\n' % (os.getenv('COPYRIGHT_HOLDERS'),)) -if os.getenv('COPYRIGHT_HOLDERS_SUBSTITUTION') != os.getenv('PACKAGE_NAME'): - f.write('QT_TRANSLATE_NOOP("bitcoin-core", "%s"),\n' % (os.getenv('COPYRIGHT_HOLDERS_SUBSTITUTION'),)) messages.sort(key=operator.itemgetter(0)) for (msgid, msgstr) in messages: if msgid != EMPTY: diff --git a/src/Makefile.am b/src/Makefile.am index 39e8d3d68974..477b1300bc65 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -143,7 +143,6 @@ BITCOIN_CORE_H = \ interfaces/wallet.h \ key.h \ key_io.h \ - keystore.h \ dbwrapper.h \ limitedmap.h \ logging.h \ @@ -151,6 +150,7 @@ BITCOIN_CORE_H = \ merkleblock.h \ miner.h \ net.h \ + net_permissions.h \ net_processing.h \ netaddress.h \ netbase.h \ @@ -175,14 +175,17 @@ BITCOIN_CORE_H = \ rpc/blockchain.h \ rpc/client.h \ rpc/protocol.h \ - rpc/server.h \ rpc/rawtransaction_util.h \ rpc/register.h \ + rpc/request.h \ + rpc/server.h \ rpc/util.h \ scheduler.h \ script/descriptor.h \ + script/keyorigin.h \ script/sigcache.h \ script/sign.h \ + script/signingprovider.h \ script/standard.h \ shutdown.h \ streams.h \ @@ -208,8 +211,10 @@ BITCOIN_CORE_H = \ util/memory.h \ util/moneystr.h \ util/rbf.h \ + util/string.h \ util/threadnames.h \ util/time.h \ + util/translation.h \ util/url.h \ util/validation.h \ validation.h \ @@ -307,7 +312,7 @@ libbitcoin_server_a_SOURCES += dummywallet.cpp endif if ENABLE_ZMQ -libbitcoin_zmq_a_CPPFLAGS = $(BITCOIN_INCLUDES) $(ZMQ_CFLAGS) +libbitcoin_zmq_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(ZMQ_CFLAGS) libbitcoin_zmq_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_zmq_a_SOURCES = \ zmq/zmqabstractnotifier.cpp \ @@ -351,6 +356,8 @@ crypto_libbitcoin_crypto_base_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) crypto_libbitcoin_crypto_base_a_SOURCES = \ crypto/aes.cpp \ crypto/aes.h \ + crypto/chacha_poly_aead.h \ + crypto/chacha_poly_aead.cpp \ crypto/chacha20.h \ crypto/chacha20.cpp \ crypto/common.h \ @@ -446,10 +453,10 @@ libbitcoin_common_a_SOURCES = \ core_write.cpp \ key.cpp \ key_io.cpp \ - keystore.cpp \ merkleblock.cpp \ netaddress.cpp \ netbase.cpp \ + net_permissions.cpp \ outputtype.cpp \ policy/feerate.cpp \ policy/policy.cpp \ @@ -460,6 +467,7 @@ libbitcoin_common_a_SOURCES = \ scheduler.cpp \ script/descriptor.cpp \ script/sign.cpp \ + script/signingprovider.cpp \ script/standard.cpp \ versionbitsinfo.cpp \ warnings.cpp \ @@ -481,7 +489,7 @@ libbitcoin_util_a_SOURCES = \ interfaces/handler.cpp \ logging.cpp \ random.cpp \ - rpc/protocol.cpp \ + rpc/request.cpp \ support/cleanse.cpp \ sync.cpp \ threadinterrupt.cpp \ @@ -494,6 +502,7 @@ libbitcoin_util_a_SOURCES = \ util/rbf.cpp \ util/threadnames.cpp \ util/strencodings.cpp \ + util/string.cpp \ util/time.cpp \ util/url.cpp \ util/validation.cpp \ @@ -613,7 +622,7 @@ bitcoin_wallet_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(CRYPTO_LIBS) $(EVENT_PTHREAD # bitcoinconsensus library # if BUILD_BITCOIN_LIBS include_HEADERS = script/bitcoinconsensus.h -libbitcoinconsensus_la_SOURCES = $(crypto_libbitcoin_crypto_base_a_SOURCES) $(libbitcoin_consensus_a_SOURCES) +libbitcoinconsensus_la_SOURCES = support/cleanse.cpp $(crypto_libbitcoin_crypto_base_a_SOURCES) $(libbitcoin_consensus_a_SOURCES) if GLIBC_BACK_COMPAT libbitcoinconsensus_la_SOURCES += compat/glibc_compat.cpp diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include index c6162b5caa97..e421b377a0ce 100644 --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -18,15 +18,19 @@ bench_bench_bitcoin_SOURCES = \ bench/block_assemble.cpp \ bench/checkblock.cpp \ bench/checkqueue.cpp \ + bench/data.h \ + bench/data.cpp \ bench/duplicate_inputs.cpp \ bench/examples.cpp \ bench/rollingbloom.cpp \ bench/chacha20.cpp \ + bench/chacha_poly_aead.cpp \ bench/crypto_hash.cpp \ bench/ccoins_caching.cpp \ bench/gcs_filter.cpp \ bench/merkle_root.cpp \ bench/mempool_eviction.cpp \ + bench/rpc_blockchain.cpp \ bench/rpc_mempool.cpp \ bench/util_time.cpp \ bench/verify_script.cpp \ @@ -76,7 +80,7 @@ CLEAN_BITCOIN_BENCH = bench/*.gcda bench/*.gcno $(GENERATED_BENCH_FILES) CLEANFILES += $(CLEAN_BITCOIN_BENCH) -bench/checkblock.cpp: bench/data/block413567.raw.h +bench/data.cpp: bench/data/block413567.raw.h bitcoin_bench: $(BENCH_BINARY) @@ -89,7 +93,7 @@ bitcoin_bench_clean : FORCE %.raw.h: %.raw @$(MKDIR_P) $(@D) @{ \ - echo "static unsigned const char $(*F)[] = {" && \ + echo "static unsigned const char $(*F)_raw[] = {" && \ $(HEXDUMP) -v -e '8/1 "0x%02x, "' -e '"\n"' $< | $(SED) -e 's/0x ,//g' && \ echo "};"; \ } > "$@.new" && mv -f "$@.new" "$@" diff --git a/src/Makefile.qt.include b/src/Makefile.qt.include index c4c08487f349..6d8faf3883f1 100644 --- a/src/Makefile.qt.include +++ b/src/Makefile.qt.include @@ -250,8 +250,6 @@ BITCOIN_QT_H = \ RES_ICONS = \ qt/res/icons/add.png \ qt/res/icons/address-book.png \ - qt/res/icons/about.png \ - qt/res/icons/about_qt.png \ qt/res/icons/bitcoin.ico \ qt/res/icons/bitcoin_testnet.ico \ qt/res/icons/bitcoin.png \ @@ -261,13 +259,11 @@ RES_ICONS = \ qt/res/icons/clock3.png \ qt/res/icons/clock4.png \ qt/res/icons/clock5.png \ - qt/res/icons/configure.png \ qt/res/icons/connect0.png \ qt/res/icons/connect1.png \ qt/res/icons/connect2.png \ qt/res/icons/connect3.png \ qt/res/icons/connect4.png \ - qt/res/icons/debugwindow.png \ qt/res/icons/edit.png \ qt/res/icons/editcopy.png \ qt/res/icons/editpaste.png \ @@ -275,21 +271,16 @@ RES_ICONS = \ qt/res/icons/eye.png \ qt/res/icons/eye_minus.png \ qt/res/icons/eye_plus.png \ - qt/res/icons/filesave.png \ qt/res/icons/fontbigger.png \ qt/res/icons/fontsmaller.png \ qt/res/icons/hd_disabled.png \ qt/res/icons/hd_enabled.png \ qt/res/icons/history.png \ - qt/res/icons/info.png \ - qt/res/icons/key.png \ qt/res/icons/lock_closed.png \ qt/res/icons/lock_open.png \ qt/res/icons/network_disabled.png \ - qt/res/icons/open.png \ qt/res/icons/overview.png \ qt/res/icons/proxy.png \ - qt/res/icons/quit.png \ qt/res/icons/receive.png \ qt/res/icons/remove.png \ qt/res/icons/send.png \ @@ -302,8 +293,7 @@ RES_ICONS = \ qt/res/icons/tx_input.png \ qt/res/icons/tx_output.png \ qt/res/icons/tx_mined.png \ - qt/res/icons/warning.png \ - qt/res/icons/verify.png + qt/res/icons/warning.png BITCOIN_QT_BASE_CPP = \ qt/bantablemodel.cpp \ @@ -447,7 +437,7 @@ SECONDARY: $(QT_QM) $(srcdir)/qt/bitcoinstrings.cpp: $(libbitcoin_server_a_SOURCES) $(libbitcoin_wallet_a_SOURCES) $(libbitcoin_common_a_SOURCES) $(libbitcoin_zmq_a_SOURCES) $(libbitcoin_consensus_a_SOURCES) $(libbitcoin_util_a_SOURCES) @test -n $(XGETTEXT) || echo "xgettext is required for updating translations" - $(AM_V_GEN) cd $(srcdir); XGETTEXT=$(XGETTEXT) PACKAGE_NAME="$(PACKAGE_NAME)" COPYRIGHT_HOLDERS="$(COPYRIGHT_HOLDERS)" COPYRIGHT_HOLDERS_SUBSTITUTION="$(COPYRIGHT_HOLDERS_SUBSTITUTION)" $(PYTHON) ../share/qt/extract_strings_qt.py $^ + $(AM_V_GEN) cd $(srcdir); XGETTEXT=$(XGETTEXT) COPYRIGHT_HOLDERS="$(COPYRIGHT_HOLDERS)" $(PYTHON) ../share/qt/extract_strings_qt.py $^ translate: $(srcdir)/qt/bitcoinstrings.cpp $(QT_FORMS_UI) $(QT_FORMS_UI) $(BITCOIN_QT_BASE_CPP) qt/bitcoin.cpp $(BITCOIN_QT_WINDOWS_CPP) $(BITCOIN_QT_WALLET_CPP) $(BITCOIN_QT_H) $(BITCOIN_MM) @test -n $(LUPDATE) || echo "lupdate is required for updating translations" diff --git a/src/addrdb.cpp b/src/addrdb.cpp index c6083f5554be..db936486b64f 100644 --- a/src/addrdb.cpp +++ b/src/addrdb.cpp @@ -44,18 +44,30 @@ bool SerializeFileDB(const std::string& prefix, const fs::path& path, const Data fs::path pathTmp = GetDataDir() / tmpfn; FILE *file = fsbridge::fopen(pathTmp, "wb"); CAutoFile fileout(file, SER_DISK, CLIENT_VERSION); - if (fileout.IsNull()) + if (fileout.IsNull()) { + fileout.fclose(); + remove(pathTmp); return error("%s: Failed to open file %s", __func__, pathTmp.string()); + } // Serialize - if (!SerializeDB(fileout, data)) return false; - if (!FileCommit(fileout.Get())) + if (!SerializeDB(fileout, data)) { + fileout.fclose(); + remove(pathTmp); + return false; + } + if (!FileCommit(fileout.Get())) { + fileout.fclose(); + remove(pathTmp); return error("%s: Failed to flush file %s", __func__, pathTmp.string()); + } fileout.fclose(); // replace existing file, if any, with new file - if (!RenameOver(pathTmp, path)) + if (!RenameOver(pathTmp, path)) { + remove(pathTmp); return error("%s: Rename-into-place failed", __func__); + } return true; } diff --git a/src/banman.cpp b/src/banman.cpp index 47d64a8f31c0..37fca7dd82aa 100644 --- a/src/banman.cpp +++ b/src/banman.cpp @@ -9,12 +9,13 @@ #include #include #include +#include BanMan::BanMan(fs::path ban_file, CClientUIInterface* client_interface, int64_t default_ban_time) : m_client_interface(client_interface), m_ban_db(std::move(ban_file)), m_default_ban_time(default_ban_time) { - if (m_client_interface) m_client_interface->InitMessage(_("Loading banlist...")); + if (m_client_interface) m_client_interface->InitMessage(_("Loading banlist...").translated); int64_t n_start = GetTimeMillis(); m_is_dirty = false; diff --git a/src/bench/bench_bitcoin.cpp b/src/bench/bench_bitcoin.cpp index 8eea96d93004..d0d7c03ee177 100644 --- a/src/bench/bench_bitcoin.cpp +++ b/src/bench/bench_bitcoin.cpp @@ -21,14 +21,14 @@ static void SetupBenchArgs() { SetupHelpOptions(gArgs); - gArgs.AddArg("-list", "List benchmarks without executing them. Can be combined with -scaling and -filter", false, OptionsCategory::OPTIONS); - gArgs.AddArg("-evals=", strprintf("Number of measurement evaluations to perform. (default: %u)", DEFAULT_BENCH_EVALUATIONS), false, OptionsCategory::OPTIONS); - gArgs.AddArg("-filter=", strprintf("Regular expression filter to select benchmark by name (default: %s)", DEFAULT_BENCH_FILTER), false, OptionsCategory::OPTIONS); - gArgs.AddArg("-scaling=", strprintf("Scaling factor for benchmark's runtime (default: %u)", DEFAULT_BENCH_SCALING), false, OptionsCategory::OPTIONS); - gArgs.AddArg("-printer=(console|plot)", strprintf("Choose printer format. console: print data to console. plot: Print results as HTML graph (default: %s)", DEFAULT_BENCH_PRINTER), false, OptionsCategory::OPTIONS); - gArgs.AddArg("-plot-plotlyurl=", strprintf("URL to use for plotly.js (default: %s)", DEFAULT_PLOT_PLOTLYURL), false, OptionsCategory::OPTIONS); - gArgs.AddArg("-plot-width=", strprintf("Plot width in pixel (default: %u)", DEFAULT_PLOT_WIDTH), false, OptionsCategory::OPTIONS); - gArgs.AddArg("-plot-height=", strprintf("Plot height in pixel (default: %u)", DEFAULT_PLOT_HEIGHT), false, OptionsCategory::OPTIONS); + gArgs.AddArg("-list", "List benchmarks without executing them. Can be combined with -scaling and -filter", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + gArgs.AddArg("-evals=", strprintf("Number of measurement evaluations to perform. (default: %u)", DEFAULT_BENCH_EVALUATIONS), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + gArgs.AddArg("-filter=", strprintf("Regular expression filter to select benchmark by name (default: %s)", DEFAULT_BENCH_FILTER), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + gArgs.AddArg("-scaling=", strprintf("Scaling factor for benchmark's runtime (default: %u)", DEFAULT_BENCH_SCALING), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + gArgs.AddArg("-printer=(console|plot)", strprintf("Choose printer format. console: print data to console. plot: Print results as HTML graph (default: %s)", DEFAULT_BENCH_PRINTER), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + gArgs.AddArg("-plot-plotlyurl=", strprintf("URL to use for plotly.js (default: %s)", DEFAULT_PLOT_PLOTLYURL), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + gArgs.AddArg("-plot-width=", strprintf("Plot width in pixel (default: %u)", DEFAULT_PLOT_WIDTH), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + gArgs.AddArg("-plot-height=", strprintf("Plot height in pixel (default: %u)", DEFAULT_PLOT_HEIGHT), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); } int main(int argc, char** argv) diff --git a/src/bench/ccoins_caching.cpp b/src/bench/ccoins_caching.cpp index 1041a2230359..39cab092cf6f 100644 --- a/src/bench/ccoins_caching.cpp +++ b/src/bench/ccoins_caching.cpp @@ -5,7 +5,7 @@ #include #include #include -#include +#include