diff --git a/README.md b/README.md index 54a46bc..06a9153 100644 --- a/README.md +++ b/README.md @@ -23,15 +23,15 @@ We vendor Bitcoin Core’s `libbitcoinkernel` using a Git subtree targeting the ```bash git subtree pull \ --prefix bitcoinkernel/bitcoin \ - https://github.com/TheCharlatan/bitcoin \ - kernelApi --squash + https://github.com/bitcoin/bitcoin \ + master --squash ``` ## Requirements - CMake - C++17 compiler (e.g. GCC/Clang) - Boost -- JDK 21+ with FFM support +- JDK 25+ with FFM support Refer to Bitcoin Core [docs](https://github.com/bitcoin/bitcoin/blob/master/doc/build-unix.md) for dependency specifics. @@ -50,7 +50,6 @@ Refer to Bitcoin Core [docs](https://github.com/bitcoin/bitcoin/blob/master/doc/ ## References -- Upstream PR of C header API library -> https://github.com/bitcoin/bitcoin/pull/30595 - [rust-bitcoinkernel wrapper](https://github.com/yuvicc/rust-bitcoinkernel) ## Current Status of the Library diff --git a/bitcoinkernel/bitcoin/.github/ISSUE_TEMPLATE/bug.yml b/bitcoinkernel/bitcoin/.github/ISSUE_TEMPLATE/bug.yml index 83922b5..7c894ab 100644 --- a/bitcoinkernel/bitcoin/.github/ISSUE_TEMPLATE/bug.yml +++ b/bitcoinkernel/bitcoin/.github/ISSUE_TEMPLATE/bug.yml @@ -78,7 +78,7 @@ body: id: os attributes: label: Operating system and version - placeholder: e.g. "MacOS Ventura 13.2" or "Ubuntu 22.04 LTS" + placeholder: e.g. "MacOS 26.0" or "Ubuntu 26.04 LTS" validations: required: true - type: textarea @@ -90,4 +90,3 @@ body: e.g. OS/CPU and disk type, network connectivity validations: required: false - diff --git a/bitcoinkernel/bitcoin/.github/actions/configure-docker/action.yml b/bitcoinkernel/bitcoin/.github/actions/configure-docker/action.yml index 09a1e6f..814f2dd 100644 --- a/bitcoinkernel/bitcoin/.github/actions/configure-docker/action.yml +++ b/bitcoinkernel/bitcoin/.github/actions/configure-docker/action.yml @@ -4,12 +4,21 @@ inputs: cache-provider: description: 'gha or cirrus cache provider' required: true - options: - - gh - - cirrus runs: using: 'composite' steps: + - name: Check inputs + shell: bash + run: | + # We expect only gha or cirrus as inputs to cache-provider + case "${{ inputs.cache-provider }}" in + gha|cirrus) + ;; + *) + echo "::warning title=Unknown input to configure docker action::Provided value was ${{ inputs.cache-provider }}" + ;; + esac + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 with: @@ -22,8 +31,12 @@ runs: uses: actions/github-script@v6 with: script: | - core.exportVariable('ACTIONS_CACHE_URL', process.env['ACTIONS_CACHE_URL']) - core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env['ACTIONS_RUNTIME_TOKEN']) + Object.keys(process.env).forEach(function (key) { + if (key.startsWith('ACTIONS_')) { + core.info(`Exporting ${key}`); + core.exportVariable(key, process.env[key]); + } + }); - name: Construct docker build cache args shell: bash @@ -44,8 +57,8 @@ runs: # Always optimistically --cache‑from in case a cache blob exists args=(--cache-from "type=gha${url_args:+,${url_args}},scope=${CONTAINER_NAME}") - # If this is a push to the default branch, also add --cache‑to to save the cache - if [[ ${{ github.event_name }} == "push" && ${{ github.ref_name }} == ${{ github.event.repository.default_branch }} ]]; then + # Only add --cache-to when using the Cirrus cache provider and pushing to the default branch. + if [[ ${{ inputs.cache-provider }} == 'cirrus' && ${{ github.event_name }} == "push" && ${{ github.ref_name }} == ${{ github.event.repository.default_branch }} ]]; then args+=(--cache-to "type=gha${url_args:+,${url_args}},mode=max,ignore-error=true,scope=${CONTAINER_NAME}") fi diff --git a/bitcoinkernel/bitcoin/.github/actions/configure-environment/action.yml b/bitcoinkernel/bitcoin/.github/actions/configure-environment/action.yml index aae5016..e2a26b7 100644 --- a/bitcoinkernel/bitcoin/.github/actions/configure-environment/action.yml +++ b/bitcoinkernel/bitcoin/.github/actions/configure-environment/action.yml @@ -17,7 +17,7 @@ runs: - name: Set cache hashes shell: bash run: | - echo "DEPENDS_HASH=$(git ls-tree HEAD depends "ci/test/$FILE_ENV" | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV + echo "DEPENDS_HASH=$(git ls-tree HEAD depends "$FILE_ENV" | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV echo "PREVIOUS_RELEASES_HASH=$(git ls-tree HEAD test/get_previous_releases.py | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV - name: Get container name diff --git a/bitcoinkernel/bitcoin/.github/ci-test-each-commit-exec.py b/bitcoinkernel/bitcoin/.github/ci-test-each-commit-exec.py index 959a157..b81241b 100755 --- a/bitcoinkernel/bitcoin/.github/ci-test-each-commit-exec.py +++ b/bitcoinkernel/bitcoin/.github/ci-test-each-commit-exec.py @@ -21,11 +21,12 @@ def main(): run(["git", "log", "-1"]) num_procs = int(run(["nproc"], stdout=subprocess.PIPE).stdout) + build_dir = "ci_build" run([ "cmake", "-B", - "build", + build_dir, "-Werror=dev", # Use clang++, because it is a bit faster and uses less memory than g++ "-DCMAKE_C_COMPILER=clang", @@ -37,28 +38,23 @@ def main(): "-DAPPEND_CFLAGS='-O3 -g2'", "-DCMAKE_BUILD_TYPE=Debug", "-DWERROR=ON", - "-DWITH_ZMQ=ON", - "-DBUILD_GUI=ON", - "-DBUILD_BENCH=ON", - "-DBUILD_FUZZ_BINARY=ON", - "-DWITH_USDT=ON", - "-DBUILD_KERNEL_LIB=ON", - "-DBUILD_KERNEL_TEST=ON", + "--preset=dev-mode", + # Tolerate unused member functions in intermediate commits in a pull request "-DCMAKE_CXX_FLAGS=-Wno-error=unused-member-function", ]) - run(["cmake", "--build", "build", "-j", str(num_procs)]) + run(["cmake", "--build", build_dir, "-j", str(num_procs)]) run([ "ctest", "--output-on-failure", "--stop-on-failure", "--test-dir", - "build", + build_dir, "-j", str(num_procs), ]) run([ sys.executable, - "./build/test/functional/test_runner.py", + f"./{build_dir}/test/functional/test_runner.py", "-j", str(num_procs * 2), "--combinedlogslen=99999999", diff --git a/bitcoinkernel/bitcoin/.github/workflows/ci.yml b/bitcoinkernel/bitcoin/.github/workflows/ci.yml index 39dc017..e4c5c8a 100644 --- a/bitcoinkernel/bitcoin/.github/workflows/ci.yml +++ b/bitcoinkernel/bitcoin/.github/workflows/ci.yml @@ -35,6 +35,14 @@ jobs: outputs: provider: ${{ steps.runners.outputs.provider }} steps: + - name: Annotate with pull request number + # This annotation is machine-readable and can be used to assign a check + # run to its corresponding pull request. Running in one check run is + # sufficient for each check suite. + run: | + if [ "${{ github.event_name }}" = "pull_request" ]; then + echo "::notice title=debug_pull_request_number_str::${{ github.event.number }}" + fi - id: runners run: | if [[ "${REPO_USE_CIRRUS_RUNNERS}" == "${{ github.repository }}" ]]; then @@ -157,11 +165,6 @@ jobs: brew install --quiet python@3 || brew link --overwrite python@3 brew install --quiet coreutils ninja pkgconf gnu-getopt ccache boost libevent zeromq qt@6 qrencode capnp - - name: Install Python packages - run: | - git clone -b v2.1.0 https://github.com/capnproto/pycapnp - pip3 install ./pycapnp -C force-bundled-libcapnp=True --break-system-packages - - name: Set Ccache directory run: echo "CCACHE_DIR=${RUNNER_TEMP}/ccache_dir" >> "$GITHUB_ENV" @@ -202,7 +205,7 @@ jobs: job-type: [standard, fuzz] include: - job-type: standard - generate-options: '-DBUILD_GUI=ON -DWITH_ZMQ=ON -DBUILD_BENCH=ON -DBUILD_KERNEL_LIB=ON -DBUILD_UTIL_CHAINSTATE=ON -DBUILD_KERNEL_TEST=OFF -DWERROR=ON' + generate-options: '-DBUILD_GUI=ON -DWITH_ZMQ=ON -DBUILD_BENCH=ON -DBUILD_KERNEL_LIB=ON -DBUILD_UTIL_CHAINSTATE=ON -DWERROR=ON' job-name: 'Windows native, VS 2022' - job-type: fuzz generate-options: '-DVCPKG_MANIFEST_NO_DEFAULT_FEATURES=ON -DVCPKG_MANIFEST_FEATURES="wallet" -DBUILD_GUI=OFF -DBUILD_FOR_FUZZING=ON -DWERROR=ON' @@ -292,8 +295,6 @@ jobs: - name: Run test suite if: matrix.job-type == 'standard' working-directory: build - env: - QT_PLUGIN_PATH: '${{ github.workspace }}\build\vcpkg_installed\x64-windows\Qt6\plugins' run: | ctest --output-on-failure --stop-on-failure -j $NUMBER_OF_PROCESSORS -C Release @@ -381,7 +382,7 @@ jobs: - *CHECKOUT - name: Download built executables - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v5 with: name: x86_64-w64-mingw32-executables-${{ github.run_id }} @@ -433,8 +434,7 @@ jobs: echo "PREVIOUS_RELEASES_DIR=${{ runner.temp }}/previous_releases" >> "$GITHUB_ENV" - name: Get previous releases - working-directory: test - run: ./get_previous_releases.py --target-dir $PREVIOUS_RELEASES_DIR + run: ./test/get_previous_releases.py --target-dir $PREVIOUS_RELEASES_DIR - name: Run functional tests env: @@ -466,18 +466,24 @@ jobs: file-env: './ci/test/00_setup_env_arm.sh' provider: 'gha' - - name: 'ASan + LSan + UBSan + integer, no depends, USDT' + - name: 'ASan + LSan + UBSan + integer' cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-md' # has to match container in ci/test/00_setup_env_native_asan.sh for tracing tools fallback-runner: 'ubuntu-24.04' timeout-minutes: 120 file-env: './ci/test/00_setup_env_native_asan.sh' - - name: 'macOS-cross, gui, no tests' + - name: 'macOS-cross to arm64' cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-sm' fallback-runner: 'ubuntu-24.04' timeout-minutes: 120 file-env: './ci/test/00_setup_env_mac_cross.sh' + - name: 'macOS-cross to x86_64' + cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-sm' + fallback-runner: 'ubuntu-24.04' + timeout-minutes: 120 + file-env: './ci/test/00_setup_env_mac_cross_intel.sh' + - name: 'No wallet, libbitcoinkernel' cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-sm' fallback-runner: 'ubuntu-24.04' @@ -496,17 +502,23 @@ jobs: timeout-minutes: 240 file-env: './ci/test/00_setup_env_native_fuzz.sh' - - name: 'previous releases, depends DEBUG' + - name: 'Valgrind, fuzz' + cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-md' + fallback-runner: 'ubuntu-24.04' + timeout-minutes: 240 + file-env: './ci/test/00_setup_env_native_fuzz_with_valgrind.sh' + + - name: 'previous releases' cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-md' fallback-runner: 'ubuntu-24.04' timeout-minutes: 120 file-env: './ci/test/00_setup_env_native_previous_releases.sh' - - name: 'CentOS, depends, gui' - cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-lg' + - name: 'Alpine (musl), depends, gui' + cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-md' fallback-runner: 'ubuntu-24.04' timeout-minutes: 120 - file-env: './ci/test/00_setup_env_native_centos.sh' + file-env: './ci/test/00_setup_env_native_alpine_musl.sh' - name: 'tidy' cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-md' @@ -520,6 +532,12 @@ jobs: timeout-minutes: 120 file-env: './ci/test/00_setup_env_native_tsan.sh' + - name: 'MSan, fuzz' + cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-md' + fallback-runner: 'ubuntu-24.04' + timeout-minutes: 150 + file-env: './ci/test/00_setup_env_native_fuzz_with_msan.sh' + - name: 'MSan, depends' cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-lg' fallback-runner: 'ubuntu-24.04' @@ -548,7 +566,7 @@ jobs: run: sed -i "s|\${INSTALL_BCC_TRACING_TOOLS}|true|g" ./ci/test/00_setup_env_native_asan.sh - name: Set mmap_rnd_bits - if: ${{ env.CONTAINER_NAME == 'ci_native_tsan' || env.CONTAINER_NAME == 'ci_native_msan' }} + if: ${{ env.CONTAINER_NAME == 'ci_native_tsan' || env.CONTAINER_NAME == 'ci_native_msan' || env.CONTAINER_NAME == 'ci_native_fuzz_msan' }} # Prevents crashes due to high ASLR entropy run: sudo sysctl -w vm.mmap_rnd_bits=28 @@ -561,7 +579,7 @@ jobs: lint: name: 'lint' needs: runners - runs-on: ${{ needs.runners.outputs.use-cirrus-runners == 'true' && 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-xs' || 'ubuntu-24.04' }} + runs-on: ${{ needs.runners.outputs.provider == 'cirrus' && 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-xs' || 'ubuntu-24.04' }} if: ${{ vars.SKIP_BRANCH_PUSH != 'true' || github.event_name == 'pull_request' }} timeout-minutes: 20 env: diff --git a/bitcoinkernel/bitcoin/CMakeLists.txt b/bitcoinkernel/bitcoin/CMakeLists.txt index 2e5196c..2d21f6e 100644 --- a/bitcoinkernel/bitcoin/CMakeLists.txt +++ b/bitcoinkernel/bitcoin/CMakeLists.txt @@ -463,6 +463,8 @@ else() try_append_cxx_flags("-Wself-assign" TARGET warn_interface SKIP_LINK) try_append_cxx_flags("-Wbidi-chars=any" TARGET warn_interface SKIP_LINK) try_append_cxx_flags("-Wundef" TARGET warn_interface SKIP_LINK) + try_append_cxx_flags("-Wleading-whitespace=spaces" TARGET warn_interface SKIP_LINK) + try_append_cxx_flags("-Wtrailing-whitespace=any" TARGET warn_interface SKIP_LINK) # Some compilers (gcc) ignore unknown -Wno-* options, but warn about all # unknown options if any other warning is produced. Test the -Wfoo case, and @@ -493,8 +495,8 @@ try_append_cxx_flags("-fmacro-prefix-map=A=B" TARGET core_interface SKIP_LINK IF_CHECK_PASSED "-fmacro-prefix-map=${PROJECT_SOURCE_DIR}/src=." ) -# Currently all versions of gcc are subject to a class of bugs, see the -# gccbug_90348 test case (only reproduces on GCC 11 and earlier) and +# GCC versions 13.2 (and earlier) are subject to a class of bugs, see +# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90348 and the meta bug # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=111843. To work around that, set # -fstack-reuse=none for all gcc builds. (Only gcc understands this flag). try_append_cxx_flags("-fstack-reuse=none" TARGET core_interface) @@ -632,7 +634,6 @@ if(CMAKE_SYSTEM_NAME STREQUAL "NetBSD") set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) else() set(CMAKE_SKIP_BUILD_RPATH TRUE) - set(CMAKE_SKIP_INSTALL_RPATH TRUE) endif() add_subdirectory(test) add_subdirectory(doc) diff --git a/bitcoinkernel/bitcoin/ci/README.md b/bitcoinkernel/bitcoin/ci/README.md index 81e048c..a729128 100644 --- a/bitcoinkernel/bitcoin/ci/README.md +++ b/bitcoinkernel/bitcoin/ci/README.md @@ -20,6 +20,14 @@ requires `bash`, `docker`, and `python3` to be installed. To run on different ar sudo apt install bash docker.io python3 qemu-user-static ``` +For some sanitizer builds, the kernel's address-space layout randomization +(ASLR) entropy can cause sanitizer shadow memory mappings to fail. When running +the CI locally you may need to reduce that entropy by running: + +``` +sudo sysctl -w vm.mmap_rnd_bits=28 +``` + It is recommended to run the ci system in a clean env. To run the test stage with a specific configuration, diff --git a/bitcoinkernel/bitcoin/ci/lint/01_install.sh b/bitcoinkernel/bitcoin/ci/lint/01_install.sh index 0d8cca9..fc42267 100755 --- a/bitcoinkernel/bitcoin/ci/lint/01_install.sh +++ b/bitcoinkernel/bitcoin/ci/lint/01_install.sh @@ -18,7 +18,8 @@ ${CI_RETRY_EXE} apt-get update # - curl/xz-utils (to install shellcheck) # - git (used in many lint scripts) # - gpg (used by verify-commits) -${CI_RETRY_EXE} apt-get install -y cargo curl xz-utils git gpg +# - moreutils (used by scripted-diff) +${CI_RETRY_EXE} apt-get install -y cargo curl xz-utils git gpg moreutils PYTHON_PATH="/python_build" if [ ! -d "${PYTHON_PATH}/bin" ]; then diff --git a/bitcoinkernel/bitcoin/ci/test/00_setup_env.sh b/bitcoinkernel/bitcoin/ci/test/00_setup_env.sh index a4293fb..a8f8d0e 100755 --- a/bitcoinkernel/bitcoin/ci/test/00_setup_env.sh +++ b/bitcoinkernel/bitcoin/ci/test/00_setup_env.sh @@ -64,7 +64,7 @@ export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_SCRATCH_DIR/out} # The folder for previous release binaries. # This folder exists only on the ci guest, and on the ci host as a volume. export PREVIOUS_RELEASES_DIR=${PREVIOUS_RELEASES_DIR:-$BASE_ROOT_DIR/prev_releases} -export CI_BASE_PACKAGES=${CI_BASE_PACKAGES:-build-essential pkgconf curl ca-certificates ccache python3 rsync git procps bison e2fsprogs cmake ninja-build} +export CI_BASE_PACKAGES=${CI_BASE_PACKAGES:-build-essential pkgconf curl ca-certificates ccache python3-dev rsync git procps bison e2fsprogs cmake ninja-build} export GOAL=${GOAL:-install} export DIR_QA_ASSETS=${DIR_QA_ASSETS:-${BASE_SCRATCH_DIR}/qa-assets} export CI_RETRY_EXE=${CI_RETRY_EXE:-"retry --"} diff --git a/bitcoinkernel/bitcoin/ci/test/00_setup_env_arm.sh b/bitcoinkernel/bitcoin/ci/test/00_setup_env_arm.sh index ce01978..12ec71b 100755 --- a/bitcoinkernel/bitcoin/ci/test/00_setup_env_arm.sh +++ b/bitcoinkernel/bitcoin/ci/test/00_setup_env_arm.sh @@ -19,4 +19,7 @@ export GOAL="install" export CI_LIMIT_STACK_SIZE=1 # -Wno-psabi is to disable ABI warnings: "note: parameter passing for argument of type ... changed in GCC 7.1" # This could be removed once the ABI change warning does not show up by default -export BITCOIN_CONFIG="-DREDUCE_EXPORTS=ON -DCMAKE_CXX_FLAGS='-Wno-psabi -Wno-error=maybe-uninitialized'" +export BITCOIN_CONFIG=" \ + -DREDUCE_EXPORTS=ON \ + -DCMAKE_CXX_FLAGS='-Wno-psabi -Wno-error=maybe-uninitialized' \ +" diff --git a/bitcoinkernel/bitcoin/ci/test/00_setup_env_mac_cross.sh b/bitcoinkernel/bitcoin/ci/test/00_setup_env_mac_cross.sh index a0d9082..c7de033 100755 --- a/bitcoinkernel/bitcoin/ci/test/00_setup_env_mac_cross.sh +++ b/bitcoinkernel/bitcoin/ci/test/00_setup_env_mac_cross.sh @@ -9,12 +9,12 @@ export LC_ALL=C.UTF-8 export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks} export CONTAINER_NAME=ci_macos_cross -export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:24.04" -export HOST=x86_64-apple-darwin +export CI_IMAGE_NAME_TAG="mirror.gcr.io/debian:trixie" # Check that https://packages.debian.org/trixie/clang (version 19, similar to guix) can cross-compile +export HOST=arm64-apple-darwin export PACKAGES="clang lld llvm zip" export XCODE_VERSION=15.0 export XCODE_BUILD_ID=15A240d export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false export GOAL="deploy" -export BITCOIN_CONFIG="-DBUILD_GUI=ON -DREDUCE_EXPORTS=ON" +export BITCOIN_CONFIG="-DBUILD_GUI=ON -DBUILD_KERNEL_LIB=ON -DREDUCE_EXPORTS=ON" diff --git a/bitcoinkernel/bitcoin/ci/test/00_setup_env_mac_cross_intel.sh b/bitcoinkernel/bitcoin/ci/test/00_setup_env_mac_cross_intel.sh new file mode 100755 index 0000000..535b801 --- /dev/null +++ b/bitcoinkernel/bitcoin/ci/test/00_setup_env_mac_cross_intel.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2019-present The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks} + +export CONTAINER_NAME=ci_macos_cross_intel +export CI_IMAGE_NAME_TAG="mirror.gcr.io/debian:trixie" # Check that https://packages.debian.org/trixie/clang (version 19, similar to guix) can cross-compile +export HOST=x86_64-apple-darwin +export PACKAGES="clang lld llvm zip" +export XCODE_VERSION=15.0 +export XCODE_BUILD_ID=15A240d +export RUN_UNIT_TESTS=false +export RUN_FUNCTIONAL_TESTS=false +export GOAL="deploy" +export BITCOIN_CONFIG="-DBUILD_GUI=ON -DREDUCE_EXPORTS=ON" diff --git a/bitcoinkernel/bitcoin/ci/test/00_setup_env_mac_native.sh b/bitcoinkernel/bitcoin/ci/test/00_setup_env_mac_native.sh index 3584246..41b97b0 100755 --- a/bitcoinkernel/bitcoin/ci/test/00_setup_env_mac_native.sh +++ b/bitcoinkernel/bitcoin/ci/test/00_setup_env_mac_native.sh @@ -6,13 +6,11 @@ export LC_ALL=C.UTF-8 -# Homebrew's python@3.12 is marked as externally managed (PEP 668). -# Therefore, `--break-system-packages` is needed. export CONTAINER_NAME="ci_mac_native" # macos does not use a container, but the env var is needed for logging -export PIP_PACKAGES="--break-system-packages zmq" +export PIP_PACKAGES="--break-system-packages pycapnp zmq" export GOAL="install deploy" export CMAKE_GENERATOR="Ninja" -export BITCOIN_CONFIG="-DBUILD_GUI=ON -DWITH_ZMQ=ON -DBUILD_KERNEL_LIB=ON -DBUILD_KERNEL_TEST=ON -DREDUCE_EXPORTS=ON -DCMAKE_EXE_LINKER_FLAGS='-Wl,-stack_size -Wl,0x80000'" +export BITCOIN_CONFIG="-DBUILD_GUI=ON -DWITH_ZMQ=ON -DBUILD_KERNEL_LIB=ON -DBUILD_UTIL_CHAINSTATE=ON -DBUILD_KERNEL_TEST=ON -DREDUCE_EXPORTS=ON -DCMAKE_EXE_LINKER_FLAGS='-Wl,-stack_size -Wl,0x80000'" export CI_OS_NAME="macos" export NO_DEPENDS=1 export OSX_SDK="" diff --git a/bitcoinkernel/bitcoin/ci/test/00_setup_env_mac_native_fuzz.sh b/bitcoinkernel/bitcoin/ci/test/00_setup_env_mac_native_fuzz.sh index a8010c7..ea074e4 100755 --- a/bitcoinkernel/bitcoin/ci/test/00_setup_env_mac_native_fuzz.sh +++ b/bitcoinkernel/bitcoin/ci/test/00_setup_env_mac_native_fuzz.sh @@ -8,7 +8,7 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME="ci_mac_native_fuzz" # macos does not use a container, but the env var is needed for logging export CMAKE_GENERATOR="Ninja" -export BITCOIN_CONFIG="-DBUILD_FOR_FUZZING=ON -DCMAKE_EXE_LINKER_FLAGS='-Wl,-stack_size -Wl,0x80000'" +export BITCOIN_CONFIG="-DBUILD_FOR_FUZZING=ON -DCMAKE_EXE_LINKER_FLAGS='-Wl,-stack_size -Wl,0x80000' -DAPPEND_CPPFLAGS='-D_LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE_DEBUG'" export CI_OS_NAME="macos" export NO_DEPENDS=1 export OSX_SDK="" diff --git a/bitcoinkernel/bitcoin/ci/test/00_setup_env_native_centos.sh b/bitcoinkernel/bitcoin/ci/test/00_setup_env_native_alpine_musl.sh similarity index 57% rename from bitcoinkernel/bitcoin/ci/test/00_setup_env_native_centos.sh rename to bitcoinkernel/bitcoin/ci/test/00_setup_env_native_alpine_musl.sh index 998ddaf..2de2a16 100755 --- a/bitcoinkernel/bitcoin/ci/test/00_setup_env_native_centos.sh +++ b/bitcoinkernel/bitcoin/ci/test/00_setup_env_native_alpine_musl.sh @@ -6,10 +6,10 @@ export LC_ALL=C.UTF-8 -export CONTAINER_NAME=ci_native_centos -export CI_IMAGE_NAME_TAG="quay.io/centos/centos:stream10" -export CI_BASE_PACKAGES="gcc-c++ glibc-devel libstdc++-devel ccache make ninja-build git python3 python3-pip which patch xz procps-ng rsync coreutils bison e2fsprogs cmake dash" -export PIP_PACKAGES="pyzmq pycapnp" +export CONTAINER_NAME=ci_native_alpine_musl +export CI_IMAGE_NAME_TAG="mirror.gcr.io/alpine:3.22" +export CI_BASE_PACKAGES="build-base musl-dev pkgconf curl ccache make ninja git python3-dev py3-pip which patch xz procps rsync util-linux bison e2fsprogs cmake dash linux-headers" +export PIP_PACKAGES="--break-system-packages pyzmq pycapnp" export DEP_OPTS="DEBUG=1" export GOAL="install" export BITCOIN_CONFIG="\ diff --git a/bitcoinkernel/bitcoin/ci/test/00_setup_env_native_asan.sh b/bitcoinkernel/bitcoin/ci/test/00_setup_env_native_asan.sh index e785f6a..0e73224 100755 --- a/bitcoinkernel/bitcoin/ci/test/00_setup_env_native_asan.sh +++ b/bitcoinkernel/bitcoin/ci/test/00_setup_env_native_asan.sh @@ -20,18 +20,19 @@ fi export CONTAINER_NAME=ci_native_asan export APT_LLVM_V="21" -export PACKAGES="systemtap-sdt-dev clang-${APT_LLVM_V} llvm-${APT_LLVM_V} libclang-rt-${APT_LLVM_V}-dev python3-zmq qt6-base-dev qt6-tools-dev qt6-l10n-tools libevent-dev libboost-dev libzmq3-dev libqrencode-dev libsqlite3-dev ${BPFCC_PACKAGE} libcapnp-dev capnproto python3-pip" +export PACKAGES="systemtap-sdt-dev clang-${APT_LLVM_V} llvm-${APT_LLVM_V} libclang-rt-${APT_LLVM_V}-dev mold python3-zmq qt6-base-dev qt6-tools-dev qt6-l10n-tools libevent-dev libboost-dev libzmq3-dev libqrencode-dev libsqlite3-dev ${BPFCC_PACKAGE} libcapnp-dev capnproto python3-pip" export PIP_PACKAGES="--break-system-packages pycapnp" export NO_DEPENDS=1 export GOAL="install" export CI_LIMIT_STACK_SIZE=1 export BITCOIN_CONFIG="\ - -DWITH_USDT=ON -DWITH_ZMQ=ON -DBUILD_GUI=ON \ + --preset=dev-mode \ -DSANITIZERS=address,float-divide-by-zero,integer,undefined \ -DCMAKE_C_COMPILER=clang \ -DCMAKE_CXX_COMPILER=clang++ \ -DCMAKE_C_FLAGS='-ftrivial-auto-var-init=pattern' \ -DCMAKE_CXX_FLAGS='-ftrivial-auto-var-init=pattern' \ + -DCMAKE_EXE_LINKER_FLAGS='-fuse-ld=mold' \ -DAPPEND_CXXFLAGS='-std=c++23' \ -DAPPEND_CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' \ " diff --git a/bitcoinkernel/bitcoin/ci/test/00_setup_env_native_fuzz_with_msan.sh b/bitcoinkernel/bitcoin/ci/test/00_setup_env_native_fuzz_with_msan.sh index 0d90605..7529bbc 100755 --- a/bitcoinkernel/bitcoin/ci/test/00_setup_env_native_fuzz_with_msan.sh +++ b/bitcoinkernel/bitcoin/ci/test/00_setup_env_native_fuzz_with_msan.sh @@ -25,7 +25,7 @@ export BITCOIN_CONFIG="\ -DCMAKE_C_FLAGS_DEBUG='' \ -DCMAKE_CXX_FLAGS_DEBUG='' \ -DBUILD_FOR_FUZZING=ON \ - -DSANITIZERS=fuzzer,memory \ + -DSANITIZERS=memory \ -DAPPEND_CPPFLAGS='-DBOOST_MULTI_INDEX_ENABLE_SAFE_MODE -U_FORTIFY_SOURCE' \ " export USE_INSTRUMENTED_LIBCPP="MemoryWithOrigins" diff --git a/bitcoinkernel/bitcoin/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh b/bitcoinkernel/bitcoin/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh index b1e0c65..1041777 100755 --- a/bitcoinkernel/bitcoin/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh +++ b/bitcoinkernel/bitcoin/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh @@ -7,10 +7,10 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_nowallet_libbitcoinkernel -export CI_IMAGE_NAME_TAG="mirror.gcr.io/debian:bookworm" -# Use minimum supported python3.10 (or best-effort 3.11) and clang-16, see doc/dependencies.md -export PACKAGES="python3-zmq python3-pip clang-16 llvm-16 libc++abi-16-dev libc++-16-dev" +export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:24.04" +# Use minimum supported python3.10 (or best-effort 3.12) and clang-17, see doc/dependencies.md +export PACKAGES="python3-zmq python3-pip clang-17 llvm-17 libc++abi-17-dev libc++-17-dev" export PIP_PACKAGES="--break-system-packages pycapnp" -export DEP_OPTS="NO_WALLET=1 CC=clang-16 CXX='clang++-16 -stdlib=libc++'" +export DEP_OPTS="NO_WALLET=1 CC=clang-17 CXX='clang++-17 -stdlib=libc++'" export GOAL="install" export BITCOIN_CONFIG="-DREDUCE_EXPORTS=ON -DBUILD_UTIL_CHAINSTATE=ON -DBUILD_KERNEL_LIB=ON -DBUILD_KERNEL_TEST=ON -DBUILD_SHARED_LIBS=ON" diff --git a/bitcoinkernel/bitcoin/ci/test/00_setup_env_native_previous_releases.sh b/bitcoinkernel/bitcoin/ci/test/00_setup_env_native_previous_releases.sh index d9f8d3b..a44a51c 100755 --- a/bitcoinkernel/bitcoin/ci/test/00_setup_env_native_previous_releases.sh +++ b/bitcoinkernel/bitcoin/ci/test/00_setup_env_native_previous_releases.sh @@ -8,15 +8,16 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_previous_releases export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:22.04" -# Use minimum supported python3.10 and gcc-11, see doc/dependencies.md -export PACKAGES="gcc-11 g++-11 python3-zmq" -export DEP_OPTS="CC=gcc-11 CXX=g++-11" +# Use minimum supported python3.10 and gcc-12, see doc/dependencies.md +export PACKAGES="gcc-12 g++-12 python3-zmq" +export DEP_OPTS="CC=gcc-12 CXX=g++-12" export TEST_RUNNER_EXTRA="--previous-releases --coverage --extended --exclude feature_dbcrash" # Run extended tests so that coverage does not fail, but exclude the very slow dbcrash export GOAL="install" export CI_LIMIT_STACK_SIZE=1 export DOWNLOAD_PREVIOUS_RELEASES="true" export BITCOIN_CONFIG="\ - -DWITH_ZMQ=ON -DBUILD_GUI=ON -DREDUCE_EXPORTS=ON \ + --preset=dev-mode \ + -DREDUCE_EXPORTS=ON \ -DCMAKE_BUILD_TYPE=Debug \ -DCMAKE_C_FLAGS='-funsigned-char' \ -DCMAKE_C_FLAGS_DEBUG='-g2 -O2' \ diff --git a/bitcoinkernel/bitcoin/ci/test/00_setup_env_native_tidy.sh b/bitcoinkernel/bitcoin/ci/test/00_setup_env_native_tidy.sh index c14e59b..04f3e2e 100755 --- a/bitcoinkernel/bitcoin/ci/test/00_setup_env_native_tidy.sh +++ b/bitcoinkernel/bitcoin/ci/test/00_setup_env_native_tidy.sh @@ -8,7 +8,7 @@ export LC_ALL=C.UTF-8 export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:24.04" export CONTAINER_NAME=ci_native_tidy -export TIDY_LLVM_V="20" +export TIDY_LLVM_V="21" export APT_LLVM_V="${TIDY_LLVM_V}" export PACKAGES="clang-${TIDY_LLVM_V} libclang-${TIDY_LLVM_V}-dev llvm-${TIDY_LLVM_V}-dev libomp-${TIDY_LLVM_V}-dev clang-tidy-${TIDY_LLVM_V} jq libevent-dev libboost-dev libzmq3-dev systemtap-sdt-dev qt6-base-dev qt6-tools-dev qt6-l10n-tools libqrencode-dev libsqlite3-dev libcapnp-dev capnproto" export NO_DEPENDS=1 @@ -19,7 +19,7 @@ export RUN_CHECK_DEPS=true export RUN_TIDY=true export GOAL="install" export BITCOIN_CONFIG="\ - -DWITH_ZMQ=ON -DBUILD_GUI=ON -DBUILD_BENCH=ON -DWITH_USDT=ON \ + --preset dev-mode \ -DCMAKE_C_COMPILER=clang-${TIDY_LLVM_V} \ -DCMAKE_CXX_COMPILER=clang++-${TIDY_LLVM_V} \ -DCMAKE_C_FLAGS_RELWITHDEBINFO='-O0 -g0' \ diff --git a/bitcoinkernel/bitcoin/ci/test/00_setup_env_win64.sh b/bitcoinkernel/bitcoin/ci/test/00_setup_env_win64.sh index 71e1e36..110db1a 100755 --- a/bitcoinkernel/bitcoin/ci/test/00_setup_env_win64.sh +++ b/bitcoinkernel/bitcoin/ci/test/00_setup_env_win64.sh @@ -8,11 +8,11 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_win64 export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:24.04" # Check that https://packages.ubuntu.com/noble/g++-mingw-w64-x86-64-posix (version 13.x, similar to guix) can cross-compile -export CI_IMAGE_PLATFORM="linux/amd64" export HOST=x86_64-w64-mingw32 export PACKAGES="g++-mingw-w64-x86-64-posix nsis" export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false export GOAL="deploy" export BITCOIN_CONFIG="-DREDUCE_EXPORTS=ON -DBUILD_GUI_TESTS=OFF -DBUILD_KERNEL_LIB=ON -DBUILD_KERNEL_TEST=ON \ --DCMAKE_CXX_FLAGS='-Wno-error=maybe-uninitialized'" + -DCMAKE_CXX_FLAGS='-Wno-error=maybe-uninitialized' \ +" diff --git a/bitcoinkernel/bitcoin/ci/test/01_base_install.sh b/bitcoinkernel/bitcoin/ci/test/01_base_install.sh index ce46098..9bb67aa 100755 --- a/bitcoinkernel/bitcoin/ci/test/01_base_install.sh +++ b/bitcoinkernel/bitcoin/ci/test/01_base_install.sh @@ -32,16 +32,17 @@ if [ -n "${APT_LLVM_V}" ]; then ) fi -if [[ $CI_IMAGE_NAME_TAG == *centos* ]]; then - bash -c "dnf -y install epel-release" - # The ninja-build package is available in the CRB repository. - bash -c "dnf -y --allowerasing --enablerepo crb install $CI_BASE_PACKAGES $PACKAGES" +if [[ $CI_IMAGE_NAME_TAG == *alpine* ]]; then + ${CI_RETRY_EXE} apk update + # shellcheck disable=SC2086 + ${CI_RETRY_EXE} apk add --no-cache $CI_BASE_PACKAGES $PACKAGES elif [ "$CI_OS_NAME" != "macos" ]; then if [[ -n "${APPEND_APT_SOURCES_LIST}" ]]; then echo "${APPEND_APT_SOURCES_LIST}" >> /etc/apt/sources.list fi ${CI_RETRY_EXE} apt-get update - ${CI_RETRY_EXE} bash -c "apt-get install --no-install-recommends --no-upgrade -y $PACKAGES $CI_BASE_PACKAGES" + # shellcheck disable=SC2086 + ${CI_RETRY_EXE} apt-get install --no-install-recommends --no-upgrade -y $PACKAGES $CI_BASE_PACKAGES fi if [ -n "${APT_LLVM_V}" ]; then @@ -56,7 +57,7 @@ if [ -n "$PIP_PACKAGES" ]; then fi if [[ -n "${USE_INSTRUMENTED_LIBCPP}" ]]; then - ${CI_RETRY_EXE} git clone --depth=1 https://github.com/llvm/llvm-project -b "llvmorg-21.1.1" /llvm-project + ${CI_RETRY_EXE} git clone --depth=1 https://github.com/llvm/llvm-project -b "llvmorg-21.1.5" /llvm-project cmake -G Ninja -B /cxx_build/ \ -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" \ diff --git a/bitcoinkernel/bitcoin/ci/test/02_run_container.py b/bitcoinkernel/bitcoin/ci/test/02_run_container.py index 513ecac..4d0bed2 100755 --- a/bitcoinkernel/bitcoin/ci/test/02_run_container.py +++ b/bitcoinkernel/bitcoin/ci/test/02_run_container.py @@ -3,16 +3,19 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or https://opensource.org/license/mit/. +from pathlib import Path import os import shlex import subprocess import sys +import time def run(cmd, **kwargs): print("+ " + shlex.join(cmd), flush=True) + kwargs.setdefault("check", True) try: - return subprocess.run(cmd, check=True, **kwargs) + return subprocess.run(cmd, **kwargs) except Exception as e: sys.exit(e) @@ -36,8 +39,8 @@ def main(): # Append $USER to /tmp/env to support multi-user systems and $CONTAINER_NAME # to allow support starting multiple runs simultaneously by the same user. env_file = "/tmp/env-{u}-{c}".format( - u=os.getenv("USER"), - c=os.getenv("CONTAINER_NAME"), + u=os.environ["USER"], + c=os.environ["CONTAINER_NAME"], ) with open(env_file, "w", encoding="utf8") as file: for k, v in os.environ.items(): @@ -45,6 +48,112 @@ def main(): file.write(f"{k}={v}\n") run(["cat", env_file]) + if not os.getenv("DANGER_RUN_CI_ON_HOST"): + CI_IMAGE_LABEL = "bitcoin-ci-test" + + # Use buildx unconditionally + # Using buildx is required to properly load the correct driver, for use with registry caching. Neither build, nor BUILDKIT=1 currently do this properly + cmd_build = ["docker", "buildx", "build"] + cmd_build += [ + f"--file={os.environ['BASE_READ_ONLY_DIR']}/ci/test_imagefile", + f"--build-arg=CI_IMAGE_NAME_TAG={os.environ['CI_IMAGE_NAME_TAG']}", + f"--build-arg=FILE_ENV={os.environ['FILE_ENV']}", + f"--build-arg=BASE_ROOT_DIR={os.environ['BASE_ROOT_DIR']}", + f"--platform={os.environ['CI_IMAGE_PLATFORM']}", + f"--label={CI_IMAGE_LABEL}", + f"--tag={os.environ['CONTAINER_NAME']}", + ] + cmd_build += shlex.split(os.getenv("DOCKER_BUILD_CACHE_ARG", "")) + cmd_build += [os.environ["BASE_READ_ONLY_DIR"]] + + print(f"Building {os.environ['CONTAINER_NAME']} image tag to run in") + if run(cmd_build, check=False).returncode != 0: + print(f"Retry building {os.environ['CONTAINER_NAME']} image tag after failure") + time.sleep(3) + run(cmd_build) + + for suffix in ["ccache", "depends", "depends_sources", "previous_releases"]: + run(["docker", "volume", "create", f"{os.environ['CONTAINER_NAME']}_{suffix}"], check=False) + + CI_CCACHE_MOUNT = f"type=volume,src={os.environ['CONTAINER_NAME']}_ccache,dst={os.environ['CCACHE_DIR']}" + CI_DEPENDS_MOUNT = f"type=volume,src={os.environ['CONTAINER_NAME']}_depends,dst={os.environ['DEPENDS_DIR']}/built" + CI_DEPENDS_SOURCES_MOUNT = f"type=volume,src={os.environ['CONTAINER_NAME']}_depends_sources,dst={os.environ['DEPENDS_DIR']}/sources" + CI_PREVIOUS_RELEASES_MOUNT = f"type=volume,src={os.environ['CONTAINER_NAME']}_previous_releases,dst={os.environ['PREVIOUS_RELEASES_DIR']}" + CI_BUILD_MOUNT = [] + + if os.getenv("DANGER_CI_ON_HOST_FOLDERS"): + # ensure the directories exist + for create_dir in [ + os.environ["CCACHE_DIR"], + f"{os.environ['DEPENDS_DIR']}/built", + f"{os.environ['DEPENDS_DIR']}/sources", + os.environ["PREVIOUS_RELEASES_DIR"], + os.environ["BASE_BUILD_DIR"], # Unset by default, must be defined externally + ]: + Path(create_dir).mkdir(parents=True, exist_ok=True) + + CI_CCACHE_MOUNT = f"type=bind,src={os.environ['CCACHE_DIR']},dst={os.environ['CCACHE_DIR']}" + CI_DEPENDS_MOUNT = f"type=bind,src={os.environ['DEPENDS_DIR']}/built,dst={os.environ['DEPENDS_DIR']}/built" + CI_DEPENDS_SOURCES_MOUNT = f"type=bind,src={os.environ['DEPENDS_DIR']}/sources,dst={os.environ['DEPENDS_DIR']}/sources" + CI_PREVIOUS_RELEASES_MOUNT = f"type=bind,src={os.environ['PREVIOUS_RELEASES_DIR']},dst={os.environ['PREVIOUS_RELEASES_DIR']}" + CI_BUILD_MOUNT = [f"--mount=type=bind,src={os.environ['BASE_BUILD_DIR']},dst={os.environ['BASE_BUILD_DIR']}"] + + if os.getenv("DANGER_CI_ON_HOST_CCACHE_FOLDER"): + if not os.path.isdir(os.environ["CCACHE_DIR"]): + print(f"Error: Directory '{os.environ['CCACHE_DIR']}' must be created in advance.") + sys.exit(1) + CI_CCACHE_MOUNT = f"type=bind,src={os.environ['CCACHE_DIR']},dst={os.environ['CCACHE_DIR']}" + + run(["docker", "network", "create", "--ipv6", "--subnet", "1111:1111::/112", "ci-ip6net"], check=False) + + if os.getenv("RESTART_CI_DOCKER_BEFORE_RUN"): + print("Restart docker before run to stop and clear all containers started with --rm") + run(["podman", "container", "rm", "--force", "--all"]) # Similar to "systemctl restart docker" + + # Still prune everything in case the filtered pruning doesn't work, or if labels were not set + # on a previous run. Belt and suspenders approach, should be fine to remove in the future. + # Prune images used by --external containers (e.g. build containers) when + # using podman. + print("Prune all dangling images") + run(["podman", "image", "prune", "--force", "--external"]) + + print(f"Prune all dangling {CI_IMAGE_LABEL} images") + # When detecting podman-docker, `--external` should be added. + run(["docker", "image", "prune", "--force", "--filter", f"label={CI_IMAGE_LABEL}"]) + + cmd_run = ["docker", "run", "--rm", "--interactive", "--detach", "--tty"] + cmd_run += [ + "--cap-add=LINUX_IMMUTABLE", + *shlex.split(os.getenv("CI_CONTAINER_CAP", "")), + f"--mount=type=bind,src={os.environ['BASE_READ_ONLY_DIR']},dst={os.environ['BASE_READ_ONLY_DIR']},readonly", + f"--mount={CI_CCACHE_MOUNT}", + f"--mount={CI_DEPENDS_MOUNT}", + f"--mount={CI_DEPENDS_SOURCES_MOUNT}", + f"--mount={CI_PREVIOUS_RELEASES_MOUNT}", + *CI_BUILD_MOUNT, + f"--env-file={env_file}", + f"--name={os.environ['CONTAINER_NAME']}", + "--network=ci-ip6net", + f"--platform={os.environ['CI_IMAGE_PLATFORM']}", + os.environ["CONTAINER_NAME"], + ] + + container_id = run( + cmd_run, + stdout=subprocess.PIPE, + text=True, + ).stdout.strip() + os.environ["CI_CONTAINER_ID"] = container_id + + # GNU getopt is required for the CI_RETRY_EXE script + if os.getenv("CI_OS_NAME") == "macos": + prefix = run( + ["brew", "--prefix", "gnu-getopt"], + stdout=subprocess.PIPE, + text=True, + ).stdout.strip() + os.environ["IN_GETOPT_BIN"] = f"{prefix}/bin/getopt" + run(["./ci/test/02_run_container.sh"]) # run the remainder diff --git a/bitcoinkernel/bitcoin/ci/test/02_run_container.sh b/bitcoinkernel/bitcoin/ci/test/02_run_container.sh index e91f875..251b01c 100755 --- a/bitcoinkernel/bitcoin/ci/test/02_run_container.sh +++ b/bitcoinkernel/bitcoin/ci/test/02_run_container.sh @@ -5,100 +5,10 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 -export CI_IMAGE_LABEL="bitcoin-ci-test" set -o errexit -o pipefail -o xtrace if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then - # Env vars during the build can not be changed. For example, a modified - # $MAKEJOBS is ignored in the build process. Use --cpuset-cpus as an - # approximation to respect $MAKEJOBS somewhat, if cpuset is available. - MAYBE_CPUSET="" - if [ "$HAVE_CGROUP_CPUSET" ]; then - MAYBE_CPUSET="--cpuset-cpus=$( python3 -c "import random;P=$( nproc );M=min(P,int('$MAKEJOBS'.lstrip('-j')));print(','.join(map(str,sorted(random.sample(range(P),M)))))" )" - fi - echo "Creating $CI_IMAGE_NAME_TAG container to run in" - - # Use buildx unconditionally - # Using buildx is required to properly load the correct driver, for use with registry caching. Neither build, nor BUILDKIT=1 currently do this properly - # shellcheck disable=SC2086 - docker buildx build \ - --file "${BASE_READ_ONLY_DIR}/ci/test_imagefile" \ - --build-arg "CI_IMAGE_NAME_TAG=${CI_IMAGE_NAME_TAG}" \ - --build-arg "FILE_ENV=${FILE_ENV}" \ - --build-arg "BASE_ROOT_DIR=${BASE_ROOT_DIR}" \ - $MAYBE_CPUSET \ - --platform="${CI_IMAGE_PLATFORM}" \ - --label="${CI_IMAGE_LABEL}" \ - --tag="${CONTAINER_NAME}" \ - $DOCKER_BUILD_CACHE_ARG \ - "${BASE_READ_ONLY_DIR}" - - docker volume create "${CONTAINER_NAME}_ccache" || true - docker volume create "${CONTAINER_NAME}_depends" || true - docker volume create "${CONTAINER_NAME}_depends_sources" || true - docker volume create "${CONTAINER_NAME}_previous_releases" || true - - CI_CCACHE_MOUNT="type=volume,src=${CONTAINER_NAME}_ccache,dst=$CCACHE_DIR" - CI_DEPENDS_MOUNT="type=volume,src=${CONTAINER_NAME}_depends,dst=$DEPENDS_DIR/built" - CI_DEPENDS_SOURCES_MOUNT="type=volume,src=${CONTAINER_NAME}_depends_sources,dst=$DEPENDS_DIR/sources" - CI_PREVIOUS_RELEASES_MOUNT="type=volume,src=${CONTAINER_NAME}_previous_releases,dst=$PREVIOUS_RELEASES_DIR" - CI_BUILD_MOUNT="" - - if [ "$DANGER_CI_ON_HOST_FOLDERS" ]; then - # ensure the directories exist - mkdir -p "${CCACHE_DIR}" - mkdir -p "${DEPENDS_DIR}/built" - mkdir -p "${DEPENDS_DIR}/sources" - mkdir -p "${PREVIOUS_RELEASES_DIR}" - mkdir -p "${BASE_BUILD_DIR}" # Unset by default, must be defined externally - - CI_CCACHE_MOUNT="type=bind,src=${CCACHE_DIR},dst=$CCACHE_DIR" - CI_DEPENDS_MOUNT="type=bind,src=${DEPENDS_DIR}/built,dst=$DEPENDS_DIR/built" - CI_DEPENDS_SOURCES_MOUNT="type=bind,src=${DEPENDS_DIR}/sources,dst=$DEPENDS_DIR/sources" - CI_PREVIOUS_RELEASES_MOUNT="type=bind,src=${PREVIOUS_RELEASES_DIR},dst=$PREVIOUS_RELEASES_DIR" - CI_BUILD_MOUNT="--mount type=bind,src=${BASE_BUILD_DIR},dst=${BASE_BUILD_DIR}" - fi - - if [ "$DANGER_CI_ON_HOST_CCACHE_FOLDER" ]; then - if [ ! -d "${CCACHE_DIR}" ]; then - echo "Error: Directory '${CCACHE_DIR}' must be created in advance." - exit 1 - fi - CI_CCACHE_MOUNT="type=bind,src=${CCACHE_DIR},dst=${CCACHE_DIR}" - fi - - docker network create --ipv6 --subnet 1111:1111::/112 ci-ip6net || true - - if [ -n "${RESTART_CI_DOCKER_BEFORE_RUN}" ] ; then - echo "Restart docker before run to stop and clear all containers started with --rm" - podman container rm --force --all # Similar to "systemctl restart docker" - - # Still prune everything in case the filtered pruning doesn't work, or if labels were not set - # on a previous run. Belt and suspenders approach, should be fine to remove in the future. - # Prune images used by --external containers (e.g. build containers) when - # using podman. - echo "Prune all dangling images" - podman image prune --force --external - fi - echo "Prune all dangling $CI_IMAGE_LABEL images" - # When detecting podman-docker, `--external` should be added. - docker image prune --force --filter "label=$CI_IMAGE_LABEL" - - # shellcheck disable=SC2086 - CI_CONTAINER_ID=$(docker run --cap-add LINUX_IMMUTABLE $CI_CONTAINER_CAP --rm --interactive --detach --tty \ - --mount "type=bind,src=$BASE_READ_ONLY_DIR,dst=$BASE_READ_ONLY_DIR,readonly" \ - --mount "${CI_CCACHE_MOUNT}" \ - --mount "${CI_DEPENDS_MOUNT}" \ - --mount "${CI_DEPENDS_SOURCES_MOUNT}" \ - --mount "${CI_PREVIOUS_RELEASES_MOUNT}" \ - ${CI_BUILD_MOUNT} \ - --env-file /tmp/env-$USER-$CONTAINER_NAME \ - --name "$CONTAINER_NAME" \ - --network ci-ip6net \ - --platform="${CI_IMAGE_PLATFORM}" \ - "$CONTAINER_NAME") - export CI_CONTAINER_ID export CI_EXEC_CMD_PREFIX="docker exec ${CI_CONTAINER_ID}" else echo "Running on host system without docker wrapper" @@ -107,11 +17,6 @@ else mkdir -p "${PREVIOUS_RELEASES_DIR}" fi -if [ "$CI_OS_NAME" == "macos" ]; then - IN_GETOPT_BIN="$(brew --prefix gnu-getopt)/bin/getopt" - export IN_GETOPT_BIN -fi - CI_EXEC () { $CI_EXEC_CMD_PREFIX bash -c "export PATH=\"/path_with space:${BINS_SCRATCH_DIR}:${BASE_ROOT_DIR}/ci/retry:\$PATH\" && cd \"${BASE_ROOT_DIR}\" && $*" } diff --git a/bitcoinkernel/bitcoin/ci/test/03_test_script.sh b/bitcoinkernel/bitcoin/ci/test/03_test_script.sh index cee09cd..09da802 100755 --- a/bitcoinkernel/bitcoin/ci/test/03_test_script.sh +++ b/bitcoinkernel/bitcoin/ci/test/03_test_script.sh @@ -41,7 +41,10 @@ echo "=== BEGIN env ===" env echo "=== END env ===" -( +# Don't apply patches in the tidy job, because it relies on the `git diff` +# command to detect IWYU errors. It is safe to skip this patch in the tidy job +# because it doesn't run a UB detector. +if [ "$RUN_TIDY" != "true" ]; then # compact->outputs[i].file_size is uninitialized memory, so reading it is UB. # The statistic bytes_written is only used for logging, which is disabled in # CI, so as a temporary minimal fix to work around UB and CI failures, leave @@ -62,7 +65,7 @@ echo "=== END env ===" mutex_.Lock(); stats_[compact->compaction->level() + 1].Add(stats); EOF -) +fi if [ "$RUN_FUZZ_TESTS" = "true" ]; then export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_corpora/ @@ -99,8 +102,8 @@ else fi if [ -z "$NO_DEPENDS" ]; then - if [[ $CI_IMAGE_NAME_TAG == *centos* ]]; then - SHELL_OPTS="CONFIG_SHELL=/bin/dash" + if [[ $CI_IMAGE_NAME_TAG == *alpine* ]]; then + SHELL_OPTS="CONFIG_SHELL=/usr/bin/dash" else SHELL_OPTS="CONFIG_SHELL=" fi @@ -211,14 +214,30 @@ if [ "${RUN_TIDY}" = "true" ]; then false fi + # TODO: Consider enforcing IWYU across the entire codebase. + FILES_WITH_ENFORCED_IWYU="/src/(crypto|index)/.*\\.cpp" + jq --arg patterns "$FILES_WITH_ENFORCED_IWYU" 'map(select(.file | test($patterns)))' "${BASE_BUILD_DIR}/compile_commands.json" > "${BASE_BUILD_DIR}/compile_commands_iwyu_errors.json" + jq --arg patterns "$FILES_WITH_ENFORCED_IWYU" 'map(select(.file | test($patterns) | not))' "${BASE_BUILD_DIR}/compile_commands.json" > "${BASE_BUILD_DIR}/compile_commands_iwyu_warnings.json" + cd "${BASE_ROOT_DIR}" - python3 "/include-what-you-use/iwyu_tool.py" \ - -p "${BASE_BUILD_DIR}" "${MAKEJOBS}" \ - -- -Xiwyu --cxx17ns -Xiwyu --mapping_file="${BASE_ROOT_DIR}/contrib/devtools/iwyu/bitcoin.core.imp" \ - -Xiwyu --max_line_length=160 \ - 2>&1 | tee /tmp/iwyu_ci.out - cd "${BASE_ROOT_DIR}/src" - python3 "/include-what-you-use/fix_includes.py" --nosafe_headers < /tmp/iwyu_ci.out + + run_iwyu() { + mv "${BASE_BUILD_DIR}/$1" "${BASE_BUILD_DIR}/compile_commands.json" + python3 "/include-what-you-use/iwyu_tool.py" \ + -p "${BASE_BUILD_DIR}" "${MAKEJOBS}" \ + -- -Xiwyu --cxx17ns -Xiwyu --mapping_file="${BASE_ROOT_DIR}/contrib/devtools/iwyu/bitcoin.core.imp" \ + -Xiwyu --max_line_length=160 \ + 2>&1 | tee /tmp/iwyu_ci.out + python3 "/include-what-you-use/fix_includes.py" --nosafe_headers < /tmp/iwyu_ci.out + } + + run_iwyu "compile_commands_iwyu_errors.json" + if ! ( git --no-pager diff --exit-code ); then + echo "^^^ ⚠️ Failure generated from IWYU" + false + fi + + run_iwyu "compile_commands_iwyu_warnings.json" git --no-pager diff fi diff --git a/bitcoinkernel/bitcoin/ci/test_imagefile b/bitcoinkernel/bitcoin/ci/test_imagefile index f9cf318..a0e1714 100644 --- a/bitcoinkernel/bitcoin/ci/test_imagefile +++ b/bitcoinkernel/bitcoin/ci/test_imagefile @@ -17,4 +17,7 @@ ENV BASE_ROOT_DIR=${BASE_ROOT_DIR} COPY ./ci/retry/retry /usr/bin/retry COPY ./ci/test/00_setup_env.sh ./${FILE_ENV} ./ci/test/01_base_install.sh /ci_container_base/ci/test/ +# Bash is required, so install it when missing +RUN sh -c "bash -c 'true' || ( apk update && apk add --no-cache bash )" + RUN ["bash", "-c", "cd /ci_container_base/ && set -o errexit && source ./ci/test/00_setup_env.sh && ./ci/test/01_base_install.sh"] diff --git a/bitcoinkernel/bitcoin/cmake/windows-app.manifest.in b/bitcoinkernel/bitcoin/cmake/windows-app.manifest.in index c3bd333..e0dc2eb 100644 --- a/bitcoinkernel/bitcoin/cmake/windows-app.manifest.in +++ b/bitcoinkernel/bitcoin/cmake/windows-app.manifest.in @@ -1,10 +1,15 @@ - + + + + UTF-8 + + diff --git a/bitcoinkernel/bitcoin/contrib/devtools/README.md b/bitcoinkernel/bitcoin/contrib/devtools/README.md index 366cd4a..3309057 100644 --- a/bitcoinkernel/bitcoin/contrib/devtools/README.md +++ b/bitcoinkernel/bitcoin/contrib/devtools/README.md @@ -137,7 +137,7 @@ BUILDDIR=$PWD/my-build-dir contrib/devtools/gen-manpages.py headerssync-params.py ===================== -A script to generate optimal parameters for the headerssync module (src/headerssync.cpp). It takes no command-line +A script to generate optimal parameters for the headerssync module (stored in src/kernel/chainparams.cpp). It takes no command-line options, as all its configuration is set at the top of the file. It runs many times faster inside PyPy. Invocation: ```bash diff --git a/bitcoinkernel/bitcoin/contrib/devtools/deterministic-fuzz-coverage/src/main.rs b/bitcoinkernel/bitcoin/contrib/devtools/deterministic-fuzz-coverage/src/main.rs index 3eeb121..362ab69 100644 --- a/bitcoinkernel/bitcoin/contrib/devtools/deterministic-fuzz-coverage/src/main.rs +++ b/bitcoinkernel/bitcoin/contrib/devtools/deterministic-fuzz-coverage/src/main.rs @@ -36,7 +36,7 @@ fn sanity_check(corpora_dir: &Path, fuzz_exe: &Path) -> AppResult { let output = Command::new(tool).arg("--help").output(); match output { Ok(output) if output.status.success() => {} - _ => Err(exit_help(&format!("The tool {} is not installed", tool)))?, + _ => Err(exit_help(&format!("The tool {tool} is not installed")))?, } } if !corpora_dir.is_dir() { @@ -274,7 +274,7 @@ fn main() -> ExitCode { match app() { Ok(()) => ExitCode::SUCCESS, Err(err) => { - eprintln!("⚠️\n{}", err); + eprintln!("⚠️\n{err}"); ExitCode::FAILURE } } diff --git a/bitcoinkernel/bitcoin/contrib/devtools/deterministic-unittest-coverage/src/main.rs b/bitcoinkernel/bitcoin/contrib/devtools/deterministic-unittest-coverage/src/main.rs index 047c8d2..95d2c08 100644 --- a/bitcoinkernel/bitcoin/contrib/devtools/deterministic-unittest-coverage/src/main.rs +++ b/bitcoinkernel/bitcoin/contrib/devtools/deterministic-unittest-coverage/src/main.rs @@ -32,7 +32,7 @@ fn sanity_check(test_exe: &Path) -> AppResult { let output = Command::new(tool).arg("--help").output(); match output { Ok(output) if output.status.success() => {} - _ => Err(exit_help(&format!("The tool {} is not installed", tool)))?, + _ => Err(exit_help(&format!("The tool {tool} is not installed")))?, } } if !test_exe.exists() { @@ -142,7 +142,7 @@ fn main() -> ExitCode { match app() { Ok(()) => ExitCode::SUCCESS, Err(err) => { - eprintln!("⚠️\n{}", err); + eprintln!("⚠️\n{err}"); ExitCode::FAILURE } } diff --git a/bitcoinkernel/bitcoin/contrib/devtools/headerssync-params.py b/bitcoinkernel/bitcoin/contrib/devtools/headerssync-params.py index ece1a78..82bdaad 100755 --- a/bitcoinkernel/bitcoin/contrib/devtools/headerssync-params.py +++ b/bitcoinkernel/bitcoin/contrib/devtools/headerssync-params.py @@ -5,8 +5,8 @@ """Script to find the optimal parameters for the headerssync module through simulation.""" -from math import log, exp, sqrt from datetime import datetime, timedelta +from math import log, exp, sqrt import random # Parameters: @@ -337,15 +337,15 @@ def analyze(when): attack_volume = NET_HEADER_SIZE * MINCHAINWORK_HEADERS # And report them. print() - print("Optimal configuration:") - print() - print("//! Store one header commitment per HEADER_COMMITMENT_PERIOD blocks.") - print(f"constexpr size_t HEADER_COMMITMENT_PERIOD{{{period}}};") + print(f"Given current min chainwork headers of {MINCHAINWORK_HEADERS}, the optimal parameters for low") + print(f"memory usage on mainchain for release until {TIME:%Y-%m-%d} is:") print() - print("//! Only feed headers to validation once this many headers on top have been") - print("//! received and validated against commitments.") - print(f"constexpr size_t REDOWNLOAD_BUFFER_SIZE{{{bufsize}}};" + print(f" // Generated by headerssync-params.py on {datetime.today():%Y-%m-%d}.") + print( " m_headers_sync_params = HeadersSyncParams{") + print(f" .commitment_period = {period},") + print(f" .redownload_buffer_size = {bufsize}," f" // {bufsize}/{period} = ~{bufsize/period:.1f} commitments") + print( " };") print() print("Properties:") print(f"- Per-peer memory for mainchain sync: {mem_mainchain / 8192:.3f} KiB") diff --git a/bitcoinkernel/bitcoin/contrib/devtools/iwyu/bitcoin.core.imp b/bitcoinkernel/bitcoin/contrib/devtools/iwyu/bitcoin.core.imp index c4c4ba4..c3195e5 100644 --- a/bitcoinkernel/bitcoin/contrib/devtools/iwyu/bitcoin.core.imp +++ b/bitcoinkernel/bitcoin/contrib/devtools/iwyu/bitcoin.core.imp @@ -1,3 +1,16 @@ -# Nothing for now. [ + # Compiler intrinsics. + # See: https://github.com/include-what-you-use/include-what-you-use/issues/1764. + { "include": [ "", "private", "", "public" ] }, + { "include": [ "", "private", "", "public" ] }, + { "include": [ "", "private", "", "public" ] }, + + # libc symbols. + { "symbol": ["AT_HWCAP", "private", "", "public"] }, + { "symbol": ["AT_HWCAP2", "private", "", "public"] }, + + # Fixed in https://github.com/include-what-you-use/include-what-you-use/pull/1706. + { "symbol": ["SEEK_CUR", "private", "", "public"] }, + { "symbol": ["SEEK_END", "private", "", "public"] }, + { "symbol": ["SEEK_SET", "private", "", "public"] }, ] diff --git a/bitcoinkernel/bitcoin/contrib/guix/INSTALL.md b/bitcoinkernel/bitcoin/contrib/guix/INSTALL.md index f9a79f6..833d540 100644 --- a/bitcoinkernel/bitcoin/contrib/guix/INSTALL.md +++ b/bitcoinkernel/bitcoin/contrib/guix/INSTALL.md @@ -64,14 +64,12 @@ Please refer to fanquake's instructions ## Option 4: Using a distribution-maintained package -Note that this section is based on the distro packaging situation at the time of -writing (July 2021). Guix is expected to be more widely packaged over time. For -an up-to-date view on Guix's package status/version across distros, please see: -https://repology.org/project/guix/versions +For an up-to-date view on Guix's package status/version across +distros, please see: https://repology.org/project/guix/versions ### Debian / Ubuntu -Guix is available as a distribution package in [Debian +Guix is available as a distribution package in various versions of [Debian ](https://packages.debian.org/search?keywords=guix) and [Ubuntu ](https://packages.ubuntu.com/search?keywords=guix). diff --git a/bitcoinkernel/bitcoin/contrib/guix/libexec/build.sh b/bitcoinkernel/bitcoin/contrib/guix/libexec/build.sh index bbf6c64..09cc058 100755 --- a/bitcoinkernel/bitcoin/contrib/guix/libexec/build.sh +++ b/bitcoinkernel/bitcoin/contrib/guix/libexec/build.sh @@ -206,7 +206,7 @@ mkdir -p "$OUTDIR" ########################### # CONFIGFLAGS -CONFIGFLAGS="-DREDUCE_EXPORTS=ON -DBUILD_BENCH=OFF -DBUILD_GUI_TESTS=OFF -DBUILD_FUZZ_BINARY=OFF" +CONFIGFLAGS="-DREDUCE_EXPORTS=ON -DBUILD_BENCH=OFF -DBUILD_GUI_TESTS=OFF -DBUILD_FUZZ_BINARY=OFF -DCMAKE_SKIP_RPATH=TRUE" # CFLAGS HOST_CFLAGS="-O2 -g" @@ -225,10 +225,15 @@ esac # LDFLAGS case "$HOST" in - *linux*) HOST_LDFLAGS="-Wl,--as-needed -Wl,--dynamic-linker=$glibc_dynamic_linker -static-libstdc++ -Wl,-O2" ;; + *linux*) HOST_LDFLAGS="-Wl,--as-needed -Wl,--dynamic-linker=$glibc_dynamic_linker -Wl,-O2" ;; *mingw*) HOST_LDFLAGS="-Wl,--no-insert-timestamp" ;; esac +# EXE FLAGS +case "$HOST" in + *linux*) CMAKE_EXE_LINKER_FLAGS="-DCMAKE_EXE_LINKER_FLAGS=${HOST_LDFLAGS} -static-libstdc++ -static-libgcc" ;; +esac + mkdir -p "$DISTSRC" ( cd "$DISTSRC" @@ -243,7 +248,8 @@ mkdir -p "$DISTSRC" --toolchain "${BASEPREFIX}/${HOST}/toolchain.cmake" \ -DWITH_CCACHE=OFF \ -Werror=dev \ - ${CONFIGFLAGS} + ${CONFIGFLAGS} \ + "${CMAKE_EXE_LINKER_FLAGS}" # Build Bitcoin Core cmake --build build -j "$JOBS" ${V:+--verbose} diff --git a/bitcoinkernel/bitcoin/contrib/guix/libexec/prelude.bash b/bitcoinkernel/bitcoin/contrib/guix/libexec/prelude.bash index d25c371..b7c13cc 100644 --- a/bitcoinkernel/bitcoin/contrib/guix/libexec/prelude.bash +++ b/bitcoinkernel/bitcoin/contrib/guix/libexec/prelude.bash @@ -71,7 +71,7 @@ fi time-machine() { # shellcheck disable=SC2086 guix time-machine --url=https://codeberg.org/guix/guix.git \ - --commit=53396a22afc04536ddf75d8f82ad2eafa5082725 \ + --commit=5cb84f2013c5b1e48a7d0e617032266f1e6059e2 \ --cores="$JOBS" \ --keep-failed \ --fallback \ diff --git a/bitcoinkernel/bitcoin/contrib/guix/manifest.scm b/bitcoinkernel/bitcoin/contrib/guix/manifest.scm index a230eea..011ba0d 100644 --- a/bitcoinkernel/bitcoin/contrib/guix/manifest.scm +++ b/bitcoinkernel/bitcoin/contrib/guix/manifest.scm @@ -18,7 +18,7 @@ ((gnu packages python-build) #:select (python-poetry-core)) ((gnu packages python-crypto) #:select (python-asn1crypto)) ((gnu packages python-science) #:select (python-scikit-build-core)) - ((gnu packages python-xyz) #:select (python-pydantic-2 python-pydantic-core)) + ((gnu packages python-xyz) #:select (python-pydantic-2)) ((gnu packages tls) #:select (openssl)) ((gnu packages version-control) #:select (git-minimal)) (guix build-system cmake) @@ -99,10 +99,10 @@ chain for " target " development.")) (define* (make-bitcoin-cross-toolchain target #:key - (base-gcc-for-libc linux-base-gcc) + (base-gcc-for-libc (gcc-libgcc-patches linux-base-gcc)) (base-kernel-headers base-linux-kernel-headers) (base-libc glibc-2.31) - (base-gcc linux-base-gcc)) + (base-gcc (gcc-libgcc-patches linux-base-gcc))) "Convenience wrapper around MAKE-CROSS-TOOLCHAIN with default values desirable for building Bitcoin Core release binaries." (make-cross-toolchain target @@ -111,7 +111,7 @@ desirable for building Bitcoin Core release binaries." base-libc base-gcc)) -(define (gcc-mingw-patches gcc) +(define (gcc-libgcc-patches gcc) (package-with-extra-patches gcc (search-our-patches "gcc-remap-guix-store.patch"))) @@ -128,10 +128,10 @@ desirable for building Bitcoin Core release binaries." (let* ((xbinutils (binutils-mingw-patches (cross-binutils target))) (machine (substring target 0 (string-index target #\-))) (pthreads-xlibc (winpthreads-patches (make-mingw-w64 machine - #:xgcc (cross-gcc target #:xgcc (gcc-mingw-patches base-gcc)) + #:xgcc (cross-gcc target #:xgcc (gcc-libgcc-patches base-gcc)) #:with-winpthreads? #t))) (pthreads-xgcc (cross-gcc target - #:xgcc (gcc-mingw-patches mingw-w64-base-gcc) + #:xgcc (gcc-libgcc-patches mingw-w64-base-gcc) #:xbinutils xbinutils #:libc pthreads-xlibc))) ;; Define a meta-package that propagates the resulting XBINUTILS, XLIBC, and @@ -175,7 +175,6 @@ chain for " target " development.")) (native-inputs (list cmake-minimal ninja python-scikit-build-core - python-pydantic-core python-pydantic-2)) (arguments (list @@ -439,6 +438,7 @@ inspecting signatures in Mach-O binaries.") "--enable-standard-branch-protection=yes", "--enable-cet=yes", "--disable-gcov", + "--disable-libsanitizer", building-on))) ((#:phases phases) `(modify-phases ,phases @@ -455,7 +455,7 @@ inspecting signatures in Mach-O binaries.") (define-public glibc-2.31 (let ((commit "7b27c450c34563a28e634cccb399cd415e71ebfe")) (package - (inherit glibc) ;; 2.35 + (inherit glibc) ;; 2.39 (version "2.31") (source (origin (method git-fetch) @@ -466,7 +466,8 @@ inspecting signatures in Mach-O binaries.") (sha256 (base32 "017qdpr5id7ddb4lpkzj2li1abvw916m3fc6n7nw28z4h5qbv2n0")) - (patches (search-our-patches "glibc-guix-prefix.patch")))) + (patches (search-our-patches "glibc-guix-prefix.patch" + "glibc-riscv-jumptarget.patch")))) (arguments (substitute-keyword-arguments (package-arguments glibc) ((#:configure-flags flags) @@ -566,9 +567,9 @@ inspecting signatures in Mach-O binaries.") (list gcc-toolchain-13 "static") (make-bitcoin-cross-toolchain target))) ((string-contains target "darwin") - (list clang-toolchain-18 - lld-18 - (make-lld-wrapper lld-18 #:lld-as-ld? #t) + (list clang-toolchain-19 + lld-19 + (make-lld-wrapper lld-19 #:lld-as-ld? #t) python-signapple zip)) (else '()))))) diff --git a/bitcoinkernel/bitcoin/contrib/guix/patches/gcc-remap-guix-store.patch b/bitcoinkernel/bitcoin/contrib/guix/patches/gcc-remap-guix-store.patch index a8b41d4..4144bcd 100644 --- a/bitcoinkernel/bitcoin/contrib/guix/patches/gcc-remap-guix-store.patch +++ b/bitcoinkernel/bitcoin/contrib/guix/patches/gcc-remap-guix-store.patch @@ -1,4 +1,4 @@ -Without ffile-prefix-map, the debug symbols will contain paths for the +Without -ffile-prefix-map, the debug symbols will contain paths for the guix store which will include the hashes of each package. However, the hash for the same package will differ when on different architectures. In order to be reproducible regardless of the architecture used to build @@ -6,7 +6,7 @@ the package, map all guix store prefixes to something fixed, e.g. /usr. --- a/libgcc/Makefile.in +++ b/libgcc/Makefile.in -@@ -854,7 +854,7 @@ endif +@@ -857,7 +857,7 @@ endif # libgcc_eh.a, only LIB2ADDEH matters. If we do, only LIB2ADDEHSTATIC and # LIB2ADDEHSHARED matter. (Usually all three are identical.) @@ -15,6 +15,15 @@ the package, map all guix store prefixes to something fixed, e.g. /usr. ifeq ($(enable_shared),yes) +@@ -880,7 +880,7 @@ endif + # Build LIBUNWIND. Use -fno-exceptions so that the unwind library does + # not generate calls to __gcc_personality_v0. + +-c_flags := -fno-exceptions ++c_flags := -fno-exceptions $(shell find /gnu/store -maxdepth 1 -mindepth 1 -type d -exec echo -n " -ffile-prefix-map={}=/usr" \;) + + libunwind-objects += $(addsuffix $(objext),$(basename $(notdir $(LIBUNWIND)))) + -- 2.37.0 diff --git a/bitcoinkernel/bitcoin/contrib/guix/patches/glibc-riscv-jumptarget.patch b/bitcoinkernel/bitcoin/contrib/guix/patches/glibc-riscv-jumptarget.patch new file mode 100644 index 0000000..7029594 --- /dev/null +++ b/bitcoinkernel/bitcoin/contrib/guix/patches/glibc-riscv-jumptarget.patch @@ -0,0 +1,57 @@ +commit 68389203832ab39dd0dbaabbc4059e7fff51c29b +Author: Fangrui Song +Date: Thu Oct 28 11:39:49 2021 -0700 + + riscv: Fix incorrect jal with HIDDEN_JUMPTARGET + + A non-local STV_DEFAULT defined symbol is by default preemptible in a + shared object. j/jal cannot target a preemptible symbol. On other + architectures, such a jump instruction either causes PLT [BZ #18822], or + if short-ranged, sometimes rejected by the linker (but not by GNU ld's + riscv port [ld PR/28509]). + + Use HIDDEN_JUMPTARGET to target a non-preemptible symbol instead. + + With this patch, ld.so and libc.so can be linked with LLD if source + files are compiled/assembled with -mno-relax/-Wa,-mno-relax. + + Acked-by: Palmer Dabbelt + Reviewed-by: Adhemerval Zanella + +Can be dropped when we are using glibc 2.35 or later. + +diff --git a/sysdeps/riscv/setjmp.S b/sysdeps/riscv/setjmp.S +index 0b92016b31..bec7ff80f4 100644 +--- a/sysdeps/riscv/setjmp.S ++++ b/sysdeps/riscv/setjmp.S +@@ -21,7 +21,7 @@ + + ENTRY (_setjmp) + li a1, 0 +- j __sigsetjmp ++ j HIDDEN_JUMPTARGET (__sigsetjmp) + END (_setjmp) + ENTRY (setjmp) + li a1, 1 +diff --git a/sysdeps/unix/sysv/linux/riscv/setcontext.S b/sysdeps/unix/sysv/linux/riscv/setcontext.S +index 9510518750..e44a68aad4 100644 +--- a/sysdeps/unix/sysv/linux/riscv/setcontext.S ++++ b/sysdeps/unix/sysv/linux/riscv/setcontext.S +@@ -95,6 +95,7 @@ LEAF (__setcontext) + 99: j __syscall_error + + END (__setcontext) ++libc_hidden_def (__setcontext) + weak_alias (__setcontext, setcontext) + + LEAF (__start_context) +@@ -108,7 +109,7 @@ LEAF (__start_context) + /* Invoke subsequent context if present, else exit(0). */ + mv a0, s2 + beqz s2, 1f +- jal __setcontext +-1: j exit ++ jal HIDDEN_JUMPTARGET (__setcontext) ++1: j HIDDEN_JUMPTARGET (exit) + + END (__start_context) diff --git a/bitcoinkernel/bitcoin/contrib/guix/symbol-check.py b/bitcoinkernel/bitcoin/contrib/guix/symbol-check.py index 7804a2c..249c53a 100755 --- a/bitcoinkernel/bitcoin/contrib/guix/symbol-check.py +++ b/bitcoinkernel/bitcoin/contrib/guix/symbol-check.py @@ -16,23 +16,18 @@ # Debian 11 (Bullseye) EOL: 2026. https://wiki.debian.org/LTS # -# - libgcc version 10.2.1 (https://packages.debian.org/bullseye/libgcc-s1) # - libc version 2.31 (https://packages.debian.org/source/bullseye/glibc) # # Ubuntu 20.04 (Focal) EOL: 2030. https://wiki.ubuntu.com/ReleaseTeam # -# - libgcc version 10.5.0 (https://packages.ubuntu.com/focal/libgcc1) # - libc version 2.31 (https://packages.ubuntu.com/focal/libc6) # # CentOS Stream 9 EOL: 2027. https://www.centos.org/cl-vs-cs/#end-of-life # -# - libgcc version 12.2.1 (https://mirror.stream.centos.org/9-stream/AppStream/x86_64/os/Packages/) # - libc version 2.34 (https://mirror.stream.centos.org/9-stream/AppStream/x86_64/os/Packages/) # -# See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html for more info. MAX_VERSIONS = { -'GCC': (7,0,0), 'GLIBC': { lief.ELF.ARCH.X86_64: (2,31), lief.ELF.ARCH.ARM: (2,31), @@ -40,7 +35,6 @@ lief.ELF.ARCH.PPC64: (2,31), lief.ELF.ARCH.RISCV: (2,31), }, -'LIBATOMIC': (1,0), 'V': (0,5,0), # xkb (bitcoin-qt only) } @@ -93,11 +87,9 @@ # Allowed NEEDED libraries ELF_ALLOWED_LIBRARIES = { # bitcoind and bitcoin-qt -'libgcc_s.so.1', # GCC base support 'libc.so.6', # C library 'libpthread.so.0', # threading 'libm.so.6', # math library -'libatomic.so.1', 'ld-linux-x86-64.so.2', # 64-bit dynamic linker 'ld-linux.so.2', # 32-bit dynamic linker 'ld-linux-aarch64.so.1', # 64-bit ARM dynamic linker @@ -258,7 +250,7 @@ def check_MACHO_sdk(binary) -> bool: return False def check_MACHO_lld(binary) -> bool: - if binary.build_version.tools[0].version == [18, 1, 8]: + if binary.build_version.tools[0].version == [19, 1, 4]: return True return False diff --git a/bitcoinkernel/bitcoin/depends/Makefile b/bitcoinkernel/bitcoin/depends/Makefile index ed54eac..fde108d 100644 --- a/bitcoinkernel/bitcoin/depends/Makefile +++ b/bitcoinkernel/bitcoin/depends/Makefile @@ -1,4 +1,6 @@ .NOTPARALLEL : +# Disable builtin variables, rules and suffixes. +MAKEFLAGS += --no-builtin-rules --no-builtin-variables # Pattern rule to print variables, e.g. make print-top_srcdir print-%: FORCE diff --git a/bitcoinkernel/bitcoin/depends/README.md b/bitcoinkernel/bitcoin/depends/README.md index 9457910..71110e9 100644 --- a/bitcoinkernel/bitcoin/depends/README.md +++ b/bitcoinkernel/bitcoin/depends/README.md @@ -123,7 +123,7 @@ Common `host-platform-triplet`s for cross compilation are: - `i686-pc-linux-gnu` for Linux x86 32 bit - `x86_64-pc-linux-gnu` for Linux x86 64 bit - `x86_64-w64-mingw32` for Win64 -- `x86_64-apple-darwin` for macOS +- `x86_64-apple-darwin` for Intel macOS - `arm64-apple-darwin` for ARM macOS - `arm-linux-gnueabihf` for Linux ARM 32 bit - `aarch64-linux-gnu` for Linux ARM 64 bit diff --git a/bitcoinkernel/bitcoin/depends/funcs.mk b/bitcoinkernel/bitcoin/depends/funcs.mk index 6c38334..60e7889 100644 --- a/bitcoinkernel/bitcoin/depends/funcs.mk +++ b/bitcoinkernel/bitcoin/depends/funcs.mk @@ -37,7 +37,7 @@ endef define fetch_file ( $(call fetch_file_inner,$(1),$(2),$(3),$(4),$(5)) || \ - $(call fetch_file_inner,$(1),$(FALLBACK_DOWNLOAD_PATH),$(3),$(4),$(5))) + $(call fetch_file_inner,$(1),$(FALLBACK_DOWNLOAD_PATH),$(4),$(4),$(5))) endef # Shell script to create a source tarball in $(1)_source from local directory @@ -108,7 +108,7 @@ $(1)_prefixbin:=$($($(1)_type)_prefix)/bin/ $(1)_all_sources=$($(1)_file_name) $($(1)_extra_sources) #stamps -$(1)_fetched=$(SOURCES_PATH)/download-stamps/.stamp_fetched-$(1)-$($(1)_version)-$($(1)_sha256_hash).hash +$(1)_fetched=$(SOURCES_PATH)/download-stamps/.stamp_fetched-$(1)-$($(1)_file_name).hash $(1)_extracted=$$($(1)_extract_dir)/.stamp_extracted $(1)_preprocessed=$$($(1)_extract_dir)/.stamp_preprocessed $(1)_cleaned=$$($(1)_extract_dir)/.stamp_cleaned diff --git a/bitcoinkernel/bitcoin/depends/packages/libevent.mk b/bitcoinkernel/bitcoin/depends/packages/libevent.mk index e9153b7..33eeed5 100644 --- a/bitcoinkernel/bitcoin/depends/packages/libevent.mk +++ b/bitcoinkernel/bitcoin/depends/packages/libevent.mk @@ -16,7 +16,7 @@ define $(package)_set_vars $(package)_config_opts+=-DEVENT__DISABLE_SAMPLES=ON -DEVENT__DISABLE_REGRESS=ON $(package)_config_opts+=-DEVENT__DISABLE_TESTS=ON -DEVENT__LIBRARY_TYPE=STATIC $(package)_cflags += -fdebug-prefix-map=$($(package)_extract_dir)=/usr -fmacro-prefix-map=$($(package)_extract_dir)=/usr - $(package)_cppflags += -D_GNU_SOURCE -D_FORTIFY_SOURCE=3 + $(package)_cppflags += -D_GNU_SOURCE -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=3 $(package)_cppflags_mingw32=-D_WIN32_WINNT=0x0A00 endef diff --git a/bitcoinkernel/bitcoin/depends/packages/native_qt.mk b/bitcoinkernel/bitcoin/depends/packages/native_qt.mk index 38469d6..3428a1f 100644 --- a/bitcoinkernel/bitcoin/depends/packages/native_qt.mk +++ b/bitcoinkernel/bitcoin/depends/packages/native_qt.mk @@ -6,7 +6,6 @@ $(package)_file_name=$(qt_details_qtbase_file_name) $(package)_sha256_hash=$(qt_details_qtbase_sha256_hash) $(package)_patches_path := $(qt_details_patches_path) $(package)_patches := dont_hardcode_pwd.patch -$(package)_patches += qtbase-moc-ignore-gcc-macro.patch $(package)_patches += qtbase_skip_tools.patch $(package)_patches += rcc_hardcode_timestamp.patch $(package)_patches += qttools_skip_dependencies.patch @@ -131,7 +130,6 @@ endef define $(package)_preprocess_cmds patch -p1 -i $($(package)_patch_dir)/dont_hardcode_pwd.patch && \ - patch -p1 -i $($(package)_patch_dir)/qtbase-moc-ignore-gcc-macro.patch && \ patch -p1 -i $($(package)_patch_dir)/qtbase_skip_tools.patch && \ patch -p1 -i $($(package)_patch_dir)/rcc_hardcode_timestamp.patch && \ patch -p1 -i $($(package)_patch_dir)/qttools_skip_dependencies.patch diff --git a/bitcoinkernel/bitcoin/depends/packages/qrencode.mk b/bitcoinkernel/bitcoin/depends/packages/qrencode.mk index 9aae62f..e3f6140 100644 --- a/bitcoinkernel/bitcoin/depends/packages/qrencode.mk +++ b/bitcoinkernel/bitcoin/depends/packages/qrencode.mk @@ -1,9 +1,8 @@ package=qrencode $(package)_version=4.1.1 -$(package)_download_path=https://github.com/fukuchi/libqrencode/archive/refs/tags/ -$(package)_download_file=v$($(package)_version).tar.gz +$(package)_download_path=https://fukuchi.org/works/qrencode/ $(package)_file_name=$(package)-$($(package)_version).tar.gz -$(package)_sha256_hash=5385bc1b8c2f20f3b91d258bf8ccc8cf62023935df2d2676b5b67049f31a049c +$(package)_sha256_hash=da448ed4f52aba6bcb0cd48cac0dd51b8692bccc4cd127431402fca6f8171e8e $(package)_patches=cmake_fixups.patch define $(package)_set_vars diff --git a/bitcoinkernel/bitcoin/depends/packages/qt.mk b/bitcoinkernel/bitcoin/depends/packages/qt.mk index 9ac4088..4a564c4 100644 --- a/bitcoinkernel/bitcoin/depends/packages/qt.mk +++ b/bitcoinkernel/bitcoin/depends/packages/qt.mk @@ -11,8 +11,6 @@ $(package)_linux_dependencies := freetype fontconfig libxcb libxkbcommon libxcb_ $(package)_freebsd_dependencies := $($(package)_linux_dependencies) $(package)_patches_path := $(qt_details_patches_path) $(package)_patches := dont_hardcode_pwd.patch -$(package)_patches += qtbase-moc-ignore-gcc-macro.patch -$(package)_patches += qtbase_avoid_native_float16.patch $(package)_patches += qtbase_avoid_qmain.patch $(package)_patches += qtbase_platformsupport.patch $(package)_patches += qtbase_plugins_cocoa.patch @@ -256,8 +254,6 @@ endif define $(package)_preprocess_cmds patch -p1 -i $($(package)_patch_dir)/dont_hardcode_pwd.patch && \ - patch -p1 -i $($(package)_patch_dir)/qtbase-moc-ignore-gcc-macro.patch && \ - patch -p1 -i $($(package)_patch_dir)/qtbase_avoid_native_float16.patch && \ patch -p1 -i $($(package)_patch_dir)/qtbase_avoid_qmain.patch && \ patch -p1 -i $($(package)_patch_dir)/qtbase_platformsupport.patch && \ patch -p1 -i $($(package)_patch_dir)/qtbase_plugins_cocoa.patch && \ diff --git a/bitcoinkernel/bitcoin/depends/patches/qt/memory_resource.patch b/bitcoinkernel/bitcoin/depends/patches/qt/memory_resource.patch deleted file mode 100644 index 14e2512..0000000 --- a/bitcoinkernel/bitcoin/depends/patches/qt/memory_resource.patch +++ /dev/null @@ -1,16 +0,0 @@ -Fix unusable memory_resource on macos - -See https://bugreports.qt.io/browse/QTBUG-117484 -and https://bugreports.qt.io/browse/QTBUG-114316 - ---- a/qtbase/src/corelib/tools/qduplicatetracker_p.h -+++ b/qtbase/src/corelib/tools/qduplicatetracker_p.h -@@ -52,7 +52,7 @@ - - #include - --#if QT_HAS_INCLUDE() && __cplusplus > 201402L -+#ifdef __cpp_lib_memory_resource - # include - # include - #else diff --git a/bitcoinkernel/bitcoin/depends/patches/qt/qtbase-moc-ignore-gcc-macro.patch b/bitcoinkernel/bitcoin/depends/patches/qt/qtbase-moc-ignore-gcc-macro.patch deleted file mode 100644 index e5e7b12..0000000 --- a/bitcoinkernel/bitcoin/depends/patches/qt/qtbase-moc-ignore-gcc-macro.patch +++ /dev/null @@ -1,17 +0,0 @@ -The moc executable loops through headers on CPLUS_INCLUDE_PATH and stumbles -on the GCC internal _GLIBCXX_VISIBILITY macro. Tell it to ignore it as it is -not supposed to be looking there to begin with. - -Upstream report: https://bugreports.qt.io/browse/QTBUG-83160 - -diff --git a/qtbase/src/tools/moc/main.cpp b/qtbase/src/tools/moc/main.cpp ---- a/qtbase/src/tools/moc/main.cpp -+++ b/qtbase/src/tools/moc/main.cpp -@@ -186,6 +186,7 @@ int runMoc(int argc, char **argv) - dummyVariadicFunctionMacro.arguments += Symbol(0, PP_IDENTIFIER, "__VA_ARGS__"); - pp.macros["__attribute__"] = dummyVariadicFunctionMacro; - pp.macros["__declspec"] = dummyVariadicFunctionMacro; -+ pp.macros["_GLIBCXX_VISIBILITY"] = dummyVariadicFunctionMacro; - - QString filename; - QString output; diff --git a/bitcoinkernel/bitcoin/depends/patches/qt/qtbase_avoid_native_float16.patch b/bitcoinkernel/bitcoin/depends/patches/qt/qtbase_avoid_native_float16.patch deleted file mode 100644 index ec03d14..0000000 --- a/bitcoinkernel/bitcoin/depends/patches/qt/qtbase_avoid_native_float16.patch +++ /dev/null @@ -1,37 +0,0 @@ -Avoid using native float16 types in Qt's qfloat16 - -Using native float16 types may introduce compatibility issues -in release binaries for Linux platforms. - -See: https://gcc.gnu.org/git/?p=gcc.git;a=commit;h=e42d2d2a20f2bb59928bc895ec9f46503a1b5c73 - -This patch can be dropped once the minimum required libgcc version -is updated to 12.x or newer. - - ---- a/qtbase/src/corelib/global/qtypes.h -+++ b/qtbase/src/corelib/global/qtypes.h -@@ -258,23 +258,8 @@ using qsizetype = QIntegerForSizeof::Signed; - - // Define a native float16 type - namespace QtPrivate { --#if defined(__STDCPP_FLOAT16_T__) --# define QFLOAT16_IS_NATIVE 1 --using NativeFloat16Type = std::float16_t; --#elif defined(Q_CC_CLANG) && defined(__FLT16_MAX__) && 0 --// disabled due to https://github.com/llvm/llvm-project/issues/56963 --# define QFLOAT16_IS_NATIVE 1 --using NativeFloat16Type = decltype(__FLT16_MAX__); --#elif defined(Q_CC_GNU_ONLY) && defined(__FLT16_MAX__) && defined(__ARM_FP16_FORMAT_IEEE) --# define QFLOAT16_IS_NATIVE 1 --using NativeFloat16Type = __fp16; --#elif defined(Q_CC_GNU_ONLY) && defined(__FLT16_MAX__) && defined(__SSE2__) --# define QFLOAT16_IS_NATIVE 1 --using NativeFloat16Type = _Float16; --#else - # define QFLOAT16_IS_NATIVE 0 - using NativeFloat16Type = void; --#endif - } // QtPrivate - - #endif // __cplusplus diff --git a/bitcoinkernel/bitcoin/doc/build-freebsd.md b/bitcoinkernel/bitcoin/doc/build-freebsd.md index fee20f8..b5940c0 100644 --- a/bitcoinkernel/bitcoin/doc/build-freebsd.md +++ b/bitcoinkernel/bitcoin/doc/build-freebsd.md @@ -101,5 +101,5 @@ cmake -B build -DENABLE_WALLET=OFF ```bash cmake --build build # Append "-j N" for N parallel jobs. -ctest --test-dir build # Append "-j N" for N parallel tests. Some tests are disabled if Python 3 is not available. +ctest --test-dir build # Append "-j N" for N parallel tests. ``` diff --git a/bitcoinkernel/bitcoin/doc/build-netbsd.md b/bitcoinkernel/bitcoin/doc/build-netbsd.md index f263177..103e625 100644 --- a/bitcoinkernel/bitcoin/doc/build-netbsd.md +++ b/bitcoinkernel/bitcoin/doc/build-netbsd.md @@ -34,7 +34,7 @@ cmake -B build SQLite is required for the wallet: ```bash -pkgin sqlite3 +pkgin install sqlite3 ``` To build Bitcoin Core without the wallet, use `-DENABLE_WALLET=OFF`. @@ -42,7 +42,7 @@ To build Bitcoin Core without the wallet, use `-DENABLE_WALLET=OFF`. Cap'n Proto is needed for IPC functionality (see [multiprocess.md](multiprocess.md)): ```bash -pkgin capnproto +pkgin install capnproto ``` Compile with `-DENABLE_IPC=OFF` if you do not need IPC functionality. @@ -84,7 +84,7 @@ Otherwise, if you don't need QR encoding support, use the `-DWITH_QRENCODE=OFF` Bitcoin Core can provide notifications via ZeroMQ. If the package is installed, support will be compiled in. ```bash -pkgin zeromq +pkgin install zeromq ``` #### Test Suite Dependencies @@ -115,5 +115,5 @@ Build and run the tests: ```bash cmake --build build # Append "-j N" for N parallel jobs. -ctest --test-dir build # Append "-j N" for N parallel tests. Some tests are disabled if Python 3 is not available. +ctest --test-dir build # Append "-j N" for N parallel tests. ``` diff --git a/bitcoinkernel/bitcoin/doc/build-openbsd.md b/bitcoinkernel/bitcoin/doc/build-openbsd.md index 99d4ec2..fdeafe8 100644 --- a/bitcoinkernel/bitcoin/doc/build-openbsd.md +++ b/bitcoinkernel/bitcoin/doc/build-openbsd.md @@ -93,7 +93,7 @@ Run `cmake -B build -LH` to see the full list of available options. ```bash cmake --build build # Append "-j N" for N parallel jobs. -ctest --test-dir build # Append "-j N" for N parallel tests. Some tests are disabled if Python 3 is not available. +ctest --test-dir build # Append "-j N" for N parallel tests. ``` ## Resource limits diff --git a/bitcoinkernel/bitcoin/doc/build-osx.md b/bitcoinkernel/bitcoin/doc/build-osx.md index e1bf037..001c018 100644 --- a/bitcoinkernel/bitcoin/doc/build-osx.md +++ b/bitcoinkernel/bitcoin/doc/build-osx.md @@ -170,7 +170,7 @@ Run the following in your terminal to compile Bitcoin Core: ``` bash cmake --build build # Append "-j N" here for N parallel jobs. -ctest --test-dir build # Append "-j N" for N parallel tests. Some tests are disabled if Python 3 is not available. +ctest --test-dir build # Append "-j N" for N parallel tests. ``` ### 3. Deploy (optional) diff --git a/bitcoinkernel/bitcoin/doc/build-unix.md b/bitcoinkernel/bitcoin/doc/build-unix.md index d9bbb35..e499aa9 100644 --- a/bitcoinkernel/bitcoin/doc/build-unix.md +++ b/bitcoinkernel/bitcoin/doc/build-unix.md @@ -46,10 +46,14 @@ Finally, clang (often less resource hungry) can be used instead of gcc, which is #### Dependency Build Instructions -Build requirements: +Build requirements for the latest Debian "stable" release, or the latest Ubuntu LTS release: sudo apt-get install build-essential cmake pkgconf python3 +For Debian "oldstable", or earlier Ubuntu LTS releases, you may need to pick a +later compiler version, according to the [dependencies](/doc/dependencies.md) +documentation. + Now, you can either build from self-compiled [depends](#dependencies) or install the required dependencies: sudo apt-get install libevent-dev libboost-dev diff --git a/bitcoinkernel/bitcoin/doc/build-windows-msvc.md b/bitcoinkernel/bitcoin/doc/build-windows-msvc.md index 2d85d22..cf84cdd 100644 --- a/bitcoinkernel/bitcoin/doc/build-windows-msvc.md +++ b/bitcoinkernel/bitcoin/doc/build-windows-msvc.md @@ -50,12 +50,14 @@ CMake will put the resulting object files, libraries, and executables into a ded In the following instructions, the "Debug" configuration can be specified instead of the "Release" one. +Run `cmake -B build -LH` to see the full list of available options. + ### 4. Building with Static Linking with GUI ``` cmake -B build --preset vs2022-static # It might take a while if the vcpkg binary cache is unpopulated or invalidated. cmake --build build --config Release # Append "-j N" for N parallel jobs. -ctest --test-dir build --build-config Release # Append "-j N" for N parallel tests. Some tests are disabled if Python 3 is not available. +ctest --test-dir build --build-config Release # Append "-j N" for N parallel tests. cmake --install build --config Release # Optional. ``` @@ -64,7 +66,7 @@ cmake --install build --config Release # Optional. ``` cmake -B build --preset vs2022 -DBUILD_GUI=OFF # It might take a while if the vcpkg binary cache is unpopulated or invalidated. cmake --build build --config Release # Append "-j N" for N parallel jobs. -ctest --test-dir build --build-config Release # Append "-j N" for N parallel tests. Some tests are disabled if Python 3 is not available. +ctest --test-dir build --build-config Release # Append "-j N" for N parallel tests. ``` ### 6. vcpkg-specific Issues and Workarounds diff --git a/bitcoinkernel/bitcoin/doc/build-windows.md b/bitcoinkernel/bitcoin/doc/build-windows.md index 84db7fa..ce822dd 100644 --- a/bitcoinkernel/bitcoin/doc/build-windows.md +++ b/bitcoinkernel/bitcoin/doc/build-windows.md @@ -49,6 +49,9 @@ Build using: gmake -C depends HOST=x86_64-w64-mingw32 # Append "-j N" for N parallel jobs. cmake -B build --toolchain depends/x86_64-w64-mingw32/toolchain.cmake + +Run `cmake -B build -LH` to see the full list of available options. + cmake --build build # Append "-j N" for N parallel jobs. ## Depends system diff --git a/bitcoinkernel/bitcoin/doc/dependencies.md b/bitcoinkernel/bitcoin/doc/dependencies.md index 93916aa..3ff306f 100644 --- a/bitcoinkernel/bitcoin/doc/dependencies.md +++ b/bitcoinkernel/bitcoin/doc/dependencies.md @@ -10,8 +10,8 @@ Bitcoin Core requires one of the following compilers. | Dependency | Minimum required | | --- | --- | -| [Clang](https://clang.llvm.org) | [16.0](https://github.com/bitcoin/bitcoin/pull/30263) | -| [GCC](https://gcc.gnu.org) | [11.1](https://github.com/bitcoin/bitcoin/pull/29091) | +| [Clang](https://clang.llvm.org) | [17.0](https://github.com/bitcoin/bitcoin/pull/33555) | +| [GCC](https://gcc.gnu.org) | [12.1](https://github.com/bitcoin/bitcoin/pull/33842) | ## Required diff --git a/bitcoinkernel/bitcoin/doc/developer-notes.md b/bitcoinkernel/bitcoin/doc/developer-notes.md index 220343a..106533b 100644 --- a/bitcoinkernel/bitcoin/doc/developer-notes.md +++ b/bitcoinkernel/bitcoin/doc/developer-notes.md @@ -17,7 +17,7 @@ Do not submit patches solely to modify the style of existing code. [clang-format-diff script](/contrib/devtools/README.md#clang-format-diffpy) tool to clean up patches automatically before submission. - Braces on new lines for classes, functions, methods. - - Braces on the same line for everything else. + - Braces on the same line for everything else (including structs). - 4 space indentation (no tabs) for every block except namespaces. - No indentation for `public`/`protected`/`private` or for `namespace`. - No extra spaces inside parentheses; don't do `( this )`. @@ -1119,6 +1119,19 @@ to check a subtree directory for consistency with its upstream repository. The tool instructions also include a list of the subtrees managed by Bitcoin Core. +To fully verify or update a subtree, add it as a remote: + +```sh +git remote add libmultiprocess https://github.com/bitcoin-core/libmultiprocess.git +``` + +To update the subtree: + +```sh +git fetch libmultiprocess +git subtree pull --prefix=src/ipc/libmultiprocess libmultiprocess master --squash +``` + The ultimate upstream of the few externally managed subtrees are: - src/leveldb @@ -1350,7 +1363,7 @@ A few guidelines for introducing and reviewing new RPC interfaces: A few guidelines for modifying existing RPC interfaces: -- It's preferable to avoid changing an RPC in a backward-incompatible manner, but in that case, add an associated `-deprecatedrpc=` option to retain previous RPC behavior during the deprecation period. Backward-incompatible changes include: data type changes (e.g. from `{"warnings":""}` to `{"warnings":[]}`, changing a value from a string to a number, etc.), logical meaning changes of a value, or key name changes (e.g. `{"warning":""}` to `{"warnings":""}`). Adding a key to an object is generally considered backward-compatible. Include a release note that refers the user to the RPC help for details of feature deprecation and re-enabling previous behavior. [Example RPC help](https://github.com/bitcoin/bitcoin/blob/94f0adcc/src/rpc/blockchain.cpp#L1316-L1323). +- It's preferable to avoid changing an RPC in a backward-incompatible manner, but in that case, add an associated `-deprecatedrpc=` option to retain previous RPC behavior during the deprecation period. Backward-incompatible changes include: data type changes (e.g. from `{"warnings":""}` to `{"warnings":[]}`, changing a value from a string to a number, etc.), logical meaning changes of a value, key name changes (e.g. `{"warning":""}` to `{"warnings":""}`), or removing a key from an object. Adding a key to an object is generally considered backward-compatible. Include a release note that refers the user to the RPC help for details of feature deprecation and re-enabling previous behavior. [Example RPC help](https://github.com/bitcoin/bitcoin/blob/94f0adcc/src/rpc/blockchain.cpp#L1316-L1323). - *Rationale*: Changes in RPC JSON structure can break downstream application compatibility. Implementation of `deprecatedrpc` provides a grace period for downstream applications to migrate. Release notes provide notification to downstream users. diff --git a/bitcoinkernel/bitcoin/doc/fuzzing.md b/bitcoinkernel/bitcoin/doc/fuzzing.md index 8a024e5..f4333a5 100644 --- a/bitcoinkernel/bitcoin/doc/fuzzing.md +++ b/bitcoinkernel/bitcoin/doc/fuzzing.md @@ -21,6 +21,8 @@ See [further](#run-without-sanitizers-for-increased-throughput) for more informa There is also a runner script to execute all fuzz targets. Refer to `./build_fuzz/test/fuzz/test_runner.py --help` for more details. +For source-based coverage reports, see [developer notes](/doc/developer-notes.md#compiling-for-fuzz-coverage). + ## Overview of Bitcoin Core fuzzing [Google](https://github.com/google/fuzzing/) has a good overview of fuzzing in general, with contributions from key architects of some of the most-used fuzzers. [This paper](https://agroce.github.io/bitcoin_report.pdf) includes an external overview of the status of Bitcoin Core fuzzing, as of summer 2021. [John Regehr](https://blog.regehr.org/archives/1687) provides good advice on writing code that assists fuzzers in finding bugs, which is useful for developers to keep in mind. diff --git a/bitcoinkernel/bitcoin/doc/i2p.md b/bitcoinkernel/bitcoin/doc/i2p.md index b769a74..624b651 100644 --- a/bitcoinkernel/bitcoin/doc/i2p.md +++ b/bitcoinkernel/bitcoin/doc/i2p.md @@ -166,3 +166,13 @@ In most cases, the default router settings should work fine. Please see the "General Guidance for Developers" section in https://geti2p.net/en/docs/api/samv3 if you are developing a downstream application that may be bundling I2P with Bitcoin. + +## Privacy recommendations + +- Operating a node that listens on multiple networks (e.g. IPv4 and I2P) can help + strengthen the Bitcoin network, as nodes in this configuration (i.e. bridge nodes) increase + the cost and complexity of launching eclipse and partition attacks. However, under certain + conditions, an adversary that can connect to your node on multiple networks may be + able to correlate those identities by observing shared runtime characteristics. It + is not recommended to expose your node over multiple networks if you require + unlinkability across those identities. diff --git a/bitcoinkernel/bitcoin/doc/release-notes-33555.md b/bitcoinkernel/bitcoin/doc/release-notes-33555.md new file mode 100644 index 0000000..a11af34 --- /dev/null +++ b/bitcoinkernel/bitcoin/doc/release-notes-33555.md @@ -0,0 +1,5 @@ +Build System +------------ + +- The minimum supported Clang compiler version has been raised to 17.0 + (#33555). diff --git a/bitcoinkernel/bitcoin/doc/release-notes-33842.md b/bitcoinkernel/bitcoin/doc/release-notes-33842.md new file mode 100644 index 0000000..ad15f50 --- /dev/null +++ b/bitcoinkernel/bitcoin/doc/release-notes-33842.md @@ -0,0 +1,4 @@ +Build System +------------ + +- The minimum supported GCC compiler version has been raised to 12.1 (#33842). diff --git a/bitcoinkernel/bitcoin/doc/release-notes-empty-template.md b/bitcoinkernel/bitcoin/doc/release-notes-empty-template.md index 7f419cb..86dafb3 100644 --- a/bitcoinkernel/bitcoin/doc/release-notes-empty-template.md +++ b/bitcoinkernel/bitcoin/doc/release-notes-empty-template.md @@ -35,8 +35,8 @@ wallet versions of Bitcoin Core are generally supported. Compatibility ============== -Bitcoin Core is supported and tested on operating systems using the -Linux Kernel 3.17+, macOS 14+, and Windows 10+. Bitcoin +Bitcoin Core is supported and tested on the following operating systems or newer: +Linux Kernel 3.17, macOS 14, and Windows 10 (version 1903). Bitcoin Core should also work on most other Unix-like systems but is not as frequently tested on them. It is not recommended to use Bitcoin Core on unsupported systems. diff --git a/bitcoinkernel/bitcoin/doc/release-notes/release-notes-28.3.md b/bitcoinkernel/bitcoin/doc/release-notes/release-notes-28.3.md new file mode 100644 index 0000000..eefeedf --- /dev/null +++ b/bitcoinkernel/bitcoin/doc/release-notes/release-notes-28.3.md @@ -0,0 +1,110 @@ +Bitcoin Core version 28.3 is now available from: + + + +This release includes various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + + +To receive security and update notifications, please subscribe to: + + + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes in some cases), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on macOS) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the data directory needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems +using the Linux Kernel 3.17+, macOS 11.0+, and Windows 7 and newer. Bitcoin +Core should also work on most other UNIX-like systems but is not as +frequently tested on them. It is not recommended to use Bitcoin Core on +unsupported systems. + +Notable changes +=============== + +### Mempool & Policy + + The minimum block feerate (`-blockmintxfee`) has been changed to 1 satoshi per kvB. It can still be changed using the +configuration option. + +- The default minimum relay feerate (`-minrelaytxfee`) and incremental relay feerate (`-incrementalrelayfee`) have been +changed to 100 satoshis per kvB. They can still be changed using their respective configuration options, but it is +recommended to change both together if you decide to do so. + - Other minimum feerates (e.g. the dust feerate, the minimum returned by the fee estimator, and all feerates used by + the wallet) remain unchanged. The mempool minimum feerate still changes in response to high volume. + - Note that unless these lower defaults are widely adopted across the network, transactions created with lower fee + rates are not guaranteed to propagate or confirm. The wallet feerates remain unchanged; `-mintxfee` must be changed + before attempting to create transactions with lower feerates using the wallet. + +- #33106 policy: lower the default blockmintxfee, incrementalrelayfee, minrelaytxfee +- #33504 mempool: Do not enforce TRUC checks on reorg + +### P2P + +- #33395 net: do not apply whitelist permissions to onion inbounds + +### Test + +- #32765 test: Fix list index out of range error in feature_bip68_sequence.py +- #33001 test: Do not pass tests on unhandled exceptions +- #30125 test: improve BDB parser (handle internal/overflow pages, support all page sizes) +- #30948 test: Add missing sync_mempools() to fill_mempool() +- #30784 test: add BulkTransaction helper to unit test transaction utils + +### Build + +- #32678 guix: warn and abort when SOURCE_DATE_EPOCH is set +- #32943 depends: Force CMAKE_EXPORT_NO_PACKAGE_REGISTRY=TRUE +- #33073 guix: warn SOURCE_DATE_EPOCH set in guix-codesign +- #33563 build: fix depends Qt download link + +### Doc + +- #32776 doc: taproot became always active in v24.0 +- #32777 doc: fix Transifex 404s +- #33070 doc/zmq: fix unix socket path example +- #33133 rpc: fix getpeerinfo ping duration unit docs +- #33236 doc: Remove wrong and redundant doxygen tag + +### Misc + +- #33340 Fix benchmark CSV output +- #33482 contrib: fix macOS deployment with no translations +- #33581 ci: Properly include $FILE_ENV in DEPENDS_HASH + +Credits +======= + +Thanks to everyone who directly contributed to this release: +- 0xB10C +- amisha +- Ava Chow +- fanquake +- glozow +- Hennadii Stepanov +- MarcoFalke +- Martin Zumsande +- romanz +- Sjors Provoost +- theStack +- Vasil Dimov +- willcl-ark +- zaidmstrr + +As well as to everyone that helped with translations on +[Transifex](https://explore.transifex.com/bitcoin/bitcoin/). diff --git a/bitcoinkernel/bitcoin/doc/release-notes/release-notes-29.2.md b/bitcoinkernel/bitcoin/doc/release-notes/release-notes-29.2.md new file mode 100644 index 0000000..b981b8a --- /dev/null +++ b/bitcoinkernel/bitcoin/doc/release-notes/release-notes-29.2.md @@ -0,0 +1,91 @@ +Bitcoin Core version 29.2 is now available from: + + + +This release includes various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + + +To receive security and update notifications, please subscribe to: + + + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes in some cases), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on macOS) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the data directory needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and tested on operating systems using the +Linux Kernel 3.17+, macOS 13+, and Windows 10+. Bitcoin +Core should also work on most other Unix-like systems but is not as +frequently tested on them. It is not recommended to use Bitcoin Core on +unsupported systems. + +Notable changes +=============== + +### P2P + +- #32646 p2p: Add witness mutation check inside FillBlock +- #33296 net: check for empty header before calling FillBlock +- #33395 net: do not apply whitelist permissions to onion inbounds + +### Mempool + +- #33504 mempool: Do not enforce TRUC checks on reorg + +### RPC + +- #33446 rpc: fix getblock(header) returns target for tip + +### CI + +- #32989 ci: Migrate CI to hosted Cirrus Runners +- #32999 ci: Use APT_LLVM_V in msan task +- #33099 ci: allow for any libc++ intrumentation & use it for TSAN +- #33258 ci: use LLVM 21 +- #33364 ci: always use tag for LLVM checkout + +### Doc + +- #33484 doc: rpc: fix case typo in `finalizepsbt` help + +### Misc + +- #33310 trace: Workaround GCC bug compiling with old systemtap +- #33340 Fix benchmark CSV output +- #33482 contrib: fix macOS deployment with no translations + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- Amisha Chhajed +- Eugene Siegel +- fanquake +- Greg Sanders +- Hennadii Stepanov +- Luke Dashjr +- MarcoFalke +- Martin Zumsande +- Sebastian Falbesoner +- Sjors Provoost +- Vasil Dimov +- Will Clark + +As well as to everyone that helped with translations on +[Transifex](https://explore.transifex.com/bitcoin/bitcoin/). diff --git a/bitcoinkernel/bitcoin/doc/release-notes/release-notes-30.0.md b/bitcoinkernel/bitcoin/doc/release-notes/release-notes-30.0.md new file mode 100644 index 0000000..a855d10 --- /dev/null +++ b/bitcoinkernel/bitcoin/doc/release-notes/release-notes-30.0.md @@ -0,0 +1,438 @@ +v30.0 Release Notes +=================== + +Bitcoin Core version v30.0 is now available from: + + + +This release includes new features, various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + + +To receive security and update notifications, please subscribe to: + + + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes in some cases), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on macOS) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the data directory needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and tested on operating systems using the +Linux Kernel 3.17+, macOS 13+, and Windows 10+. Bitcoin +Core should also work on most other Unix-like systems but is not as +frequently tested on them. It is not recommended to use Bitcoin Core on +unsupported systems. + +Notable changes +=============== + +Policy +------ + +- The maximum number of potentially executed legacy signature operations in a + single standard transaction is now limited to 2500. Signature operations in all + previous output scripts, in all input scripts, as well as all P2SH redeem + scripts (if there are any) are counted toward the limit. The new limit is + assumed to not affect any known typically formed standard transactions. The + change was done to prepare for a possible BIP54 deployment in the future. (#32521) + +- `-datacarriersize` is increased to 100,000 by default, which effectively uncaps + the limit (as the maximum transaction size limit will be hit first). It can be + overridden with `-datacarriersize=83` to revert to the limit enforced in previous + versions. (#32406) + +- Multiple data carrier (OP_RETURN) outputs in a transaction are now permitted for + relay and mining. The `-datacarriersize` limit applies to the aggregate size of + the scriptPubKeys across all such outputs in a transaction, not including the + scriptPubKey size itself. (#32406) + +- The minimum block feerate (`-blockmintxfee`) has been changed to 0.001 satoshi per + vB. It can still be changed using the configuration option. This option can be used + by miners to set a minimum feerate on packages added to block templates. (#33106) + +- The default minimum relay feerate (`-minrelaytxfee`) and incremental relay feerate + (`-incrementalrelayfee`) have been changed to 0.1 satoshis per vB. They can still + be changed using their respective configuration options, but it is recommended to + change both together if you decide to do so. (#33106) + + Other minimum feerates (e.g. the dust feerate, the minimum returned by the fee + estimator, and all feerates used by the wallet) remain unchanged. The mempool minimum + feerate still changes in response to high volume. + + Note that unless these lower defaults are widely adopted across the network, transactions + created with lower fee rates are not guaranteed to propagate or confirm. The wallet + feerates remain unchanged; `-mintxfee` must be changed before attempting to create + transactions with lower feerates using the wallet. (#33106) + +P2P and network changes +----------------------- + +- Opportunistic 1-parent-1-child package relay has been improved to handle + situations when the child already has unconfirmed parent(s) in the mempool. + This means that 1p1c packages can be accepted and propagate, even if they are + connected to broader topologies: multi-parent-1-child (where only 1 parent + requires fee-bumping), grandparent-parent-child (where only parent requires + fee-bumping) etc. (#31385) + +- The transaction orphanage, which holds transactions with missing inputs temporarily + while the node attempts to fetch its parents, now has improved Denial of Service protections. + Previously, it enforced a maximum number of unique transactions (default 100, + configurable using `-maxorphantx`). Now, its limits are as follows: the number of + entries (unique by wtxid and peer), plus each unique transaction's input count divided + by 10, must not exceed 3,000. The total weight of unique transactions must not exceed + `404,000` Wu multiplied by the number of peers. (#31829) + +- The `-maxorphantx` option no longer has any effect, since the orphanage no longer + limits the number of unique transactions. Users should remove this configuration + option if they were using it, as the setting will cause an error in future versions + when it is no longer recognized. (#31829) + +New `bitcoin` command +--------------------- + +- A new `bitcoin` command line tool has been added to make features more discoverable + and convenient to use. The `bitcoin` tool just calls other executables and does not + implement any functionality on its own. Specifically `bitcoin node` is a synonym for + `bitcoind`, `bitcoin gui` is a synonym for `bitcoin-qt`, and `bitcoin rpc` is a synonym + for `bitcoin-cli -named`. Other commands and options can be listed with `bitcoin help`. + The new `bitcoin` command is an alternative to calling other commands directly, but it + doesn't replace them, and there are no plans to deprecate existing commands. (#31375) + +External Signing +---------------- + +- Support for external signing on Windows has been re-enabled. (#29868) + +IPC Mining Interface +-------------------- + +- The new `bitcoin` command does support one new feature: an (experimental) IPC Mining + Interface that allows the node to work with Stratum v2 or other mining client software, + see (#31098). When the node is started with `bitcoin -m node -ipcbind=unix` it will + listen on a unix socket for IPC client connections, allowing clients to request block + templates and submit mined blocks. The `-m` option launches a new internal binary + (`bitcoin-node` instead of `bitcoind`) and is currently required but will become optional + in the future (with [#33229](https://github.com/bitcoin/bitcoin/pull/33229)). + +- IPC connectivity introduces new dependencies (see [multiprocess.md](https://github.com/bitcoin/bitcoin/blob/master/doc/multiprocess.md)), + which can be turned off with the `-DENABLE_IPC=OFF` build option if you do not intend + to use IPC. (#31802) + +Install changes +--------------- + +- The `test_bitcoin` executable is now installed in `libexec/` instead of `bin/`. + It can still be executed directly, or accessed through the new `bitcoin` command + as `bitcoin test`. The `libexec/` directory also contains new `bitcoin-node` and + `bitcoin-gui` binaries which support IPC features and are called through the + `bitcoin` tool. In source builds only, `test_bitcoin-qt`, `bench_bitcoin`, and + `bitcoin-chainstate` are also now installed to `libexec/` instead of `bin/` and + can be accessed through the new `bitcoin` command. See `bitcoin help` output for + details. (#31679) + +- On Windows, the installer no longer adds a “(64-bit)” suffix to entries in the + Start Menu (#32132), and it now automatically removes obsolete artifacts during + upgrades (#33422). + +Indexes +------- + +- The implementation of coinstatsindex was changed to prevent an overflow bug that + could already be observed on the default Signet. The new version of the index will + need to be synced from scratch when starting the upgraded node for the first time. + + The new version is stored in `/indexes/coinstatsindex/` in contrast to the old version + which was stored at `/indexes/coinstats/`. The old version of the index is not deleted + by the upgraded node in case the user chooses to downgrade their node in the future. + If the user does not plan to downgrade it is safe for them to remove `/indexes/coinstats/` + from their datadir. A future release of Bitcoin Core may remove the old version of the + index automatically. (#30469) + +Logging +------- +- Unconditional logging to disk is now rate limited by giving each source location + a quota of 1MiB per hour. Unconditional logging is any logging with a log level + higher than debug, that is `info`, `warning`, and `error`. All logs will be + prefixed with `[*]` if there is at least one source location that is currently + being suppressed. (#32604) + +- When `-logsourcelocations` is enabled, the log output now contains the entire + function signature instead of just the function name. (#32604) + +Updated RPCs +------------ + +- The `-paytxfee` startup option and the `settxfee` RPC are now deprecated and + will be removed in Bitcoin Core 31.0. They allowed the user to set a static fee + rate for wallet transactions, which could potentially lead to overpaying or underpaying. + Users should instead rely on fee estimation or specify a fee rate per transaction + using the `fee_rate` argument in RPCs such as `fundrawtransaction`, `sendtoaddress`, + `send`, `sendall`, and `sendmany`. (#31278) + +- Any RPC in which one of the parameters is a descriptor will throw an error + if the provided descriptor contains a whitespace at the beginning or the end + of the public key within a fragment - e.g. `pk( KEY)` or `pk(KEY )`. (#31603) + +- The `submitpackage` RPC, which allows submissions of child-with-parents + packages, no longer requires that all unconfirmed parents be present. The + package may contain other in-mempool ancestors as well. (#31385) + +- The `waitfornewblock` RPC now takes an optional `current_tip` argument. It + is also no longer hidden. (#30635) + +- The `waitforblock` and `waitforblockheight` RPCs are no longer hidden. (#30635) + +- The `psbtbumpfee` and `bumpfee` RPCs allow a replacement under fullrbf and no + longer require BIP-125 signalling. (#31953) + +- Transaction Script validation errors used to return the reason for the error + prefixed by either `mandatory-script-verify-flag-failed` if it was a consensus + error, or `non-mandatory-script-verify-flag` (without "-failed") if it was a + standardness error. This has been changed to `block-script-verify-flag-failed` + and `mempool-script-verify-flag-failed` for all block and mempool errors + respectively. (#33183) + +- The `getmininginfo` RPC now returns "blockmintxfee" result specifying the value of + `-blockmintxfee` configuration. (#33189) + +- The `getmempoolinfo` RPC now returns an additional "permitbaremultisig" and + "maxdatacarriersize" field, reflecting the `-permitbaremultisig` and `-datacarriersize` + config values. (#29954) + +Changes to wallet-related RPCs can be found in the Wallet section below. + +New RPCs +-------- + +- A new REST API endpoint (`/rest/spenttxouts/BLOCKHASH`) has been introduced for + efficiently fetching spent transaction outputs using the block's undo data (#32540). + +Build System +------------ + +Updated settings +---------------- + +- The `-maxmempool` and `-dbcache` startup parameters are now capped on 32-bit systems + to 500MB and 1GiB respectively. (#32530) + +- The `-natpmp` option is now set to `1` by default. This means nodes with `-listen` + enabled (the default) but running behind a firewall, such as a local network router, + will be reachable if the firewall/router supports any of the `PCP` or `NAT-PMP` + protocols. (#33004) + +- The `-upnp` setting has now been fully removed. Use `-natpmp` instead. (#32500) + +- Previously, `-proxy` specified the proxy for all networks (except I2P which + uses `-i2psam`) and only the Tor proxy could have been specified separately + via `-onion`. Now, the syntax of `-proxy` has been extended and it is possible + to specify separately the proxy for IPv4, IPv6, Tor and CJDNS by appending `=` + followed by the network name, for example `-proxy=127.0.0.1:5555=ipv6` + configures a proxy only for IPv6. The `-proxy` option can be used multiple + times to define different proxies for different networks, such as + `-proxy=127.0.0.1:4444=ipv4 -proxy=10.0.0.1:6666=ipv6`. Later settings + override earlier ones for the same network; this can be used to remove an + earlier all-networks proxy and use direct connections only for a given + network, for example `-proxy=127.0.0.1:5555 -proxy=0=cjdns`. (#32425) + +- The `-blockmaxweight` startup option has been updated to be debug-only. + It is still available to users, but now hidden from the default `-help` text + and shown only in `-help-debug` (#32654). + +Changes to GUI or wallet related settings can be found in the GUI or Wallet section below. + +Wallet +------ + +- BDB legacy wallets can no longer be created or loaded. They can be migrated + to the new descriptor wallet format. Refer to the `migratewallet` RPC for more + details. + +- The legacy wallet removal drops redundant options in the bitcoin-wallet tool, + such as `-withinternalbdb`, `-legacy`, and `-descriptors`. Moreover, the + legacy-only RPCs `addmultisigaddress`, `dumpprivkey`, `dumpwallet`, + `importaddress`, `importmulti`, `importprivkey`, `importpubkey`, + `importwallet`, `newkeypool`, `sethdseed`, and `upgradewallet`, are removed. + (#32944, #28710, #32438, #31250) + +- Support has been added for spending TRUC transactions received by the + wallet, as well as creating TRUC transactions. The wallet ensures that + TRUC policy rules are being met. The wallet will throw an error if the + user is trying to spend TRUC utxos with utxos of other versions. + Additionally, the wallet will treat unconfirmed TRUC sibling + transactions as mempool conflicts. The wallet will also ensure that + transactions spending TRUC utxos meet the required size restrictions. (#32896) + +- Since descriptor wallets do not allow mixing watchonly and non-watchonly descriptors, + the `include_watchonly` option (and its variants in naming) are removed from all RPCs + that had it. (#32618) + +- The `iswatchonly` field is removed from any RPCs that returned it. (#32618) + +- `unloadwallet` - Return RPC_INVALID_PARAMETER when both the RPC wallet endpoint + and wallet_name parameters are unspecified. Previously the RPC failed with a JSON + parsing error. (#32845) + +- `getdescriptoractivity` - Mark blockhashes and scanobjects arguments as required, + so the user receives a clear help message when either is missing. As in `unloadwallet`, + previously the RPC failed with a JSON parsing error. (#32845) + +- `getwalletinfo` - Removes the fields `balance`, `immature_balance` and + `unconfirmed_balance`. (#32721) + +- `getunconfirmedbalance` - Removes this RPC command. You can query the `getbalances` + RPC and inspect the `["mine"]["untrusted_pending"]` entry within the JSON + response. (#32721) + +- The following RPCs now contain a `version` parameter that allows + the user to create transactions of any standard version number (1-3): + - `createrawtransaction` + - `createpsbt` + - `send` + - `sendall` + - `walletcreatefundedpsbt` + (#32896) + +GUI changes +----------- + +- The GUI has been migrated from Qt 5 to Qt 6. On Windows, dark mode is now supported. + On macOS, the Metal backend is now used. (#30997) + +- A transaction's fee bump is allowed under fullrbf and no longer requires + BIP-125 signalling. (#31953) + +- Custom column widths in the Transactions tab are reset as a side-effect of legacy + wallet removal. (#32459) + +Low-level changes +================= + +- Logs now include which peer sent us a header. Additionally there are fewer + redundant header log messages. A side-effect of this change is that for + some untypical cases new headers aren't logged anymore, e.g. a direct + `BLOCK` message with a previously unknown header and `submitheader` RPC. (#27826) + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- 0xb10c +- amisha +- Andrew Toth +- Anthony Towns +- Antoine Poinsot +- Ava Chow +- benthecarman +- Brandon Odiwuor +- brunoerg +- Bue-von-hon +- Bufo +- Chandra Pratap +- Chris Stewart +- Cory Fields +- Daniel Pfeifer +- Daniela Brozzoni +- David Gumberg +- deadmanoz +- dennsikl +- dergoegge +- enoch +- Ethan Heilman +- Eugene Siegel +- Eunovo +- Eval EXEC +- Fabian Jahr +- fanquake +- Florian Schmaus +- fuder.eth +- furszy +- glozow +- Greg Sanders +- Hao Xu +- Haoran Peng +- Haowen Liu +- Hennadii Stepanov +- Hodlinator +- hoffman +- ishaanam +- ismaelsadeeq +- Jameson Lopp +- janb84 +- Jiri Jakes +- John Bampton +- Jon Atack +- josibake +- jurraca +- kevkevin +- kevkevinpal +- kilavvy +- Kristaps Kaupe +- l0rinc +- laanwj +- leopardracer +- Lőrinc +- Luis Schwab +- Luke Dashjr +- MarcoFalke +- marcofleon +- Martin Zumsande +- Matt Corallo +- Matthew Zipkin +- Max Edwards +- monlovesmango +- Murch +- naiyoma +- nervana21 +- Nicola Leonardo Susca +- Novo +- pablomartin4btc +- Peter Todd +- Pieter Wuille +- Pol Espinasa +- Prabhat Verma +- rkrux +- Roman Zeyde +- Ryan Ofsky +- Saikiran +- Salvatore Ingala +- Sebastian Falbesoner +- Sergi Delgado Segura +- Shunsuke Shimizu +- Sjors Provoost +- stickies-v +- stratospher +- stringintech +- strmfos +- stutxo +- tdb3 +- TheCharlatan +- Tomás Andróil +- UdjinM6 +- Vasil Dimov +- VolodymyrBg +- w0xlt +- will +- willcl-ark +- William Casarin +- woltx +- yancy +- zaidmstrr + +As well as to everyone that helped with translations on +[Transifex](https://explore.transifex.com/bitcoin/bitcoin/). diff --git a/bitcoinkernel/bitcoin/doc/release-process.md b/bitcoinkernel/bitcoin/doc/release-process.md index 9159b8c..272f36e 100644 --- a/bitcoinkernel/bitcoin/doc/release-process.md +++ b/bitcoinkernel/bitcoin/doc/release-process.md @@ -53,7 +53,7 @@ Release Process - Set `MINCHAINWORK_HEADERS` to the height used for the `nMinimumChainWork` calculation above. - Check that the other variables still look reasonable. - Run the script. It works fine in CPython, but PyPy is much faster (seconds instead of minutes): `pypy3 contrib/devtools/headerssync-params.py`. - - Paste the output defining `HEADER_COMMITMENT_PERIOD` and `REDOWNLOAD_BUFFER_SIZE` into the top of [`src/headerssync.cpp`](/src/headerssync.cpp). + - Paste the output defining the header `commitment_period` and `redownload_buffer_size` into the mainnet section of [`src/kernel/chainparams.cpp`](/src/kernel/chainparams.cpp). - Clear the release notes and move them to the wiki (see "Write the release notes" below). - Translations on Transifex: - Pull translations from Transifex into the master branch. diff --git a/bitcoinkernel/bitcoin/doc/tor.md b/bitcoinkernel/bitcoin/doc/tor.md index 839c02e..e9db555 100644 --- a/bitcoinkernel/bitcoin/doc/tor.md +++ b/bitcoinkernel/bitcoin/doc/tor.md @@ -238,3 +238,10 @@ for normal IPv4/IPv6 communication, use: Otherwise it is trivial to link them, which may reduce privacy. Onion services created automatically (as in section 2) always have only one port open. +- Operating a node that listens on multiple networks (e.g. IPv4 and Tor) can help + strengthen the Bitcoin network, as nodes in this configuration (i.e. bridge nodes) increase + the cost and complexity of launching eclipse and partition attacks. However, under certain + conditions, an adversary that can connect to your node on multiple networks may be + able to correlate those identities by observing shared runtime characteristics. It + is not recommended to expose your node over multiple networks if you require + unlinkability across those identities. diff --git a/bitcoinkernel/bitcoin/src/.clang-format b/bitcoinkernel/bitcoin/src/.clang-format index f20e5ee..096c6f8 100644 --- a/bitcoinkernel/bitcoin/src/.clang-format +++ b/bitcoinkernel/bitcoin/src/.clang-format @@ -1,49 +1,222 @@ Language: Cpp AccessModifierOffset: -4 -AlignAfterOpenBracket: true -AlignEscapedNewlinesLeft: true -AlignTrailingComments: true +AlignAfterOpenBracket: Align +AlignArrayOfStructures: None +AlignConsecutiveAssignments: + Enabled: false + AcrossEmptyLines: false + AcrossComments: false + AlignCompound: false + PadOperators: true +AlignConsecutiveBitFields: + Enabled: false + AcrossEmptyLines: false + AcrossComments: false + AlignCompound: false + PadOperators: false +AlignConsecutiveDeclarations: + Enabled: false + AcrossEmptyLines: false + AcrossComments: false + AlignCompound: false + PadOperators: false +AlignConsecutiveMacros: + Enabled: false + AcrossEmptyLines: false + AcrossComments: false + AlignCompound: false + PadOperators: false +AlignEscapedNewlines: Left +AlignOperands: Align +AlignTrailingComments: + Kind: Always + OverEmptyLines: 0 +AllowAllArgumentsOnNextLine: true AllowAllParametersOfDeclarationOnNextLine: true -AllowShortBlocksOnASingleLine: false +AllowShortBlocksOnASingleLine: Never AllowShortCaseLabelsOnASingleLine: true +AllowShortEnumsOnASingleLine: true AllowShortFunctionsOnASingleLine: All -AllowShortIfStatementsOnASingleLine: true +AllowShortIfStatementsOnASingleLine: WithoutElse +AllowShortLambdasOnASingleLine: All AllowShortLoopsOnASingleLine: false +AlwaysBreakAfterDefinitionReturnType: None +AlwaysBreakAfterReturnType: None AlwaysBreakBeforeMultilineStrings: false -AlwaysBreakTemplateDeclarations: true +AlwaysBreakTemplateDeclarations: Yes +AttributeMacros: + - __capability BinPackArguments: true BinPackParameters: true -BreakBeforeBinaryOperators: false -BreakBeforeBraces: Custom +BitFieldColonSpacing: Both BraceWrapping: - AfterClass: true - AfterFunction: true + AfterCaseLabel: false + AfterClass: true + AfterControlStatement: Never + AfterEnum: false + AfterExternBlock: false + AfterFunction: true + AfterNamespace: false + AfterObjCDeclaration: false + AfterStruct: false + AfterUnion: false + BeforeCatch: false + BeforeElse: false + BeforeLambdaBody: false + BeforeWhile: false + IndentBraces: false + SplitEmptyFunction: true + SplitEmptyRecord: true + SplitEmptyNamespace: true +BreakAfterAttributes: Never +BreakAfterJavaFieldAnnotations: false +BreakArrays: true +BreakBeforeBinaryOperators: None +BreakBeforeConceptDeclarations: Always +BreakBeforeBraces: Custom +BreakBeforeInlineASMColon: OnlyMultiline BreakBeforeTernaryOperators: false -BreakConstructorInitializersBeforeComma: false +BreakConstructorInitializers: BeforeColon +BreakInheritanceList: BeforeColon +BreakStringLiterals: true ColumnLimit: 0 CommentPragmas: '^ IWYU pragma:' -ConstructorInitializerAllOnOneLineOrOnePerLine: false +CompactNamespaces: false ConstructorInitializerIndentWidth: 4 ContinuationIndentWidth: 4 Cpp11BracedListStyle: true DerivePointerAlignment: false DisableFormat: false +EmptyLineAfterAccessModifier: Never +EmptyLineBeforeAccessModifier: LogicalBlock +ExperimentalAutoDetectBinPacking: false +FixNamespaceComments: true +ForEachMacros: + - foreach + - Q_FOREACH + - BOOST_FOREACH +IfMacros: + - KJ_IF_MAYBE +IncludeBlocks: Preserve +IncludeCategories: + - Regex: '^"(llvm|llvm-c|clang|clang-c)/' + Priority: 2 + SortPriority: 0 + CaseSensitive: false + - Regex: '^(<|"(gtest|gmock|isl|json)/)' + Priority: 3 + SortPriority: 0 + CaseSensitive: false + - Regex: '.*' + Priority: 1 + SortPriority: 0 + CaseSensitive: false +IncludeIsMainRegex: '(Test)?$' +IncludeIsMainSourceRegex: '' +IndentAccessModifiers: false +IndentCaseBlocks: false IndentCaseLabels: false -IndentFunctionDeclarationAfterType: false +IndentExternBlock: AfterExternBlock +IndentGotoLabels: true +IndentPPDirectives: None +IndentRequiresClause: true IndentWidth: 4 +IndentWrappedFunctionNames: false +InsertBraces: false +InsertNewlineAtEOF: false +InsertTrailingCommas: None +IntegerLiteralSeparator: + Binary: 0 + BinaryMinDigits: 0 + Decimal: 0 + DecimalMinDigits: 0 + Hex: 0 + HexMinDigits: 0 +JavaScriptQuotes: Leave +JavaScriptWrapImports: true KeepEmptyLinesAtTheStartOfBlocks: false +LambdaBodyIndentation: Signature +LineEnding: DeriveLF +MacroBlockBegin: '' +MacroBlockEnd: '' MaxEmptyLinesToKeep: 2 NamespaceIndentation: None +ObjCBinPackProtocolList: Auto +ObjCBlockIndentWidth: 2 +ObjCBreakBeforeNestedBlockParam: true +ObjCSpaceAfterProperty: false +ObjCSpaceBeforeProtocolList: true +PackConstructorInitializers: BinPack +PenaltyBreakAssignment: 2 +PenaltyBreakBeforeFirstCallParameter: 19 +PenaltyBreakComment: 300 +PenaltyBreakFirstLessLess: 120 +PenaltyBreakOpenParenthesis: 0 +PenaltyBreakString: 1000 +PenaltyBreakTemplateDeclaration: 10 +PenaltyExcessCharacter: 1000000 +PenaltyIndentedWhitespace: 0 +PenaltyReturnTypeOnItsOwnLine: 60 PointerAlignment: Left +PPIndentWidth: -1 +QualifierAlignment: Leave +ReferenceAlignment: Pointer +ReflowComments: true +RemoveBracesLLVM: false +RemoveSemicolon: false +RequiresClausePosition: OwnLine +RequiresExpressionIndentation: OuterScope +SeparateDefinitionBlocks: Leave +ShortNamespaceLines: 1 +SortIncludes: CaseSensitive +SortJavaStaticImport: Before +SortUsingDeclarations: LexicographicNumeric +SpaceAfterCStyleCast: false +SpaceAfterLogicalNot: false +SpaceAfterTemplateKeyword: true +SpaceAroundPointerQualifiers: Default SpaceBeforeAssignmentOperators: true +SpaceBeforeCaseColon: false +SpaceBeforeCpp11BracedList: false +SpaceBeforeCtorInitializerColon: true +SpaceBeforeInheritanceColon: true SpaceBeforeParens: ControlStatements +SpaceBeforeParensOptions: + AfterControlStatements: true + AfterForeachMacros: true + AfterFunctionDefinitionName: false + AfterFunctionDeclarationName: false + AfterIfMacros: true + AfterOverloadedOperator: false + AfterRequiresInClause: false + AfterRequiresInExpression: false + BeforeNonEmptyParentheses: false +SpaceBeforeRangeBasedForLoopColon: true +SpaceBeforeSquareBrackets: false +SpaceInEmptyBlock: false SpaceInEmptyParentheses: false SpacesBeforeTrailingComments: 1 -SpacesInAngles: false +SpacesInAngles: Never +SpacesInConditionalStatement: false SpacesInContainerLiterals: true SpacesInCStyleCastParentheses: false +SpacesInLineCommentPrefix: + Minimum: 1 + Maximum: -1 SpacesInParentheses: false -BreakBeforeConceptDeclarations: Always -RequiresExpressionIndentation: OuterScope -Standard: c++20 +SpacesInSquareBrackets: false +Standard: c++20 +StatementAttributeLikeMacros: + - Q_EMIT +StatementMacros: + - Q_UNUSED + - QT_REQUIRE_VERSION +TabWidth: 8 UseTab: Never +WhitespaceSensitiveMacros: + - BOOST_PP_STRINGIZE + - CF_SWIFT_NAME + - NS_SWIFT_NAME + - PP_STRINGIZE + - STRINGIZE +... diff --git a/bitcoinkernel/bitcoin/src/CMakeLists.txt b/bitcoinkernel/bitcoin/src/CMakeLists.txt index 81c8577..aed4061 100644 --- a/bitcoinkernel/bitcoin/src/CMakeLists.txt +++ b/bitcoinkernel/bitcoin/src/CMakeLists.txt @@ -238,8 +238,8 @@ add_library(bitcoin_node STATIC EXCLUDE_FROM_ALL node/warnings.cpp noui.cpp policy/ephemeral_policy.cpp - policy/fees.cpp - policy/fees_args.cpp + policy/fees/block_policy_estimator.cpp + policy/fees/block_policy_estimator_args.cpp policy/packages.cpp policy/rbf.cpp policy/settings.cpp @@ -411,6 +411,7 @@ if(BUILD_KERNEL_LIB) add_executable(bitcoin-chainstate bitcoin-chainstate.cpp ) + add_windows_application_manifest(bitcoin-chainstate) # TODO: The `SKIP_BUILD_RPATH` property setting can be deleted # in the future after reordering Guix script commands to # perform binary checks after the installation step. diff --git a/bitcoinkernel/bitcoin/src/arith_uint256.h b/bitcoinkernel/bitcoin/src/arith_uint256.h index 5cefff5..69e0aff 100644 --- a/bitcoinkernel/bitcoin/src/arith_uint256.h +++ b/bitcoinkernel/bitcoin/src/arith_uint256.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2022 The Bitcoin Core developers +// Copyright (c) 2009-present The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -21,7 +21,7 @@ class uint_error : public std::runtime_error { }; /** Template base class for unsigned big integers. */ -template +template class base_uint { protected: @@ -29,9 +29,9 @@ class base_uint static constexpr int WIDTH = BITS / 32; /** Big integer represented with 32-bit digits, least-significant first. */ uint32_t pn[WIDTH]; -public: - base_uint() +public: + constexpr base_uint() { for (int i = 0; i < WIDTH; i++) pn[i] = 0; @@ -40,7 +40,7 @@ class base_uint base_uint(const base_uint& b) = default; base_uint& operator=(const base_uint& b) = default; - base_uint(uint64_t b) + constexpr base_uint(uint64_t b) { pn[0] = (unsigned int)b; pn[1] = (unsigned int)(b >> 32); @@ -227,11 +227,12 @@ class base_uint }; /** 256-bit unsigned big integer. */ -class arith_uint256 : public base_uint<256> { +class arith_uint256 : public base_uint<256> +{ public: - arith_uint256() = default; - arith_uint256(const base_uint<256>& b) : base_uint<256>(b) {} - arith_uint256(uint64_t b) : base_uint<256>(b) {} + constexpr arith_uint256() = default; + constexpr arith_uint256(const base_uint& b) : base_uint(b) {} + constexpr arith_uint256(uint64_t b) : base_uint(b) {} /** * The "compact" format is a representation of a whole diff --git a/bitcoinkernel/bitcoin/src/bench/CMakeLists.txt b/bitcoinkernel/bitcoin/src/bench/CMakeLists.txt index 0bf469c..e0e03b1 100644 --- a/bitcoinkernel/bitcoin/src/bench/CMakeLists.txt +++ b/bitcoinkernel/bitcoin/src/bench/CMakeLists.txt @@ -56,6 +56,8 @@ add_executable(bench_bitcoin verify_script.cpp ) +add_windows_application_manifest(bench_bitcoin) + include(TargetDataSources) target_raw_data_sources(bench_bitcoin NAMESPACE benchmark::data data/block413567.raw diff --git a/bitcoinkernel/bitcoin/src/bench/bench.cpp b/bitcoinkernel/bitcoin/src/bench/bench.cpp index 26daff5..9cc1f30 100644 --- a/bitcoinkernel/bitcoin/src/bench/bench.cpp +++ b/bitcoinkernel/bitcoin/src/bench/bench.cpp @@ -56,7 +56,7 @@ void GenerateTemplateResults(const std::vector& bench // nothing to write, bail out return; } - std::ofstream fout{file}; + std::ofstream fout{file.std_path()}; if (fout.is_open()) { ankerl::nanobench::render(tpl, benchmarkResults, fout); std::cout << "Created " << file << std::endl; diff --git a/bitcoinkernel/bitcoin/src/bench/descriptors.cpp b/bitcoinkernel/bitcoin/src/bench/descriptors.cpp index c454566..ac5580a 100644 --- a/bitcoinkernel/bitcoin/src/bench/descriptors.cpp +++ b/bitcoinkernel/bitcoin/src/bench/descriptors.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -19,7 +20,7 @@ static void ExpandDescriptor(benchmark::Bench& bench) { ECC_Context ecc_context{}; - const auto desc_str = "sh(wsh(multi(16,03669b8afcec803a0d323e9a17f3ea8e68e8abe5a278020a929adbec52421adbd0,0260b2003c386519fc9eadf2b5cf124dd8eea4c4e68d5e154050a9346ea98ce600,0362a74e399c39ed5593852a30147f2959b56bb827dfa3e60e464b02ccf87dc5e8,0261345b53de74a4d721ef877c255429961b7e43714171ac06168d7e08c542a8b8,02da72e8b46901a65d4374fe6315538d8f368557dda3a1dcf9ea903f3afe7314c8,0318c82dd0b53fd3a932d16e0ba9e278fcc937c582d5781be626ff16e201f72286,0297ccef1ef99f9d73dec9ad37476ddb232f1238aff877af19e72ba04493361009,02e502cfd5c3f972fe9a3e2a18827820638f96b6f347e54d63deb839011fd5765d,03e687710f0e3ebe81c1037074da939d409c0025f17eb86adb9427d28f0f7ae0e9,02c04d3a5274952acdbc76987f3184b346a483d43be40874624b29e3692c1df5af,02ed06e0f418b5b43a7ec01d1d7d27290fa15f75771cb69b642a51471c29c84acd,036d46073cbb9ffee90473f3da429abc8de7f8751199da44485682a989a4bebb24,02f5d1ff7c9029a80a4e36b9a5497027ef7f3e73384a4a94fbfe7c4e9164eec8bc,02e41deffd1b7cce11cde209a781adcffdabd1b91c0ba0375857a2bfd9302419f3,02d76625f7956a7fc505ab02556c23ee72d832f1bac391bcd2d3abce5710a13d06,0399eb0a5487515802dc14544cf10b3666623762fbed2ec38a3975716e2c29c232)))"; + constexpr std::string_view desc_str{"sh(wsh(multi(16,03669b8afcec803a0d323e9a17f3ea8e68e8abe5a278020a929adbec52421adbd0,0260b2003c386519fc9eadf2b5cf124dd8eea4c4e68d5e154050a9346ea98ce600,0362a74e399c39ed5593852a30147f2959b56bb827dfa3e60e464b02ccf87dc5e8,0261345b53de74a4d721ef877c255429961b7e43714171ac06168d7e08c542a8b8,02da72e8b46901a65d4374fe6315538d8f368557dda3a1dcf9ea903f3afe7314c8,0318c82dd0b53fd3a932d16e0ba9e278fcc937c582d5781be626ff16e201f72286,0297ccef1ef99f9d73dec9ad37476ddb232f1238aff877af19e72ba04493361009,02e502cfd5c3f972fe9a3e2a18827820638f96b6f347e54d63deb839011fd5765d,03e687710f0e3ebe81c1037074da939d409c0025f17eb86adb9427d28f0f7ae0e9,02c04d3a5274952acdbc76987f3184b346a483d43be40874624b29e3692c1df5af,02ed06e0f418b5b43a7ec01d1d7d27290fa15f75771cb69b642a51471c29c84acd,036d46073cbb9ffee90473f3da429abc8de7f8751199da44485682a989a4bebb24,02f5d1ff7c9029a80a4e36b9a5497027ef7f3e73384a4a94fbfe7c4e9164eec8bc,02e41deffd1b7cce11cde209a781adcffdabd1b91c0ba0375857a2bfd9302419f3,02d76625f7956a7fc505ab02556c23ee72d832f1bac391bcd2d3abce5710a13d06,0399eb0a5487515802dc14544cf10b3666623762fbed2ec38a3975716e2c29c232)))"}; const std::pair range = {0, 1000}; FlatSigningProvider provider; std::string error; diff --git a/bitcoinkernel/bitcoin/src/bitcoin-chainstate.cpp b/bitcoinkernel/bitcoin/src/bitcoin-chainstate.cpp index 4a4bd87..310ca95 100644 --- a/bitcoinkernel/bitcoin/src/bitcoin-chainstate.cpp +++ b/bitcoinkernel/bitcoin/src/bitcoin-chainstate.cpp @@ -9,15 +9,6 @@ #include #include -#ifdef WIN32 -// clang-format off -#include -// clang-format on -#include -#include -#include -#endif - using namespace btck; std::vector hex_string_to_byte_vec(std::string_view hex) @@ -46,7 +37,7 @@ class KernelLog } }; -class TestValidationInterface : public ValidationInterface +class TestValidationInterface : public ValidationInterface { public: TestValidationInterface() = default; @@ -103,7 +94,7 @@ class TestValidationInterface : public ValidationInterface +class TestKernelNotifications : public KernelNotifications { public: void BlockTipHandler(SynchronizationState, const BlockTreeEntry, double) override @@ -149,22 +140,6 @@ int main(int argc, char* argv[]) << " BREAK IN FUTURE VERSIONS. DO NOT USE ON YOUR ACTUAL DATADIR." << std::endl; return 1; } - -#ifdef WIN32 - int win_argc; - wchar_t** wargv = CommandLineToArgvW(GetCommandLineW(), &win_argc); - std::vector utf8_args(win_argc); - std::vector win_argv(win_argc); - std::wstring_convert, wchar_t> utf8_cvt; - for (int i = 0; i < win_argc; i++) { - utf8_args[i] = utf8_cvt.to_bytes(wargv[i]); - win_argv[i] = &utf8_args[i][0]; - } - LocalFree(wargv); - argc = win_argc; - argv = win_argv.data(); -#endif - std::filesystem::path abs_datadir{std::filesystem::absolute(argv[1])}; std::filesystem::create_directories(abs_datadir); @@ -176,7 +151,9 @@ int main(int argc, char* argv[]) .always_print_category_levels = true, }; - Logger logger{std::make_unique(KernelLog{}), logging_options}; + logging_set_options(logging_options); + + Logger logger{std::make_unique()}; ContextOptions options{}; ChainParams params{ChainType::MAINNET}; diff --git a/bitcoinkernel/bitcoin/src/bitcoin-cli.cpp b/bitcoinkernel/bitcoin/src/bitcoin-cli.cpp index 56f2a90..279aa89 100644 --- a/bitcoinkernel/bitcoin/src/bitcoin-cli.cpp +++ b/bitcoinkernel/bitcoin/src/bitcoin-cli.cpp @@ -1328,10 +1328,6 @@ static int CommandLineRPC(int argc, char *argv[]) MAIN_FUNCTION { -#ifdef WIN32 - common::WinCmdLineArgs winArgs; - std::tie(argc, argv) = winArgs.get(); -#endif SetupEnvironment(); if (!SetupNetworking()) { tfm::format(std::cerr, "Error: Initializing networking failed\n"); diff --git a/bitcoinkernel/bitcoin/src/bitcoin-wallet.cpp b/bitcoinkernel/bitcoin/src/bitcoin-wallet.cpp index c871523..6811f8c 100644 --- a/bitcoinkernel/bitcoin/src/bitcoin-wallet.cpp +++ b/bitcoinkernel/bitcoin/src/bitcoin-wallet.cpp @@ -94,10 +94,6 @@ static std::optional WalletAppInit(ArgsManager& args, int argc, char* argv[ MAIN_FUNCTION { ArgsManager& args = gArgs; -#ifdef WIN32 - common::WinCmdLineArgs winArgs; - std::tie(argc, argv) = winArgs.get(); -#endif int exit_status; std::unique_ptr init = interfaces::MakeWalletInit(argc, argv, exit_status); diff --git a/bitcoinkernel/bitcoin/src/bitcoin.cpp b/bitcoinkernel/bitcoin/src/bitcoin.cpp index c1a5fce..a8ebab9 100644 --- a/bitcoinkernel/bitcoin/src/bitcoin.cpp +++ b/bitcoinkernel/bitcoin/src/bitcoin.cpp @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -61,6 +62,8 @@ static void ExecCommand(const std::vector& args, std::string_view a int main(int argc, char* argv[]) { + SetupEnvironment(); + try { CommandLine cmd{ParseCommandLine(argc, argv)}; if (cmd.show_version) { @@ -209,7 +212,7 @@ static void ExecCommand(const std::vector& args, std::string_view w // Try to resolve any symlinks and figure out the directory containing the wrapper executable. std::error_code ec; - fs::path wrapper_dir{fs::weakly_canonical(wrapper_path, ec)}; + auto wrapper_dir{fs::weakly_canonical(wrapper_path, ec)}; if (wrapper_dir.empty()) wrapper_dir = wrapper_path; // Restore previous path if weakly_canonical failed. wrapper_dir = wrapper_dir.parent_path(); @@ -225,7 +228,7 @@ static void ExecCommand(const std::vector& args, std::string_view w // If wrapper is installed in a bin/ directory, look for target executable // in libexec/ - (wrapper_dir.filename() == "bin" && try_exec(fs::path{wrapper_dir.parent_path()} / "libexec" / arg0.filename())) || + (wrapper_dir.filename() == "bin" && try_exec(wrapper_dir.parent_path() / "libexec" / arg0.filename())) || #ifdef WIN32 // Otherwise check the "daemon" subdirectory in a windows install. (!wrapper_dir.empty() && try_exec(wrapper_dir / "daemon" / arg0.filename())) || diff --git a/bitcoinkernel/bitcoin/src/bitcoind.cpp b/bitcoinkernel/bitcoin/src/bitcoind.cpp index a4373da..37cacb2 100644 --- a/bitcoinkernel/bitcoin/src/bitcoind.cpp +++ b/bitcoinkernel/bitcoin/src/bitcoind.cpp @@ -259,11 +259,6 @@ static bool AppInit(NodeContext& node) MAIN_FUNCTION { -#ifdef WIN32 - common::WinCmdLineArgs winArgs; - std::tie(argc, argv) = winArgs.get(); -#endif - NodeContext node; int exit_status; std::unique_ptr init = interfaces::MakeNodeInit(node, argc, argv, exit_status); diff --git a/bitcoinkernel/bitcoin/src/blockfilter.cpp b/bitcoinkernel/bitcoin/src/blockfilter.cpp index 5e6702c..94bcc49 100644 --- a/bitcoinkernel/bitcoin/src/blockfilter.cpp +++ b/bitcoinkernel/bitcoin/src/blockfilter.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include @@ -151,7 +152,8 @@ const std::string& BlockFilterTypeName(BlockFilterType filter_type) return it != g_filter_types.end() ? it->second : unknown_retval; } -bool BlockFilterTypeByName(const std::string& name, BlockFilterType& filter_type) { +bool BlockFilterTypeByName(std::string_view name, BlockFilterType& filter_type) +{ for (const auto& entry : g_filter_types) { if (entry.second == name) { filter_type = entry.first; diff --git a/bitcoinkernel/bitcoin/src/blockfilter.h b/bitcoinkernel/bitcoin/src/blockfilter.h index 8eab4af..b70afb9 100644 --- a/bitcoinkernel/bitcoin/src/blockfilter.h +++ b/bitcoinkernel/bitcoin/src/blockfilter.h @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -99,7 +100,7 @@ enum class BlockFilterType : uint8_t const std::string& BlockFilterTypeName(BlockFilterType filter_type); /** Find a filter type by its human-readable name. */ -bool BlockFilterTypeByName(const std::string& name, BlockFilterType& filter_type); +bool BlockFilterTypeByName(std::string_view name, BlockFilterType& filter_type); /** Get a list of known filter types. */ const std::set& AllBlockFilterTypes(); diff --git a/bitcoinkernel/bitcoin/src/chain.cpp b/bitcoinkernel/bitcoin/src/chain.cpp index 3dd2263..94be718 100644 --- a/bitcoinkernel/bitcoin/src/chain.cpp +++ b/bitcoinkernel/bitcoin/src/chain.cpp @@ -6,12 +6,6 @@ #include #include #include -#include - -std::string CBlockFileInfo::ToString() const -{ - return strprintf("CBlockFileInfo(blocks=%u, size=%u, heights=%u...%u, time=%s...%s)", nBlocks, nSize, nHeightFirst, nHeightLast, FormatISO8601Date(nTimeFirst), FormatISO8601Date(nTimeLast)); -} std::string CBlockIndex::ToString() const { diff --git a/bitcoinkernel/bitcoin/src/chain.h b/bitcoinkernel/bitcoin/src/chain.h index 68aa612..64c8635 100644 --- a/bitcoinkernel/bitcoin/src/chain.h +++ b/bitcoinkernel/bitcoin/src/chain.h @@ -35,6 +35,9 @@ static constexpr int64_t MAX_FUTURE_BLOCK_TIME = 2 * 60 * 60; * MAX_FUTURE_BLOCK_TIME. */ static constexpr int64_t TIMESTAMP_WINDOW = MAX_FUTURE_BLOCK_TIME; +//! Init values for CBlockIndex nSequenceId when loaded from disk +static constexpr int32_t SEQ_ID_BEST_CHAIN_FROM_DISK = 0; +static constexpr int32_t SEQ_ID_INIT_FROM_DISK = 1; /** * Maximum gap between node time and block time used @@ -44,47 +47,6 @@ static constexpr int64_t TIMESTAMP_WINDOW = MAX_FUTURE_BLOCK_TIME; */ static constexpr int64_t MAX_BLOCK_TIME_GAP = 90 * 60; -class CBlockFileInfo -{ -public: - unsigned int nBlocks{}; //!< number of blocks stored in file - unsigned int nSize{}; //!< number of used bytes of block file - unsigned int nUndoSize{}; //!< number of used bytes in the undo file - unsigned int nHeightFirst{}; //!< lowest height of block in file - unsigned int nHeightLast{}; //!< highest height of block in file - uint64_t nTimeFirst{}; //!< earliest time of block in file - uint64_t nTimeLast{}; //!< latest time of block in file - - SERIALIZE_METHODS(CBlockFileInfo, obj) - { - READWRITE(VARINT(obj.nBlocks)); - READWRITE(VARINT(obj.nSize)); - READWRITE(VARINT(obj.nUndoSize)); - READWRITE(VARINT(obj.nHeightFirst)); - READWRITE(VARINT(obj.nHeightLast)); - READWRITE(VARINT(obj.nTimeFirst)); - READWRITE(VARINT(obj.nTimeLast)); - } - - CBlockFileInfo() = default; - - std::string ToString() const; - - /** update statistics (does not update nSize) */ - void AddBlock(unsigned int nHeightIn, uint64_t nTimeIn) - { - if (nBlocks == 0 || nHeightFirst > nHeightIn) - nHeightFirst = nHeightIn; - if (nBlocks == 0 || nTimeFirst > nTimeIn) - nTimeFirst = nTimeIn; - nBlocks++; - if (nHeightIn > nHeightLast) - nHeightLast = nHeightIn; - if (nTimeIn > nTimeLast) - nTimeLast = nTimeIn; - } -}; - enum BlockStatus : uint32_t { //! Unused. BLOCK_VALID_UNKNOWN = 0, @@ -191,7 +153,9 @@ class CBlockIndex uint32_t nNonce{0}; //! (memory only) Sequential id assigned to distinguish order in which blocks are received. - int32_t nSequenceId{0}; + //! Initialized to SEQ_ID_INIT_FROM_DISK{1} when loading blocks from disk, except for blocks + //! belonging to the best chain which overwrite it to SEQ_ID_BEST_CHAIN_FROM_DISK{0}. + int32_t nSequenceId{SEQ_ID_INIT_FROM_DISK}; //! (memory only) Maximum nTime in the chain up to and including this block. unsigned int nTimeMax{0}; diff --git a/bitcoinkernel/bitcoin/src/cluster_linearize.h b/bitcoinkernel/bitcoin/src/cluster_linearize.h index bec44d9..73c8a03 100644 --- a/bitcoinkernel/bitcoin/src/cluster_linearize.h +++ b/bitcoinkernel/bitcoin/src/cluster_linearize.h @@ -12,6 +12,7 @@ #include #include +#include #include #include #include @@ -332,6 +333,17 @@ class DepGraph } return true; } + + /** Reduce memory usage if possible. No observable effect. */ + void Compact() noexcept + { + entries.shrink_to_fit(); + } + + size_t DynamicMemoryUsage() const noexcept + { + return memusage::DynamicUsage(entries); + } }; /** A set of transactions together with their aggregate feerate. */ diff --git a/bitcoinkernel/bitcoin/src/coins.cpp b/bitcoinkernel/bitcoin/src/coins.cpp index 24a102b..554a3eb 100644 --- a/bitcoinkernel/bitcoin/src/coins.cpp +++ b/bitcoinkernel/bitcoin/src/coins.cpp @@ -76,9 +76,6 @@ void CCoinsViewCache::AddCoin(const COutPoint &outpoint, Coin&& coin, bool possi bool inserted; std::tie(it, inserted) = cacheCoins.emplace(std::piecewise_construct, std::forward_as_tuple(outpoint), std::tuple<>()); bool fresh = false; - if (!inserted) { - cachedCoinsUsage -= it->second.coin.DynamicMemoryUsage(); - } if (!possible_overwrite) { if (!it->second.coin.IsSpent()) { throw std::logic_error("Attempted to overwrite an unspent coin (when possible_overwrite is false)"); @@ -98,6 +95,9 @@ void CCoinsViewCache::AddCoin(const COutPoint &outpoint, Coin&& coin, bool possi // DIRTY, then it can be marked FRESH. fresh = !it->second.IsDirty(); } + if (!inserted) { + cachedCoinsUsage -= it->second.coin.DynamicMemoryUsage(); + } it->second.coin = std::move(coin); CCoinsCacheEntry::SetDirty(*it, m_sentinel); if (fresh) CCoinsCacheEntry::SetFresh(*it, m_sentinel); @@ -111,9 +111,12 @@ void CCoinsViewCache::AddCoin(const COutPoint &outpoint, Coin&& coin, bool possi } void CCoinsViewCache::EmplaceCoinInternalDANGER(COutPoint&& outpoint, Coin&& coin) { - cachedCoinsUsage += coin.DynamicMemoryUsage(); + const auto mem_usage{coin.DynamicMemoryUsage()}; auto [it, inserted] = cacheCoins.try_emplace(std::move(outpoint), std::move(coin)); - if (inserted) CCoinsCacheEntry::SetDirty(*it, m_sentinel); + if (inserted) { + CCoinsCacheEntry::SetDirty(*it, m_sentinel); + cachedCoinsUsage += mem_usage; + } } void AddCoins(CCoinsViewCache& cache, const CTransaction &tx, int nHeight, bool check_for_overwrite) { @@ -195,6 +198,7 @@ bool CCoinsViewCache::BatchWrite(CoinsViewCacheCursor& cursor, const uint256 &ha // and mark it as dirty. itUs = cacheCoins.try_emplace(it->first).first; CCoinsCacheEntry& entry{itUs->second}; + assert(entry.coin.DynamicMemoryUsage() == 0); if (cursor.WillErase(*it)) { // Since this entry will be erased, // we can move the coin into us instead of copying it @@ -248,19 +252,19 @@ bool CCoinsViewCache::BatchWrite(CoinsViewCacheCursor& cursor, const uint256 &ha } bool CCoinsViewCache::Flush() { - auto cursor{CoinsViewCacheCursor(cachedCoinsUsage, m_sentinel, cacheCoins, /*will_erase=*/true)}; + auto cursor{CoinsViewCacheCursor(m_sentinel, cacheCoins, /*will_erase=*/true)}; bool fOk = base->BatchWrite(cursor, hashBlock); if (fOk) { cacheCoins.clear(); ReallocateCache(); + cachedCoinsUsage = 0; } - cachedCoinsUsage = 0; return fOk; } bool CCoinsViewCache::Sync() { - auto cursor{CoinsViewCacheCursor(cachedCoinsUsage, m_sentinel, cacheCoins, /*will_erase=*/false)}; + auto cursor{CoinsViewCacheCursor(m_sentinel, cacheCoins, /*will_erase=*/false)}; bool fOk = base->BatchWrite(cursor, hashBlock); if (fOk) { if (m_sentinel.second.Next() != &m_sentinel) { @@ -345,8 +349,8 @@ void CCoinsViewCache::SanityCheck() const assert(recomputed_usage == cachedCoinsUsage); } -static const size_t MIN_TRANSACTION_OUTPUT_WEIGHT = WITNESS_SCALE_FACTOR * ::GetSerializeSize(CTxOut()); -static const size_t MAX_OUTPUTS_PER_BLOCK = MAX_BLOCK_WEIGHT / MIN_TRANSACTION_OUTPUT_WEIGHT; +static const uint64_t MIN_TRANSACTION_OUTPUT_WEIGHT{WITNESS_SCALE_FACTOR * ::GetSerializeSize(CTxOut())}; +static const uint64_t MAX_OUTPUTS_PER_BLOCK{MAX_BLOCK_WEIGHT / MIN_TRANSACTION_OUTPUT_WEIGHT}; const Coin& AccessByTxid(const CCoinsViewCache& view, const Txid& txid) { diff --git a/bitcoinkernel/bitcoin/src/coins.h b/bitcoinkernel/bitcoin/src/coins.h index 6725d5a..2fcc764 100644 --- a/bitcoinkernel/bitcoin/src/coins.h +++ b/bitcoinkernel/bitcoin/src/coins.h @@ -271,11 +271,10 @@ struct CoinsViewCacheCursor //! This is an optimization compared to erasing all entries as the cursor iterates them when will_erase is set. //! Calling CCoinsMap::clear() afterwards is faster because a CoinsCachePair cannot be coerced back into a //! CCoinsMap::iterator to be erased, and must therefore be looked up again by key in the CCoinsMap before being erased. - CoinsViewCacheCursor(size_t& usage LIFETIMEBOUND, - CoinsCachePair& sentinel LIFETIMEBOUND, - CCoinsMap& map LIFETIMEBOUND, - bool will_erase) noexcept - : m_usage(usage), m_sentinel(sentinel), m_map(map), m_will_erase(will_erase) {} + CoinsViewCacheCursor(CoinsCachePair& sentinel LIFETIMEBOUND, + CCoinsMap& map LIFETIMEBOUND, + bool will_erase) noexcept + : m_sentinel(sentinel), m_map(map), m_will_erase(will_erase) {} inline CoinsCachePair* Begin() const noexcept { return m_sentinel.second.Next(); } inline CoinsCachePair* End() const noexcept { return &m_sentinel; } @@ -288,7 +287,7 @@ struct CoinsViewCacheCursor // Otherwise, clear the state of the entry. if (!m_will_erase) { if (current.second.coin.IsSpent()) { - m_usage -= current.second.coin.DynamicMemoryUsage(); + assert(current.second.coin.DynamicMemoryUsage() == 0); // scriptPubKey was already cleared in SpendCoin m_map.erase(current.first); } else { current.second.SetClean(); @@ -299,7 +298,6 @@ struct CoinsViewCacheCursor inline bool WillErase(CoinsCachePair& current) const noexcept { return m_will_erase || current.second.coin.IsSpent(); } private: - size_t& m_usage; CoinsCachePair& m_sentinel; CCoinsMap& m_map; bool m_will_erase; diff --git a/bitcoinkernel/bitcoin/src/common/args.cpp b/bitcoinkernel/bitcoin/src/common/args.cpp index d44cd43..50b9902 100644 --- a/bitcoinkernel/bitcoin/src/common/args.cpp +++ b/bitcoinkernel/bitcoin/src/common/args.cpp @@ -19,8 +19,6 @@ #include #ifdef WIN32 -#include -#include #include #endif @@ -879,30 +877,3 @@ void ArgsManager::LogArgs() const } logArgsPrefix("Command-line arg:", "", m_settings.command_line_options); } - -namespace common { -#ifdef WIN32 -WinCmdLineArgs::WinCmdLineArgs() -{ - wchar_t** wargv = CommandLineToArgvW(GetCommandLineW(), &argc); - std::wstring_convert, wchar_t> utf8_cvt; - argv = new char*[argc]; - args.resize(argc); - for (int i = 0; i < argc; i++) { - args[i] = utf8_cvt.to_bytes(wargv[i]); - argv[i] = &*args[i].begin(); - } - LocalFree(wargv); -} - -WinCmdLineArgs::~WinCmdLineArgs() -{ - delete[] argv; -} - -std::pair WinCmdLineArgs::get() -{ - return std::make_pair(argc, argv); -} -#endif -} // namespace common diff --git a/bitcoinkernel/bitcoin/src/common/args.h b/bitcoinkernel/bitcoin/src/common/args.h index d907ad7..1b9233e 100644 --- a/bitcoinkernel/bitcoin/src/common/args.h +++ b/bitcoinkernel/bitcoin/src/common/args.h @@ -480,21 +480,4 @@ std::string HelpMessageGroup(const std::string& message); */ std::string HelpMessageOpt(const std::string& option, const std::string& message); -namespace common { -#ifdef WIN32 -class WinCmdLineArgs -{ -public: - WinCmdLineArgs(); - ~WinCmdLineArgs(); - std::pair get(); - -private: - int argc; - char** argv; - std::vector args; -}; -#endif -} // namespace common - #endif // BITCOIN_COMMON_ARGS_H diff --git a/bitcoinkernel/bitcoin/src/common/config.cpp b/bitcoinkernel/bitcoin/src/common/config.cpp index 42f5e28..7216022 100644 --- a/bitcoinkernel/bitcoin/src/common/config.cpp +++ b/bitcoinkernel/bitcoin/src/common/config.cpp @@ -135,7 +135,7 @@ bool ArgsManager::ReadConfigFiles(std::string& error, bool ignore_invalid_keys) error = strprintf("Config file \"%s\" is a directory.", fs::PathToString(conf_path)); return false; } - stream = std::ifstream{conf_path}; + stream = std::ifstream{conf_path.std_path()}; // If the file is explicitly specified, it must be readable if (IsArgSet("-conf") && !stream.good()) { error = strprintf("specified config file \"%s\" could not be opened.", fs::PathToString(conf_path)); @@ -187,7 +187,7 @@ bool ArgsManager::ReadConfigFiles(std::string& error, bool ignore_invalid_keys) error = strprintf("Included config file \"%s\" is a directory.", fs::PathToString(include_conf_path)); return false; } - std::ifstream conf_file_stream{include_conf_path}; + std::ifstream conf_file_stream{include_conf_path.std_path()}; if (conf_file_stream.good()) { if (!ReadConfigStream(conf_file_stream, conf_file_name, error, ignore_invalid_keys)) { return false; diff --git a/bitcoinkernel/bitcoin/src/common/messages.cpp b/bitcoinkernel/bitcoin/src/common/messages.cpp index fc0b4a8..983c982 100644 --- a/bitcoinkernel/bitcoin/src/common/messages.cpp +++ b/bitcoinkernel/bitcoin/src/common/messages.cpp @@ -6,7 +6,7 @@ #include #include -#include +#include #include #include #include @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -91,7 +92,7 @@ std::string InvalidEstimateModeErrorMessage() return "Invalid estimate_mode parameter, must be one of: \"" + FeeModes("\", \"") + "\""; } -bool FeeModeFromString(const std::string& mode_string, FeeEstimateMode& fee_estimate_mode) +bool FeeModeFromString(std::string_view mode_string, FeeEstimateMode& fee_estimate_mode) { auto searchkey = ToUpper(mode_string); for (const auto& pair : FeeModeMap()) { diff --git a/bitcoinkernel/bitcoin/src/common/messages.h b/bitcoinkernel/bitcoin/src/common/messages.h index 5827fcc..5d33eda 100644 --- a/bitcoinkernel/bitcoin/src/common/messages.h +++ b/bitcoinkernel/bitcoin/src/common/messages.h @@ -12,6 +12,7 @@ #define BITCOIN_COMMON_MESSAGES_H #include +#include struct bilingual_str; @@ -23,7 +24,7 @@ enum class TransactionError; namespace common { enum class PSBTError; -bool FeeModeFromString(const std::string& mode_string, FeeEstimateMode& fee_estimate_mode); +bool FeeModeFromString(std::string_view mode_string, FeeEstimateMode& fee_estimate_mode); std::string StringForFeeReason(FeeReason reason); std::string FeeModes(const std::string& delimiter); std::string FeeModeInfo(std::pair& mode); diff --git a/bitcoinkernel/bitcoin/src/common/settings.cpp b/bitcoinkernel/bitcoin/src/common/settings.cpp index 046afca..fbe531c 100644 --- a/bitcoinkernel/bitcoin/src/common/settings.cpp +++ b/bitcoinkernel/bitcoin/src/common/settings.cpp @@ -78,7 +78,7 @@ bool ReadSettings(const fs::path& path, std::map& va if (!fs::exists(path)) return true; std::ifstream file; - file.open(path); + file.open(path.std_path()); if (!file.is_open()) { errors.emplace_back(strprintf("%s. Please check permissions.", fs::PathToString(path))); return false; @@ -133,7 +133,7 @@ bool WriteSettings(const fs::path& path, out.pushKVEnd(value.first, value.second); } std::ofstream file; - file.open(path); + file.open(path.std_path()); if (file.fail()) { errors.emplace_back(strprintf("Error: Unable to open settings file %s for writing", fs::PathToString(path))); return false; diff --git a/bitcoinkernel/bitcoin/src/common/system.cpp b/bitcoinkernel/bitcoin/src/common/system.cpp index 35a6f44..3833763 100644 --- a/bitcoinkernel/bitcoin/src/common/system.cpp +++ b/bitcoinkernel/bitcoin/src/common/system.cpp @@ -12,6 +12,7 @@ #include #ifdef WIN32 +#include #include #include #include @@ -83,6 +84,7 @@ void SetupEnvironment() setenv("LC_ALL", "C.UTF-8", 1); } #elif defined(WIN32) + assert(GetACP() == CP_UTF8); // Set the default input/output charset is utf-8 SetConsoleCP(CP_UTF8); SetConsoleOutputCP(CP_UTF8); diff --git a/bitcoinkernel/bitcoin/src/consensus/tx_verify.cpp b/bitcoinkernel/bitcoin/src/consensus/tx_verify.cpp index 9d09872..c38b964 100644 --- a/bitcoinkernel/bitcoin/src/consensus/tx_verify.cpp +++ b/bitcoinkernel/bitcoin/src/consensus/tx_verify.cpp @@ -156,7 +156,7 @@ int64_t GetTransactionSigOpCost(const CTransaction& tx, const CCoinsViewCache& i const Coin& coin = inputs.AccessCoin(tx.vin[i].prevout); assert(!coin.IsSpent()); const CTxOut &prevout = coin.out; - nSigOps += CountWitnessSigOps(tx.vin[i].scriptSig, prevout.scriptPubKey, &tx.vin[i].scriptWitness, flags); + nSigOps += CountWitnessSigOps(tx.vin[i].scriptSig, prevout.scriptPubKey, tx.vin[i].scriptWitness, flags); } return nSigOps; } diff --git a/bitcoinkernel/bitcoin/src/core_io.h b/bitcoinkernel/bitcoin/src/core_io.h index ce2e8f6..1874c93 100644 --- a/bitcoinkernel/bitcoin/src/core_io.h +++ b/bitcoinkernel/bitcoin/src/core_io.h @@ -21,6 +21,7 @@ class SigningProvider; class uint256; class UniValue; class CTxUndo; +class CTxOut; /** * Verbose level for block's transaction @@ -46,6 +47,6 @@ std::string FormatScript(const CScript& script); std::string EncodeHexTx(const CTransaction& tx); std::string SighashToStr(unsigned char sighash_type); void ScriptToUniv(const CScript& script, UniValue& out, bool include_hex = true, bool include_address = false, const SigningProvider* provider = nullptr); -void TxToUniv(const CTransaction& tx, const uint256& block_hash, UniValue& entry, bool include_hex = true, const CTxUndo* txundo = nullptr, TxVerbosity verbosity = TxVerbosity::SHOW_DETAILS); +void TxToUniv(const CTransaction& tx, const uint256& block_hash, UniValue& entry, bool include_hex = true, const CTxUndo* txundo = nullptr, TxVerbosity verbosity = TxVerbosity::SHOW_DETAILS, std::function is_change_func = {}); #endif // BITCOIN_CORE_IO_H diff --git a/bitcoinkernel/bitcoin/src/core_write.cpp b/bitcoinkernel/bitcoin/src/core_write.cpp index 14836f5..d805cfd 100644 --- a/bitcoinkernel/bitcoin/src/core_write.cpp +++ b/bitcoinkernel/bitcoin/src/core_write.cpp @@ -168,7 +168,7 @@ void ScriptToUniv(const CScript& script, UniValue& out, bool include_hex, bool i out.pushKV("type", GetTxnOutputType(type)); } -void TxToUniv(const CTransaction& tx, const uint256& block_hash, UniValue& entry, bool include_hex, const CTxUndo* txundo, TxVerbosity verbosity) +void TxToUniv(const CTransaction& tx, const uint256& block_hash, UniValue& entry, bool include_hex, const CTxUndo* txundo, TxVerbosity verbosity, std::function is_change_func) { CHECK_NONFATAL(verbosity >= TxVerbosity::SHOW_DETAILS); @@ -246,6 +246,11 @@ void TxToUniv(const CTransaction& tx, const uint256& block_hash, UniValue& entry UniValue o(UniValue::VOBJ); ScriptToUniv(txout.scriptPubKey, /*out=*/o, /*include_hex=*/true, /*include_address=*/true); out.pushKV("scriptPubKey", std::move(o)); + + if (is_change_func && is_change_func(txout)) { + out.pushKV("ischange", true); + } + vout.push_back(std::move(out)); if (have_undo) { diff --git a/bitcoinkernel/bitcoin/src/crypto/chacha20.cpp b/bitcoinkernel/bitcoin/src/crypto/chacha20.cpp index 9106405..6bdffe6 100644 --- a/bitcoinkernel/bitcoin/src/crypto/chacha20.cpp +++ b/bitcoinkernel/bitcoin/src/crypto/chacha20.cpp @@ -8,11 +8,10 @@ #include #include #include -#include #include #include -#include +#include #define QUARTERROUND(a,b,c,d) \ a += b; d = std::rotl(d ^ a, 16); \ diff --git a/bitcoinkernel/bitcoin/src/crypto/chacha20.h b/bitcoinkernel/bitcoin/src/crypto/chacha20.h index 6653c2c..1503562 100644 --- a/bitcoinkernel/bitcoin/src/crypto/chacha20.h +++ b/bitcoinkernel/bitcoin/src/crypto/chacha20.h @@ -5,12 +5,11 @@ #ifndef BITCOIN_CRYPTO_CHACHA20_H #define BITCOIN_CRYPTO_CHACHA20_H -#include - #include #include #include -#include +#include +#include #include // classes for ChaCha20 256-bit stream cipher developed by Daniel J. Bernstein diff --git a/bitcoinkernel/bitcoin/src/crypto/chacha20poly1305.h b/bitcoinkernel/bitcoin/src/crypto/chacha20poly1305.h index a6c3a66..9a863dd 100644 --- a/bitcoinkernel/bitcoin/src/crypto/chacha20poly1305.h +++ b/bitcoinkernel/bitcoin/src/crypto/chacha20poly1305.h @@ -7,10 +7,10 @@ #include #include +#include #include #include -#include /** The AEAD_CHACHA20_POLY1305 authenticated encryption algorithm from RFC8439 section 2.8. */ class AEADChaCha20Poly1305 diff --git a/bitcoinkernel/bitcoin/src/crypto/hex_base.cpp b/bitcoinkernel/bitcoin/src/crypto/hex_base.cpp index e2eccf5..7098aba 100644 --- a/bitcoinkernel/bitcoin/src/crypto/hex_base.cpp +++ b/bitcoinkernel/bitcoin/src/crypto/hex_base.cpp @@ -5,8 +5,10 @@ #include #include +#include #include #include +#include namespace { diff --git a/bitcoinkernel/bitcoin/src/crypto/hex_base.h b/bitcoinkernel/bitcoin/src/crypto/hex_base.h index 704b19f..9975f7f 100644 --- a/bitcoinkernel/bitcoin/src/crypto/hex_base.h +++ b/bitcoinkernel/bitcoin/src/crypto/hex_base.h @@ -9,6 +9,7 @@ #include #include +#include #include /** diff --git a/bitcoinkernel/bitcoin/src/crypto/hkdf_sha256_32.cpp b/bitcoinkernel/bitcoin/src/crypto/hkdf_sha256_32.cpp index e622494..3cd2cf8 100644 --- a/bitcoinkernel/bitcoin/src/crypto/hkdf_sha256_32.cpp +++ b/bitcoinkernel/bitcoin/src/crypto/hkdf_sha256_32.cpp @@ -4,8 +4,9 @@ #include +#include + #include -#include CHKDF_HMAC_SHA256_L32::CHKDF_HMAC_SHA256_L32(const unsigned char* ikm, size_t ikmlen, const std::string& salt) { diff --git a/bitcoinkernel/bitcoin/src/crypto/hkdf_sha256_32.h b/bitcoinkernel/bitcoin/src/crypto/hkdf_sha256_32.h index f9f343b..7c5d5a7 100644 --- a/bitcoinkernel/bitcoin/src/crypto/hkdf_sha256_32.h +++ b/bitcoinkernel/bitcoin/src/crypto/hkdf_sha256_32.h @@ -5,10 +5,8 @@ #ifndef BITCOIN_CRYPTO_HKDF_SHA256_32_H #define BITCOIN_CRYPTO_HKDF_SHA256_32_H -#include - -#include -#include +#include +#include /** A rfc5869 HKDF implementation with HMAC_SHA256 and fixed key output length of 32 bytes (L=32) */ class CHKDF_HMAC_SHA256_L32 diff --git a/bitcoinkernel/bitcoin/src/crypto/hmac_sha256.cpp b/bitcoinkernel/bitcoin/src/crypto/hmac_sha256.cpp index 0346520..a95ef70 100644 --- a/bitcoinkernel/bitcoin/src/crypto/hmac_sha256.cpp +++ b/bitcoinkernel/bitcoin/src/crypto/hmac_sha256.cpp @@ -4,6 +4,8 @@ #include +#include + #include CHMAC_SHA256::CHMAC_SHA256(const unsigned char* key, size_t keylen) diff --git a/bitcoinkernel/bitcoin/src/crypto/hmac_sha256.h b/bitcoinkernel/bitcoin/src/crypto/hmac_sha256.h index ea8cee8..a26947d 100644 --- a/bitcoinkernel/bitcoin/src/crypto/hmac_sha256.h +++ b/bitcoinkernel/bitcoin/src/crypto/hmac_sha256.h @@ -7,8 +7,7 @@ #include -#include -#include +#include /** A hasher class for HMAC-SHA-256. */ class CHMAC_SHA256 diff --git a/bitcoinkernel/bitcoin/src/crypto/hmac_sha512.cpp b/bitcoinkernel/bitcoin/src/crypto/hmac_sha512.cpp index 245ed29..f37e709 100644 --- a/bitcoinkernel/bitcoin/src/crypto/hmac_sha512.cpp +++ b/bitcoinkernel/bitcoin/src/crypto/hmac_sha512.cpp @@ -4,6 +4,8 @@ #include +#include + #include CHMAC_SHA512::CHMAC_SHA512(const unsigned char* key, size_t keylen) diff --git a/bitcoinkernel/bitcoin/src/crypto/hmac_sha512.h b/bitcoinkernel/bitcoin/src/crypto/hmac_sha512.h index d6bebc0..dfae8d0 100644 --- a/bitcoinkernel/bitcoin/src/crypto/hmac_sha512.h +++ b/bitcoinkernel/bitcoin/src/crypto/hmac_sha512.h @@ -7,8 +7,7 @@ #include -#include -#include +#include /** A hasher class for HMAC-SHA-512. */ class CHMAC_SHA512 diff --git a/bitcoinkernel/bitcoin/src/crypto/muhash.cpp b/bitcoinkernel/bitcoin/src/crypto/muhash.cpp index 02c38c4..c04f58e 100644 --- a/bitcoinkernel/bitcoin/src/crypto/muhash.cpp +++ b/bitcoinkernel/bitcoin/src/crypto/muhash.cpp @@ -7,11 +7,12 @@ #include #include #include +#include +#include #include #include -#include -#include +#include #include namespace { diff --git a/bitcoinkernel/bitcoin/src/crypto/muhash.h b/bitcoinkernel/bitcoin/src/crypto/muhash.h index f340937..5e2be32 100644 --- a/bitcoinkernel/bitcoin/src/crypto/muhash.h +++ b/bitcoinkernel/bitcoin/src/crypto/muhash.h @@ -6,9 +6,12 @@ #define BITCOIN_CRYPTO_MUHASH_H #include -#include +#include #include +#include + +class uint256; class Num3072 { diff --git a/bitcoinkernel/bitcoin/src/crypto/poly1305.cpp b/bitcoinkernel/bitcoin/src/crypto/poly1305.cpp index c9c066b..d6762a1 100644 --- a/bitcoinkernel/bitcoin/src/crypto/poly1305.cpp +++ b/bitcoinkernel/bitcoin/src/crypto/poly1305.cpp @@ -5,8 +5,6 @@ #include #include -#include - namespace poly1305_donna { // Based on the public domain implementation by Andrew Moon diff --git a/bitcoinkernel/bitcoin/src/crypto/poly1305.h b/bitcoinkernel/bitcoin/src/crypto/poly1305.h index 494f7cd..af4aede 100644 --- a/bitcoinkernel/bitcoin/src/crypto/poly1305.h +++ b/bitcoinkernel/bitcoin/src/crypto/poly1305.h @@ -8,8 +8,9 @@ #include #include +#include #include -#include +#include #define POLY1305_BLOCK_SIZE 16 diff --git a/bitcoinkernel/bitcoin/src/crypto/sha256.cpp b/bitcoinkernel/bitcoin/src/crypto/sha256.cpp index d75d01b..8ad9a85 100644 --- a/bitcoinkernel/bitcoin/src/crypto/sha256.cpp +++ b/bitcoinkernel/bitcoin/src/crypto/sha256.cpp @@ -10,7 +10,7 @@ #include #if !defined(DISABLE_OPTIMIZED_SHA256) -#include +#include // IWYU pragma: keep #if defined(__linux__) && defined(ENABLE_ARM_SHANI) #include diff --git a/bitcoinkernel/bitcoin/src/crypto/sha256_arm_shani.cpp b/bitcoinkernel/bitcoin/src/crypto/sha256_arm_shani.cpp index 2ea1d9c..a778e87 100644 --- a/bitcoinkernel/bitcoin/src/crypto/sha256_arm_shani.cpp +++ b/bitcoinkernel/bitcoin/src/crypto/sha256_arm_shani.cpp @@ -13,7 +13,6 @@ #include #include #include -#include #include namespace { diff --git a/bitcoinkernel/bitcoin/src/crypto/sha256_sse4.cpp b/bitcoinkernel/bitcoin/src/crypto/sha256_sse4.cpp index d0a3493..2d37d12 100644 --- a/bitcoinkernel/bitcoin/src/crypto/sha256_sse4.cpp +++ b/bitcoinkernel/bitcoin/src/crypto/sha256_sse4.cpp @@ -5,11 +5,11 @@ // This is a translation to GCC extended asm syntax from YASM code by Intel // (available at the bottom of this file). +#if defined(__x86_64__) || defined(__amd64__) + #include #include -#if defined(__x86_64__) || defined(__amd64__) - namespace sha256_sse4 { void Transform(uint32_t* s, const unsigned char* chunk, size_t blocks) @@ -969,27 +969,27 @@ void Transform(uint32_t* s, const unsigned char* chunk, size_t blocks) /* ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Copyright (c) 2012, Intel Corporation -; -; All rights reserved. -; +; Copyright (c) 2012, Intel Corporation +; +; All rights reserved. +; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions are -; met: -; +; met: +; ; * Redistributions of source code must retain the above copyright -; notice, this list of conditions and the following disclaimer. -; +; notice, this list of conditions and the following disclaimer. +; ; * Redistributions in binary form must reproduce the above copyright ; notice, this list of conditions and the following disclaimer in the ; documentation and/or other materials provided with the -; distribution. -; +; distribution. +; ; * Neither the name of the Intel Corporation nor the names of its ; contributors may be used to endorse or promote products derived from -; this software without specific prior written permission. -; -; +; this software without specific prior written permission. +; +; ; THIS SOFTWARE IS PROVIDED BY INTEL CORPORATION "AS IS" AND ANY ; EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE ; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR @@ -1020,15 +1020,15 @@ void Transform(uint32_t* s, const unsigned char* chunk, size_t blocks) ; This code schedules 1 blocks at a time, with 4 lanes per block ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -%define MOVDQ movdqu ;; assume buffers not aligned +%define MOVDQ movdqu ;; assume buffers not aligned ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Define Macros ; addm [mem], reg ; Add reg to mem using reg-mem add and store %macro addm 2 - add %2, %1 - mov %1, %2 + add %2, %1 + mov %1, %2 %endm ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -1054,31 +1054,31 @@ void Transform(uint32_t* s, const unsigned char* chunk, size_t blocks) %define XTMP4 xmm8 %define XFER xmm9 -%define SHUF_00BA xmm10 ; shuffle xBxA -> 00BA -%define SHUF_DC00 xmm11 ; shuffle xDxC -> DC00 -%define BYTE_FLIP_MASK xmm12 - +%define SHUF_00BA xmm10 ; shuffle xBxA -> 00BA +%define SHUF_DC00 xmm11 ; shuffle xDxC -> DC00 +%define BYTE_FLIP_MASK xmm12 + %ifdef LINUX -%define NUM_BLKS rdx ; 3rd arg -%define CTX rsi ; 2nd arg -%define INP rdi ; 1st arg - -%define SRND rdi ; clobbers INP -%define c ecx -%define d r8d -%define e edx +%define NUM_BLKS rdx ; 3rd arg +%define CTX rsi ; 2nd arg +%define INP rdi ; 1st arg + +%define SRND rdi ; clobbers INP +%define c ecx +%define d r8d +%define e edx %else -%define NUM_BLKS r8 ; 3rd arg -%define CTX rdx ; 2nd arg -%define INP rcx ; 1st arg - -%define SRND rcx ; clobbers INP -%define c edi -%define d esi -%define e r8d - +%define NUM_BLKS r8 ; 3rd arg +%define CTX rdx ; 2nd arg +%define INP rcx ; 1st arg + +%define SRND rcx ; clobbers INP +%define c edi +%define d esi +%define e r8d + %endif -%define TBL rbp +%define TBL rbp %define a eax %define b ebx @@ -1092,22 +1092,22 @@ void Transform(uint32_t* s, const unsigned char* chunk, size_t blocks) -_INP_END_SIZE equ 8 -_INP_SIZE equ 8 -_XFER_SIZE equ 8 +_INP_END_SIZE equ 8 +_INP_SIZE equ 8 +_XFER_SIZE equ 8 %ifdef LINUX -_XMM_SAVE_SIZE equ 0 +_XMM_SAVE_SIZE equ 0 %else -_XMM_SAVE_SIZE equ 7*16 +_XMM_SAVE_SIZE equ 7*16 %endif ; STACK_SIZE plus pushes must be an odd multiple of 8 -_ALIGN_SIZE equ 8 +_ALIGN_SIZE equ 8 -_INP_END equ 0 -_INP equ _INP_END + _INP_END_SIZE -_XFER equ _INP + _INP_SIZE -_XMM_SAVE equ _XFER + _XFER_SIZE + _ALIGN_SIZE -STACK_SIZE equ _XMM_SAVE + _XMM_SAVE_SIZE +_INP_END equ 0 +_INP equ _INP_END + _INP_END_SIZE +_XFER equ _INP + _INP_SIZE +_XMM_SAVE equ _XFER + _XFER_SIZE + _ALIGN_SIZE +STACK_SIZE equ _XMM_SAVE + _XMM_SAVE_SIZE ; rotate_Xs ; Rotate values of symbols X0...X3 @@ -1134,168 +1134,168 @@ STACK_SIZE equ _XMM_SAVE + _XMM_SAVE_SIZE %endm %macro FOUR_ROUNDS_AND_SCHED 0 - ;; compute s0 four at a time and s1 two at a time - ;; compute W[-16] + W[-7] 4 at a time - movdqa XTMP0, X3 - mov y0, e ; y0 = e - ror y0, (25-11) ; y0 = e >> (25-11) - mov y1, a ; y1 = a - palignr XTMP0, X2, 4 ; XTMP0 = W[-7] - ror y1, (22-13) ; y1 = a >> (22-13) - xor y0, e ; y0 = e ^ (e >> (25-11)) - mov y2, f ; y2 = f - ror y0, (11-6) ; y0 = (e >> (11-6)) ^ (e >> (25-6)) - movdqa XTMP1, X1 - xor y1, a ; y1 = a ^ (a >> (22-13) - xor y2, g ; y2 = f^g - paddd XTMP0, X0 ; XTMP0 = W[-7] + W[-16] - xor y0, e ; y0 = e ^ (e >> (11-6)) ^ (e >> (25-6)) - and y2, e ; y2 = (f^g)&e - ror y1, (13-2) ; y1 = (a >> (13-2)) ^ (a >> (22-2)) - ;; compute s0 - palignr XTMP1, X0, 4 ; XTMP1 = W[-15] - xor y1, a ; y1 = a ^ (a >> (13-2)) ^ (a >> (22-2)) - ror y0, 6 ; y0 = S1 = (e>>6) & (e>>11) ^ (e>>25) - xor y2, g ; y2 = CH = ((f^g)&e)^g - movdqa XTMP2, XTMP1 ; XTMP2 = W[-15] - ror y1, 2 ; y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22) - add y2, y0 ; y2 = S1 + CH - add y2, [rsp + _XFER + 0*4] ; y2 = k + w + S1 + CH - movdqa XTMP3, XTMP1 ; XTMP3 = W[-15] - mov y0, a ; y0 = a - add h, y2 ; h = h + S1 + CH + k + w - mov y2, a ; y2 = a - pslld XTMP1, (32-7) - or y0, c ; y0 = a|c - add d, h ; d = d + h + S1 + CH + k + w - and y2, c ; y2 = a&c - psrld XTMP2, 7 - and y0, b ; y0 = (a|c)&b - add h, y1 ; h = h + S1 + CH + k + w + S0 - por XTMP1, XTMP2 ; XTMP1 = W[-15] ror 7 - or y0, y2 ; y0 = MAJ = (a|c)&b)|(a&c) - add h, y0 ; h = h + S1 + CH + k + w + S0 + MAJ + ;; compute s0 four at a time and s1 two at a time + ;; compute W[-16] + W[-7] 4 at a time + movdqa XTMP0, X3 + mov y0, e ; y0 = e + ror y0, (25-11) ; y0 = e >> (25-11) + mov y1, a ; y1 = a + palignr XTMP0, X2, 4 ; XTMP0 = W[-7] + ror y1, (22-13) ; y1 = a >> (22-13) + xor y0, e ; y0 = e ^ (e >> (25-11)) + mov y2, f ; y2 = f + ror y0, (11-6) ; y0 = (e >> (11-6)) ^ (e >> (25-6)) + movdqa XTMP1, X1 + xor y1, a ; y1 = a ^ (a >> (22-13) + xor y2, g ; y2 = f^g + paddd XTMP0, X0 ; XTMP0 = W[-7] + W[-16] + xor y0, e ; y0 = e ^ (e >> (11-6)) ^ (e >> (25-6)) + and y2, e ; y2 = (f^g)&e + ror y1, (13-2) ; y1 = (a >> (13-2)) ^ (a >> (22-2)) + ;; compute s0 + palignr XTMP1, X0, 4 ; XTMP1 = W[-15] + xor y1, a ; y1 = a ^ (a >> (13-2)) ^ (a >> (22-2)) + ror y0, 6 ; y0 = S1 = (e>>6) & (e>>11) ^ (e>>25) + xor y2, g ; y2 = CH = ((f^g)&e)^g + movdqa XTMP2, XTMP1 ; XTMP2 = W[-15] + ror y1, 2 ; y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22) + add y2, y0 ; y2 = S1 + CH + add y2, [rsp + _XFER + 0*4] ; y2 = k + w + S1 + CH + movdqa XTMP3, XTMP1 ; XTMP3 = W[-15] + mov y0, a ; y0 = a + add h, y2 ; h = h + S1 + CH + k + w + mov y2, a ; y2 = a + pslld XTMP1, (32-7) + or y0, c ; y0 = a|c + add d, h ; d = d + h + S1 + CH + k + w + and y2, c ; y2 = a&c + psrld XTMP2, 7 + and y0, b ; y0 = (a|c)&b + add h, y1 ; h = h + S1 + CH + k + w + S0 + por XTMP1, XTMP2 ; XTMP1 = W[-15] ror 7 + or y0, y2 ; y0 = MAJ = (a|c)&b)|(a&c) + add h, y0 ; h = h + S1 + CH + k + w + S0 + MAJ ROTATE_ARGS - movdqa XTMP2, XTMP3 ; XTMP2 = W[-15] - mov y0, e ; y0 = e - mov y1, a ; y1 = a - movdqa XTMP4, XTMP3 ; XTMP4 = W[-15] - ror y0, (25-11) ; y0 = e >> (25-11) - xor y0, e ; y0 = e ^ (e >> (25-11)) - mov y2, f ; y2 = f - ror y1, (22-13) ; y1 = a >> (22-13) - pslld XTMP3, (32-18) - xor y1, a ; y1 = a ^ (a >> (22-13) - ror y0, (11-6) ; y0 = (e >> (11-6)) ^ (e >> (25-6)) - xor y2, g ; y2 = f^g - psrld XTMP2, 18 - ror y1, (13-2) ; y1 = (a >> (13-2)) ^ (a >> (22-2)) - xor y0, e ; y0 = e ^ (e >> (11-6)) ^ (e >> (25-6)) - and y2, e ; y2 = (f^g)&e - ror y0, 6 ; y0 = S1 = (e>>6) & (e>>11) ^ (e>>25) - pxor XTMP1, XTMP3 - xor y1, a ; y1 = a ^ (a >> (13-2)) ^ (a >> (22-2)) - xor y2, g ; y2 = CH = ((f^g)&e)^g - psrld XTMP4, 3 ; XTMP4 = W[-15] >> 3 - add y2, y0 ; y2 = S1 + CH - add y2, [rsp + _XFER + 1*4] ; y2 = k + w + S1 + CH - ror y1, 2 ; y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22) - pxor XTMP1, XTMP2 ; XTMP1 = W[-15] ror 7 ^ W[-15] ror 18 - mov y0, a ; y0 = a - add h, y2 ; h = h + S1 + CH + k + w - mov y2, a ; y2 = a - pxor XTMP1, XTMP4 ; XTMP1 = s0 - or y0, c ; y0 = a|c - add d, h ; d = d + h + S1 + CH + k + w - and y2, c ; y2 = a&c - ;; compute low s1 - pshufd XTMP2, X3, 11111010b ; XTMP2 = W[-2] {BBAA} - and y0, b ; y0 = (a|c)&b - add h, y1 ; h = h + S1 + CH + k + w + S0 - paddd XTMP0, XTMP1 ; XTMP0 = W[-16] + W[-7] + s0 - or y0, y2 ; y0 = MAJ = (a|c)&b)|(a&c) - add h, y0 ; h = h + S1 + CH + k + w + S0 + MAJ + movdqa XTMP2, XTMP3 ; XTMP2 = W[-15] + mov y0, e ; y0 = e + mov y1, a ; y1 = a + movdqa XTMP4, XTMP3 ; XTMP4 = W[-15] + ror y0, (25-11) ; y0 = e >> (25-11) + xor y0, e ; y0 = e ^ (e >> (25-11)) + mov y2, f ; y2 = f + ror y1, (22-13) ; y1 = a >> (22-13) + pslld XTMP3, (32-18) + xor y1, a ; y1 = a ^ (a >> (22-13) + ror y0, (11-6) ; y0 = (e >> (11-6)) ^ (e >> (25-6)) + xor y2, g ; y2 = f^g + psrld XTMP2, 18 + ror y1, (13-2) ; y1 = (a >> (13-2)) ^ (a >> (22-2)) + xor y0, e ; y0 = e ^ (e >> (11-6)) ^ (e >> (25-6)) + and y2, e ; y2 = (f^g)&e + ror y0, 6 ; y0 = S1 = (e>>6) & (e>>11) ^ (e>>25) + pxor XTMP1, XTMP3 + xor y1, a ; y1 = a ^ (a >> (13-2)) ^ (a >> (22-2)) + xor y2, g ; y2 = CH = ((f^g)&e)^g + psrld XTMP4, 3 ; XTMP4 = W[-15] >> 3 + add y2, y0 ; y2 = S1 + CH + add y2, [rsp + _XFER + 1*4] ; y2 = k + w + S1 + CH + ror y1, 2 ; y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22) + pxor XTMP1, XTMP2 ; XTMP1 = W[-15] ror 7 ^ W[-15] ror 18 + mov y0, a ; y0 = a + add h, y2 ; h = h + S1 + CH + k + w + mov y2, a ; y2 = a + pxor XTMP1, XTMP4 ; XTMP1 = s0 + or y0, c ; y0 = a|c + add d, h ; d = d + h + S1 + CH + k + w + and y2, c ; y2 = a&c + ;; compute low s1 + pshufd XTMP2, X3, 11111010b ; XTMP2 = W[-2] {BBAA} + and y0, b ; y0 = (a|c)&b + add h, y1 ; h = h + S1 + CH + k + w + S0 + paddd XTMP0, XTMP1 ; XTMP0 = W[-16] + W[-7] + s0 + or y0, y2 ; y0 = MAJ = (a|c)&b)|(a&c) + add h, y0 ; h = h + S1 + CH + k + w + S0 + MAJ ROTATE_ARGS - movdqa XTMP3, XTMP2 ; XTMP3 = W[-2] {BBAA} - mov y0, e ; y0 = e - mov y1, a ; y1 = a - ror y0, (25-11) ; y0 = e >> (25-11) - movdqa XTMP4, XTMP2 ; XTMP4 = W[-2] {BBAA} - xor y0, e ; y0 = e ^ (e >> (25-11)) - ror y1, (22-13) ; y1 = a >> (22-13) - mov y2, f ; y2 = f - xor y1, a ; y1 = a ^ (a >> (22-13) - ror y0, (11-6) ; y0 = (e >> (11-6)) ^ (e >> (25-6)) - psrlq XTMP2, 17 ; XTMP2 = W[-2] ror 17 {xBxA} - xor y2, g ; y2 = f^g - psrlq XTMP3, 19 ; XTMP3 = W[-2] ror 19 {xBxA} - xor y0, e ; y0 = e ^ (e >> (11-6)) ^ (e >> (25-6)) - and y2, e ; y2 = (f^g)&e - psrld XTMP4, 10 ; XTMP4 = W[-2] >> 10 {BBAA} - ror y1, (13-2) ; y1 = (a >> (13-2)) ^ (a >> (22-2)) - xor y1, a ; y1 = a ^ (a >> (13-2)) ^ (a >> (22-2)) - xor y2, g ; y2 = CH = ((f^g)&e)^g - ror y0, 6 ; y0 = S1 = (e>>6) & (e>>11) ^ (e>>25) - pxor XTMP2, XTMP3 - add y2, y0 ; y2 = S1 + CH - ror y1, 2 ; y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22) - add y2, [rsp + _XFER + 2*4] ; y2 = k + w + S1 + CH - pxor XTMP4, XTMP2 ; XTMP4 = s1 {xBxA} - mov y0, a ; y0 = a - add h, y2 ; h = h + S1 + CH + k + w - mov y2, a ; y2 = a - pshufb XTMP4, SHUF_00BA ; XTMP4 = s1 {00BA} - or y0, c ; y0 = a|c - add d, h ; d = d + h + S1 + CH + k + w - and y2, c ; y2 = a&c - paddd XTMP0, XTMP4 ; XTMP0 = {..., ..., W[1], W[0]} - and y0, b ; y0 = (a|c)&b - add h, y1 ; h = h + S1 + CH + k + w + S0 - ;; compute high s1 - pshufd XTMP2, XTMP0, 01010000b ; XTMP2 = W[-2] {DDCC} - or y0, y2 ; y0 = MAJ = (a|c)&b)|(a&c) - add h, y0 ; h = h + S1 + CH + k + w + S0 + MAJ + movdqa XTMP3, XTMP2 ; XTMP3 = W[-2] {BBAA} + mov y0, e ; y0 = e + mov y1, a ; y1 = a + ror y0, (25-11) ; y0 = e >> (25-11) + movdqa XTMP4, XTMP2 ; XTMP4 = W[-2] {BBAA} + xor y0, e ; y0 = e ^ (e >> (25-11)) + ror y1, (22-13) ; y1 = a >> (22-13) + mov y2, f ; y2 = f + xor y1, a ; y1 = a ^ (a >> (22-13) + ror y0, (11-6) ; y0 = (e >> (11-6)) ^ (e >> (25-6)) + psrlq XTMP2, 17 ; XTMP2 = W[-2] ror 17 {xBxA} + xor y2, g ; y2 = f^g + psrlq XTMP3, 19 ; XTMP3 = W[-2] ror 19 {xBxA} + xor y0, e ; y0 = e ^ (e >> (11-6)) ^ (e >> (25-6)) + and y2, e ; y2 = (f^g)&e + psrld XTMP4, 10 ; XTMP4 = W[-2] >> 10 {BBAA} + ror y1, (13-2) ; y1 = (a >> (13-2)) ^ (a >> (22-2)) + xor y1, a ; y1 = a ^ (a >> (13-2)) ^ (a >> (22-2)) + xor y2, g ; y2 = CH = ((f^g)&e)^g + ror y0, 6 ; y0 = S1 = (e>>6) & (e>>11) ^ (e>>25) + pxor XTMP2, XTMP3 + add y2, y0 ; y2 = S1 + CH + ror y1, 2 ; y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22) + add y2, [rsp + _XFER + 2*4] ; y2 = k + w + S1 + CH + pxor XTMP4, XTMP2 ; XTMP4 = s1 {xBxA} + mov y0, a ; y0 = a + add h, y2 ; h = h + S1 + CH + k + w + mov y2, a ; y2 = a + pshufb XTMP4, SHUF_00BA ; XTMP4 = s1 {00BA} + or y0, c ; y0 = a|c + add d, h ; d = d + h + S1 + CH + k + w + and y2, c ; y2 = a&c + paddd XTMP0, XTMP4 ; XTMP0 = {..., ..., W[1], W[0]} + and y0, b ; y0 = (a|c)&b + add h, y1 ; h = h + S1 + CH + k + w + S0 + ;; compute high s1 + pshufd XTMP2, XTMP0, 01010000b ; XTMP2 = W[-2] {DDCC} + or y0, y2 ; y0 = MAJ = (a|c)&b)|(a&c) + add h, y0 ; h = h + S1 + CH + k + w + S0 + MAJ ROTATE_ARGS - movdqa XTMP3, XTMP2 ; XTMP3 = W[-2] {DDCC} - mov y0, e ; y0 = e - ror y0, (25-11) ; y0 = e >> (25-11) - mov y1, a ; y1 = a - movdqa X0, XTMP2 ; X0 = W[-2] {DDCC} - ror y1, (22-13) ; y1 = a >> (22-13) - xor y0, e ; y0 = e ^ (e >> (25-11)) - mov y2, f ; y2 = f - ror y0, (11-6) ; y0 = (e >> (11-6)) ^ (e >> (25-6)) - psrlq XTMP2, 17 ; XTMP2 = W[-2] ror 17 {xDxC} - xor y1, a ; y1 = a ^ (a >> (22-13) - xor y2, g ; y2 = f^g - psrlq XTMP3, 19 ; XTMP3 = W[-2] ror 19 {xDxC} - xor y0, e ; y0 = e ^ (e >> (11-6)) ^ (e >> (25-6)) - and y2, e ; y2 = (f^g)&e - ror y1, (13-2) ; y1 = (a >> (13-2)) ^ (a >> (22-2)) - psrld X0, 10 ; X0 = W[-2] >> 10 {DDCC} - xor y1, a ; y1 = a ^ (a >> (13-2)) ^ (a >> (22-2)) - ror y0, 6 ; y0 = S1 = (e>>6) & (e>>11) ^ (e>>25) - xor y2, g ; y2 = CH = ((f^g)&e)^g - pxor XTMP2, XTMP3 - ror y1, 2 ; y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22) - add y2, y0 ; y2 = S1 + CH - add y2, [rsp + _XFER + 3*4] ; y2 = k + w + S1 + CH - pxor X0, XTMP2 ; X0 = s1 {xDxC} - mov y0, a ; y0 = a - add h, y2 ; h = h + S1 + CH + k + w - mov y2, a ; y2 = a - pshufb X0, SHUF_DC00 ; X0 = s1 {DC00} - or y0, c ; y0 = a|c - add d, h ; d = d + h + S1 + CH + k + w - and y2, c ; y2 = a&c - paddd X0, XTMP0 ; X0 = {W[3], W[2], W[1], W[0]} - and y0, b ; y0 = (a|c)&b - add h, y1 ; h = h + S1 + CH + k + w + S0 - or y0, y2 ; y0 = MAJ = (a|c)&b)|(a&c) - add h, y0 ; h = h + S1 + CH + k + w + S0 + MAJ + movdqa XTMP3, XTMP2 ; XTMP3 = W[-2] {DDCC} + mov y0, e ; y0 = e + ror y0, (25-11) ; y0 = e >> (25-11) + mov y1, a ; y1 = a + movdqa X0, XTMP2 ; X0 = W[-2] {DDCC} + ror y1, (22-13) ; y1 = a >> (22-13) + xor y0, e ; y0 = e ^ (e >> (25-11)) + mov y2, f ; y2 = f + ror y0, (11-6) ; y0 = (e >> (11-6)) ^ (e >> (25-6)) + psrlq XTMP2, 17 ; XTMP2 = W[-2] ror 17 {xDxC} + xor y1, a ; y1 = a ^ (a >> (22-13) + xor y2, g ; y2 = f^g + psrlq XTMP3, 19 ; XTMP3 = W[-2] ror 19 {xDxC} + xor y0, e ; y0 = e ^ (e >> (11-6)) ^ (e >> (25-6)) + and y2, e ; y2 = (f^g)&e + ror y1, (13-2) ; y1 = (a >> (13-2)) ^ (a >> (22-2)) + psrld X0, 10 ; X0 = W[-2] >> 10 {DDCC} + xor y1, a ; y1 = a ^ (a >> (13-2)) ^ (a >> (22-2)) + ror y0, 6 ; y0 = S1 = (e>>6) & (e>>11) ^ (e>>25) + xor y2, g ; y2 = CH = ((f^g)&e)^g + pxor XTMP2, XTMP3 + ror y1, 2 ; y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22) + add y2, y0 ; y2 = S1 + CH + add y2, [rsp + _XFER + 3*4] ; y2 = k + w + S1 + CH + pxor X0, XTMP2 ; X0 = s1 {xDxC} + mov y0, a ; y0 = a + add h, y2 ; h = h + S1 + CH + k + w + mov y2, a ; y2 = a + pshufb X0, SHUF_DC00 ; X0 = s1 {DC00} + or y0, c ; y0 = a|c + add d, h ; d = d + h + S1 + CH + k + w + and y2, c ; y2 = a&c + paddd X0, XTMP0 ; X0 = {W[3], W[2], W[1], W[0]} + and y0, b ; y0 = (a|c)&b + add h, y1 ; h = h + S1 + CH + k + w + S0 + or y0, y2 ; y0 = MAJ = (a|c)&b)|(a&c) + add h, y0 ; h = h + S1 + CH + k + w + S0 + MAJ ROTATE_ARGS rotate_Xs @@ -1303,34 +1303,34 @@ rotate_Xs ;; input is [rsp + _XFER + %1 * 4] %macro DO_ROUND 1 - mov y0, e ; y0 = e - ror y0, (25-11) ; y0 = e >> (25-11) - mov y1, a ; y1 = a - xor y0, e ; y0 = e ^ (e >> (25-11)) - ror y1, (22-13) ; y1 = a >> (22-13) - mov y2, f ; y2 = f - xor y1, a ; y1 = a ^ (a >> (22-13) - ror y0, (11-6) ; y0 = (e >> (11-6)) ^ (e >> (25-6)) - xor y2, g ; y2 = f^g - xor y0, e ; y0 = e ^ (e >> (11-6)) ^ (e >> (25-6)) - ror y1, (13-2) ; y1 = (a >> (13-2)) ^ (a >> (22-2)) - and y2, e ; y2 = (f^g)&e - xor y1, a ; y1 = a ^ (a >> (13-2)) ^ (a >> (22-2)) - ror y0, 6 ; y0 = S1 = (e>>6) & (e>>11) ^ (e>>25) - xor y2, g ; y2 = CH = ((f^g)&e)^g - add y2, y0 ; y2 = S1 + CH - ror y1, 2 ; y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22) - add y2, [rsp + _XFER + %1 * 4] ; y2 = k + w + S1 + CH - mov y0, a ; y0 = a - add h, y2 ; h = h + S1 + CH + k + w - mov y2, a ; y2 = a - or y0, c ; y0 = a|c - add d, h ; d = d + h + S1 + CH + k + w - and y2, c ; y2 = a&c - and y0, b ; y0 = (a|c)&b - add h, y1 ; h = h + S1 + CH + k + w + S0 - or y0, y2 ; y0 = MAJ = (a|c)&b)|(a&c) - add h, y0 ; h = h + S1 + CH + k + w + S0 + MAJ + mov y0, e ; y0 = e + ror y0, (25-11) ; y0 = e >> (25-11) + mov y1, a ; y1 = a + xor y0, e ; y0 = e ^ (e >> (25-11)) + ror y1, (22-13) ; y1 = a >> (22-13) + mov y2, f ; y2 = f + xor y1, a ; y1 = a ^ (a >> (22-13) + ror y0, (11-6) ; y0 = (e >> (11-6)) ^ (e >> (25-6)) + xor y2, g ; y2 = f^g + xor y0, e ; y0 = e ^ (e >> (11-6)) ^ (e >> (25-6)) + ror y1, (13-2) ; y1 = (a >> (13-2)) ^ (a >> (22-2)) + and y2, e ; y2 = (f^g)&e + xor y1, a ; y1 = a ^ (a >> (13-2)) ^ (a >> (22-2)) + ror y0, 6 ; y0 = S1 = (e>>6) & (e>>11) ^ (e>>25) + xor y2, g ; y2 = CH = ((f^g)&e)^g + add y2, y0 ; y2 = S1 + CH + ror y1, 2 ; y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22) + add y2, [rsp + _XFER + %1 * 4] ; y2 = k + w + S1 + CH + mov y0, a ; y0 = a + add h, y2 ; h = h + S1 + CH + k + w + mov y2, a ; y2 = a + or y0, c ; y0 = a|c + add d, h ; d = d + h + S1 + CH + k + w + and y2, c ; y2 = a&c + and y0, b ; y0 = (a|c)&b + add h, y1 ; h = h + S1 + CH + k + w + S0 + or y0, y2 ; y0 = MAJ = (a|c)&b)|(a&c) + add h, y0 ; h = h + S1 + CH + k + w + S0 + MAJ ROTATE_ARGS %endm @@ -1344,166 +1344,166 @@ section .text global sha256_sse4 align 32 sha256_sse4: - push rbx + push rbx %ifndef LINUX - push rsi - push rdi + push rsi + push rdi %endif - push rbp - push r13 - push r14 - push r15 + push rbp + push r13 + push r14 + push r15 - sub rsp,STACK_SIZE + sub rsp,STACK_SIZE %ifndef LINUX - movdqa [rsp + _XMM_SAVE + 0*16],xmm6 - movdqa [rsp + _XMM_SAVE + 1*16],xmm7 - movdqa [rsp + _XMM_SAVE + 2*16],xmm8 - movdqa [rsp + _XMM_SAVE + 3*16],xmm9 - movdqa [rsp + _XMM_SAVE + 4*16],xmm10 - movdqa [rsp + _XMM_SAVE + 5*16],xmm11 - movdqa [rsp + _XMM_SAVE + 6*16],xmm12 + movdqa [rsp + _XMM_SAVE + 0*16],xmm6 + movdqa [rsp + _XMM_SAVE + 1*16],xmm7 + movdqa [rsp + _XMM_SAVE + 2*16],xmm8 + movdqa [rsp + _XMM_SAVE + 3*16],xmm9 + movdqa [rsp + _XMM_SAVE + 4*16],xmm10 + movdqa [rsp + _XMM_SAVE + 5*16],xmm11 + movdqa [rsp + _XMM_SAVE + 6*16],xmm12 %endif - shl NUM_BLKS, 6 ; convert to bytes - jz done_hash - add NUM_BLKS, INP ; pointer to end of data - mov [rsp + _INP_END], NUM_BLKS + shl NUM_BLKS, 6 ; convert to bytes + jz done_hash + add NUM_BLKS, INP ; pointer to end of data + mov [rsp + _INP_END], NUM_BLKS ;; load initial digest - mov a,[4*0 + CTX] - mov b,[4*1 + CTX] - mov c,[4*2 + CTX] - mov d,[4*3 + CTX] - mov e,[4*4 + CTX] - mov f,[4*5 + CTX] - mov g,[4*6 + CTX] - mov h,[4*7 + CTX] - - movdqa BYTE_FLIP_MASK, [PSHUFFLE_BYTE_FLIP_MASK wrt rip] - movdqa SHUF_00BA, [_SHUF_00BA wrt rip] - movdqa SHUF_DC00, [_SHUF_DC00 wrt rip] + mov a,[4*0 + CTX] + mov b,[4*1 + CTX] + mov c,[4*2 + CTX] + mov d,[4*3 + CTX] + mov e,[4*4 + CTX] + mov f,[4*5 + CTX] + mov g,[4*6 + CTX] + mov h,[4*7 + CTX] + + movdqa BYTE_FLIP_MASK, [PSHUFFLE_BYTE_FLIP_MASK wrt rip] + movdqa SHUF_00BA, [_SHUF_00BA wrt rip] + movdqa SHUF_DC00, [_SHUF_DC00 wrt rip] loop0: - lea TBL,[K256 wrt rip] + lea TBL,[K256 wrt rip] ;; byte swap first 16 dwords - COPY_XMM_AND_BSWAP X0, [INP + 0*16], BYTE_FLIP_MASK - COPY_XMM_AND_BSWAP X1, [INP + 1*16], BYTE_FLIP_MASK - COPY_XMM_AND_BSWAP X2, [INP + 2*16], BYTE_FLIP_MASK - COPY_XMM_AND_BSWAP X3, [INP + 3*16], BYTE_FLIP_MASK - - mov [rsp + _INP], INP + COPY_XMM_AND_BSWAP X0, [INP + 0*16], BYTE_FLIP_MASK + COPY_XMM_AND_BSWAP X1, [INP + 1*16], BYTE_FLIP_MASK + COPY_XMM_AND_BSWAP X2, [INP + 2*16], BYTE_FLIP_MASK + COPY_XMM_AND_BSWAP X3, [INP + 3*16], BYTE_FLIP_MASK + + mov [rsp + _INP], INP ;; schedule 48 input dwords, by doing 3 rounds of 16 each - mov SRND, 3 + mov SRND, 3 align 16 loop1: - movdqa XFER, [TBL + 0*16] - paddd XFER, X0 - movdqa [rsp + _XFER], XFER + movdqa XFER, [TBL + 0*16] + paddd XFER, X0 + movdqa [rsp + _XFER], XFER FOUR_ROUNDS_AND_SCHED - movdqa XFER, [TBL + 1*16] - paddd XFER, X0 - movdqa [rsp + _XFER], XFER + movdqa XFER, [TBL + 1*16] + paddd XFER, X0 + movdqa [rsp + _XFER], XFER FOUR_ROUNDS_AND_SCHED - movdqa XFER, [TBL + 2*16] - paddd XFER, X0 - movdqa [rsp + _XFER], XFER + movdqa XFER, [TBL + 2*16] + paddd XFER, X0 + movdqa [rsp + _XFER], XFER FOUR_ROUNDS_AND_SCHED - movdqa XFER, [TBL + 3*16] - paddd XFER, X0 - movdqa [rsp + _XFER], XFER - add TBL, 4*16 + movdqa XFER, [TBL + 3*16] + paddd XFER, X0 + movdqa [rsp + _XFER], XFER + add TBL, 4*16 FOUR_ROUNDS_AND_SCHED - sub SRND, 1 - jne loop1 + sub SRND, 1 + jne loop1 - mov SRND, 2 + mov SRND, 2 loop2: - paddd X0, [TBL + 0*16] - movdqa [rsp + _XFER], X0 - DO_ROUND 0 - DO_ROUND 1 - DO_ROUND 2 - DO_ROUND 3 - paddd X1, [TBL + 1*16] - movdqa [rsp + _XFER], X1 - add TBL, 2*16 - DO_ROUND 0 - DO_ROUND 1 - DO_ROUND 2 - DO_ROUND 3 - - movdqa X0, X2 - movdqa X1, X3 - - sub SRND, 1 - jne loop2 - - addm [4*0 + CTX],a - addm [4*1 + CTX],b - addm [4*2 + CTX],c - addm [4*3 + CTX],d - addm [4*4 + CTX],e - addm [4*5 + CTX],f - addm [4*6 + CTX],g - addm [4*7 + CTX],h - - mov INP, [rsp + _INP] - add INP, 64 - cmp INP, [rsp + _INP_END] - jne loop0 + paddd X0, [TBL + 0*16] + movdqa [rsp + _XFER], X0 + DO_ROUND 0 + DO_ROUND 1 + DO_ROUND 2 + DO_ROUND 3 + paddd X1, [TBL + 1*16] + movdqa [rsp + _XFER], X1 + add TBL, 2*16 + DO_ROUND 0 + DO_ROUND 1 + DO_ROUND 2 + DO_ROUND 3 + + movdqa X0, X2 + movdqa X1, X3 + + sub SRND, 1 + jne loop2 + + addm [4*0 + CTX],a + addm [4*1 + CTX],b + addm [4*2 + CTX],c + addm [4*3 + CTX],d + addm [4*4 + CTX],e + addm [4*5 + CTX],f + addm [4*6 + CTX],g + addm [4*7 + CTX],h + + mov INP, [rsp + _INP] + add INP, 64 + cmp INP, [rsp + _INP_END] + jne loop0 done_hash: %ifndef LINUX - movdqa xmm6,[rsp + _XMM_SAVE + 0*16] - movdqa xmm7,[rsp + _XMM_SAVE + 1*16] - movdqa xmm8,[rsp + _XMM_SAVE + 2*16] - movdqa xmm9,[rsp + _XMM_SAVE + 3*16] - movdqa xmm10,[rsp + _XMM_SAVE + 4*16] - movdqa xmm11,[rsp + _XMM_SAVE + 5*16] - movdqa xmm12,[rsp + _XMM_SAVE + 6*16] + movdqa xmm6,[rsp + _XMM_SAVE + 0*16] + movdqa xmm7,[rsp + _XMM_SAVE + 1*16] + movdqa xmm8,[rsp + _XMM_SAVE + 2*16] + movdqa xmm9,[rsp + _XMM_SAVE + 3*16] + movdqa xmm10,[rsp + _XMM_SAVE + 4*16] + movdqa xmm11,[rsp + _XMM_SAVE + 5*16] + movdqa xmm12,[rsp + _XMM_SAVE + 6*16] %endif - add rsp, STACK_SIZE + add rsp, STACK_SIZE - pop r15 - pop r14 - pop r13 - pop rbp + pop r15 + pop r14 + pop r13 + pop rbp %ifndef LINUX - pop rdi - pop rsi + pop rdi + pop rsi %endif - pop rbx + pop rbx + + ret - ret - section .data align 64 K256: - dd 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 - dd 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5 - dd 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3 - dd 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174 - dd 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc - dd 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da - dd 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7 - dd 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967 - dd 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13 - dd 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85 - dd 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3 - dd 0xd192e819,0xd6990624,0xf40e3585,0x106aa070 - dd 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5 - dd 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 - dd 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 - dd 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 + dd 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 + dd 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5 + dd 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3 + dd 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174 + dd 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc + dd 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da + dd 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7 + dd 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967 + dd 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13 + dd 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85 + dd 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3 + dd 0xd192e819,0xd6990624,0xf40e3585,0x106aa070 + dd 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5 + dd 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 + dd 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 + dd 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 PSHUFFLE_BYTE_FLIP_MASK: ddq 0x0c0d0e0f08090a0b0405060700010203 diff --git a/bitcoinkernel/bitcoin/src/crypto/sha256_x86_shani.cpp b/bitcoinkernel/bitcoin/src/crypto/sha256_x86_shani.cpp index 5ee957a..daccdc5 100644 --- a/bitcoinkernel/bitcoin/src/crypto/sha256_x86_shani.cpp +++ b/bitcoinkernel/bitcoin/src/crypto/sha256_x86_shani.cpp @@ -8,6 +8,7 @@ #if defined(ENABLE_SSE41) && defined(ENABLE_X86_SHANI) +#include #include #include diff --git a/bitcoinkernel/bitcoin/src/crypto/sha3.cpp b/bitcoinkernel/bitcoin/src/crypto/sha3.cpp index 4c61bce..f684447 100644 --- a/bitcoinkernel/bitcoin/src/crypto/sha3.cpp +++ b/bitcoinkernel/bitcoin/src/crypto/sha3.cpp @@ -9,9 +9,9 @@ #include #include -#include #include -#include +#include +#include #include void KeccakF(uint64_t (&st)[25]) diff --git a/bitcoinkernel/bitcoin/src/crypto/sha3.h b/bitcoinkernel/bitcoin/src/crypto/sha3.h index 7a538e4..91cdb5c 100644 --- a/bitcoinkernel/bitcoin/src/crypto/sha3.h +++ b/bitcoinkernel/bitcoin/src/crypto/sha3.h @@ -5,10 +5,9 @@ #ifndef BITCOIN_CRYPTO_SHA3_H #define BITCOIN_CRYPTO_SHA3_H -#include - #include #include +#include //! The Keccak-f[1600] transform. void KeccakF(uint64_t (&st)[25]); diff --git a/bitcoinkernel/bitcoin/src/crypto/siphash.cpp b/bitcoinkernel/bitcoin/src/crypto/siphash.cpp index dda28a6..1a9eb77 100644 --- a/bitcoinkernel/bitcoin/src/crypto/siphash.cpp +++ b/bitcoinkernel/bitcoin/src/crypto/siphash.cpp @@ -4,7 +4,11 @@ #include +#include + #include +#include +#include #define SIPROUND do { \ v0 += v1; v1 = std::rotl(v1, 13); v1 ^= v0; \ diff --git a/bitcoinkernel/bitcoin/src/crypto/siphash.h b/bitcoinkernel/bitcoin/src/crypto/siphash.h index 4c6a39e..8d41a08 100644 --- a/bitcoinkernel/bitcoin/src/crypto/siphash.h +++ b/bitcoinkernel/bitcoin/src/crypto/siphash.h @@ -6,9 +6,9 @@ #define BITCOIN_CRYPTO_SIPHASH_H #include +#include -#include -#include +class uint256; /** SipHash-2-4 */ class CSipHasher diff --git a/bitcoinkernel/bitcoin/src/dbwrapper.cpp b/bitcoinkernel/bitcoin/src/dbwrapper.cpp index fe5f9cb..cb9abee 100644 --- a/bitcoinkernel/bitcoin/src/dbwrapper.cpp +++ b/bitcoinkernel/bitcoin/src/dbwrapper.cpp @@ -274,7 +274,7 @@ CDBWrapper::~CDBWrapper() DBContext().options.env = nullptr; } -bool CDBWrapper::WriteBatch(CDBBatch& batch, bool fSync) +void CDBWrapper::WriteBatch(CDBBatch& batch, bool fSync) { const bool log_memory = LogAcceptCategory(BCLog::LEVELDB, BCLog::Level::Debug); double mem_before = 0; @@ -288,7 +288,6 @@ bool CDBWrapper::WriteBatch(CDBBatch& batch, bool fSync) LogDebug(BCLog::LEVELDB, "WriteBatch memory usage: db=%s, before=%.1fMiB, after=%.1fMiB\n", m_name, mem_before, mem_after); } - return true; } size_t CDBWrapper::DynamicMemoryUsage() const diff --git a/bitcoinkernel/bitcoin/src/dbwrapper.h b/bitcoinkernel/bitcoin/src/dbwrapper.h index b9b98bd..c770df8 100644 --- a/bitcoinkernel/bitcoin/src/dbwrapper.h +++ b/bitcoinkernel/bitcoin/src/dbwrapper.h @@ -230,11 +230,11 @@ class CDBWrapper } template - bool Write(const K& key, const V& value, bool fSync = false) + void Write(const K& key, const V& value, bool fSync = false) { CDBBatch batch(*this); batch.Write(key, value); - return WriteBatch(batch, fSync); + WriteBatch(batch, fSync); } //! @returns filesystem path to the on-disk data. @@ -255,14 +255,14 @@ class CDBWrapper } template - bool Erase(const K& key, bool fSync = false) + void Erase(const K& key, bool fSync = false) { CDBBatch batch(*this); batch.Erase(key); - return WriteBatch(batch, fSync); + WriteBatch(batch, fSync); } - bool WriteBatch(CDBBatch& batch, bool fSync = false); + void WriteBatch(CDBBatch& batch, bool fSync = false); // Get an estimate of LevelDB memory usage (in bytes). size_t DynamicMemoryUsage() const; diff --git a/bitcoinkernel/bitcoin/src/flatfile.h b/bitcoinkernel/bitcoin/src/flatfile.h index 3edb0b8..3ec4eaf 100644 --- a/bitcoinkernel/bitcoin/src/flatfile.h +++ b/bitcoinkernel/bitcoin/src/flatfile.h @@ -13,16 +13,16 @@ struct FlatFilePos { - int nFile{-1}; - unsigned int nPos{0}; + int32_t nFile{-1}; + uint32_t nPos{0}; SERIALIZE_METHODS(FlatFilePos, obj) { READWRITE(VARINT_MODE(obj.nFile, VarIntMode::NONNEGATIVE_SIGNED), VARINT(obj.nPos)); } FlatFilePos() = default; - FlatFilePos(int nFileIn, unsigned int nPosIn) : - nFile(nFileIn), - nPos(nPosIn) + FlatFilePos(int32_t nFileIn, uint32_t nPosIn) + : nFile{nFileIn}, + nPos{nPosIn} {} friend bool operator==(const FlatFilePos &a, const FlatFilePos &b) { diff --git a/bitcoinkernel/bitcoin/src/headerssync.cpp b/bitcoinkernel/bitcoin/src/headerssync.cpp index ae7187f..e52a8cd 100644 --- a/bitcoinkernel/bitcoin/src/headerssync.cpp +++ b/bitcoinkernel/bitcoin/src/headerssync.cpp @@ -1,32 +1,26 @@ -// Copyright (c) 2022 The Bitcoin Core developers +// Copyright (c) 2022-present The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include + #include #include #include #include #include -// The two constants below are computed using the simulation script in -// contrib/devtools/headerssync-params.py. - -//! Store one header commitment per HEADER_COMMITMENT_PERIOD blocks. -constexpr size_t HEADER_COMMITMENT_PERIOD{632}; - -//! Only feed headers to validation once this many headers on top have been -//! received and validated against commitments. -constexpr size_t REDOWNLOAD_BUFFER_SIZE{15009}; // 15009/632 = ~23.7 commitments - -// Our memory analysis assumes 48 bytes for a CompressedHeader (so we should -// re-calculate parameters if we compress further) +// Our memory analysis in headerssync-params.py assumes this many bytes for a +// CompressedHeader (we should re-calculate parameters if we compress further). static_assert(sizeof(CompressedHeader) == 48); HeadersSyncState::HeadersSyncState(NodeId id, const Consensus::Params& consensus_params, - const CBlockIndex* chain_start, const arith_uint256& minimum_required_work) : - m_commit_offset(FastRandomContext().randrange(HEADER_COMMITMENT_PERIOD)), + const HeadersSyncParams& params, const CBlockIndex* chain_start, + const arith_uint256& minimum_required_work) : + m_commit_offset((assert(params.commitment_period > 0), // HeadersSyncParams field must be initialized to non-zero. + FastRandomContext().randrange(params.commitment_period))), m_id(id), m_consensus_params(consensus_params), + m_params(params), m_chain_start(chain_start), m_minimum_required_work(minimum_required_work), m_current_chain_work(chain_start->nChainWork), @@ -41,7 +35,9 @@ HeadersSyncState::HeadersSyncState(NodeId id, const Consensus::Params& consensus // exceeds this bound, because it's not possible for a consensus-valid // chain to be longer than this (at the current time -- in the future we // could try again, if necessary, to sync a longer chain). - m_max_commitments = 6*(Ticks(NodeClock::now() - NodeSeconds{std::chrono::seconds{chain_start->GetMedianTimePast()}}) + MAX_FUTURE_BLOCK_TIME) / HEADER_COMMITMENT_PERIOD; + const auto max_seconds_since_start{(Ticks(NodeClock::now() - NodeSeconds{std::chrono::seconds{chain_start->GetMedianTimePast()}})) + + MAX_FUTURE_BLOCK_TIME}; + m_max_commitments = 6 * max_seconds_since_start / m_params.commitment_period; LogDebug(BCLog::NET, "Initial headers sync started with peer=%d: height=%i, max_commitments=%i, min_work=%s\n", m_id, m_current_height, m_max_commitments, m_minimum_required_work.ToString()); } @@ -66,8 +62,8 @@ void HeadersSyncState::Finalize() /** Process the next batch of headers received from our peer. * Validate and store commitments, and compare total chainwork to our target to * see if we can switch to REDOWNLOAD mode. */ -HeadersSyncState::ProcessingResult HeadersSyncState::ProcessNextHeaders(const - std::vector& received_headers, const bool full_headers_message) +HeadersSyncState::ProcessingResult HeadersSyncState::ProcessNextHeaders( + std::span received_headers, const bool full_headers_message) { ProcessingResult ret; @@ -137,7 +133,7 @@ HeadersSyncState::ProcessingResult HeadersSyncState::ProcessNextHeaders(const return ret; } -bool HeadersSyncState::ValidateAndStoreHeadersCommitments(const std::vector& headers) +bool HeadersSyncState::ValidateAndStoreHeadersCommitments(std::span headers) { // The caller should not give us an empty set of headers. Assume(headers.size() > 0); @@ -193,7 +189,7 @@ bool HeadersSyncState::ValidateAndProcessSingleHeader(const CBlockHeader& curren return false; } - if (next_height % HEADER_COMMITMENT_PERIOD == m_commit_offset) { + if (next_height % m_params.commitment_period == m_commit_offset) { // Add a commitment. m_header_commitments.push_back(m_hasher(current.GetHash()) & 1); if (m_header_commitments.size() > m_max_commitments) { @@ -254,7 +250,7 @@ bool HeadersSyncState::ValidateAndStoreRedownloadedHeader(const CBlockHeader& he // it's possible our peer has extended its chain between our first sync and // our second, and we don't want to return failure after we've seen our // target blockhash just because we ran out of commitments. - if (!m_process_all_remaining_headers && next_height % HEADER_COMMITMENT_PERIOD == m_commit_offset) { + if (!m_process_all_remaining_headers && next_height % m_params.commitment_period == m_commit_offset) { if (m_header_commitments.size() == 0) { LogDebug(BCLog::NET, "Initial headers sync aborted with peer=%d: commitment overrun at height=%i (redownload phase)\n", m_id, next_height); // Somehow our peer managed to feed us a different chain and @@ -285,7 +281,7 @@ std::vector HeadersSyncState::PopHeadersReadyForAcceptance() Assume(m_download_state == State::REDOWNLOAD); if (m_download_state != State::REDOWNLOAD) return ret; - while (m_redownloaded_headers.size() > REDOWNLOAD_BUFFER_SIZE || + while (m_redownloaded_headers.size() > m_params.redownload_buffer_size || (m_redownloaded_headers.size() > 0 && m_process_all_remaining_headers)) { ret.emplace_back(m_redownloaded_headers.front().GetFullHeader(m_redownload_buffer_first_prev_hash)); m_redownloaded_headers.pop_front(); diff --git a/bitcoinkernel/bitcoin/src/headerssync.h b/bitcoinkernel/bitcoin/src/headerssync.h index 56380c6..9e3af58 100644 --- a/bitcoinkernel/bitcoin/src/headerssync.h +++ b/bitcoinkernel/bitcoin/src/headerssync.h @@ -136,7 +136,8 @@ class HeadersSyncState { * minimum_required_work: amount of chain work required to accept the chain */ HeadersSyncState(NodeId id, const Consensus::Params& consensus_params, - const CBlockIndex* chain_start, const arith_uint256& minimum_required_work); + const HeadersSyncParams& params, const CBlockIndex* chain_start, + const arith_uint256& minimum_required_work); /** Result data structure for ProcessNextHeaders. */ struct ProcessingResult { @@ -165,7 +166,7 @@ class HeadersSyncState { * ProcessingResult.request_more: if true, the caller is suggested to call * NextHeadersRequestLocator and send a getheaders message using it. */ - ProcessingResult ProcessNextHeaders(const std::vector& + ProcessingResult ProcessNextHeaders(std::span received_headers, bool full_headers_message); /** Issue the next GETHEADERS message to our peer. @@ -179,8 +180,8 @@ class HeadersSyncState { /** The (secret) offset on the heights for which to create commitments. * * m_header_commitments entries are created at any height h for which - * (h % HEADER_COMMITMENT_PERIOD) == m_commit_offset. */ - const unsigned m_commit_offset; + * (h % m_params.commitment_period) == m_commit_offset. */ + const size_t m_commit_offset; private: /** Clear out all download state that might be in progress (freeing any used @@ -195,7 +196,7 @@ class HeadersSyncState { * processed headers. * On failure, this invokes Finalize() and returns false. */ - bool ValidateAndStoreHeadersCommitments(const std::vector& headers); + bool ValidateAndStoreHeadersCommitments(std::span headers); /** In PRESYNC, process and update state for a single header */ bool ValidateAndProcessSingleHeader(const CBlockHeader& current); @@ -214,6 +215,9 @@ class HeadersSyncState { /** We use the consensus params in our anti-DoS calculations */ const Consensus::Params& m_consensus_params; + /** Parameters that impact memory usage for a given chain, especially when attacked. */ + const HeadersSyncParams m_params; + /** Store the last block in our block index that the peer's chain builds from */ const CBlockIndex* m_chain_start{nullptr}; diff --git a/bitcoinkernel/bitcoin/src/index/base.cpp b/bitcoinkernel/bitcoin/src/index/base.cpp index 82259ac..d5c4044 100644 --- a/bitcoinkernel/bitcoin/src/index/base.cpp +++ b/bitcoinkernel/bitcoin/src/index/base.cpp @@ -2,10 +2,13 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include -#include #include + +#include +#include +#include #include +#include #include #include #include @@ -13,20 +16,31 @@ #include #include #include +#include +#include #include +#include #include +#include #include #include +#include +#include #include #include +#include -#include +#include +#include +#include #include #include +#include #include #include #include #include +#include constexpr uint8_t DB_BEST_BLOCK{'B'}; @@ -261,7 +275,7 @@ bool BaseIndex::Commit() ok = CustomCommit(batch); if (ok) { GetDB().WriteBestBlock(batch, GetLocator(*m_chain, m_best_block_index.load()->GetBlockHash())); - ok = GetDB().WriteBatch(batch); + GetDB().WriteBatch(batch); } } if (!ok) { diff --git a/bitcoinkernel/bitcoin/src/index/base.h b/bitcoinkernel/bitcoin/src/index/base.h index 4131b06..5a6f0a4 100644 --- a/bitcoinkernel/bitcoin/src/index/base.h +++ b/bitcoinkernel/bitcoin/src/index/base.h @@ -5,29 +5,41 @@ #ifndef BITCOIN_INDEX_BASE_H #define BITCOIN_INDEX_BASE_H +#include #include #include -#include -#include +#include +#include +#include +#include #include #include +#include +#include +#include +#include #include +#include class CBlock; class CBlockIndex; class Chainstate; -class ChainstateManager; -namespace interfaces { -class Chain; -} // namespace interfaces +struct CBlockLocator; struct IndexSummary { std::string name; bool synced{false}; int best_block_height{0}; uint256 best_block_hash; }; +namespace interfaces { +struct BlockRef; +} +namespace util { +template +struct ConstevalFormatString; +} /** * Base class for indices of blockchain data. This implements diff --git a/bitcoinkernel/bitcoin/src/index/blockfilterindex.cpp b/bitcoinkernel/bitcoin/src/index/blockfilterindex.cpp index 2ccae3a..4bf2f5e 100644 --- a/bitcoinkernel/bitcoin/src/index/blockfilterindex.cpp +++ b/bitcoinkernel/bitcoin/src/index/blockfilterindex.cpp @@ -2,19 +2,39 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include +#include -#include +#include +#include #include #include +#include #include -#include +#include +#include +#include #include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + /* The index database stores three items for each block: the disk location of the encoded filter, * its dSHA256 hash, and the header. Those belonging to blocks on the active chain are indexed by * height, and those belonging to blocks that have been reorganized out of the active chain are @@ -202,9 +222,9 @@ size_t BlockFilterIndex::WriteFilterToDisk(FlatFilePos& pos, const BlockFilter& { assert(filter.GetFilterType() == GetFilterType()); - size_t data_size = + uint64_t data_size{ GetSerializeSize(filter.GetBlockHash()) + - GetSerializeSize(filter.GetEncodedFilter()); + GetSerializeSize(filter.GetEncodedFilter())}; // If writing the filter would overflow the file, flush and move to the next one. if (pos.nPos + data_size > MAX_FLTR_FILE_SIZE) { @@ -291,9 +311,7 @@ bool BlockFilterIndex::Write(const BlockFilter& filter, uint32_t block_height, c value.second.header = filter_header; value.second.pos = m_next_filter_pos; - if (!m_db->Write(DBHeightKey(block_height), value)) { - return false; - } + m_db->Write(DBHeightKey(block_height), value); m_next_filter_pos.nPos += bytes_written; return true; @@ -338,7 +356,7 @@ bool BlockFilterIndex::CustomRemove(const interfaces::BlockInfo& block) // But since this creates new references to the filter, the position should get updated here // atomically as well in case Commit fails. batch.Write(DB_FILTER_POS, m_next_filter_pos); - if (!m_db->WriteBatch(batch)) return false; + m_db->WriteBatch(batch); // Update cached header to the previous block hash m_last_header = *Assert(ReadFilterHeader(block.height - 1, *Assert(block.prev_hash))); diff --git a/bitcoinkernel/bitcoin/src/index/blockfilterindex.h b/bitcoinkernel/bitcoin/src/index/blockfilterindex.h index 983cf32..b1ebd55 100644 --- a/bitcoinkernel/bitcoin/src/index/blockfilterindex.h +++ b/bitcoinkernel/bitcoin/src/index/blockfilterindex.h @@ -6,13 +6,24 @@ #define BITCOIN_INDEX_BLOCKFILTERINDEX_H #include -#include -#include #include #include +#include +#include +#include #include +#include +#include +#include +#include +#include #include +#include + +class BlockFilter; +class CBlockIndex; +enum class BlockFilterType : uint8_t; static const char* const DEFAULT_BLOCKFILTERINDEX = "0"; diff --git a/bitcoinkernel/bitcoin/src/index/coinstatsindex.cpp b/bitcoinkernel/bitcoin/src/index/coinstatsindex.cpp index af798e2..72bc09c 100644 --- a/bitcoinkernel/bitcoin/src/index/coinstatsindex.cpp +++ b/bitcoinkernel/bitcoin/src/index/coinstatsindex.cpp @@ -2,20 +2,39 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include + #include +#include #include #include #include +#include #include -#include +#include +#include +#include +#include #include #include -#include +#include +#include +#include