From 3c0887c03d7c400194a8e444333fcbf0eebae985 Mon Sep 17 00:00:00 2001 From: yuvicc Date: Sun, 16 Nov 2025 19:36:10 +0530 Subject: [PATCH 1/5] first update readme.md since github.com/bitcoin/bitcoin/pull/30595 has been merged --- README.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 54a46bc..06a9153 100644 --- a/README.md +++ b/README.md @@ -23,15 +23,15 @@ We vendor Bitcoin Core’s `libbitcoinkernel` using a Git subtree targeting the ```bash git subtree pull \ --prefix bitcoinkernel/bitcoin \ - https://github.com/TheCharlatan/bitcoin \ - kernelApi --squash + https://github.com/bitcoin/bitcoin \ + master --squash ``` ## Requirements - CMake - C++17 compiler (e.g. GCC/Clang) - Boost -- JDK 21+ with FFM support +- JDK 25+ with FFM support Refer to Bitcoin Core [docs](https://github.com/bitcoin/bitcoin/blob/master/doc/build-unix.md) for dependency specifics. @@ -50,7 +50,6 @@ Refer to Bitcoin Core [docs](https://github.com/bitcoin/bitcoin/blob/master/doc/ ## References -- Upstream PR of C header API library -> https://github.com/bitcoin/bitcoin/pull/30595 - [rust-bitcoinkernel wrapper](https://github.com/yuvicc/rust-bitcoinkernel) ## Current Status of the Library From f620a2206b18608dad6d61b4f8457ae85c2ded4c Mon Sep 17 00:00:00 2001 From: yuvicc Date: Sun, 16 Nov 2025 19:41:35 +0530 Subject: [PATCH 2/5] Squashed 'bitcoinkernel/bitcoin/' changes from 81cec737e6..c66e988754 c66e988754 Merge bitcoin/bitcoin#33865: cmake: Specify Windows plugin path in `test_bitcoin-qt` property e221b25246 Merge bitcoin/bitcoin#33860: depends: drop Qt patches dfde31f2ec Merge bitcoin/bitcoin#33864: scripted-diff: fix leftover references to `policy/fees.h` 0dd8d5c237 cmake: Specify Windows plugin path in `test_bitcoin-qt` property b0a3887154 scripted-diff: fix leftover references to `policy/fees.h` 48d4b936e0 Merge bitcoin/bitcoin#33511: init: Fix Ctrl-C shutdown hangs during wait calls 3c3c6adb72 Merge bitcoin/bitcoin#33745: mining: check witness commitment in submitBlock e652b69b8d Merge bitcoin/bitcoin#33003: test: add option to skip large re-org test in feature_block 3789215f73 Merge bitcoin/bitcoin#33724: refactor: Return uint64_t from GetSerializeSize d4e2a45833 Merge bitcoin/bitcoin#33750: doc: document fingerprinting risk when operating node on multiple networks 47618446a0 Merge bitcoin/bitcoin#33853: kernel: Allow null arguments for serialized data 3e9aca6f1b depends: drop qtbase-moc-ignore-gcc-macro.patch qt patch 0da5a82700 depends: drop unused qt patch d0da953773 Merge bitcoin/bitcoin#32482: build: add `-W*-whitespace` f450761f83 Merge bitcoin/bitcoin#33842: build: Bump g++ minimum supported version to 12 fa9f29a4a7 doc: Recommend latest Debian stable or Ubuntu LTS fa1711ee0d doc: Add GCC-12 min release notes faa8be75c9 ci: Enable experimental kernel stuff in G++-12 task (previous releases) fabce97b30 test: Remove gccbug_90348 test case fa3854e432 test: Remove unused fs::create_directories test fa9dacdbde util: [refactor] Remove unused create_directories workaround 138726a6f8 Merge bitcoin/bitcoin#33850: depends: drop qtbase_avoid_native_float16 qt patch 1c3d5c8ffd Merge bitcoin/bitcoin#33840: test: [refactor] Use reference over ptr to chainman 40dcbf580d build: add -Wtrailing-whitespace=any a3ac59a431 ci: Enable experimental kernel stuff in ASan task 5b89956eeb kernel: Allow null arguments for serialized data 169f93d2ac depends: drop qtbase_avoid_native_float16 qt patch d7659cd7e6 build: add -Wleading-whitespace=spaces d86650220a cmake: Disable `-Wtrailing-whitespace` warnings for RCC-generated files aabc5ca6ed cmake: Switch from AUTORCC to `qt6_add_resources` 25ae14c339 subprocess: replace tab with space 0c2b9dadd5 scripted-diff: remove whitespace in sha256_sse4.cpp 4da084fbc9 scripted-diff: change whitespace to spaces in univalue e6caf150b3 ci: add moreutils to lint job a7e8067610 Merge bitcoin/bitcoin#33181: guix: build for Linux HOSTS with `-static-libgcc` b354d1ce5c Merge bitcoin/bitcoin#33820: kernel: trim Chain interface fa807f78ae build: Bump g++ minimum supported version to 12 a4e96cae7d Merge bitcoin/bitcoin#33042: refactor: inline constant return values from `dbwrapper` write methods 8c2710b041 Merge bitcoin/bitcoin#32517: rpc: add "ischange: true" to decoded tx outputs in wallet gettransaction response 1fe851a478 Merge bitcoin/bitcoin#32180: p2p: Advance pindexLastCommonBlock early after connecting blocks 5f0303b93f Merge bitcoin/bitcoin#33443: log: reduce excessive "rolling back/forward" messages during block replay f4903dddc9 Merge bitcoin/bitcoin#33433: Bugfix: QA: rpc_bind: Skip nonloopback test if no such address is found 7a4901c902 test, refactor: Fix `-Warray-bounds` warning 06e9458869 Merge bitcoin/bitcoin#32856: Update `minisketch` subtree 66978a1a95 kernel: remove btck_chain_get_tip 4dd7e6dc48 kernel: remove btck_chain_get_genesis faf2759c8c test: [refactor] Use reference over ptr to chainman 490cb056f6 Merge bitcoin/bitcoin#33785: util: Allow `Assert` (et al.) in contexts without __func__ dcd0099a76 Merge bitcoin/bitcoin#33826: scripted-diff: Remove obsolete comment 01adbbcd9c Merge bitcoin/bitcoin#33827: doc: Correct `pkgin` command usage on NetBSD eec21bc7c8 Merge bitcoin/bitcoin#33536: doc: reference fuzz coverage steps in quick-start 035f934e02 Merge bitcoin/bitcoin#33823: ci: Use cmake --preset=dev-mode in test-each-commit task ddd2afac10 Merge bitcoin/bitcoin#33676: interfaces: enable cancelling running `waitNext` calls dee7eec643 doc: mention coverage build in quickstart section 0698c6b494 doc: Correct `pkgin` command usage on NetBSD 36724205fc scripted-diff: Remove obsolete comment ca1ce52a0f Merge bitcoin/bitcoin#33825: refactor: Add missing include in bitcoinkernel_wrapper.h fa1e8d8bad refactor: Add missing include in bitcoinkernel_wrapper.h fa6db67369 ci: [refactor] Extract build_dir constant in ci-test-each-commit-exec.py fa95e6cdc1 ci: Use cmake --preset=dev-mode in test-each-commit task 513a0da2e0 Merge bitcoin/bitcoin#33818: ci: Extend tidy job to cover kernel code 5d0a40d607 ci: Extend tidy job to cover kernel code 93e79181da Merge bitcoin/bitcoin#33786: script: remove dead code in `CountWitnessSigOps` 3c4bec6223 Merge bitcoin/bitcoin#33782: test: remove obsolete `get_{key,multisig}` helpers from wallet_util.py 4b12beedae Merge bitcoin/bitcoin#33793: test: move create_malleated_version() to messages.py for reuse 0b45e6db10 Merge bitcoin/bitcoin#33789: doc: add cmake help option in Windows build docs 2b9c351198 Merge bitcoin/bitcoin#33768: refactor: remove dead branches in `SingletonClusterImpl` fad6efd3be refactor: Use STR_INTERNAL_BUG macro where possible fada379589 doc: Remove unused bugprone-lambda-function-name suppression f06c6e1898 guix: build for Linux HOSTS with -static-libgcc 1bdf4695b0 guix: patch store paths out of libunwind 078a72c35f guix: move static-libc++ into CMAKE_EXE_LINKER_FLAGS flags 2bd155e6ee test: move create_malleated_version() to messages.py for reuse 5c5704e730 Merge bitcoin/bitcoin#33791: kernel: Use enumeration type for flags argument ed5720509f kernel: Use enumeration type for flags argument 50d106a4d6 Merge bitcoin/bitcoin#33781: clang-tidy: Remove no longer needed NOLINT ffd7ca3c46 Merge bitcoin/bitcoin#33780: guix: disable libsanitizer in Linux GCC build 9577daa3b8 doc: Add cmake help option in Windows build instructions fae1d99651 refactor: Use const reference to std::source_location fa5fbcd615 util: Allow Assert() in contexts without __func__ 24bcad3d4d refactor: remove dead code in `CountWitnessSigOps` 33389f1144 Merge bitcoin-core/gui#899: Modernize custom filtering ec8516ceb7 test: remove obsolete `get_{key,multisig}` helpers from wallet_util.py 038849e2e0 clang-tidy: Remove no longer needed NOLINT 5c41fa2918 guix: disable libsanitizer in Linux GCC build 4da01123df Merge bitcoin/bitcoin#30595: kernel: Introduce C header API 96614fff63 Merge bitcoin/bitcoin#33714: random: scope environ extern to macOS, BSDs and Illumos 4e9bd579d3 Merge bitcoin/bitcoin#33045: depends: disable variables, rules and suffixes. 5ffa63d681 Merge bitcoin/bitcoin#33626: ci: run native fuzz with MSAN job 75baff98fc Merge bitcoin/bitcoin#33744: ci: Fix lint runner selection (and docker cache) 2593ed1b5f Merge bitcoin/bitcoin#33574: doc: update Guix INSTALL.md 1cd8d9fe5c Merge bitcoin/bitcoin#33445: ci: Update Clang in "tidy" job 56329beaee Merge bitcoin/bitcoin#32301: test: cover invalid codesep positions for signature in taproot 1e6e32fa8a ci: run native fuzz with MSAN job 3784d15bcd ci: use LLVM libcxx 21.1.5 6c7a34f3b0 kernel: Add Purpose section to header documentation 7e9f00bcc1 kernel: Allowing reducing exports 7990463b10 kernel: Add pure kernel bitcoin-chainstate 36ec9a3ea2 Kernel: Add functions for working with outpoints 5eec7fa96a kernel: Add block hash type and block tree utility functions to C header f5d5d1213c kernel: Add function to read block undo data from disk to C header 09d0f62638 kernel: Add functions to read block from disk to C header a263a4caf2 kernel: Add function for copying block data to C header b30e15f432 kernel: Add functions for the block validation state to C header aa262da7bc kernel: Add validation interface to C header d27e27758d kernel: Add interrupt function to C header 1976b13be9 kernel: Add import blocks function to C header a747ca1f51 kernel: Add chainstate load options for in-memory dbs in C header 070e77732c kernel: Add options for reindexing in C header ad80abc73d kernel: Add block validation to C header cb1590b05e kernel: Add chainstate loading when instantiating a ChainstateManager e2c1bd3d71 kernel: Add chainstate manager option for setting worker threads 65571c36a2 kernel: Add chainstate manager object to C header c62f657ba3 kernel: Add notifications context option to C header 9e1bac4585 kernel: Add chain params context option to C header 337ea860df kernel: Add kernel library context object 28d679bad9 kernel: Add logging to kernel library C header 2cf136dec4 kernel: Introduce initial kernel C header API e15e8cbada qt: Modernize custom filtering 745eb053a4 Merge bitcoin-core/gui#901: Add createwallet, createwalletdescriptor, and migratewallet to history filter 52b1595850 depends: disable builtin variables 8b5a28fa78 depends: disable builtin rules and suffixes. 7632e0ba31 ci: fix configure docker action inputs 746d36cc80 Merge bitcoin/bitcoin#33754: ci: gha: Set debug_pull_request_number_str annotation 2d23820ee1 refactor: remove dead branches in `SingletonClusterImpl` e346ecae83 Add eclipse, partitioning, and fingerprinting note to i2p.md 25c45bb0d0 Merge bitcoin/bitcoin#33567: node: change a tx-relay on/off flag to enum 422b468229 Merge bitcoin/bitcoin#33683: refactor/doc: Add blockman param to GetTransaction doc comment da6f041e39 Merge bitcoin/bitcoin#31645: [IBD] coins: increase default UTXO flush batch size to 32 MiB 832a57673a Merge bitcoin/bitcoin#33749: test: ipc: resolve symlinks in `which capnp` 3cd4263bf6 Merge bitcoin/bitcoin#33753: test: Format strings in `test_runner` 78d4d36730 test: Format strings in `*.rs` fa9d0f994b ci: gha: Set debug_pull_request_number_str annotation 6eaa00fe20 test: clarify submitBlock() mutates the template 862bd43283 mining: ensure witness commitment check in submitBlock 00d1b6ef4b doc: clarify UpdateUncommittedBlockStructures 305384a037 Merge bitcoin/bitcoin#33746: ci: Add missing python3-dev package for riscv64 8eda7210eb Merge bitcoin/bitcoin#33743: fuzz: refactor memcpy to std::ranges::copy to work around ubsan warn 19a6a3e75e Add eclipse, partitioning, and fingerprinting note in tor.md 51093d6ae1 test: resolve symlinks in which result for capnp fa6c0bedd3 refactor: Return uint64_t from GetSerializeSize fad0c8680e refactor: Use uint64_t over size_t for serialized-size values fa4f388fc9 refactor: Use fixed size ints over (un)signed ints for serialized values 6f359695c3 Merge bitcoin/bitcoin#33698: test: Use same rpc timeout for authproxy and cli c281bb6837 Merge bitcoin/bitcoin#32924: test: add valid tx test with minimum-sized ECDSA signature (8 bytes DER-encoded) facf8b771a ci: Add missing python3-dev package for riscv64 b4d0288c46 doc: update Guix INSTALL.md 0b3b8a3be1 ci: fix lint docker caching fa4b52bd16 fuzz: refactor memcpy to std::ranges::copy to work around ubsan warn 72511fd02e Merge bitcoin/bitcoin#33555: build: Bump clang minimum supported version to 17 060bb55508 rpc: add decoded tx details to gettransaction with extra wallet fields ad1c3bdba5 [move only] move DecodeTxDoc() to a common util file for sharing d633db5416 rpc: add "ischange: true" in wallet gettransaction decoded tx output 79d6f458e2 random: scope environ extern to macOS, BSDs and Illumos 292ea0eb89 Merge bitcoin/bitcoin#33677: ci: Retry image building once on failure dd82c6c5d0 Merge bitcoin/bitcoin#33693: ci: use pycapnp 2.2.1 3bb30658e6 Merge bitcoin/bitcoin#32380: Modernize use of UTF-8 in Windows code 5a58d4915e Merge bitcoin/bitcoin#33546: test: add functional test for `TestShell` (matching doc example) 1abc8fa308 Merge bitcoin/bitcoin#33218: refactor: rename `fees.{h,cpp}` to `fees/block_policy_estimator.{h,cpp}` de15e52f09 Merge bitcoin/bitcoin#32867: doc: mention key removal in rpc interface modification 5d784bebaf clang-tidy: Disable `ArrayBound` check in src/ipc and src/test 5efdb0ef30 ci: Update Clang in "tidy" job fa01f38e53 move-only: Move CBlockFileInfo to kernel namespace fa2bbc9e4c refactor: [rpc] Remove cast when reporting serialized size fa364af89b test: Remove outdated comment 24434c1284 Merge bitcoin/bitcoin#31308: ci, iwyu: Treat warnings as errors for specific directories 27cd7f5049 Merge bitcoin/bitcoin#33185: guix: update time-machine to 5cb84f2013c5b1e48a7d0e617032266f1e6059e2 80bb7012be Merge bitcoin/bitcoin#31514: wallet: allow label for non-ranged external descriptor (if `internal=false`) & disallow label for ranged descriptors 5e1f626ac3 Merge bitcoin/bitcoin#32504: test: descriptor: cover invalid multi/multi_a cases dcb56fd4cb interfaces: add interruptWait method 56e9703968 Merge bitcoin/bitcoin#29640: Fix tiebreak when loading blocks from disk (and add tests for comparing chain ties) 53b34c80c6 ci: use pycapnp 2.2.1 in mac native job 865432869c ci: remove Python version comment from mac config 9bd9ec00b2 Merge bitcoin/bitcoin#33688: test: Update BIP324 test vectors 1a7fb5eeee fees: return current block height in estimateSmartFee ab49480d9b fees: rename fees_args to block_policy_estimator_args 06db08a435 fees: refactor: rename fees to block_policy_estimator 6dfdd7e034 fees: refactor: rename policy_fee_tests.cpp to feerounder_tests.cpp f54ffb4bc1 Merge bitcoin/bitcoin#32813: clang-format: make formatting deterministic for different formatter versions 1916c51cd8 Merge bitcoin/bitcoin#33210: fuzz: enhance wallet_fees by mocking mempool stuff 0eb554728c Merge bitcoin/bitcoin#33336: log: print every script verification state change c6c4edf324 Merge bitcoin/bitcoin#32983: rpc: refactor: use string_view in Arg/MaybeArg 00ad998d95 Merge bitcoin/bitcoin#33252: p2p: add `DifferenceFormatter` fuzz target and invariant check 1a1f46c228 refactor/doc: Add blockman param to `GetTransaction` doc comment and reorder out param 66667d6512 test: Use same rpc timeout for authproxy and cli 5555bce994 ci: Document why IN_GETOPT_BIN env var is needed on macOS fabe516440 ci: Export the container id in python script f6ba97cea1 Merge bitcoin/bitcoin#33666: ci: Drop libFuzzer from msan fuzz task af78d36512 Merge bitcoin/bitcoin#32588: util: Abort on failing CHECK_NONFATAL in debug builds 51877f2fc5 test: Update BIP324 test vectors 161864a038 Merge bitcoin/bitcoin#32579: p2p: Correct unrealistic headerssync unit test behavior 70a6fb5e5a Merge bitcoin/bitcoin#33172: test: p2p block malleability 99cb2054bd Merge bitcoin/bitcoin#33600: refactor: Construct g_verify_flag_names on first use 211bf6c975 Merge bitcoin/bitcoin#33566: miner: fix empty mempool case for waitNext() 944e5ff848 doc: mention key removal in rpc interface modification d32f9525e4 Merge bitcoin/bitcoin#33679: test: set number of RPC server threads to 2 1c85d06232 Merge bitcoin/bitcoin#32266: depends: Avoid `warning: "_FORTIFY_SOURCE" redefined` for `libevent` 11684c9ce2 Merge bitcoin/bitcoin#33674: ci: Doc ASLR workaround for sanitizer tasks e9cd45e3d3 test: set number of RPC server threads to 2 fa6aa9f42f ci: Retry image building once on failure fa4dbe04d7 ci: Allow overwriting check option in run() helper fa8e4de5c3 ci: Use os.environ[key] access when value must be set 7d27af98c7 Merge bitcoin/bitcoin#33461: ci: add Valgrind fuzz 1569bcc387 Merge bitcoin/bitcoin#33639: ci: Only write docker build images to Cirrus cache 98c4994d0f Merge bitcoin/bitcoin#33570: randomenv: Fix MinGW dllimport warning for `environ` c211d18322 Merge bitcoin/bitcoin#33670: test: Use unassigned p2p_port instead of hardcoded 60000 in p2p_i2p_ports.py e4b04630bc ci: add Valgrind fuzz 3fee0754a2 Merge bitcoin/bitcoin#33550: Fix windows libc++ `fs::path` `fstream` compile errors fa0e36156c ci: Doc ASLR workaround for sanitizer tasks fa20275db3 test: Use unassigned p2p_port instead of hardcoded 60000 in p2p_i2p_ports.py c862936d16 Merge bitcoin/bitcoin#33370: ci: use Mold linker for asan-lsan-ubsan-integer-no-depends-usdt workflow fabe0e07de ci: Only write docker build images to Cirrus cache fab64a5d6f ci: Move buildx command to python script fa72a2bd5c ci: Remove unused MAYBE_CPUSET fa70e23de7 ci: Drop libFuzzer from msan fuzz task abe7cbfe1a Merge bitcoin/bitcoin#33470: build: Move CMAKE_SKIP_INSTALL_RPATH from CMake to Guix script 689ec28d1d Merge bitcoin/bitcoin#33633: test: [move-only] binary utils to utils.py 0eeae4d174 Merge bitcoin/bitcoin#33625: Update secp256k1 subtree to latest master 4b41f99d57 build: Move CMAKE_SKIP_INSTALL_RPATH from CMake to Guix script d30f149360 Merge bitcoin/bitcoin#33630: doc: correct topology requirements in submitpackage helptext 3d22282564 [doc] correct topology requirements in submitpackage helptext e744fd1249 Merge bitcoin/bitcoin#33641: Update leveldb subtree to latest master 4371740beb Merge bitcoin/bitcoin#33642: doc: archive release notes for v28.3 ceea24b921 doc: archive release notes for v28.3 54ffe3de5b Update leveldb subtree to latest master f21162d819 Squashed 'src/leveldb/' changes from aba469ad6a..cad64b151d e14451ac87 Merge bitcoin/bitcoin#33469: TxGraph: change m_excluded_clusters f76e1ae389 Merge bitcoin/bitcoin#32313: coins: fix `cachedCoinsUsage` accounting in `CCoinsViewCache` 59c4898994 guix: remove python-pydantic-core input from LIEF 9f2a6927d3 guix: use Clang & LLVM 19 for macOS build 9570ddbec9 guix: update time-machine to 5cb84f2013c5b1e48a7d0e617032266f1e6059e2 7b5cc276aa guix: patch around riscv issue with newer (2.40+) binutils 91b5cbaabb ci: use Debian Trixie for macOS cross job fa75ef4328 test: Move export_env_build_path to util.py fa9f495308 test: Move get_binary_paths and Binaries to util.py 40e7d4cd0d Merge bitcoin/bitcoin#33549: ci: Add macOS cross task for arm64-apple-darwin ea17618c11 Merge bitcoin/bitcoin#33480: ci: Turn CentOS config into Alpine musl config b1f8a13702 Merge bitcoin/bitcoin#33624: test: P2SH sig ops are only counted with `SCRIPT_VERIFY_P2SH` 879c21045e Update secp256k1 subtree to latest master 3cbf7cb3e6 Squashed 'src/secp256k1/' changes from b9313c6e1a..d543c0d917 2f7a50f67c Merge bitcoin/bitcoin#33462: ci: add libcpp hardening flags to macOS fuzz job 07a926474b node: change a tx-relay on/off flag to enum 48aa0e98d0 Merge bitcoin/bitcoin#29675: wallet: Be able to receive and spend inputs involving MuSig2 aggregate keys db4bde0b03 Merge bitcoin/bitcoin#33517: multiprocess: Fix high overhead from message logging 3a10d700bc test: P2SH sig ops are only counted with `SCRIPT_VERIFY_P2SH` flag 9314113b29 Merge bitcoin/bitcoin#33610: doc: archive release notes for v29.2 9b43428c96 TxGraph: change m_excluded_clusters 6e1adbbaa1 Merge bitcoin/bitcoin#33612: test: change log rate limit version gate fdcf67de80 Merge bitcoin/bitcoin#33157: cluster mempool: control/optimize TxGraph memory usage 7b544341c0 test: change log rate limit version gate from 299900 to 290100 9610b0d1e2 randomenv: Fix MinGW dllimport warning for `environ` 6c4fe401e9 Merge bitcoin/bitcoin#33508: ci: fix buildx gha cache authentication on forks 8f7673257a miner: fix empty mempool case for waitNext() c11a3dcc88 doc: archive release notes for v29.2 64a7c7cbb9 Merge bitcoin/bitcoin#33558: ci: Use native platform for win-cross task 93b56e95c0 Merge bitcoin/bitcoin#33601: doc: archive release notes for v30.0 c235aa468b Update minisketch subtree to latest upstream 4543a3bde2 Squashed 'src/minisketch/' changes from ea8f66b1ea..d1bd01e189 563747971b Merge bitcoin/bitcoin#33580: depends: Use $(package)_file_name when downloading from the fallback 24d861da78 coins: only adjust `cachedCoinsUsage` on `EmplaceCoinInternalDANGER` insert d7c9d6c291 coins: fix `cachedCoinsUsage` accounting to prevent underflow 39cf8bb3d0 refactor: remove redundant usage tracking from `CoinsViewCacheCursor` 67cff8bec9 refactor: assert newly-created parent cache entry has zero memory usage 023cd5a546 txgraph: add SingletonClusterImpl (mem optimization) e346250732 txgraph: give Clusters a range of intended tx counts (preparation) e93b0f09cc txgraph: abstract out creation of empty Clusters (refactor) 6baf12621f txgraph: comment fixes (doc fix) 726b995739 txgraph: make Cluster an abstract class (refactor) 2602d89edd txgraph: avoid accessing other Cluster internals (refactor) 04c808ac4c txgraph: expose memory usage estimate function (feature) 7680bb8fd4 txgraph: keep track of Cluster memory usage (preparation) 4ba562e5f4 txgraph: keep data structures compact (mem optimization) bb5cb222ae depgraph: add memory usage control (feature) b1637a90de txgraph: avoid holes in DepGraph positions (mem optimization) 2b1d302508 txgraph: move some sanity checks from Cluster to TxGraphImpl (refactor) d40302fbaf txgraph: Make level of Cluster implicit (optimization) 8d6e49158e doc: archive release notes for v30.0 4e352efa2c qt: add createwallet, createwalletdescriptor, and migratewallet to history filter 0626b90f50 multiprocess: align our logging with libmultiprocess's 9d068225ee multiprocess: update multiprocess EventLoop construction to use options d2987102dd Merge bitcoin/bitcoin#33573: doc: bump the template macOS version f6567527d8 doc: bump the template macOS version faa9d10c84 refactor: Construct g_verify_flag_names on first use becf150013 Merge bitcoin/bitcoin#33518: Update libmultiprocess subtree to support reduced logging 01cc20f330 test: improve coverage for a resolved stalling situation 9af6daf07e test: remove magic number when checking for blocks that have arrived 3069d66dca p2p: During block download, adjust pindexLastCommonBlock better cd1b7fa1ff Merge bitcoin/bitcoin#33577: Revert "depends: Update URL for `qrencode` package source tarball" fa0fa0f700 refactor: Revert "disable self-assign warning for tests" faed118fb3 build: Bump clang minimum supported version to 17 6b4a92b0fa Merge bitcoin/bitcoin#33568: doc: how to update a subtree 90b2884ce4 Merge bitcoin/bitcoin#33581: ci: Properly include $FILE_ENV in DEPENDS_HASH d44b860cd0 Merge bitcoin/bitcoin#33584: ci: upgrade GitHub Action to download-artifact@v5 57f7c68821 test: add functional test for `TestShell` (matching doc example) 53874f7934 doc: test: update TestShell example instructions/options b35341b9ba Update ci.yml ceeb53adcd ci: Properly include $FILE_ENV in DEPENDS_HASH 671b774d1b depends: Use $(package)_file_name when downloading from the fallback e4335a3192 Revert "depends: Update URL for `qrencode` package source tarball" a89a822e6e Revert "depends: Use hash instead of file name for package download stamp" fad5a7101c ci: Add macOS cross task for arm64 fa8c750a0a ci: Refactor get_previous_releases step in win-test-cross task e4c04f7759 ci: add libcpp hardening flags to macOS fuzz job a1226bc760 doc: how to update a subtree eda91b07fd Merge commit '0f01e1577f7c6734eb345139a12aba329ef22a5f' into pr/subtree-6 0f01e1577f Squashed 'src/ipc/libmultiprocess/' changes from 47d79db8a552..a4f929696490 fa6fd16f36 ci: Use native platform for win-cross task 53e4951a5b Switch to ANSI Windows API in `fsbridge::fopen()` function dbe770d921 Switch to ANSI Windows API in `Win32ErrorString()` function 06d0be4e22 Remove no longer necessary `WinCmdLineArgs` class f366408492 cmake: Set process code page to UTF-8 on Windows dccbb17806 Set minimum supported Windows version to 1903 (May 2019 Update) c864a4c194 Simplify fs::path by dropping filename() and make_preferred() overloads b0113afd44 Fix windows libc++ fs::path fstream compile errors b63428ac9c rpc: refactor: use more (Maybe)Arg 037830ca0d refactor: increase string_view usage b3bf18f0ba rpc: refactor: use string_view in Arg/MaybeArg 45bd891465 log: split assumevalid ancestry-failure-reason message 6c13a38ab5 log: separate script verification reasons f2ea6f04e7 refactor: untangle assumevalid decision branches 9bc298556c validation: log initial script verification state 4fad4e992c test: add assumevalid scenarios scaffold c25a5e670b init: Signal m_tip_block_cv on Ctrl-C 6a29f79006 test: Test SIGTERM handling during waitforblockheight call ac599c4a9c test: Test MuSig2 in the wallet 68ef954c4c wallet: Keep secnonces in DescriptorScriptPubKeyMan 4a273edda0 sign: Create MuSig2 signatures for known MuSig2 aggregate keys 258db93889 sign: Add CreateMuSig2AggregateSig bf69442b3f sign: Add CreateMuSig2PartialSig 512b17fc56 sign: Add CreateMuSig2Nonce 82ea67c607 musig: Add MuSig2AggregatePubkeys variant that validates the aggregate d99a081679 psbt: MuSig2 data in Fill/FromSignatureData 4d8b4f5336 signingprovider: Add musig2 secnonces c06a1dc86f Add MuSig2SecNonce class for secure allocation of musig nonces 9baff05e49 sign: Include taproot output key's KeyOriginInfo in sigdata 4b24bfeab9 pubkey: Return tweaks from BIP32 derivation bc706955d7 ci: expose all ACTIONS_* vars 444409ff2b ci: Reduce Alpine musl task to md runner size fa6b2e9efe ci: Turn centos config into alpine musl config 1fc7a81f1f log: reduce excessive messages during block replay 79b4c276e7 Bugfix: QA: rpc_bind: Skip nonloopback test if no such address is found 91ac64b0a6 log: reword `signature validations` to `script verification` in `assumevalid` log f14876213a musig: Move synthetic xpub construction to its own function f031536f2d ci: use Mold linker for asan-lsan-ubsan-integer-no-depends-usdt workflow cc5dda1de3 headerssync: Make HeadersSyncState more flexible and move constants 8fd1c2893e test(headerssync): Test returning of pow_validated_headers behavior 7b00643ef5 test(headerssync): headers_sync_chainwork test improvements 04eeb9578c doc(test): Improve comments fe896f8faa refactor(test): Store HeadersSyncState on the stack f03686892a refactor(test): Break up headers_sync_state e984618d0b refactor(headerssync): Process spans of headers a4ac9915a9 refactor(headerssync): Extract test constants ahead of breakup into functions 02d2b5a11c ci, iwyu: Treat warnings as errors for specific directories 57a3eac387 refactor: Fix includes in `index` directory bdb8eadcdc refactor: Fix includes in `crypto` directory 56f2a689a2 ci: Do not patch `leveldb` to workaround UB in "tidy" CI job 65a10fc3c5 p2p: add assertion for BlockTransactionsRequest indexes 58be359f6b fuzz: add a target for DifferenceFormatter Class 13f36c020f clang-format: regenerate configs 5ded99a7f0 fuzz: MockMempoolMinFee in wallet_fees c9a7a198d9 test: move MockMempoolMinFee to util/txmempool adf67eb21b fuzz: create FeeEstimatorTestingSetup to set fee_estimator ff10a37e99 fuzz: mock CBlockPolicyEstimator in wallet_fuzz f591c3beca fees: make estimateSmartFee/HighestTargetTracked virtual for mocking b6f8c48946 coins: increase default `dbbatchsize` to 32 MiB 8bbb7b8bf8 refactor: Extract default batch size into kernel d0e1bbad01 test: repeat block malleability test with relayable block over P2P 19273d0705 fuzz: set mempool options in wallet_fees 81e5c8385b test: cover invalid codesep positions for signature in taproot 743abbcbde refactor: inline constant return value of `BlockTreeDB::WriteBatchSync` and `BlockManager::WriteBlockIndexDB` and `BlockTreeDB::WriteFlag` e030240e90 refactor: inline constant return value of `CDBWrapper::Erase` and `BlockTreeDB::WriteReindexing` cdab9480e9 refactor: inline constant return value of `CDBWrapper::Write` d1847cf5b5 refactor: inline constant return value of `TxIndex::DB::WriteTxs` 50b63a5698 refactor: inline constant return value of `CDBWrapper::WriteBatch` fb8720f1e0 sign: Refactor Schnorr sighash computation out of CreateSchnorrSig a4cfddda64 tests: Clarify why musig derivation adds a pubkey and xpub 39a63bf2e7 descriptors: Add a doxygen comment for has_hardened output_parameter 2320184d0e descriptors: Fix meaning of any_key_parsed 0465574c12 test: Fixes send_blocks_and_test docs 09c95f21e7 test: Adds block tiebreak over restarts tests 18524b072e Make nSequenceId init value constants 8b91883a23 Set the same best tip on restart if two candidates have the same work 5370bed21e test: add functional test for complex reorgs ab145cb3b4 Updates CBlockIndexWorkComparator outdated comment fa37153288 util: Abort on failing CHECK_NONFATAL in debug builds fa0dc4bdff test: Allow testing of check failures faeb58fe66 refactor: Set G_ABORT_ON_FAILED_ASSUME when G_FUZZING_BUILD 8810642b57 test: add option to skip large re-org test in feature_block 5fa81e239a test: add valid tx test with minimum-sized ECDSA signature (8 bytes DER-encoded) 58e55b17e6 test: descriptor: cover invalid multi/multi_a cases 664657ed13 bugfix: disallow label for ranged descriptors & allow external non-ranged descriptors to have label fe71a4b139 depends: Avoid `warning: "_FORTIFY_SOURCE" redefined` for `libevent` REVERT: 81cec737e6 kernel: Fix bitcoin-chainstate for windows REVERT: 1826c485dd kernel: Add Purpose section to header documentation REVERT: d7e618aa98 kernel: Allowing reducing exports REVERT: fb7f524133 kernel: Add pure kernel bitcoin-chainstate REVERT: dd0bdf279e Kernel: Add functions for working with outpoints REVERT: eaa6abfc73 kernel: Add block hash type and block tree utility functions to C header REVERT: 824ddf2885 kernel: Add function to read block undo data from disk to C header REVERT: 76cab0768b kernel: Add functions to read block from disk to C header REVERT: e41f6f459a kernel: Add function for copying block data to C header REVERT: 39c647647a kernel: Add functions for the block validation state to C header REVERT: 8a19a9d607 kernel: Add validation interface to C header REVERT: 38a990dd48 kernel: Add interrupt function to C header REVERT: fee8f6ff38 kernel: Add import blocks function to C header REVERT: c29a6b87cc kernel: Add chainstate load options for in-memory dbs in C header REVERT: e788b3ba06 kernel: Add options for reindexing in C header REVERT: 2707fc515c kernel: Add block validation to C header REVERT: 51a24c4004 kernel: Add chainstate loading when instantiating a ChainstateManager REVERT: ea01a8caf3 kernel: Add chainstate manager option for setting worker threads REVERT: add5904e0a kernel: Add chainstate manager object to C header REVERT: 37a3395d27 kernel: Add notifications context option to C header REVERT: d838a934be kernel: Add chain params context option to C header REVERT: dc58ae9fc0 kernel: Add kernel library context object REVERT: 7744997596 kernel: Add logging to kernel library C header REVERT: dc504f57b3 kernel: Introduce initial kernel C header API git-subtree-dir: bitcoinkernel/bitcoin git-subtree-split: c66e988754391a094af93ef2a9127200d093b669 --- .github/ISSUE_TEMPLATE/bug.yml | 3 +- .github/actions/configure-docker/action.yml | 27 +- .../actions/configure-environment/action.yml | 2 +- .github/ci-test-each-commit-exec.py | 18 +- .github/workflows/ci.yml | 56 +- CMakeLists.txt | 7 +- ci/README.md | 8 + ci/lint/01_install.sh | 3 +- ci/test/00_setup_env.sh | 2 +- ci/test/00_setup_env_arm.sh | 5 +- ci/test/00_setup_env_mac_cross.sh | 6 +- ci/test/00_setup_env_mac_cross_intel.sh | 20 + ci/test/00_setup_env_mac_native.sh | 6 +- ci/test/00_setup_env_mac_native_fuzz.sh | 2 +- ....sh => 00_setup_env_native_alpine_musl.sh} | 8 +- ci/test/00_setup_env_native_asan.sh | 5 +- ci/test/00_setup_env_native_fuzz_with_msan.sh | 2 +- ...up_env_native_nowallet_libbitcoinkernel.sh | 8 +- .../00_setup_env_native_previous_releases.sh | 9 +- ci/test/00_setup_env_native_tidy.sh | 4 +- ci/test/00_setup_env_win64.sh | 4 +- ci/test/01_base_install.sh | 13 +- ci/test/02_run_container.py | 115 ++- ci/test/02_run_container.sh | 95 -- ci/test/03_test_script.sh | 41 +- ci/test_imagefile | 3 + cmake/windows-app.manifest.in | 7 +- contrib/devtools/README.md | 2 +- .../deterministic-fuzz-coverage/src/main.rs | 4 +- .../src/main.rs | 4 +- contrib/devtools/headerssync-params.py | 16 +- contrib/devtools/iwyu/bitcoin.core.imp | 15 +- contrib/guix/INSTALL.md | 8 +- contrib/guix/libexec/build.sh | 12 +- contrib/guix/libexec/prelude.bash | 2 +- contrib/guix/manifest.scm | 25 +- .../guix/patches/gcc-remap-guix-store.patch | 13 +- .../guix/patches/glibc-riscv-jumptarget.patch | 57 ++ contrib/guix/symbol-check.py | 10 +- depends/Makefile | 2 + depends/README.md | 2 +- depends/funcs.mk | 4 +- depends/packages/libevent.mk | 2 +- depends/packages/native_qt.mk | 2 - depends/packages/qrencode.mk | 5 +- depends/packages/qt.mk | 4 - depends/patches/qt/memory_resource.patch | 16 - .../qt/qtbase-moc-ignore-gcc-macro.patch | 17 - .../qt/qtbase_avoid_native_float16.patch | 37 - doc/build-freebsd.md | 2 +- doc/build-netbsd.md | 8 +- doc/build-openbsd.md | 2 +- doc/build-osx.md | 2 +- doc/build-unix.md | 6 +- doc/build-windows-msvc.md | 6 +- doc/build-windows.md | 3 + doc/dependencies.md | 4 +- doc/developer-notes.md | 17 +- doc/fuzzing.md | 2 + doc/i2p.md | 10 + doc/release-notes-33555.md | 5 + doc/release-notes-33842.md | 4 + doc/release-notes-empty-template.md | 4 +- doc/release-notes/release-notes-28.3.md | 110 ++ doc/release-notes/release-notes-29.2.md | 91 ++ doc/release-notes/release-notes-30.0.md | 438 ++++++++ doc/release-process.md | 2 +- doc/tor.md | 7 + src/.clang-format | 207 +++- src/CMakeLists.txt | 5 +- src/arith_uint256.h | 19 +- src/bench/CMakeLists.txt | 2 + src/bench/bench.cpp | 2 +- src/bench/descriptors.cpp | 3 +- src/bitcoin-chainstate.cpp | 33 +- src/bitcoin-cli.cpp | 4 - src/bitcoin-wallet.cpp | 4 - src/bitcoin.cpp | 7 +- src/bitcoind.cpp | 5 - src/blockfilter.cpp | 4 +- src/blockfilter.h | 3 +- src/chain.cpp | 6 - src/chain.h | 48 +- src/cluster_linearize.h | 12 + src/coins.cpp | 24 +- src/coins.h | 12 +- src/common/args.cpp | 29 - src/common/args.h | 17 - src/common/config.cpp | 4 +- src/common/messages.cpp | 5 +- src/common/messages.h | 3 +- src/common/settings.cpp | 4 +- src/common/system.cpp | 2 + src/consensus/tx_verify.cpp | 2 +- src/core_io.h | 3 +- src/core_write.cpp | 7 +- src/crypto/chacha20.cpp | 3 +- src/crypto/chacha20.h | 5 +- src/crypto/chacha20poly1305.h | 2 +- src/crypto/hex_base.cpp | 2 + src/crypto/hex_base.h | 1 + src/crypto/hkdf_sha256_32.cpp | 3 +- src/crypto/hkdf_sha256_32.h | 6 +- src/crypto/hmac_sha256.cpp | 2 + src/crypto/hmac_sha256.h | 3 +- src/crypto/hmac_sha512.cpp | 2 + src/crypto/hmac_sha512.h | 3 +- src/crypto/muhash.cpp | 5 +- src/crypto/muhash.h | 5 +- src/crypto/poly1305.cpp | 2 - src/crypto/poly1305.h | 3 +- src/crypto/sha256.cpp | 2 +- src/crypto/sha256_arm_shani.cpp | 1 - src/crypto/sha256_sse4.cpp | 712 ++++++------- src/crypto/sha256_x86_shani.cpp | 1 + src/crypto/sha3.cpp | 4 +- src/crypto/sha3.h | 3 +- src/crypto/siphash.cpp | 4 + src/crypto/siphash.h | 4 +- src/dbwrapper.cpp | 3 +- src/dbwrapper.h | 10 +- src/flatfile.h | 10 +- src/headerssync.cpp | 40 +- src/headerssync.h | 14 +- src/index/base.cpp | 22 +- src/index/base.h | 24 +- src/index/blockfilterindex.cpp | 42 +- src/index/blockfilterindex.h | 15 +- src/index/coinstatsindex.cpp | 30 +- src/index/coinstatsindex.h | 9 +- src/index/disktxpos.h | 4 +- src/index/txindex.cpp | 31 +- src/index/txindex.h | 9 + src/init.cpp | 12 +- src/interfaces/chain.h | 19 +- src/interfaces/mining.h | 22 +- src/ipc/.clang-tidy.in | 8 +- src/ipc/capnp/mining.capnp | 1 + src/ipc/capnp/protocol.cpp | 44 +- .../libmultiprocess/ci/configs/sanitize.bash | 1 + src/ipc/libmultiprocess/doc/design.md | 12 +- src/ipc/libmultiprocess/doc/usage.md | 4 +- .../libmultiprocess/example/calculator.cpp | 2 + src/ipc/libmultiprocess/example/example.cpp | 6 +- src/ipc/libmultiprocess/example/printer.cpp | 6 +- src/ipc/libmultiprocess/include/mp/proxy-io.h | 108 +- .../libmultiprocess/include/mp/proxy-types.h | 39 +- .../libmultiprocess/include/mp/type-context.h | 11 +- src/ipc/libmultiprocess/shell.nix | 21 +- src/ipc/libmultiprocess/src/mp/proxy.cpp | 23 +- src/ipc/libmultiprocess/src/mp/util.cpp | 7 + .../libmultiprocess/test/mp/test/foo.capnp | 2 + src/ipc/libmultiprocess/test/mp/test/foo.h | 2 + src/ipc/libmultiprocess/test/mp/test/test.cpp | 86 +- src/kernel/bitcoinkernel.cpp | 131 +-- src/kernel/bitcoinkernel.h | 171 ++-- src/kernel/bitcoinkernel_wrapper.h | 142 ++- src/kernel/caches.h | 3 + src/kernel/chainparams.cpp | 30 + src/kernel/chainparams.h | 11 + src/key.cpp | 123 +++ src/key.h | 4 + src/leveldb/README.md | 2 +- src/leveldb/db/db_test.cc | 2 +- src/leveldb/db/snapshot.h | 2 +- src/leveldb/doc/benchmark.html | 2 +- src/leveldb/doc/index.md | 2 +- src/leveldb/util/env_posix.cc | 4 +- src/leveldb/util/env_windows.cc | 2 +- src/minisketch/.github/workflows/ci.yml | 69 ++ src/minisketch/.gitignore | 3 + src/minisketch/CMakeLists.txt | 122 +++ src/minisketch/README.md | 42 +- .../cmake/PrintConfigureSummary.cmake | 112 ++ .../cmake/SystemIntrospection.cmake | 43 + src/minisketch/doc/math.md | 2 +- src/minisketch/src/CMakeLists.txt | 76 ++ src/minisketch/src/fields/clmul_common_impl.h | 2 +- src/musig.cpp | 161 ++- src/musig.h | 51 +- src/net.cpp | 11 +- src/net.h | 7 +- src/net_processing.cpp | 26 +- src/netaddress.cpp | 20 +- src/netaddress.h | 7 +- src/node/blockstorage.cpp | 34 +- src/node/blockstorage.h | 50 +- src/node/context.cpp | 2 +- src/node/interfaces.cpp | 29 +- src/node/miner.cpp | 37 +- src/node/miner.h | 6 +- src/node/transaction.cpp | 45 +- src/node/transaction.h | 13 +- src/node/types.h | 13 + src/outputtype.cpp | 2 +- src/outputtype.h | 3 +- .../block_policy_estimator.cpp} | 3 +- .../{fees.h => fees/block_policy_estimator.h} | 11 +- .../block_policy_estimator_args.cpp} | 2 +- .../block_policy_estimator_args.h} | 6 +- src/policy/policy.cpp | 2 +- src/psbt.cpp | 24 + src/psbt.h | 2 +- src/pubkey.cpp | 9 +- src/pubkey.h | 4 +- src/qt/CMakeLists.txt | 14 +- src/qt/bitcoin.cpp | 5 - src/qt/guiutil.cpp | 6 +- src/qt/recentrequeststablemodel.h | 4 +- src/qt/rpcconsole.cpp | 3 + src/qt/sendcoinsdialog.cpp | 2 +- src/qt/test/CMakeLists.txt | 14 +- src/qt/test/optiontests.cpp | 2 +- src/qt/transactionfilterproxy.cpp | 50 +- src/qt/transactionfilterproxy.h | 2 +- src/randomenv.cpp | 8 +- src/rest.cpp | 18 +- src/rpc/blockchain.cpp | 50 +- src/rpc/fees.cpp | 14 +- src/rpc/mempool.cpp | 22 +- src/rpc/mining.cpp | 4 +- src/rpc/net.cpp | 61 +- src/rpc/node.cpp | 8 +- src/rpc/output_script.cpp | 24 +- src/rpc/rawtransaction.cpp | 47 +- src/rpc/rawtransaction_util.cpp | 46 + src/rpc/rawtransaction_util.h | 4 + src/rpc/request.cpp | 4 +- src/rpc/server.cpp | 12 +- src/rpc/server.h | 2 +- src/rpc/server_util.cpp | 2 +- src/rpc/signmessage.cpp | 8 +- src/rpc/txoutproof.cpp | 2 +- src/rpc/util.cpp | 14 +- src/rpc/util.h | 10 +- src/script/descriptor.cpp | 26 +- src/script/descriptor.h | 2 +- src/script/interpreter.cpp | 64 +- src/script/interpreter.h | 4 +- src/script/sign.cpp | 283 +++++- src/script/sign.h | 17 +- src/script/signingprovider.cpp | 47 + src/script/signingprovider.h | 17 + .../actions/run-in-docker-action/action.yml | 23 +- src/secp256k1/.github/workflows/ci.yml | 276 ++--- src/secp256k1/CMakeLists.txt | 6 +- src/secp256k1/CONTRIBUTING.md | 6 +- src/secp256k1/Makefile.am | 5 +- src/secp256k1/README.md | 6 +- src/secp256k1/build-aux/m4/bitcoin_secp.m4 | 2 +- src/secp256k1/ci/linux-debian.Dockerfile | 13 +- src/secp256k1/cmake/CheckX86_64Assembly.cmake | 5 +- src/secp256k1/configure.ac | 10 +- src/secp256k1/include/secp256k1.h | 42 +- src/secp256k1/include/secp256k1_ellswift.h | 2 +- src/secp256k1/include/secp256k1_recovery.h | 12 +- src/secp256k1/src/CMakeLists.txt | 14 +- src/secp256k1/src/bench.h | 22 +- src/secp256k1/src/checkmem.h | 12 +- src/secp256k1/src/ecmult_gen_impl.h | 12 +- src/secp256k1/src/field_impl.h | 2 +- src/secp256k1/src/group_impl.h | 4 +- src/secp256k1/src/hash_impl.h | 12 +- src/secp256k1/src/modules/ecdh/bench_impl.h | 7 +- src/secp256k1/src/modules/ecdh/main_impl.h | 4 +- src/secp256k1/src/modules/ecdh/tests_impl.h | 25 +- .../src/modules/ellswift/main_impl.h | 2 +- .../src/modules/ellswift/tests_impl.h | 21 +- .../src/modules/extrakeys/tests_impl.h | 21 +- .../src/modules/musig/session_impl.h | 8 +- src/secp256k1/src/modules/musig/tests_impl.h | 78 +- .../src/modules/recovery/tests_impl.h | 25 +- .../src/modules/schnorrsig/main_impl.h | 11 +- .../src/modules/schnorrsig/tests_impl.h | 45 +- src/secp256k1/src/scalar_impl.h | 2 +- src/secp256k1/src/secp256k1.c | 4 +- src/secp256k1/src/tests.c | 338 +++--- src/secp256k1/src/tests_common.h | 42 + src/secp256k1/src/testutil.h | 8 + src/secp256k1/src/unit_test.c | 479 +++++++++ src/secp256k1/src/unit_test.h | 145 +++ src/secp256k1/src/util.h | 19 +- .../src/wycheproof/WYCHEPROOF_COPYING | 4 +- .../ecdsa_secp256k1_sha256_bitcoin_test.json | 10 +- src/secp256k1/tools/check-abi.sh | 9 +- src/serialize.h | 21 +- src/test/.clang-tidy.in | 3 + src/test/CMakeLists.txt | 6 +- src/test/arith_uint256_tests.cpp | 19 +- src/test/bip324_tests.cpp | 72 +- src/test/blockmanager_tests.cpp | 21 +- src/test/coins_tests.cpp | 40 +- src/test/compilerbug_tests.cpp | 42 - src/test/data/tx_valid.json | 4 + src/test/dbwrapper_tests.cpp | 38 +- src/test/descriptor_tests.cpp | 6 +- ...icy_fee_tests.cpp => feerounder_tests.cpp} | 4 +- src/test/flatfile_tests.cpp | 4 +- src/test/fs_tests.cpp | 44 +- src/test/fuzz/CMakeLists.txt | 4 + src/test/fuzz/block_index.cpp | 4 +- src/test/fuzz/cluster_linearize.cpp | 111 +- src/test/fuzz/coins_view.cpp | 32 +- src/test/fuzz/deserialize.cpp | 2 + src/test/fuzz/difference_formatter.cpp | 32 + src/test/fuzz/fees.cpp | 2 +- src/test/fuzz/headerssync.cpp | 14 +- src/test/fuzz/integer.cpp | 5 - src/test/fuzz/kitchen_sink.cpp | 2 +- src/test/fuzz/miniscript.cpp | 2 +- src/test/fuzz/muhash.cpp | 12 - src/test/fuzz/policy_estimator.cpp | 4 +- src/test/fuzz/policy_estimator_io.cpp | 4 +- src/test/fuzz/rpc.cpp | 6 + src/test/fuzz/script.cpp | 2 +- src/test/fuzz/txgraph.cpp | 15 + src/test/fuzz/util/net.cpp | 11 +- src/test/headers_sync_chainwork_tests.cpp | 280 +++-- src/test/kernel/block_data.h | 623 ++++-------- src/test/kernel/test_kernel.cpp | 145 ++- src/test/logging_tests.cpp | 4 +- src/test/miner_tests.cpp | 24 +- src/test/miniscript_tests.cpp | 2 +- src/test/net_tests.cpp | 2 +- src/test/policyestimator_tests.cpp | 4 +- src/test/rpc_tests.cpp | 41 +- src/test/script_assets_tests.cpp | 2 +- src/test/settings_tests.cpp | 2 +- src/test/sigopcount_tests.cpp | 3 + src/test/transaction_tests.cpp | 2 +- src/test/txpackage_tests.cpp | 6 +- src/test/util/setup_common.cpp | 36 +- src/test/util/setup_common.h | 11 - src/test/util/txmempool.cpp | 36 + src/test/util/txmempool.h | 13 + src/test/util_check_tests.cpp | 33 + src/test/util_tests.cpp | 4 +- src/txdb.cpp | 8 +- src/txdb.h | 11 +- src/txgraph.cpp | 962 ++++++++++++++---- src/txgraph.h | 7 + src/univalue/include/univalue_escapes.h | 512 +++++----- src/util/check.cpp | 21 +- src/util/check.h | 52 +- src/util/fs.cpp | 4 +- src/util/fs.h | 41 +- src/util/overloaded.h | 3 - src/util/subprocess.h | 2 +- src/util/syserror.cpp | 15 +- src/validation.cpp | 148 ++- src/validation.h | 16 +- src/wallet/coincontrol.h | 2 +- src/wallet/db.cpp | 4 +- src/wallet/dump.cpp | 4 +- src/wallet/feebumper.cpp | 2 +- src/wallet/interfaces.cpp | 2 +- src/wallet/rpc/backup.cpp | 11 +- src/wallet/rpc/coins.cpp | 3 +- src/wallet/rpc/spend.cpp | 2 +- src/wallet/rpc/transactions.cpp | 14 +- src/wallet/rpc/util.cpp | 5 +- src/wallet/rpc/util.h | 4 +- src/wallet/rpc/wallet.cpp | 5 +- src/wallet/scriptpubkeyman.cpp | 4 + src/wallet/scriptpubkeyman.h | 14 + src/wallet/spend.cpp | 2 +- src/wallet/spend.h | 2 +- src/wallet/test/fuzz/fees.cpp | 57 +- src/wallet/test/init_test_fixture.cpp | 2 +- src/wallet/test/spend_tests.cpp | 2 +- src/wallet/test/wallet_rpc_tests.cpp | 2 +- src/wallet/test/wallet_tests.cpp | 5 +- src/wallet/wallet.cpp | 35 +- src/wallet/wallet.h | 7 +- src/zmq/zmqabstractnotifier.h | 3 +- test/README.md | 4 +- test/functional/feature_assumevalid.py | 136 ++- test/functional/feature_block.py | 93 +- test/functional/feature_chain_tiebreaks.py | 151 +++ .../functional/feature_framework_testshell.py | 47 + test/functional/feature_init.py | 51 + test/functional/feature_taproot.py | 6 +- test/functional/interface_ipc.py | 62 +- test/functional/p2p_i2p_ports.py | 11 +- test/functional/p2p_ibd_stalling.py | 49 +- test/functional/p2p_orphan_handling.py | 24 +- test/functional/p2p_segwit.py | 32 +- test/functional/rpc_bind.py | 5 +- test/functional/rpc_estimatefee.py | 2 +- test/functional/rpc_misc.py | 24 +- test/functional/rpc_psbt.py | 2 +- test/functional/test-shell.md | 3 +- test/functional/test_framework/messages.py | 22 +- test/functional/test_framework/p2p.py | 4 +- test/functional/test_framework/script.py | 2 +- .../test_framework/test_framework.py | 97 +- test/functional/test_framework/test_node.py | 22 +- test/functional/test_framework/util.py | 95 ++ test/functional/test_framework/wallet_util.py | 51 - test/functional/test_runner.py | 3 + test/functional/wallet_basic.py | 11 +- test/functional/wallet_fundrawtransaction.py | 2 +- test/functional/wallet_importdescriptors.py | 13 + test/functional/wallet_musig.py | 242 +++++ test/functional/wallet_rescan_unconfirmed.py | 3 +- test/functional/wallet_send.py | 2 +- test/lint/test_runner/src/main.rs | 28 +- test/sanitizer_suppressions/ubsan | 5 - 408 files changed, 8724 insertions(+), 4247 deletions(-) create mode 100755 ci/test/00_setup_env_mac_cross_intel.sh rename ci/test/{00_setup_env_native_centos.sh => 00_setup_env_native_alpine_musl.sh} (57%) create mode 100644 contrib/guix/patches/glibc-riscv-jumptarget.patch delete mode 100644 depends/patches/qt/memory_resource.patch delete mode 100644 depends/patches/qt/qtbase-moc-ignore-gcc-macro.patch delete mode 100644 depends/patches/qt/qtbase_avoid_native_float16.patch create mode 100644 doc/release-notes-33555.md create mode 100644 doc/release-notes-33842.md create mode 100644 doc/release-notes/release-notes-28.3.md create mode 100644 doc/release-notes/release-notes-29.2.md create mode 100644 doc/release-notes/release-notes-30.0.md create mode 100644 src/minisketch/.github/workflows/ci.yml create mode 100644 src/minisketch/CMakeLists.txt create mode 100644 src/minisketch/cmake/PrintConfigureSummary.cmake create mode 100644 src/minisketch/cmake/SystemIntrospection.cmake create mode 100644 src/minisketch/src/CMakeLists.txt rename src/policy/{fees.cpp => fees/block_policy_estimator.cpp} (99%) rename src/policy/{fees.h => fees/block_policy_estimator.h} (97%) rename src/policy/{fees_args.cpp => fees/block_policy_estimator_args.cpp} (88%) rename src/policy/{fees_args.h => fees/block_policy_estimator_args.h} (65%) create mode 100644 src/secp256k1/src/tests_common.h create mode 100644 src/secp256k1/src/unit_test.c create mode 100644 src/secp256k1/src/unit_test.h create mode 100644 src/test/.clang-tidy.in delete mode 100644 src/test/compilerbug_tests.cpp rename src/test/{policy_fee_tests.cpp => feerounder_tests.cpp} (91%) create mode 100644 src/test/fuzz/difference_formatter.cpp create mode 100644 src/test/util_check_tests.cpp create mode 100755 test/functional/feature_chain_tiebreaks.py create mode 100755 test/functional/feature_framework_testshell.py create mode 100755 test/functional/wallet_musig.py diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml index 83922b5..7c894ab 100644 --- a/.github/ISSUE_TEMPLATE/bug.yml +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -78,7 +78,7 @@ body: id: os attributes: label: Operating system and version - placeholder: e.g. "MacOS Ventura 13.2" or "Ubuntu 22.04 LTS" + placeholder: e.g. "MacOS 26.0" or "Ubuntu 26.04 LTS" validations: required: true - type: textarea @@ -90,4 +90,3 @@ body: e.g. OS/CPU and disk type, network connectivity validations: required: false - diff --git a/.github/actions/configure-docker/action.yml b/.github/actions/configure-docker/action.yml index 09a1e6f..814f2dd 100644 --- a/.github/actions/configure-docker/action.yml +++ b/.github/actions/configure-docker/action.yml @@ -4,12 +4,21 @@ inputs: cache-provider: description: 'gha or cirrus cache provider' required: true - options: - - gh - - cirrus runs: using: 'composite' steps: + - name: Check inputs + shell: bash + run: | + # We expect only gha or cirrus as inputs to cache-provider + case "${{ inputs.cache-provider }}" in + gha|cirrus) + ;; + *) + echo "::warning title=Unknown input to configure docker action::Provided value was ${{ inputs.cache-provider }}" + ;; + esac + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 with: @@ -22,8 +31,12 @@ runs: uses: actions/github-script@v6 with: script: | - core.exportVariable('ACTIONS_CACHE_URL', process.env['ACTIONS_CACHE_URL']) - core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env['ACTIONS_RUNTIME_TOKEN']) + Object.keys(process.env).forEach(function (key) { + if (key.startsWith('ACTIONS_')) { + core.info(`Exporting ${key}`); + core.exportVariable(key, process.env[key]); + } + }); - name: Construct docker build cache args shell: bash @@ -44,8 +57,8 @@ runs: # Always optimistically --cache‑from in case a cache blob exists args=(--cache-from "type=gha${url_args:+,${url_args}},scope=${CONTAINER_NAME}") - # If this is a push to the default branch, also add --cache‑to to save the cache - if [[ ${{ github.event_name }} == "push" && ${{ github.ref_name }} == ${{ github.event.repository.default_branch }} ]]; then + # Only add --cache-to when using the Cirrus cache provider and pushing to the default branch. + if [[ ${{ inputs.cache-provider }} == 'cirrus' && ${{ github.event_name }} == "push" && ${{ github.ref_name }} == ${{ github.event.repository.default_branch }} ]]; then args+=(--cache-to "type=gha${url_args:+,${url_args}},mode=max,ignore-error=true,scope=${CONTAINER_NAME}") fi diff --git a/.github/actions/configure-environment/action.yml b/.github/actions/configure-environment/action.yml index aae5016..e2a26b7 100644 --- a/.github/actions/configure-environment/action.yml +++ b/.github/actions/configure-environment/action.yml @@ -17,7 +17,7 @@ runs: - name: Set cache hashes shell: bash run: | - echo "DEPENDS_HASH=$(git ls-tree HEAD depends "ci/test/$FILE_ENV" | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV + echo "DEPENDS_HASH=$(git ls-tree HEAD depends "$FILE_ENV" | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV echo "PREVIOUS_RELEASES_HASH=$(git ls-tree HEAD test/get_previous_releases.py | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV - name: Get container name diff --git a/.github/ci-test-each-commit-exec.py b/.github/ci-test-each-commit-exec.py index 959a157..b81241b 100755 --- a/.github/ci-test-each-commit-exec.py +++ b/.github/ci-test-each-commit-exec.py @@ -21,11 +21,12 @@ def main(): run(["git", "log", "-1"]) num_procs = int(run(["nproc"], stdout=subprocess.PIPE).stdout) + build_dir = "ci_build" run([ "cmake", "-B", - "build", + build_dir, "-Werror=dev", # Use clang++, because it is a bit faster and uses less memory than g++ "-DCMAKE_C_COMPILER=clang", @@ -37,28 +38,23 @@ def main(): "-DAPPEND_CFLAGS='-O3 -g2'", "-DCMAKE_BUILD_TYPE=Debug", "-DWERROR=ON", - "-DWITH_ZMQ=ON", - "-DBUILD_GUI=ON", - "-DBUILD_BENCH=ON", - "-DBUILD_FUZZ_BINARY=ON", - "-DWITH_USDT=ON", - "-DBUILD_KERNEL_LIB=ON", - "-DBUILD_KERNEL_TEST=ON", + "--preset=dev-mode", + # Tolerate unused member functions in intermediate commits in a pull request "-DCMAKE_CXX_FLAGS=-Wno-error=unused-member-function", ]) - run(["cmake", "--build", "build", "-j", str(num_procs)]) + run(["cmake", "--build", build_dir, "-j", str(num_procs)]) run([ "ctest", "--output-on-failure", "--stop-on-failure", "--test-dir", - "build", + build_dir, "-j", str(num_procs), ]) run([ sys.executable, - "./build/test/functional/test_runner.py", + f"./{build_dir}/test/functional/test_runner.py", "-j", str(num_procs * 2), "--combinedlogslen=99999999", diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 39dc017..e4c5c8a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,6 +35,14 @@ jobs: outputs: provider: ${{ steps.runners.outputs.provider }} steps: + - name: Annotate with pull request number + # This annotation is machine-readable and can be used to assign a check + # run to its corresponding pull request. Running in one check run is + # sufficient for each check suite. + run: | + if [ "${{ github.event_name }}" = "pull_request" ]; then + echo "::notice title=debug_pull_request_number_str::${{ github.event.number }}" + fi - id: runners run: | if [[ "${REPO_USE_CIRRUS_RUNNERS}" == "${{ github.repository }}" ]]; then @@ -157,11 +165,6 @@ jobs: brew install --quiet python@3 || brew link --overwrite python@3 brew install --quiet coreutils ninja pkgconf gnu-getopt ccache boost libevent zeromq qt@6 qrencode capnp - - name: Install Python packages - run: | - git clone -b v2.1.0 https://github.com/capnproto/pycapnp - pip3 install ./pycapnp -C force-bundled-libcapnp=True --break-system-packages - - name: Set Ccache directory run: echo "CCACHE_DIR=${RUNNER_TEMP}/ccache_dir" >> "$GITHUB_ENV" @@ -202,7 +205,7 @@ jobs: job-type: [standard, fuzz] include: - job-type: standard - generate-options: '-DBUILD_GUI=ON -DWITH_ZMQ=ON -DBUILD_BENCH=ON -DBUILD_KERNEL_LIB=ON -DBUILD_UTIL_CHAINSTATE=ON -DBUILD_KERNEL_TEST=OFF -DWERROR=ON' + generate-options: '-DBUILD_GUI=ON -DWITH_ZMQ=ON -DBUILD_BENCH=ON -DBUILD_KERNEL_LIB=ON -DBUILD_UTIL_CHAINSTATE=ON -DWERROR=ON' job-name: 'Windows native, VS 2022' - job-type: fuzz generate-options: '-DVCPKG_MANIFEST_NO_DEFAULT_FEATURES=ON -DVCPKG_MANIFEST_FEATURES="wallet" -DBUILD_GUI=OFF -DBUILD_FOR_FUZZING=ON -DWERROR=ON' @@ -292,8 +295,6 @@ jobs: - name: Run test suite if: matrix.job-type == 'standard' working-directory: build - env: - QT_PLUGIN_PATH: '${{ github.workspace }}\build\vcpkg_installed\x64-windows\Qt6\plugins' run: | ctest --output-on-failure --stop-on-failure -j $NUMBER_OF_PROCESSORS -C Release @@ -381,7 +382,7 @@ jobs: - *CHECKOUT - name: Download built executables - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v5 with: name: x86_64-w64-mingw32-executables-${{ github.run_id }} @@ -433,8 +434,7 @@ jobs: echo "PREVIOUS_RELEASES_DIR=${{ runner.temp }}/previous_releases" >> "$GITHUB_ENV" - name: Get previous releases - working-directory: test - run: ./get_previous_releases.py --target-dir $PREVIOUS_RELEASES_DIR + run: ./test/get_previous_releases.py --target-dir $PREVIOUS_RELEASES_DIR - name: Run functional tests env: @@ -466,18 +466,24 @@ jobs: file-env: './ci/test/00_setup_env_arm.sh' provider: 'gha' - - name: 'ASan + LSan + UBSan + integer, no depends, USDT' + - name: 'ASan + LSan + UBSan + integer' cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-md' # has to match container in ci/test/00_setup_env_native_asan.sh for tracing tools fallback-runner: 'ubuntu-24.04' timeout-minutes: 120 file-env: './ci/test/00_setup_env_native_asan.sh' - - name: 'macOS-cross, gui, no tests' + - name: 'macOS-cross to arm64' cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-sm' fallback-runner: 'ubuntu-24.04' timeout-minutes: 120 file-env: './ci/test/00_setup_env_mac_cross.sh' + - name: 'macOS-cross to x86_64' + cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-sm' + fallback-runner: 'ubuntu-24.04' + timeout-minutes: 120 + file-env: './ci/test/00_setup_env_mac_cross_intel.sh' + - name: 'No wallet, libbitcoinkernel' cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-sm' fallback-runner: 'ubuntu-24.04' @@ -496,17 +502,23 @@ jobs: timeout-minutes: 240 file-env: './ci/test/00_setup_env_native_fuzz.sh' - - name: 'previous releases, depends DEBUG' + - name: 'Valgrind, fuzz' + cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-md' + fallback-runner: 'ubuntu-24.04' + timeout-minutes: 240 + file-env: './ci/test/00_setup_env_native_fuzz_with_valgrind.sh' + + - name: 'previous releases' cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-md' fallback-runner: 'ubuntu-24.04' timeout-minutes: 120 file-env: './ci/test/00_setup_env_native_previous_releases.sh' - - name: 'CentOS, depends, gui' - cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-lg' + - name: 'Alpine (musl), depends, gui' + cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-md' fallback-runner: 'ubuntu-24.04' timeout-minutes: 120 - file-env: './ci/test/00_setup_env_native_centos.sh' + file-env: './ci/test/00_setup_env_native_alpine_musl.sh' - name: 'tidy' cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-md' @@ -520,6 +532,12 @@ jobs: timeout-minutes: 120 file-env: './ci/test/00_setup_env_native_tsan.sh' + - name: 'MSan, fuzz' + cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-md' + fallback-runner: 'ubuntu-24.04' + timeout-minutes: 150 + file-env: './ci/test/00_setup_env_native_fuzz_with_msan.sh' + - name: 'MSan, depends' cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-lg' fallback-runner: 'ubuntu-24.04' @@ -548,7 +566,7 @@ jobs: run: sed -i "s|\${INSTALL_BCC_TRACING_TOOLS}|true|g" ./ci/test/00_setup_env_native_asan.sh - name: Set mmap_rnd_bits - if: ${{ env.CONTAINER_NAME == 'ci_native_tsan' || env.CONTAINER_NAME == 'ci_native_msan' }} + if: ${{ env.CONTAINER_NAME == 'ci_native_tsan' || env.CONTAINER_NAME == 'ci_native_msan' || env.CONTAINER_NAME == 'ci_native_fuzz_msan' }} # Prevents crashes due to high ASLR entropy run: sudo sysctl -w vm.mmap_rnd_bits=28 @@ -561,7 +579,7 @@ jobs: lint: name: 'lint' needs: runners - runs-on: ${{ needs.runners.outputs.use-cirrus-runners == 'true' && 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-xs' || 'ubuntu-24.04' }} + runs-on: ${{ needs.runners.outputs.provider == 'cirrus' && 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-xs' || 'ubuntu-24.04' }} if: ${{ vars.SKIP_BRANCH_PUSH != 'true' || github.event_name == 'pull_request' }} timeout-minutes: 20 env: diff --git a/CMakeLists.txt b/CMakeLists.txt index 2e5196c..2d21f6e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -463,6 +463,8 @@ else() try_append_cxx_flags("-Wself-assign" TARGET warn_interface SKIP_LINK) try_append_cxx_flags("-Wbidi-chars=any" TARGET warn_interface SKIP_LINK) try_append_cxx_flags("-Wundef" TARGET warn_interface SKIP_LINK) + try_append_cxx_flags("-Wleading-whitespace=spaces" TARGET warn_interface SKIP_LINK) + try_append_cxx_flags("-Wtrailing-whitespace=any" TARGET warn_interface SKIP_LINK) # Some compilers (gcc) ignore unknown -Wno-* options, but warn about all # unknown options if any other warning is produced. Test the -Wfoo case, and @@ -493,8 +495,8 @@ try_append_cxx_flags("-fmacro-prefix-map=A=B" TARGET core_interface SKIP_LINK IF_CHECK_PASSED "-fmacro-prefix-map=${PROJECT_SOURCE_DIR}/src=." ) -# Currently all versions of gcc are subject to a class of bugs, see the -# gccbug_90348 test case (only reproduces on GCC 11 and earlier) and +# GCC versions 13.2 (and earlier) are subject to a class of bugs, see +# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90348 and the meta bug # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=111843. To work around that, set # -fstack-reuse=none for all gcc builds. (Only gcc understands this flag). try_append_cxx_flags("-fstack-reuse=none" TARGET core_interface) @@ -632,7 +634,6 @@ if(CMAKE_SYSTEM_NAME STREQUAL "NetBSD") set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) else() set(CMAKE_SKIP_BUILD_RPATH TRUE) - set(CMAKE_SKIP_INSTALL_RPATH TRUE) endif() add_subdirectory(test) add_subdirectory(doc) diff --git a/ci/README.md b/ci/README.md index 81e048c..a729128 100644 --- a/ci/README.md +++ b/ci/README.md @@ -20,6 +20,14 @@ requires `bash`, `docker`, and `python3` to be installed. To run on different ar sudo apt install bash docker.io python3 qemu-user-static ``` +For some sanitizer builds, the kernel's address-space layout randomization +(ASLR) entropy can cause sanitizer shadow memory mappings to fail. When running +the CI locally you may need to reduce that entropy by running: + +``` +sudo sysctl -w vm.mmap_rnd_bits=28 +``` + It is recommended to run the ci system in a clean env. To run the test stage with a specific configuration, diff --git a/ci/lint/01_install.sh b/ci/lint/01_install.sh index 0d8cca9..fc42267 100755 --- a/ci/lint/01_install.sh +++ b/ci/lint/01_install.sh @@ -18,7 +18,8 @@ ${CI_RETRY_EXE} apt-get update # - curl/xz-utils (to install shellcheck) # - git (used in many lint scripts) # - gpg (used by verify-commits) -${CI_RETRY_EXE} apt-get install -y cargo curl xz-utils git gpg +# - moreutils (used by scripted-diff) +${CI_RETRY_EXE} apt-get install -y cargo curl xz-utils git gpg moreutils PYTHON_PATH="/python_build" if [ ! -d "${PYTHON_PATH}/bin" ]; then diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh index a4293fb..a8f8d0e 100755 --- a/ci/test/00_setup_env.sh +++ b/ci/test/00_setup_env.sh @@ -64,7 +64,7 @@ export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_SCRATCH_DIR/out} # The folder for previous release binaries. # This folder exists only on the ci guest, and on the ci host as a volume. export PREVIOUS_RELEASES_DIR=${PREVIOUS_RELEASES_DIR:-$BASE_ROOT_DIR/prev_releases} -export CI_BASE_PACKAGES=${CI_BASE_PACKAGES:-build-essential pkgconf curl ca-certificates ccache python3 rsync git procps bison e2fsprogs cmake ninja-build} +export CI_BASE_PACKAGES=${CI_BASE_PACKAGES:-build-essential pkgconf curl ca-certificates ccache python3-dev rsync git procps bison e2fsprogs cmake ninja-build} export GOAL=${GOAL:-install} export DIR_QA_ASSETS=${DIR_QA_ASSETS:-${BASE_SCRATCH_DIR}/qa-assets} export CI_RETRY_EXE=${CI_RETRY_EXE:-"retry --"} diff --git a/ci/test/00_setup_env_arm.sh b/ci/test/00_setup_env_arm.sh index ce01978..12ec71b 100755 --- a/ci/test/00_setup_env_arm.sh +++ b/ci/test/00_setup_env_arm.sh @@ -19,4 +19,7 @@ export GOAL="install" export CI_LIMIT_STACK_SIZE=1 # -Wno-psabi is to disable ABI warnings: "note: parameter passing for argument of type ... changed in GCC 7.1" # This could be removed once the ABI change warning does not show up by default -export BITCOIN_CONFIG="-DREDUCE_EXPORTS=ON -DCMAKE_CXX_FLAGS='-Wno-psabi -Wno-error=maybe-uninitialized'" +export BITCOIN_CONFIG=" \ + -DREDUCE_EXPORTS=ON \ + -DCMAKE_CXX_FLAGS='-Wno-psabi -Wno-error=maybe-uninitialized' \ +" diff --git a/ci/test/00_setup_env_mac_cross.sh b/ci/test/00_setup_env_mac_cross.sh index a0d9082..c7de033 100755 --- a/ci/test/00_setup_env_mac_cross.sh +++ b/ci/test/00_setup_env_mac_cross.sh @@ -9,12 +9,12 @@ export LC_ALL=C.UTF-8 export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks} export CONTAINER_NAME=ci_macos_cross -export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:24.04" -export HOST=x86_64-apple-darwin +export CI_IMAGE_NAME_TAG="mirror.gcr.io/debian:trixie" # Check that https://packages.debian.org/trixie/clang (version 19, similar to guix) can cross-compile +export HOST=arm64-apple-darwin export PACKAGES="clang lld llvm zip" export XCODE_VERSION=15.0 export XCODE_BUILD_ID=15A240d export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false export GOAL="deploy" -export BITCOIN_CONFIG="-DBUILD_GUI=ON -DREDUCE_EXPORTS=ON" +export BITCOIN_CONFIG="-DBUILD_GUI=ON -DBUILD_KERNEL_LIB=ON -DREDUCE_EXPORTS=ON" diff --git a/ci/test/00_setup_env_mac_cross_intel.sh b/ci/test/00_setup_env_mac_cross_intel.sh new file mode 100755 index 0000000..535b801 --- /dev/null +++ b/ci/test/00_setup_env_mac_cross_intel.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2019-present The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks} + +export CONTAINER_NAME=ci_macos_cross_intel +export CI_IMAGE_NAME_TAG="mirror.gcr.io/debian:trixie" # Check that https://packages.debian.org/trixie/clang (version 19, similar to guix) can cross-compile +export HOST=x86_64-apple-darwin +export PACKAGES="clang lld llvm zip" +export XCODE_VERSION=15.0 +export XCODE_BUILD_ID=15A240d +export RUN_UNIT_TESTS=false +export RUN_FUNCTIONAL_TESTS=false +export GOAL="deploy" +export BITCOIN_CONFIG="-DBUILD_GUI=ON -DREDUCE_EXPORTS=ON" diff --git a/ci/test/00_setup_env_mac_native.sh b/ci/test/00_setup_env_mac_native.sh index 3584246..41b97b0 100755 --- a/ci/test/00_setup_env_mac_native.sh +++ b/ci/test/00_setup_env_mac_native.sh @@ -6,13 +6,11 @@ export LC_ALL=C.UTF-8 -# Homebrew's python@3.12 is marked as externally managed (PEP 668). -# Therefore, `--break-system-packages` is needed. export CONTAINER_NAME="ci_mac_native" # macos does not use a container, but the env var is needed for logging -export PIP_PACKAGES="--break-system-packages zmq" +export PIP_PACKAGES="--break-system-packages pycapnp zmq" export GOAL="install deploy" export CMAKE_GENERATOR="Ninja" -export BITCOIN_CONFIG="-DBUILD_GUI=ON -DWITH_ZMQ=ON -DBUILD_KERNEL_LIB=ON -DBUILD_KERNEL_TEST=ON -DREDUCE_EXPORTS=ON -DCMAKE_EXE_LINKER_FLAGS='-Wl,-stack_size -Wl,0x80000'" +export BITCOIN_CONFIG="-DBUILD_GUI=ON -DWITH_ZMQ=ON -DBUILD_KERNEL_LIB=ON -DBUILD_UTIL_CHAINSTATE=ON -DBUILD_KERNEL_TEST=ON -DREDUCE_EXPORTS=ON -DCMAKE_EXE_LINKER_FLAGS='-Wl,-stack_size -Wl,0x80000'" export CI_OS_NAME="macos" export NO_DEPENDS=1 export OSX_SDK="" diff --git a/ci/test/00_setup_env_mac_native_fuzz.sh b/ci/test/00_setup_env_mac_native_fuzz.sh index a8010c7..ea074e4 100755 --- a/ci/test/00_setup_env_mac_native_fuzz.sh +++ b/ci/test/00_setup_env_mac_native_fuzz.sh @@ -8,7 +8,7 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME="ci_mac_native_fuzz" # macos does not use a container, but the env var is needed for logging export CMAKE_GENERATOR="Ninja" -export BITCOIN_CONFIG="-DBUILD_FOR_FUZZING=ON -DCMAKE_EXE_LINKER_FLAGS='-Wl,-stack_size -Wl,0x80000'" +export BITCOIN_CONFIG="-DBUILD_FOR_FUZZING=ON -DCMAKE_EXE_LINKER_FLAGS='-Wl,-stack_size -Wl,0x80000' -DAPPEND_CPPFLAGS='-D_LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE_DEBUG'" export CI_OS_NAME="macos" export NO_DEPENDS=1 export OSX_SDK="" diff --git a/ci/test/00_setup_env_native_centos.sh b/ci/test/00_setup_env_native_alpine_musl.sh similarity index 57% rename from ci/test/00_setup_env_native_centos.sh rename to ci/test/00_setup_env_native_alpine_musl.sh index 998ddaf..2de2a16 100755 --- a/ci/test/00_setup_env_native_centos.sh +++ b/ci/test/00_setup_env_native_alpine_musl.sh @@ -6,10 +6,10 @@ export LC_ALL=C.UTF-8 -export CONTAINER_NAME=ci_native_centos -export CI_IMAGE_NAME_TAG="quay.io/centos/centos:stream10" -export CI_BASE_PACKAGES="gcc-c++ glibc-devel libstdc++-devel ccache make ninja-build git python3 python3-pip which patch xz procps-ng rsync coreutils bison e2fsprogs cmake dash" -export PIP_PACKAGES="pyzmq pycapnp" +export CONTAINER_NAME=ci_native_alpine_musl +export CI_IMAGE_NAME_TAG="mirror.gcr.io/alpine:3.22" +export CI_BASE_PACKAGES="build-base musl-dev pkgconf curl ccache make ninja git python3-dev py3-pip which patch xz procps rsync util-linux bison e2fsprogs cmake dash linux-headers" +export PIP_PACKAGES="--break-system-packages pyzmq pycapnp" export DEP_OPTS="DEBUG=1" export GOAL="install" export BITCOIN_CONFIG="\ diff --git a/ci/test/00_setup_env_native_asan.sh b/ci/test/00_setup_env_native_asan.sh index e785f6a..0e73224 100755 --- a/ci/test/00_setup_env_native_asan.sh +++ b/ci/test/00_setup_env_native_asan.sh @@ -20,18 +20,19 @@ fi export CONTAINER_NAME=ci_native_asan export APT_LLVM_V="21" -export PACKAGES="systemtap-sdt-dev clang-${APT_LLVM_V} llvm-${APT_LLVM_V} libclang-rt-${APT_LLVM_V}-dev python3-zmq qt6-base-dev qt6-tools-dev qt6-l10n-tools libevent-dev libboost-dev libzmq3-dev libqrencode-dev libsqlite3-dev ${BPFCC_PACKAGE} libcapnp-dev capnproto python3-pip" +export PACKAGES="systemtap-sdt-dev clang-${APT_LLVM_V} llvm-${APT_LLVM_V} libclang-rt-${APT_LLVM_V}-dev mold python3-zmq qt6-base-dev qt6-tools-dev qt6-l10n-tools libevent-dev libboost-dev libzmq3-dev libqrencode-dev libsqlite3-dev ${BPFCC_PACKAGE} libcapnp-dev capnproto python3-pip" export PIP_PACKAGES="--break-system-packages pycapnp" export NO_DEPENDS=1 export GOAL="install" export CI_LIMIT_STACK_SIZE=1 export BITCOIN_CONFIG="\ - -DWITH_USDT=ON -DWITH_ZMQ=ON -DBUILD_GUI=ON \ + --preset=dev-mode \ -DSANITIZERS=address,float-divide-by-zero,integer,undefined \ -DCMAKE_C_COMPILER=clang \ -DCMAKE_CXX_COMPILER=clang++ \ -DCMAKE_C_FLAGS='-ftrivial-auto-var-init=pattern' \ -DCMAKE_CXX_FLAGS='-ftrivial-auto-var-init=pattern' \ + -DCMAKE_EXE_LINKER_FLAGS='-fuse-ld=mold' \ -DAPPEND_CXXFLAGS='-std=c++23' \ -DAPPEND_CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' \ " diff --git a/ci/test/00_setup_env_native_fuzz_with_msan.sh b/ci/test/00_setup_env_native_fuzz_with_msan.sh index 0d90605..7529bbc 100755 --- a/ci/test/00_setup_env_native_fuzz_with_msan.sh +++ b/ci/test/00_setup_env_native_fuzz_with_msan.sh @@ -25,7 +25,7 @@ export BITCOIN_CONFIG="\ -DCMAKE_C_FLAGS_DEBUG='' \ -DCMAKE_CXX_FLAGS_DEBUG='' \ -DBUILD_FOR_FUZZING=ON \ - -DSANITIZERS=fuzzer,memory \ + -DSANITIZERS=memory \ -DAPPEND_CPPFLAGS='-DBOOST_MULTI_INDEX_ENABLE_SAFE_MODE -U_FORTIFY_SOURCE' \ " export USE_INSTRUMENTED_LIBCPP="MemoryWithOrigins" diff --git a/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh b/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh index b1e0c65..1041777 100755 --- a/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh +++ b/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh @@ -7,10 +7,10 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_nowallet_libbitcoinkernel -export CI_IMAGE_NAME_TAG="mirror.gcr.io/debian:bookworm" -# Use minimum supported python3.10 (or best-effort 3.11) and clang-16, see doc/dependencies.md -export PACKAGES="python3-zmq python3-pip clang-16 llvm-16 libc++abi-16-dev libc++-16-dev" +export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:24.04" +# Use minimum supported python3.10 (or best-effort 3.12) and clang-17, see doc/dependencies.md +export PACKAGES="python3-zmq python3-pip clang-17 llvm-17 libc++abi-17-dev libc++-17-dev" export PIP_PACKAGES="--break-system-packages pycapnp" -export DEP_OPTS="NO_WALLET=1 CC=clang-16 CXX='clang++-16 -stdlib=libc++'" +export DEP_OPTS="NO_WALLET=1 CC=clang-17 CXX='clang++-17 -stdlib=libc++'" export GOAL="install" export BITCOIN_CONFIG="-DREDUCE_EXPORTS=ON -DBUILD_UTIL_CHAINSTATE=ON -DBUILD_KERNEL_LIB=ON -DBUILD_KERNEL_TEST=ON -DBUILD_SHARED_LIBS=ON" diff --git a/ci/test/00_setup_env_native_previous_releases.sh b/ci/test/00_setup_env_native_previous_releases.sh index d9f8d3b..a44a51c 100755 --- a/ci/test/00_setup_env_native_previous_releases.sh +++ b/ci/test/00_setup_env_native_previous_releases.sh @@ -8,15 +8,16 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_previous_releases export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:22.04" -# Use minimum supported python3.10 and gcc-11, see doc/dependencies.md -export PACKAGES="gcc-11 g++-11 python3-zmq" -export DEP_OPTS="CC=gcc-11 CXX=g++-11" +# Use minimum supported python3.10 and gcc-12, see doc/dependencies.md +export PACKAGES="gcc-12 g++-12 python3-zmq" +export DEP_OPTS="CC=gcc-12 CXX=g++-12" export TEST_RUNNER_EXTRA="--previous-releases --coverage --extended --exclude feature_dbcrash" # Run extended tests so that coverage does not fail, but exclude the very slow dbcrash export GOAL="install" export CI_LIMIT_STACK_SIZE=1 export DOWNLOAD_PREVIOUS_RELEASES="true" export BITCOIN_CONFIG="\ - -DWITH_ZMQ=ON -DBUILD_GUI=ON -DREDUCE_EXPORTS=ON \ + --preset=dev-mode \ + -DREDUCE_EXPORTS=ON \ -DCMAKE_BUILD_TYPE=Debug \ -DCMAKE_C_FLAGS='-funsigned-char' \ -DCMAKE_C_FLAGS_DEBUG='-g2 -O2' \ diff --git a/ci/test/00_setup_env_native_tidy.sh b/ci/test/00_setup_env_native_tidy.sh index c14e59b..04f3e2e 100755 --- a/ci/test/00_setup_env_native_tidy.sh +++ b/ci/test/00_setup_env_native_tidy.sh @@ -8,7 +8,7 @@ export LC_ALL=C.UTF-8 export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:24.04" export CONTAINER_NAME=ci_native_tidy -export TIDY_LLVM_V="20" +export TIDY_LLVM_V="21" export APT_LLVM_V="${TIDY_LLVM_V}" export PACKAGES="clang-${TIDY_LLVM_V} libclang-${TIDY_LLVM_V}-dev llvm-${TIDY_LLVM_V}-dev libomp-${TIDY_LLVM_V}-dev clang-tidy-${TIDY_LLVM_V} jq libevent-dev libboost-dev libzmq3-dev systemtap-sdt-dev qt6-base-dev qt6-tools-dev qt6-l10n-tools libqrencode-dev libsqlite3-dev libcapnp-dev capnproto" export NO_DEPENDS=1 @@ -19,7 +19,7 @@ export RUN_CHECK_DEPS=true export RUN_TIDY=true export GOAL="install" export BITCOIN_CONFIG="\ - -DWITH_ZMQ=ON -DBUILD_GUI=ON -DBUILD_BENCH=ON -DWITH_USDT=ON \ + --preset dev-mode \ -DCMAKE_C_COMPILER=clang-${TIDY_LLVM_V} \ -DCMAKE_CXX_COMPILER=clang++-${TIDY_LLVM_V} \ -DCMAKE_C_FLAGS_RELWITHDEBINFO='-O0 -g0' \ diff --git a/ci/test/00_setup_env_win64.sh b/ci/test/00_setup_env_win64.sh index 71e1e36..110db1a 100755 --- a/ci/test/00_setup_env_win64.sh +++ b/ci/test/00_setup_env_win64.sh @@ -8,11 +8,11 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_win64 export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:24.04" # Check that https://packages.ubuntu.com/noble/g++-mingw-w64-x86-64-posix (version 13.x, similar to guix) can cross-compile -export CI_IMAGE_PLATFORM="linux/amd64" export HOST=x86_64-w64-mingw32 export PACKAGES="g++-mingw-w64-x86-64-posix nsis" export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false export GOAL="deploy" export BITCOIN_CONFIG="-DREDUCE_EXPORTS=ON -DBUILD_GUI_TESTS=OFF -DBUILD_KERNEL_LIB=ON -DBUILD_KERNEL_TEST=ON \ --DCMAKE_CXX_FLAGS='-Wno-error=maybe-uninitialized'" + -DCMAKE_CXX_FLAGS='-Wno-error=maybe-uninitialized' \ +" diff --git a/ci/test/01_base_install.sh b/ci/test/01_base_install.sh index ce46098..9bb67aa 100755 --- a/ci/test/01_base_install.sh +++ b/ci/test/01_base_install.sh @@ -32,16 +32,17 @@ if [ -n "${APT_LLVM_V}" ]; then ) fi -if [[ $CI_IMAGE_NAME_TAG == *centos* ]]; then - bash -c "dnf -y install epel-release" - # The ninja-build package is available in the CRB repository. - bash -c "dnf -y --allowerasing --enablerepo crb install $CI_BASE_PACKAGES $PACKAGES" +if [[ $CI_IMAGE_NAME_TAG == *alpine* ]]; then + ${CI_RETRY_EXE} apk update + # shellcheck disable=SC2086 + ${CI_RETRY_EXE} apk add --no-cache $CI_BASE_PACKAGES $PACKAGES elif [ "$CI_OS_NAME" != "macos" ]; then if [[ -n "${APPEND_APT_SOURCES_LIST}" ]]; then echo "${APPEND_APT_SOURCES_LIST}" >> /etc/apt/sources.list fi ${CI_RETRY_EXE} apt-get update - ${CI_RETRY_EXE} bash -c "apt-get install --no-install-recommends --no-upgrade -y $PACKAGES $CI_BASE_PACKAGES" + # shellcheck disable=SC2086 + ${CI_RETRY_EXE} apt-get install --no-install-recommends --no-upgrade -y $PACKAGES $CI_BASE_PACKAGES fi if [ -n "${APT_LLVM_V}" ]; then @@ -56,7 +57,7 @@ if [ -n "$PIP_PACKAGES" ]; then fi if [[ -n "${USE_INSTRUMENTED_LIBCPP}" ]]; then - ${CI_RETRY_EXE} git clone --depth=1 https://github.com/llvm/llvm-project -b "llvmorg-21.1.1" /llvm-project + ${CI_RETRY_EXE} git clone --depth=1 https://github.com/llvm/llvm-project -b "llvmorg-21.1.5" /llvm-project cmake -G Ninja -B /cxx_build/ \ -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" \ diff --git a/ci/test/02_run_container.py b/ci/test/02_run_container.py index 513ecac..4d0bed2 100755 --- a/ci/test/02_run_container.py +++ b/ci/test/02_run_container.py @@ -3,16 +3,19 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or https://opensource.org/license/mit/. +from pathlib import Path import os import shlex import subprocess import sys +import time def run(cmd, **kwargs): print("+ " + shlex.join(cmd), flush=True) + kwargs.setdefault("check", True) try: - return subprocess.run(cmd, check=True, **kwargs) + return subprocess.run(cmd, **kwargs) except Exception as e: sys.exit(e) @@ -36,8 +39,8 @@ def main(): # Append $USER to /tmp/env to support multi-user systems and $CONTAINER_NAME # to allow support starting multiple runs simultaneously by the same user. env_file = "/tmp/env-{u}-{c}".format( - u=os.getenv("USER"), - c=os.getenv("CONTAINER_NAME"), + u=os.environ["USER"], + c=os.environ["CONTAINER_NAME"], ) with open(env_file, "w", encoding="utf8") as file: for k, v in os.environ.items(): @@ -45,6 +48,112 @@ def main(): file.write(f"{k}={v}\n") run(["cat", env_file]) + if not os.getenv("DANGER_RUN_CI_ON_HOST"): + CI_IMAGE_LABEL = "bitcoin-ci-test" + + # Use buildx unconditionally + # Using buildx is required to properly load the correct driver, for use with registry caching. Neither build, nor BUILDKIT=1 currently do this properly + cmd_build = ["docker", "buildx", "build"] + cmd_build += [ + f"--file={os.environ['BASE_READ_ONLY_DIR']}/ci/test_imagefile", + f"--build-arg=CI_IMAGE_NAME_TAG={os.environ['CI_IMAGE_NAME_TAG']}", + f"--build-arg=FILE_ENV={os.environ['FILE_ENV']}", + f"--build-arg=BASE_ROOT_DIR={os.environ['BASE_ROOT_DIR']}", + f"--platform={os.environ['CI_IMAGE_PLATFORM']}", + f"--label={CI_IMAGE_LABEL}", + f"--tag={os.environ['CONTAINER_NAME']}", + ] + cmd_build += shlex.split(os.getenv("DOCKER_BUILD_CACHE_ARG", "")) + cmd_build += [os.environ["BASE_READ_ONLY_DIR"]] + + print(f"Building {os.environ['CONTAINER_NAME']} image tag to run in") + if run(cmd_build, check=False).returncode != 0: + print(f"Retry building {os.environ['CONTAINER_NAME']} image tag after failure") + time.sleep(3) + run(cmd_build) + + for suffix in ["ccache", "depends", "depends_sources", "previous_releases"]: + run(["docker", "volume", "create", f"{os.environ['CONTAINER_NAME']}_{suffix}"], check=False) + + CI_CCACHE_MOUNT = f"type=volume,src={os.environ['CONTAINER_NAME']}_ccache,dst={os.environ['CCACHE_DIR']}" + CI_DEPENDS_MOUNT = f"type=volume,src={os.environ['CONTAINER_NAME']}_depends,dst={os.environ['DEPENDS_DIR']}/built" + CI_DEPENDS_SOURCES_MOUNT = f"type=volume,src={os.environ['CONTAINER_NAME']}_depends_sources,dst={os.environ['DEPENDS_DIR']}/sources" + CI_PREVIOUS_RELEASES_MOUNT = f"type=volume,src={os.environ['CONTAINER_NAME']}_previous_releases,dst={os.environ['PREVIOUS_RELEASES_DIR']}" + CI_BUILD_MOUNT = [] + + if os.getenv("DANGER_CI_ON_HOST_FOLDERS"): + # ensure the directories exist + for create_dir in [ + os.environ["CCACHE_DIR"], + f"{os.environ['DEPENDS_DIR']}/built", + f"{os.environ['DEPENDS_DIR']}/sources", + os.environ["PREVIOUS_RELEASES_DIR"], + os.environ["BASE_BUILD_DIR"], # Unset by default, must be defined externally + ]: + Path(create_dir).mkdir(parents=True, exist_ok=True) + + CI_CCACHE_MOUNT = f"type=bind,src={os.environ['CCACHE_DIR']},dst={os.environ['CCACHE_DIR']}" + CI_DEPENDS_MOUNT = f"type=bind,src={os.environ['DEPENDS_DIR']}/built,dst={os.environ['DEPENDS_DIR']}/built" + CI_DEPENDS_SOURCES_MOUNT = f"type=bind,src={os.environ['DEPENDS_DIR']}/sources,dst={os.environ['DEPENDS_DIR']}/sources" + CI_PREVIOUS_RELEASES_MOUNT = f"type=bind,src={os.environ['PREVIOUS_RELEASES_DIR']},dst={os.environ['PREVIOUS_RELEASES_DIR']}" + CI_BUILD_MOUNT = [f"--mount=type=bind,src={os.environ['BASE_BUILD_DIR']},dst={os.environ['BASE_BUILD_DIR']}"] + + if os.getenv("DANGER_CI_ON_HOST_CCACHE_FOLDER"): + if not os.path.isdir(os.environ["CCACHE_DIR"]): + print(f"Error: Directory '{os.environ['CCACHE_DIR']}' must be created in advance.") + sys.exit(1) + CI_CCACHE_MOUNT = f"type=bind,src={os.environ['CCACHE_DIR']},dst={os.environ['CCACHE_DIR']}" + + run(["docker", "network", "create", "--ipv6", "--subnet", "1111:1111::/112", "ci-ip6net"], check=False) + + if os.getenv("RESTART_CI_DOCKER_BEFORE_RUN"): + print("Restart docker before run to stop and clear all containers started with --rm") + run(["podman", "container", "rm", "--force", "--all"]) # Similar to "systemctl restart docker" + + # Still prune everything in case the filtered pruning doesn't work, or if labels were not set + # on a previous run. Belt and suspenders approach, should be fine to remove in the future. + # Prune images used by --external containers (e.g. build containers) when + # using podman. + print("Prune all dangling images") + run(["podman", "image", "prune", "--force", "--external"]) + + print(f"Prune all dangling {CI_IMAGE_LABEL} images") + # When detecting podman-docker, `--external` should be added. + run(["docker", "image", "prune", "--force", "--filter", f"label={CI_IMAGE_LABEL}"]) + + cmd_run = ["docker", "run", "--rm", "--interactive", "--detach", "--tty"] + cmd_run += [ + "--cap-add=LINUX_IMMUTABLE", + *shlex.split(os.getenv("CI_CONTAINER_CAP", "")), + f"--mount=type=bind,src={os.environ['BASE_READ_ONLY_DIR']},dst={os.environ['BASE_READ_ONLY_DIR']},readonly", + f"--mount={CI_CCACHE_MOUNT}", + f"--mount={CI_DEPENDS_MOUNT}", + f"--mount={CI_DEPENDS_SOURCES_MOUNT}", + f"--mount={CI_PREVIOUS_RELEASES_MOUNT}", + *CI_BUILD_MOUNT, + f"--env-file={env_file}", + f"--name={os.environ['CONTAINER_NAME']}", + "--network=ci-ip6net", + f"--platform={os.environ['CI_IMAGE_PLATFORM']}", + os.environ["CONTAINER_NAME"], + ] + + container_id = run( + cmd_run, + stdout=subprocess.PIPE, + text=True, + ).stdout.strip() + os.environ["CI_CONTAINER_ID"] = container_id + + # GNU getopt is required for the CI_RETRY_EXE script + if os.getenv("CI_OS_NAME") == "macos": + prefix = run( + ["brew", "--prefix", "gnu-getopt"], + stdout=subprocess.PIPE, + text=True, + ).stdout.strip() + os.environ["IN_GETOPT_BIN"] = f"{prefix}/bin/getopt" + run(["./ci/test/02_run_container.sh"]) # run the remainder diff --git a/ci/test/02_run_container.sh b/ci/test/02_run_container.sh index e91f875..251b01c 100755 --- a/ci/test/02_run_container.sh +++ b/ci/test/02_run_container.sh @@ -5,100 +5,10 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 -export CI_IMAGE_LABEL="bitcoin-ci-test" set -o errexit -o pipefail -o xtrace if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then - # Env vars during the build can not be changed. For example, a modified - # $MAKEJOBS is ignored in the build process. Use --cpuset-cpus as an - # approximation to respect $MAKEJOBS somewhat, if cpuset is available. - MAYBE_CPUSET="" - if [ "$HAVE_CGROUP_CPUSET" ]; then - MAYBE_CPUSET="--cpuset-cpus=$( python3 -c "import random;P=$( nproc );M=min(P,int('$MAKEJOBS'.lstrip('-j')));print(','.join(map(str,sorted(random.sample(range(P),M)))))" )" - fi - echo "Creating $CI_IMAGE_NAME_TAG container to run in" - - # Use buildx unconditionally - # Using buildx is required to properly load the correct driver, for use with registry caching. Neither build, nor BUILDKIT=1 currently do this properly - # shellcheck disable=SC2086 - docker buildx build \ - --file "${BASE_READ_ONLY_DIR}/ci/test_imagefile" \ - --build-arg "CI_IMAGE_NAME_TAG=${CI_IMAGE_NAME_TAG}" \ - --build-arg "FILE_ENV=${FILE_ENV}" \ - --build-arg "BASE_ROOT_DIR=${BASE_ROOT_DIR}" \ - $MAYBE_CPUSET \ - --platform="${CI_IMAGE_PLATFORM}" \ - --label="${CI_IMAGE_LABEL}" \ - --tag="${CONTAINER_NAME}" \ - $DOCKER_BUILD_CACHE_ARG \ - "${BASE_READ_ONLY_DIR}" - - docker volume create "${CONTAINER_NAME}_ccache" || true - docker volume create "${CONTAINER_NAME}_depends" || true - docker volume create "${CONTAINER_NAME}_depends_sources" || true - docker volume create "${CONTAINER_NAME}_previous_releases" || true - - CI_CCACHE_MOUNT="type=volume,src=${CONTAINER_NAME}_ccache,dst=$CCACHE_DIR" - CI_DEPENDS_MOUNT="type=volume,src=${CONTAINER_NAME}_depends,dst=$DEPENDS_DIR/built" - CI_DEPENDS_SOURCES_MOUNT="type=volume,src=${CONTAINER_NAME}_depends_sources,dst=$DEPENDS_DIR/sources" - CI_PREVIOUS_RELEASES_MOUNT="type=volume,src=${CONTAINER_NAME}_previous_releases,dst=$PREVIOUS_RELEASES_DIR" - CI_BUILD_MOUNT="" - - if [ "$DANGER_CI_ON_HOST_FOLDERS" ]; then - # ensure the directories exist - mkdir -p "${CCACHE_DIR}" - mkdir -p "${DEPENDS_DIR}/built" - mkdir -p "${DEPENDS_DIR}/sources" - mkdir -p "${PREVIOUS_RELEASES_DIR}" - mkdir -p "${BASE_BUILD_DIR}" # Unset by default, must be defined externally - - CI_CCACHE_MOUNT="type=bind,src=${CCACHE_DIR},dst=$CCACHE_DIR" - CI_DEPENDS_MOUNT="type=bind,src=${DEPENDS_DIR}/built,dst=$DEPENDS_DIR/built" - CI_DEPENDS_SOURCES_MOUNT="type=bind,src=${DEPENDS_DIR}/sources,dst=$DEPENDS_DIR/sources" - CI_PREVIOUS_RELEASES_MOUNT="type=bind,src=${PREVIOUS_RELEASES_DIR},dst=$PREVIOUS_RELEASES_DIR" - CI_BUILD_MOUNT="--mount type=bind,src=${BASE_BUILD_DIR},dst=${BASE_BUILD_DIR}" - fi - - if [ "$DANGER_CI_ON_HOST_CCACHE_FOLDER" ]; then - if [ ! -d "${CCACHE_DIR}" ]; then - echo "Error: Directory '${CCACHE_DIR}' must be created in advance." - exit 1 - fi - CI_CCACHE_MOUNT="type=bind,src=${CCACHE_DIR},dst=${CCACHE_DIR}" - fi - - docker network create --ipv6 --subnet 1111:1111::/112 ci-ip6net || true - - if [ -n "${RESTART_CI_DOCKER_BEFORE_RUN}" ] ; then - echo "Restart docker before run to stop and clear all containers started with --rm" - podman container rm --force --all # Similar to "systemctl restart docker" - - # Still prune everything in case the filtered pruning doesn't work, or if labels were not set - # on a previous run. Belt and suspenders approach, should be fine to remove in the future. - # Prune images used by --external containers (e.g. build containers) when - # using podman. - echo "Prune all dangling images" - podman image prune --force --external - fi - echo "Prune all dangling $CI_IMAGE_LABEL images" - # When detecting podman-docker, `--external` should be added. - docker image prune --force --filter "label=$CI_IMAGE_LABEL" - - # shellcheck disable=SC2086 - CI_CONTAINER_ID=$(docker run --cap-add LINUX_IMMUTABLE $CI_CONTAINER_CAP --rm --interactive --detach --tty \ - --mount "type=bind,src=$BASE_READ_ONLY_DIR,dst=$BASE_READ_ONLY_DIR,readonly" \ - --mount "${CI_CCACHE_MOUNT}" \ - --mount "${CI_DEPENDS_MOUNT}" \ - --mount "${CI_DEPENDS_SOURCES_MOUNT}" \ - --mount "${CI_PREVIOUS_RELEASES_MOUNT}" \ - ${CI_BUILD_MOUNT} \ - --env-file /tmp/env-$USER-$CONTAINER_NAME \ - --name "$CONTAINER_NAME" \ - --network ci-ip6net \ - --platform="${CI_IMAGE_PLATFORM}" \ - "$CONTAINER_NAME") - export CI_CONTAINER_ID export CI_EXEC_CMD_PREFIX="docker exec ${CI_CONTAINER_ID}" else echo "Running on host system without docker wrapper" @@ -107,11 +17,6 @@ else mkdir -p "${PREVIOUS_RELEASES_DIR}" fi -if [ "$CI_OS_NAME" == "macos" ]; then - IN_GETOPT_BIN="$(brew --prefix gnu-getopt)/bin/getopt" - export IN_GETOPT_BIN -fi - CI_EXEC () { $CI_EXEC_CMD_PREFIX bash -c "export PATH=\"/path_with space:${BINS_SCRATCH_DIR}:${BASE_ROOT_DIR}/ci/retry:\$PATH\" && cd \"${BASE_ROOT_DIR}\" && $*" } diff --git a/ci/test/03_test_script.sh b/ci/test/03_test_script.sh index cee09cd..09da802 100755 --- a/ci/test/03_test_script.sh +++ b/ci/test/03_test_script.sh @@ -41,7 +41,10 @@ echo "=== BEGIN env ===" env echo "=== END env ===" -( +# Don't apply patches in the tidy job, because it relies on the `git diff` +# command to detect IWYU errors. It is safe to skip this patch in the tidy job +# because it doesn't run a UB detector. +if [ "$RUN_TIDY" != "true" ]; then # compact->outputs[i].file_size is uninitialized memory, so reading it is UB. # The statistic bytes_written is only used for logging, which is disabled in # CI, so as a temporary minimal fix to work around UB and CI failures, leave @@ -62,7 +65,7 @@ echo "=== END env ===" mutex_.Lock(); stats_[compact->compaction->level() + 1].Add(stats); EOF -) +fi if [ "$RUN_FUZZ_TESTS" = "true" ]; then export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_corpora/ @@ -99,8 +102,8 @@ else fi if [ -z "$NO_DEPENDS" ]; then - if [[ $CI_IMAGE_NAME_TAG == *centos* ]]; then - SHELL_OPTS="CONFIG_SHELL=/bin/dash" + if [[ $CI_IMAGE_NAME_TAG == *alpine* ]]; then + SHELL_OPTS="CONFIG_SHELL=/usr/bin/dash" else SHELL_OPTS="CONFIG_SHELL=" fi @@ -211,14 +214,30 @@ if [ "${RUN_TIDY}" = "true" ]; then false fi + # TODO: Consider enforcing IWYU across the entire codebase. + FILES_WITH_ENFORCED_IWYU="/src/(crypto|index)/.*\\.cpp" + jq --arg patterns "$FILES_WITH_ENFORCED_IWYU" 'map(select(.file | test($patterns)))' "${BASE_BUILD_DIR}/compile_commands.json" > "${BASE_BUILD_DIR}/compile_commands_iwyu_errors.json" + jq --arg patterns "$FILES_WITH_ENFORCED_IWYU" 'map(select(.file | test($patterns) | not))' "${BASE_BUILD_DIR}/compile_commands.json" > "${BASE_BUILD_DIR}/compile_commands_iwyu_warnings.json" + cd "${BASE_ROOT_DIR}" - python3 "/include-what-you-use/iwyu_tool.py" \ - -p "${BASE_BUILD_DIR}" "${MAKEJOBS}" \ - -- -Xiwyu --cxx17ns -Xiwyu --mapping_file="${BASE_ROOT_DIR}/contrib/devtools/iwyu/bitcoin.core.imp" \ - -Xiwyu --max_line_length=160 \ - 2>&1 | tee /tmp/iwyu_ci.out - cd "${BASE_ROOT_DIR}/src" - python3 "/include-what-you-use/fix_includes.py" --nosafe_headers < /tmp/iwyu_ci.out + + run_iwyu() { + mv "${BASE_BUILD_DIR}/$1" "${BASE_BUILD_DIR}/compile_commands.json" + python3 "/include-what-you-use/iwyu_tool.py" \ + -p "${BASE_BUILD_DIR}" "${MAKEJOBS}" \ + -- -Xiwyu --cxx17ns -Xiwyu --mapping_file="${BASE_ROOT_DIR}/contrib/devtools/iwyu/bitcoin.core.imp" \ + -Xiwyu --max_line_length=160 \ + 2>&1 | tee /tmp/iwyu_ci.out + python3 "/include-what-you-use/fix_includes.py" --nosafe_headers < /tmp/iwyu_ci.out + } + + run_iwyu "compile_commands_iwyu_errors.json" + if ! ( git --no-pager diff --exit-code ); then + echo "^^^ ⚠️ Failure generated from IWYU" + false + fi + + run_iwyu "compile_commands_iwyu_warnings.json" git --no-pager diff fi diff --git a/ci/test_imagefile b/ci/test_imagefile index f9cf318..a0e1714 100644 --- a/ci/test_imagefile +++ b/ci/test_imagefile @@ -17,4 +17,7 @@ ENV BASE_ROOT_DIR=${BASE_ROOT_DIR} COPY ./ci/retry/retry /usr/bin/retry COPY ./ci/test/00_setup_env.sh ./${FILE_ENV} ./ci/test/01_base_install.sh /ci_container_base/ci/test/ +# Bash is required, so install it when missing +RUN sh -c "bash -c 'true' || ( apk update && apk add --no-cache bash )" + RUN ["bash", "-c", "cd /ci_container_base/ && set -o errexit && source ./ci/test/00_setup_env.sh && ./ci/test/01_base_install.sh"] diff --git a/cmake/windows-app.manifest.in b/cmake/windows-app.manifest.in index c3bd333..e0dc2eb 100644 --- a/cmake/windows-app.manifest.in +++ b/cmake/windows-app.manifest.in @@ -1,10 +1,15 @@ - + + + + UTF-8 + + diff --git a/contrib/devtools/README.md b/contrib/devtools/README.md index 366cd4a..3309057 100644 --- a/contrib/devtools/README.md +++ b/contrib/devtools/README.md @@ -137,7 +137,7 @@ BUILDDIR=$PWD/my-build-dir contrib/devtools/gen-manpages.py headerssync-params.py ===================== -A script to generate optimal parameters for the headerssync module (src/headerssync.cpp). It takes no command-line +A script to generate optimal parameters for the headerssync module (stored in src/kernel/chainparams.cpp). It takes no command-line options, as all its configuration is set at the top of the file. It runs many times faster inside PyPy. Invocation: ```bash diff --git a/contrib/devtools/deterministic-fuzz-coverage/src/main.rs b/contrib/devtools/deterministic-fuzz-coverage/src/main.rs index 3eeb121..362ab69 100644 --- a/contrib/devtools/deterministic-fuzz-coverage/src/main.rs +++ b/contrib/devtools/deterministic-fuzz-coverage/src/main.rs @@ -36,7 +36,7 @@ fn sanity_check(corpora_dir: &Path, fuzz_exe: &Path) -> AppResult { let output = Command::new(tool).arg("--help").output(); match output { Ok(output) if output.status.success() => {} - _ => Err(exit_help(&format!("The tool {} is not installed", tool)))?, + _ => Err(exit_help(&format!("The tool {tool} is not installed")))?, } } if !corpora_dir.is_dir() { @@ -274,7 +274,7 @@ fn main() -> ExitCode { match app() { Ok(()) => ExitCode::SUCCESS, Err(err) => { - eprintln!("⚠️\n{}", err); + eprintln!("⚠️\n{err}"); ExitCode::FAILURE } } diff --git a/contrib/devtools/deterministic-unittest-coverage/src/main.rs b/contrib/devtools/deterministic-unittest-coverage/src/main.rs index 047c8d2..95d2c08 100644 --- a/contrib/devtools/deterministic-unittest-coverage/src/main.rs +++ b/contrib/devtools/deterministic-unittest-coverage/src/main.rs @@ -32,7 +32,7 @@ fn sanity_check(test_exe: &Path) -> AppResult { let output = Command::new(tool).arg("--help").output(); match output { Ok(output) if output.status.success() => {} - _ => Err(exit_help(&format!("The tool {} is not installed", tool)))?, + _ => Err(exit_help(&format!("The tool {tool} is not installed")))?, } } if !test_exe.exists() { @@ -142,7 +142,7 @@ fn main() -> ExitCode { match app() { Ok(()) => ExitCode::SUCCESS, Err(err) => { - eprintln!("⚠️\n{}", err); + eprintln!("⚠️\n{err}"); ExitCode::FAILURE } } diff --git a/contrib/devtools/headerssync-params.py b/contrib/devtools/headerssync-params.py index ece1a78..82bdaad 100755 --- a/contrib/devtools/headerssync-params.py +++ b/contrib/devtools/headerssync-params.py @@ -5,8 +5,8 @@ """Script to find the optimal parameters for the headerssync module through simulation.""" -from math import log, exp, sqrt from datetime import datetime, timedelta +from math import log, exp, sqrt import random # Parameters: @@ -337,15 +337,15 @@ def analyze(when): attack_volume = NET_HEADER_SIZE * MINCHAINWORK_HEADERS # And report them. print() - print("Optimal configuration:") - print() - print("//! Store one header commitment per HEADER_COMMITMENT_PERIOD blocks.") - print(f"constexpr size_t HEADER_COMMITMENT_PERIOD{{{period}}};") + print(f"Given current min chainwork headers of {MINCHAINWORK_HEADERS}, the optimal parameters for low") + print(f"memory usage on mainchain for release until {TIME:%Y-%m-%d} is:") print() - print("//! Only feed headers to validation once this many headers on top have been") - print("//! received and validated against commitments.") - print(f"constexpr size_t REDOWNLOAD_BUFFER_SIZE{{{bufsize}}};" + print(f" // Generated by headerssync-params.py on {datetime.today():%Y-%m-%d}.") + print( " m_headers_sync_params = HeadersSyncParams{") + print(f" .commitment_period = {period},") + print(f" .redownload_buffer_size = {bufsize}," f" // {bufsize}/{period} = ~{bufsize/period:.1f} commitments") + print( " };") print() print("Properties:") print(f"- Per-peer memory for mainchain sync: {mem_mainchain / 8192:.3f} KiB") diff --git a/contrib/devtools/iwyu/bitcoin.core.imp b/contrib/devtools/iwyu/bitcoin.core.imp index c4c4ba4..c3195e5 100644 --- a/contrib/devtools/iwyu/bitcoin.core.imp +++ b/contrib/devtools/iwyu/bitcoin.core.imp @@ -1,3 +1,16 @@ -# Nothing for now. [ + # Compiler intrinsics. + # See: https://github.com/include-what-you-use/include-what-you-use/issues/1764. + { "include": [ "", "private", "", "public" ] }, + { "include": [ "", "private", "", "public" ] }, + { "include": [ "", "private", "", "public" ] }, + + # libc symbols. + { "symbol": ["AT_HWCAP", "private", "", "public"] }, + { "symbol": ["AT_HWCAP2", "private", "", "public"] }, + + # Fixed in https://github.com/include-what-you-use/include-what-you-use/pull/1706. + { "symbol": ["SEEK_CUR", "private", "", "public"] }, + { "symbol": ["SEEK_END", "private", "", "public"] }, + { "symbol": ["SEEK_SET", "private", "", "public"] }, ] diff --git a/contrib/guix/INSTALL.md b/contrib/guix/INSTALL.md index f9a79f6..833d540 100644 --- a/contrib/guix/INSTALL.md +++ b/contrib/guix/INSTALL.md @@ -64,14 +64,12 @@ Please refer to fanquake's instructions ## Option 4: Using a distribution-maintained package -Note that this section is based on the distro packaging situation at the time of -writing (July 2021). Guix is expected to be more widely packaged over time. For -an up-to-date view on Guix's package status/version across distros, please see: -https://repology.org/project/guix/versions +For an up-to-date view on Guix's package status/version across +distros, please see: https://repology.org/project/guix/versions ### Debian / Ubuntu -Guix is available as a distribution package in [Debian +Guix is available as a distribution package in various versions of [Debian ](https://packages.debian.org/search?keywords=guix) and [Ubuntu ](https://packages.ubuntu.com/search?keywords=guix). diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index bbf6c64..09cc058 100755 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -206,7 +206,7 @@ mkdir -p "$OUTDIR" ########################### # CONFIGFLAGS -CONFIGFLAGS="-DREDUCE_EXPORTS=ON -DBUILD_BENCH=OFF -DBUILD_GUI_TESTS=OFF -DBUILD_FUZZ_BINARY=OFF" +CONFIGFLAGS="-DREDUCE_EXPORTS=ON -DBUILD_BENCH=OFF -DBUILD_GUI_TESTS=OFF -DBUILD_FUZZ_BINARY=OFF -DCMAKE_SKIP_RPATH=TRUE" # CFLAGS HOST_CFLAGS="-O2 -g" @@ -225,10 +225,15 @@ esac # LDFLAGS case "$HOST" in - *linux*) HOST_LDFLAGS="-Wl,--as-needed -Wl,--dynamic-linker=$glibc_dynamic_linker -static-libstdc++ -Wl,-O2" ;; + *linux*) HOST_LDFLAGS="-Wl,--as-needed -Wl,--dynamic-linker=$glibc_dynamic_linker -Wl,-O2" ;; *mingw*) HOST_LDFLAGS="-Wl,--no-insert-timestamp" ;; esac +# EXE FLAGS +case "$HOST" in + *linux*) CMAKE_EXE_LINKER_FLAGS="-DCMAKE_EXE_LINKER_FLAGS=${HOST_LDFLAGS} -static-libstdc++ -static-libgcc" ;; +esac + mkdir -p "$DISTSRC" ( cd "$DISTSRC" @@ -243,7 +248,8 @@ mkdir -p "$DISTSRC" --toolchain "${BASEPREFIX}/${HOST}/toolchain.cmake" \ -DWITH_CCACHE=OFF \ -Werror=dev \ - ${CONFIGFLAGS} + ${CONFIGFLAGS} \ + "${CMAKE_EXE_LINKER_FLAGS}" # Build Bitcoin Core cmake --build build -j "$JOBS" ${V:+--verbose} diff --git a/contrib/guix/libexec/prelude.bash b/contrib/guix/libexec/prelude.bash index d25c371..b7c13cc 100644 --- a/contrib/guix/libexec/prelude.bash +++ b/contrib/guix/libexec/prelude.bash @@ -71,7 +71,7 @@ fi time-machine() { # shellcheck disable=SC2086 guix time-machine --url=https://codeberg.org/guix/guix.git \ - --commit=53396a22afc04536ddf75d8f82ad2eafa5082725 \ + --commit=5cb84f2013c5b1e48a7d0e617032266f1e6059e2 \ --cores="$JOBS" \ --keep-failed \ --fallback \ diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index a230eea..011ba0d 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -18,7 +18,7 @@ ((gnu packages python-build) #:select (python-poetry-core)) ((gnu packages python-crypto) #:select (python-asn1crypto)) ((gnu packages python-science) #:select (python-scikit-build-core)) - ((gnu packages python-xyz) #:select (python-pydantic-2 python-pydantic-core)) + ((gnu packages python-xyz) #:select (python-pydantic-2)) ((gnu packages tls) #:select (openssl)) ((gnu packages version-control) #:select (git-minimal)) (guix build-system cmake) @@ -99,10 +99,10 @@ chain for " target " development.")) (define* (make-bitcoin-cross-toolchain target #:key - (base-gcc-for-libc linux-base-gcc) + (base-gcc-for-libc (gcc-libgcc-patches linux-base-gcc)) (base-kernel-headers base-linux-kernel-headers) (base-libc glibc-2.31) - (base-gcc linux-base-gcc)) + (base-gcc (gcc-libgcc-patches linux-base-gcc))) "Convenience wrapper around MAKE-CROSS-TOOLCHAIN with default values desirable for building Bitcoin Core release binaries." (make-cross-toolchain target @@ -111,7 +111,7 @@ desirable for building Bitcoin Core release binaries." base-libc base-gcc)) -(define (gcc-mingw-patches gcc) +(define (gcc-libgcc-patches gcc) (package-with-extra-patches gcc (search-our-patches "gcc-remap-guix-store.patch"))) @@ -128,10 +128,10 @@ desirable for building Bitcoin Core release binaries." (let* ((xbinutils (binutils-mingw-patches (cross-binutils target))) (machine (substring target 0 (string-index target #\-))) (pthreads-xlibc (winpthreads-patches (make-mingw-w64 machine - #:xgcc (cross-gcc target #:xgcc (gcc-mingw-patches base-gcc)) + #:xgcc (cross-gcc target #:xgcc (gcc-libgcc-patches base-gcc)) #:with-winpthreads? #t))) (pthreads-xgcc (cross-gcc target - #:xgcc (gcc-mingw-patches mingw-w64-base-gcc) + #:xgcc (gcc-libgcc-patches mingw-w64-base-gcc) #:xbinutils xbinutils #:libc pthreads-xlibc))) ;; Define a meta-package that propagates the resulting XBINUTILS, XLIBC, and @@ -175,7 +175,6 @@ chain for " target " development.")) (native-inputs (list cmake-minimal ninja python-scikit-build-core - python-pydantic-core python-pydantic-2)) (arguments (list @@ -439,6 +438,7 @@ inspecting signatures in Mach-O binaries.") "--enable-standard-branch-protection=yes", "--enable-cet=yes", "--disable-gcov", + "--disable-libsanitizer", building-on))) ((#:phases phases) `(modify-phases ,phases @@ -455,7 +455,7 @@ inspecting signatures in Mach-O binaries.") (define-public glibc-2.31 (let ((commit "7b27c450c34563a28e634cccb399cd415e71ebfe")) (package - (inherit glibc) ;; 2.35 + (inherit glibc) ;; 2.39 (version "2.31") (source (origin (method git-fetch) @@ -466,7 +466,8 @@ inspecting signatures in Mach-O binaries.") (sha256 (base32 "017qdpr5id7ddb4lpkzj2li1abvw916m3fc6n7nw28z4h5qbv2n0")) - (patches (search-our-patches "glibc-guix-prefix.patch")))) + (patches (search-our-patches "glibc-guix-prefix.patch" + "glibc-riscv-jumptarget.patch")))) (arguments (substitute-keyword-arguments (package-arguments glibc) ((#:configure-flags flags) @@ -566,9 +567,9 @@ inspecting signatures in Mach-O binaries.") (list gcc-toolchain-13 "static") (make-bitcoin-cross-toolchain target))) ((string-contains target "darwin") - (list clang-toolchain-18 - lld-18 - (make-lld-wrapper lld-18 #:lld-as-ld? #t) + (list clang-toolchain-19 + lld-19 + (make-lld-wrapper lld-19 #:lld-as-ld? #t) python-signapple zip)) (else '()))))) diff --git a/contrib/guix/patches/gcc-remap-guix-store.patch b/contrib/guix/patches/gcc-remap-guix-store.patch index a8b41d4..4144bcd 100644 --- a/contrib/guix/patches/gcc-remap-guix-store.patch +++ b/contrib/guix/patches/gcc-remap-guix-store.patch @@ -1,4 +1,4 @@ -Without ffile-prefix-map, the debug symbols will contain paths for the +Without -ffile-prefix-map, the debug symbols will contain paths for the guix store which will include the hashes of each package. However, the hash for the same package will differ when on different architectures. In order to be reproducible regardless of the architecture used to build @@ -6,7 +6,7 @@ the package, map all guix store prefixes to something fixed, e.g. /usr. --- a/libgcc/Makefile.in +++ b/libgcc/Makefile.in -@@ -854,7 +854,7 @@ endif +@@ -857,7 +857,7 @@ endif # libgcc_eh.a, only LIB2ADDEH matters. If we do, only LIB2ADDEHSTATIC and # LIB2ADDEHSHARED matter. (Usually all three are identical.) @@ -15,6 +15,15 @@ the package, map all guix store prefixes to something fixed, e.g. /usr. ifeq ($(enable_shared),yes) +@@ -880,7 +880,7 @@ endif + # Build LIBUNWIND. Use -fno-exceptions so that the unwind library does + # not generate calls to __gcc_personality_v0. + +-c_flags := -fno-exceptions ++c_flags := -fno-exceptions $(shell find /gnu/store -maxdepth 1 -mindepth 1 -type d -exec echo -n " -ffile-prefix-map={}=/usr" \;) + + libunwind-objects += $(addsuffix $(objext),$(basename $(notdir $(LIBUNWIND)))) + -- 2.37.0 diff --git a/contrib/guix/patches/glibc-riscv-jumptarget.patch b/contrib/guix/patches/glibc-riscv-jumptarget.patch new file mode 100644 index 0000000..7029594 --- /dev/null +++ b/contrib/guix/patches/glibc-riscv-jumptarget.patch @@ -0,0 +1,57 @@ +commit 68389203832ab39dd0dbaabbc4059e7fff51c29b +Author: Fangrui Song +Date: Thu Oct 28 11:39:49 2021 -0700 + + riscv: Fix incorrect jal with HIDDEN_JUMPTARGET + + A non-local STV_DEFAULT defined symbol is by default preemptible in a + shared object. j/jal cannot target a preemptible symbol. On other + architectures, such a jump instruction either causes PLT [BZ #18822], or + if short-ranged, sometimes rejected by the linker (but not by GNU ld's + riscv port [ld PR/28509]). + + Use HIDDEN_JUMPTARGET to target a non-preemptible symbol instead. + + With this patch, ld.so and libc.so can be linked with LLD if source + files are compiled/assembled with -mno-relax/-Wa,-mno-relax. + + Acked-by: Palmer Dabbelt + Reviewed-by: Adhemerval Zanella + +Can be dropped when we are using glibc 2.35 or later. + +diff --git a/sysdeps/riscv/setjmp.S b/sysdeps/riscv/setjmp.S +index 0b92016b31..bec7ff80f4 100644 +--- a/sysdeps/riscv/setjmp.S ++++ b/sysdeps/riscv/setjmp.S +@@ -21,7 +21,7 @@ + + ENTRY (_setjmp) + li a1, 0 +- j __sigsetjmp ++ j HIDDEN_JUMPTARGET (__sigsetjmp) + END (_setjmp) + ENTRY (setjmp) + li a1, 1 +diff --git a/sysdeps/unix/sysv/linux/riscv/setcontext.S b/sysdeps/unix/sysv/linux/riscv/setcontext.S +index 9510518750..e44a68aad4 100644 +--- a/sysdeps/unix/sysv/linux/riscv/setcontext.S ++++ b/sysdeps/unix/sysv/linux/riscv/setcontext.S +@@ -95,6 +95,7 @@ LEAF (__setcontext) + 99: j __syscall_error + + END (__setcontext) ++libc_hidden_def (__setcontext) + weak_alias (__setcontext, setcontext) + + LEAF (__start_context) +@@ -108,7 +109,7 @@ LEAF (__start_context) + /* Invoke subsequent context if present, else exit(0). */ + mv a0, s2 + beqz s2, 1f +- jal __setcontext +-1: j exit ++ jal HIDDEN_JUMPTARGET (__setcontext) ++1: j HIDDEN_JUMPTARGET (exit) + + END (__start_context) diff --git a/contrib/guix/symbol-check.py b/contrib/guix/symbol-check.py index 7804a2c..249c53a 100755 --- a/contrib/guix/symbol-check.py +++ b/contrib/guix/symbol-check.py @@ -16,23 +16,18 @@ # Debian 11 (Bullseye) EOL: 2026. https://wiki.debian.org/LTS # -# - libgcc version 10.2.1 (https://packages.debian.org/bullseye/libgcc-s1) # - libc version 2.31 (https://packages.debian.org/source/bullseye/glibc) # # Ubuntu 20.04 (Focal) EOL: 2030. https://wiki.ubuntu.com/ReleaseTeam # -# - libgcc version 10.5.0 (https://packages.ubuntu.com/focal/libgcc1) # - libc version 2.31 (https://packages.ubuntu.com/focal/libc6) # # CentOS Stream 9 EOL: 2027. https://www.centos.org/cl-vs-cs/#end-of-life # -# - libgcc version 12.2.1 (https://mirror.stream.centos.org/9-stream/AppStream/x86_64/os/Packages/) # - libc version 2.34 (https://mirror.stream.centos.org/9-stream/AppStream/x86_64/os/Packages/) # -# See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html for more info. MAX_VERSIONS = { -'GCC': (7,0,0), 'GLIBC': { lief.ELF.ARCH.X86_64: (2,31), lief.ELF.ARCH.ARM: (2,31), @@ -40,7 +35,6 @@ lief.ELF.ARCH.PPC64: (2,31), lief.ELF.ARCH.RISCV: (2,31), }, -'LIBATOMIC': (1,0), 'V': (0,5,0), # xkb (bitcoin-qt only) } @@ -93,11 +87,9 @@ # Allowed NEEDED libraries ELF_ALLOWED_LIBRARIES = { # bitcoind and bitcoin-qt -'libgcc_s.so.1', # GCC base support 'libc.so.6', # C library 'libpthread.so.0', # threading 'libm.so.6', # math library -'libatomic.so.1', 'ld-linux-x86-64.so.2', # 64-bit dynamic linker 'ld-linux.so.2', # 32-bit dynamic linker 'ld-linux-aarch64.so.1', # 64-bit ARM dynamic linker @@ -258,7 +250,7 @@ def check_MACHO_sdk(binary) -> bool: return False def check_MACHO_lld(binary) -> bool: - if binary.build_version.tools[0].version == [18, 1, 8]: + if binary.build_version.tools[0].version == [19, 1, 4]: return True return False diff --git a/depends/Makefile b/depends/Makefile index ed54eac..fde108d 100644 --- a/depends/Makefile +++ b/depends/Makefile @@ -1,4 +1,6 @@ .NOTPARALLEL : +# Disable builtin variables, rules and suffixes. +MAKEFLAGS += --no-builtin-rules --no-builtin-variables # Pattern rule to print variables, e.g. make print-top_srcdir print-%: FORCE diff --git a/depends/README.md b/depends/README.md index 9457910..71110e9 100644 --- a/depends/README.md +++ b/depends/README.md @@ -123,7 +123,7 @@ Common `host-platform-triplet`s for cross compilation are: - `i686-pc-linux-gnu` for Linux x86 32 bit - `x86_64-pc-linux-gnu` for Linux x86 64 bit - `x86_64-w64-mingw32` for Win64 -- `x86_64-apple-darwin` for macOS +- `x86_64-apple-darwin` for Intel macOS - `arm64-apple-darwin` for ARM macOS - `arm-linux-gnueabihf` for Linux ARM 32 bit - `aarch64-linux-gnu` for Linux ARM 64 bit diff --git a/depends/funcs.mk b/depends/funcs.mk index 6c38334..60e7889 100644 --- a/depends/funcs.mk +++ b/depends/funcs.mk @@ -37,7 +37,7 @@ endef define fetch_file ( $(call fetch_file_inner,$(1),$(2),$(3),$(4),$(5)) || \ - $(call fetch_file_inner,$(1),$(FALLBACK_DOWNLOAD_PATH),$(3),$(4),$(5))) + $(call fetch_file_inner,$(1),$(FALLBACK_DOWNLOAD_PATH),$(4),$(4),$(5))) endef # Shell script to create a source tarball in $(1)_source from local directory @@ -108,7 +108,7 @@ $(1)_prefixbin:=$($($(1)_type)_prefix)/bin/ $(1)_all_sources=$($(1)_file_name) $($(1)_extra_sources) #stamps -$(1)_fetched=$(SOURCES_PATH)/download-stamps/.stamp_fetched-$(1)-$($(1)_version)-$($(1)_sha256_hash).hash +$(1)_fetched=$(SOURCES_PATH)/download-stamps/.stamp_fetched-$(1)-$($(1)_file_name).hash $(1)_extracted=$$($(1)_extract_dir)/.stamp_extracted $(1)_preprocessed=$$($(1)_extract_dir)/.stamp_preprocessed $(1)_cleaned=$$($(1)_extract_dir)/.stamp_cleaned diff --git a/depends/packages/libevent.mk b/depends/packages/libevent.mk index e9153b7..33eeed5 100644 --- a/depends/packages/libevent.mk +++ b/depends/packages/libevent.mk @@ -16,7 +16,7 @@ define $(package)_set_vars $(package)_config_opts+=-DEVENT__DISABLE_SAMPLES=ON -DEVENT__DISABLE_REGRESS=ON $(package)_config_opts+=-DEVENT__DISABLE_TESTS=ON -DEVENT__LIBRARY_TYPE=STATIC $(package)_cflags += -fdebug-prefix-map=$($(package)_extract_dir)=/usr -fmacro-prefix-map=$($(package)_extract_dir)=/usr - $(package)_cppflags += -D_GNU_SOURCE -D_FORTIFY_SOURCE=3 + $(package)_cppflags += -D_GNU_SOURCE -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=3 $(package)_cppflags_mingw32=-D_WIN32_WINNT=0x0A00 endef diff --git a/depends/packages/native_qt.mk b/depends/packages/native_qt.mk index 38469d6..3428a1f 100644 --- a/depends/packages/native_qt.mk +++ b/depends/packages/native_qt.mk @@ -6,7 +6,6 @@ $(package)_file_name=$(qt_details_qtbase_file_name) $(package)_sha256_hash=$(qt_details_qtbase_sha256_hash) $(package)_patches_path := $(qt_details_patches_path) $(package)_patches := dont_hardcode_pwd.patch -$(package)_patches += qtbase-moc-ignore-gcc-macro.patch $(package)_patches += qtbase_skip_tools.patch $(package)_patches += rcc_hardcode_timestamp.patch $(package)_patches += qttools_skip_dependencies.patch @@ -131,7 +130,6 @@ endef define $(package)_preprocess_cmds patch -p1 -i $($(package)_patch_dir)/dont_hardcode_pwd.patch && \ - patch -p1 -i $($(package)_patch_dir)/qtbase-moc-ignore-gcc-macro.patch && \ patch -p1 -i $($(package)_patch_dir)/qtbase_skip_tools.patch && \ patch -p1 -i $($(package)_patch_dir)/rcc_hardcode_timestamp.patch && \ patch -p1 -i $($(package)_patch_dir)/qttools_skip_dependencies.patch diff --git a/depends/packages/qrencode.mk b/depends/packages/qrencode.mk index 9aae62f..e3f6140 100644 --- a/depends/packages/qrencode.mk +++ b/depends/packages/qrencode.mk @@ -1,9 +1,8 @@ package=qrencode $(package)_version=4.1.1 -$(package)_download_path=https://github.com/fukuchi/libqrencode/archive/refs/tags/ -$(package)_download_file=v$($(package)_version).tar.gz +$(package)_download_path=https://fukuchi.org/works/qrencode/ $(package)_file_name=$(package)-$($(package)_version).tar.gz -$(package)_sha256_hash=5385bc1b8c2f20f3b91d258bf8ccc8cf62023935df2d2676b5b67049f31a049c +$(package)_sha256_hash=da448ed4f52aba6bcb0cd48cac0dd51b8692bccc4cd127431402fca6f8171e8e $(package)_patches=cmake_fixups.patch define $(package)_set_vars diff --git a/depends/packages/qt.mk b/depends/packages/qt.mk index 9ac4088..4a564c4 100644 --- a/depends/packages/qt.mk +++ b/depends/packages/qt.mk @@ -11,8 +11,6 @@ $(package)_linux_dependencies := freetype fontconfig libxcb libxkbcommon libxcb_ $(package)_freebsd_dependencies := $($(package)_linux_dependencies) $(package)_patches_path := $(qt_details_patches_path) $(package)_patches := dont_hardcode_pwd.patch -$(package)_patches += qtbase-moc-ignore-gcc-macro.patch -$(package)_patches += qtbase_avoid_native_float16.patch $(package)_patches += qtbase_avoid_qmain.patch $(package)_patches += qtbase_platformsupport.patch $(package)_patches += qtbase_plugins_cocoa.patch @@ -256,8 +254,6 @@ endif define $(package)_preprocess_cmds patch -p1 -i $($(package)_patch_dir)/dont_hardcode_pwd.patch && \ - patch -p1 -i $($(package)_patch_dir)/qtbase-moc-ignore-gcc-macro.patch && \ - patch -p1 -i $($(package)_patch_dir)/qtbase_avoid_native_float16.patch && \ patch -p1 -i $($(package)_patch_dir)/qtbase_avoid_qmain.patch && \ patch -p1 -i $($(package)_patch_dir)/qtbase_platformsupport.patch && \ patch -p1 -i $($(package)_patch_dir)/qtbase_plugins_cocoa.patch && \ diff --git a/depends/patches/qt/memory_resource.patch b/depends/patches/qt/memory_resource.patch deleted file mode 100644 index 14e2512..0000000 --- a/depends/patches/qt/memory_resource.patch +++ /dev/null @@ -1,16 +0,0 @@ -Fix unusable memory_resource on macos - -See https://bugreports.qt.io/browse/QTBUG-117484 -and https://bugreports.qt.io/browse/QTBUG-114316 - ---- a/qtbase/src/corelib/tools/qduplicatetracker_p.h -+++ b/qtbase/src/corelib/tools/qduplicatetracker_p.h -@@ -52,7 +52,7 @@ - - #include - --#if QT_HAS_INCLUDE() && __cplusplus > 201402L -+#ifdef __cpp_lib_memory_resource - # include - # include - #else diff --git a/depends/patches/qt/qtbase-moc-ignore-gcc-macro.patch b/depends/patches/qt/qtbase-moc-ignore-gcc-macro.patch deleted file mode 100644 index e5e7b12..0000000 --- a/depends/patches/qt/qtbase-moc-ignore-gcc-macro.patch +++ /dev/null @@ -1,17 +0,0 @@ -The moc executable loops through headers on CPLUS_INCLUDE_PATH and stumbles -on the GCC internal _GLIBCXX_VISIBILITY macro. Tell it to ignore it as it is -not supposed to be looking there to begin with. - -Upstream report: https://bugreports.qt.io/browse/QTBUG-83160 - -diff --git a/qtbase/src/tools/moc/main.cpp b/qtbase/src/tools/moc/main.cpp ---- a/qtbase/src/tools/moc/main.cpp -+++ b/qtbase/src/tools/moc/main.cpp -@@ -186,6 +186,7 @@ int runMoc(int argc, char **argv) - dummyVariadicFunctionMacro.arguments += Symbol(0, PP_IDENTIFIER, "__VA_ARGS__"); - pp.macros["__attribute__"] = dummyVariadicFunctionMacro; - pp.macros["__declspec"] = dummyVariadicFunctionMacro; -+ pp.macros["_GLIBCXX_VISIBILITY"] = dummyVariadicFunctionMacro; - - QString filename; - QString output; diff --git a/depends/patches/qt/qtbase_avoid_native_float16.patch b/depends/patches/qt/qtbase_avoid_native_float16.patch deleted file mode 100644 index ec03d14..0000000 --- a/depends/patches/qt/qtbase_avoid_native_float16.patch +++ /dev/null @@ -1,37 +0,0 @@ -Avoid using native float16 types in Qt's qfloat16 - -Using native float16 types may introduce compatibility issues -in release binaries for Linux platforms. - -See: https://gcc.gnu.org/git/?p=gcc.git;a=commit;h=e42d2d2a20f2bb59928bc895ec9f46503a1b5c73 - -This patch can be dropped once the minimum required libgcc version -is updated to 12.x or newer. - - ---- a/qtbase/src/corelib/global/qtypes.h -+++ b/qtbase/src/corelib/global/qtypes.h -@@ -258,23 +258,8 @@ using qsizetype = QIntegerForSizeof::Signed; - - // Define a native float16 type - namespace QtPrivate { --#if defined(__STDCPP_FLOAT16_T__) --# define QFLOAT16_IS_NATIVE 1 --using NativeFloat16Type = std::float16_t; --#elif defined(Q_CC_CLANG) && defined(__FLT16_MAX__) && 0 --// disabled due to https://github.com/llvm/llvm-project/issues/56963 --# define QFLOAT16_IS_NATIVE 1 --using NativeFloat16Type = decltype(__FLT16_MAX__); --#elif defined(Q_CC_GNU_ONLY) && defined(__FLT16_MAX__) && defined(__ARM_FP16_FORMAT_IEEE) --# define QFLOAT16_IS_NATIVE 1 --using NativeFloat16Type = __fp16; --#elif defined(Q_CC_GNU_ONLY) && defined(__FLT16_MAX__) && defined(__SSE2__) --# define QFLOAT16_IS_NATIVE 1 --using NativeFloat16Type = _Float16; --#else - # define QFLOAT16_IS_NATIVE 0 - using NativeFloat16Type = void; --#endif - } // QtPrivate - - #endif // __cplusplus diff --git a/doc/build-freebsd.md b/doc/build-freebsd.md index fee20f8..b5940c0 100644 --- a/doc/build-freebsd.md +++ b/doc/build-freebsd.md @@ -101,5 +101,5 @@ cmake -B build -DENABLE_WALLET=OFF ```bash cmake --build build # Append "-j N" for N parallel jobs. -ctest --test-dir build # Append "-j N" for N parallel tests. Some tests are disabled if Python 3 is not available. +ctest --test-dir build # Append "-j N" for N parallel tests. ``` diff --git a/doc/build-netbsd.md b/doc/build-netbsd.md index f263177..103e625 100644 --- a/doc/build-netbsd.md +++ b/doc/build-netbsd.md @@ -34,7 +34,7 @@ cmake -B build SQLite is required for the wallet: ```bash -pkgin sqlite3 +pkgin install sqlite3 ``` To build Bitcoin Core without the wallet, use `-DENABLE_WALLET=OFF`. @@ -42,7 +42,7 @@ To build Bitcoin Core without the wallet, use `-DENABLE_WALLET=OFF`. Cap'n Proto is needed for IPC functionality (see [multiprocess.md](multiprocess.md)): ```bash -pkgin capnproto +pkgin install capnproto ``` Compile with `-DENABLE_IPC=OFF` if you do not need IPC functionality. @@ -84,7 +84,7 @@ Otherwise, if you don't need QR encoding support, use the `-DWITH_QRENCODE=OFF` Bitcoin Core can provide notifications via ZeroMQ. If the package is installed, support will be compiled in. ```bash -pkgin zeromq +pkgin install zeromq ``` #### Test Suite Dependencies @@ -115,5 +115,5 @@ Build and run the tests: ```bash cmake --build build # Append "-j N" for N parallel jobs. -ctest --test-dir build # Append "-j N" for N parallel tests. Some tests are disabled if Python 3 is not available. +ctest --test-dir build # Append "-j N" for N parallel tests. ``` diff --git a/doc/build-openbsd.md b/doc/build-openbsd.md index 99d4ec2..fdeafe8 100644 --- a/doc/build-openbsd.md +++ b/doc/build-openbsd.md @@ -93,7 +93,7 @@ Run `cmake -B build -LH` to see the full list of available options. ```bash cmake --build build # Append "-j N" for N parallel jobs. -ctest --test-dir build # Append "-j N" for N parallel tests. Some tests are disabled if Python 3 is not available. +ctest --test-dir build # Append "-j N" for N parallel tests. ``` ## Resource limits diff --git a/doc/build-osx.md b/doc/build-osx.md index e1bf037..001c018 100644 --- a/doc/build-osx.md +++ b/doc/build-osx.md @@ -170,7 +170,7 @@ Run the following in your terminal to compile Bitcoin Core: ``` bash cmake --build build # Append "-j N" here for N parallel jobs. -ctest --test-dir build # Append "-j N" for N parallel tests. Some tests are disabled if Python 3 is not available. +ctest --test-dir build # Append "-j N" for N parallel tests. ``` ### 3. Deploy (optional) diff --git a/doc/build-unix.md b/doc/build-unix.md index d9bbb35..e499aa9 100644 --- a/doc/build-unix.md +++ b/doc/build-unix.md @@ -46,10 +46,14 @@ Finally, clang (often less resource hungry) can be used instead of gcc, which is #### Dependency Build Instructions -Build requirements: +Build requirements for the latest Debian "stable" release, or the latest Ubuntu LTS release: sudo apt-get install build-essential cmake pkgconf python3 +For Debian "oldstable", or earlier Ubuntu LTS releases, you may need to pick a +later compiler version, according to the [dependencies](/doc/dependencies.md) +documentation. + Now, you can either build from self-compiled [depends](#dependencies) or install the required dependencies: sudo apt-get install libevent-dev libboost-dev diff --git a/doc/build-windows-msvc.md b/doc/build-windows-msvc.md index 2d85d22..cf84cdd 100644 --- a/doc/build-windows-msvc.md +++ b/doc/build-windows-msvc.md @@ -50,12 +50,14 @@ CMake will put the resulting object files, libraries, and executables into a ded In the following instructions, the "Debug" configuration can be specified instead of the "Release" one. +Run `cmake -B build -LH` to see the full list of available options. + ### 4. Building with Static Linking with GUI ``` cmake -B build --preset vs2022-static # It might take a while if the vcpkg binary cache is unpopulated or invalidated. cmake --build build --config Release # Append "-j N" for N parallel jobs. -ctest --test-dir build --build-config Release # Append "-j N" for N parallel tests. Some tests are disabled if Python 3 is not available. +ctest --test-dir build --build-config Release # Append "-j N" for N parallel tests. cmake --install build --config Release # Optional. ``` @@ -64,7 +66,7 @@ cmake --install build --config Release # Optional. ``` cmake -B build --preset vs2022 -DBUILD_GUI=OFF # It might take a while if the vcpkg binary cache is unpopulated or invalidated. cmake --build build --config Release # Append "-j N" for N parallel jobs. -ctest --test-dir build --build-config Release # Append "-j N" for N parallel tests. Some tests are disabled if Python 3 is not available. +ctest --test-dir build --build-config Release # Append "-j N" for N parallel tests. ``` ### 6. vcpkg-specific Issues and Workarounds diff --git a/doc/build-windows.md b/doc/build-windows.md index 84db7fa..ce822dd 100644 --- a/doc/build-windows.md +++ b/doc/build-windows.md @@ -49,6 +49,9 @@ Build using: gmake -C depends HOST=x86_64-w64-mingw32 # Append "-j N" for N parallel jobs. cmake -B build --toolchain depends/x86_64-w64-mingw32/toolchain.cmake + +Run `cmake -B build -LH` to see the full list of available options. + cmake --build build # Append "-j N" for N parallel jobs. ## Depends system diff --git a/doc/dependencies.md b/doc/dependencies.md index 93916aa..3ff306f 100644 --- a/doc/dependencies.md +++ b/doc/dependencies.md @@ -10,8 +10,8 @@ Bitcoin Core requires one of the following compilers. | Dependency | Minimum required | | --- | --- | -| [Clang](https://clang.llvm.org) | [16.0](https://github.com/bitcoin/bitcoin/pull/30263) | -| [GCC](https://gcc.gnu.org) | [11.1](https://github.com/bitcoin/bitcoin/pull/29091) | +| [Clang](https://clang.llvm.org) | [17.0](https://github.com/bitcoin/bitcoin/pull/33555) | +| [GCC](https://gcc.gnu.org) | [12.1](https://github.com/bitcoin/bitcoin/pull/33842) | ## Required diff --git a/doc/developer-notes.md b/doc/developer-notes.md index 220343a..106533b 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -17,7 +17,7 @@ Do not submit patches solely to modify the style of existing code. [clang-format-diff script](/contrib/devtools/README.md#clang-format-diffpy) tool to clean up patches automatically before submission. - Braces on new lines for classes, functions, methods. - - Braces on the same line for everything else. + - Braces on the same line for everything else (including structs). - 4 space indentation (no tabs) for every block except namespaces. - No indentation for `public`/`protected`/`private` or for `namespace`. - No extra spaces inside parentheses; don't do `( this )`. @@ -1119,6 +1119,19 @@ to check a subtree directory for consistency with its upstream repository. The tool instructions also include a list of the subtrees managed by Bitcoin Core. +To fully verify or update a subtree, add it as a remote: + +```sh +git remote add libmultiprocess https://github.com/bitcoin-core/libmultiprocess.git +``` + +To update the subtree: + +```sh +git fetch libmultiprocess +git subtree pull --prefix=src/ipc/libmultiprocess libmultiprocess master --squash +``` + The ultimate upstream of the few externally managed subtrees are: - src/leveldb @@ -1350,7 +1363,7 @@ A few guidelines for introducing and reviewing new RPC interfaces: A few guidelines for modifying existing RPC interfaces: -- It's preferable to avoid changing an RPC in a backward-incompatible manner, but in that case, add an associated `-deprecatedrpc=` option to retain previous RPC behavior during the deprecation period. Backward-incompatible changes include: data type changes (e.g. from `{"warnings":""}` to `{"warnings":[]}`, changing a value from a string to a number, etc.), logical meaning changes of a value, or key name changes (e.g. `{"warning":""}` to `{"warnings":""}`). Adding a key to an object is generally considered backward-compatible. Include a release note that refers the user to the RPC help for details of feature deprecation and re-enabling previous behavior. [Example RPC help](https://github.com/bitcoin/bitcoin/blob/94f0adcc/src/rpc/blockchain.cpp#L1316-L1323). +- It's preferable to avoid changing an RPC in a backward-incompatible manner, but in that case, add an associated `-deprecatedrpc=` option to retain previous RPC behavior during the deprecation period. Backward-incompatible changes include: data type changes (e.g. from `{"warnings":""}` to `{"warnings":[]}`, changing a value from a string to a number, etc.), logical meaning changes of a value, key name changes (e.g. `{"warning":""}` to `{"warnings":""}`), or removing a key from an object. Adding a key to an object is generally considered backward-compatible. Include a release note that refers the user to the RPC help for details of feature deprecation and re-enabling previous behavior. [Example RPC help](https://github.com/bitcoin/bitcoin/blob/94f0adcc/src/rpc/blockchain.cpp#L1316-L1323). - *Rationale*: Changes in RPC JSON structure can break downstream application compatibility. Implementation of `deprecatedrpc` provides a grace period for downstream applications to migrate. Release notes provide notification to downstream users. diff --git a/doc/fuzzing.md b/doc/fuzzing.md index 8a024e5..f4333a5 100644 --- a/doc/fuzzing.md +++ b/doc/fuzzing.md @@ -21,6 +21,8 @@ See [further](#run-without-sanitizers-for-increased-throughput) for more informa There is also a runner script to execute all fuzz targets. Refer to `./build_fuzz/test/fuzz/test_runner.py --help` for more details. +For source-based coverage reports, see [developer notes](/doc/developer-notes.md#compiling-for-fuzz-coverage). + ## Overview of Bitcoin Core fuzzing [Google](https://github.com/google/fuzzing/) has a good overview of fuzzing in general, with contributions from key architects of some of the most-used fuzzers. [This paper](https://agroce.github.io/bitcoin_report.pdf) includes an external overview of the status of Bitcoin Core fuzzing, as of summer 2021. [John Regehr](https://blog.regehr.org/archives/1687) provides good advice on writing code that assists fuzzers in finding bugs, which is useful for developers to keep in mind. diff --git a/doc/i2p.md b/doc/i2p.md index b769a74..624b651 100644 --- a/doc/i2p.md +++ b/doc/i2p.md @@ -166,3 +166,13 @@ In most cases, the default router settings should work fine. Please see the "General Guidance for Developers" section in https://geti2p.net/en/docs/api/samv3 if you are developing a downstream application that may be bundling I2P with Bitcoin. + +## Privacy recommendations + +- Operating a node that listens on multiple networks (e.g. IPv4 and I2P) can help + strengthen the Bitcoin network, as nodes in this configuration (i.e. bridge nodes) increase + the cost and complexity of launching eclipse and partition attacks. However, under certain + conditions, an adversary that can connect to your node on multiple networks may be + able to correlate those identities by observing shared runtime characteristics. It + is not recommended to expose your node over multiple networks if you require + unlinkability across those identities. diff --git a/doc/release-notes-33555.md b/doc/release-notes-33555.md new file mode 100644 index 0000000..a11af34 --- /dev/null +++ b/doc/release-notes-33555.md @@ -0,0 +1,5 @@ +Build System +------------ + +- The minimum supported Clang compiler version has been raised to 17.0 + (#33555). diff --git a/doc/release-notes-33842.md b/doc/release-notes-33842.md new file mode 100644 index 0000000..ad15f50 --- /dev/null +++ b/doc/release-notes-33842.md @@ -0,0 +1,4 @@ +Build System +------------ + +- The minimum supported GCC compiler version has been raised to 12.1 (#33842). diff --git a/doc/release-notes-empty-template.md b/doc/release-notes-empty-template.md index 7f419cb..86dafb3 100644 --- a/doc/release-notes-empty-template.md +++ b/doc/release-notes-empty-template.md @@ -35,8 +35,8 @@ wallet versions of Bitcoin Core are generally supported. Compatibility ============== -Bitcoin Core is supported and tested on operating systems using the -Linux Kernel 3.17+, macOS 14+, and Windows 10+. Bitcoin +Bitcoin Core is supported and tested on the following operating systems or newer: +Linux Kernel 3.17, macOS 14, and Windows 10 (version 1903). Bitcoin Core should also work on most other Unix-like systems but is not as frequently tested on them. It is not recommended to use Bitcoin Core on unsupported systems. diff --git a/doc/release-notes/release-notes-28.3.md b/doc/release-notes/release-notes-28.3.md new file mode 100644 index 0000000..eefeedf --- /dev/null +++ b/doc/release-notes/release-notes-28.3.md @@ -0,0 +1,110 @@ +Bitcoin Core version 28.3 is now available from: + + + +This release includes various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + + +To receive security and update notifications, please subscribe to: + + + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes in some cases), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on macOS) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the data directory needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems +using the Linux Kernel 3.17+, macOS 11.0+, and Windows 7 and newer. Bitcoin +Core should also work on most other UNIX-like systems but is not as +frequently tested on them. It is not recommended to use Bitcoin Core on +unsupported systems. + +Notable changes +=============== + +### Mempool & Policy + + The minimum block feerate (`-blockmintxfee`) has been changed to 1 satoshi per kvB. It can still be changed using the +configuration option. + +- The default minimum relay feerate (`-minrelaytxfee`) and incremental relay feerate (`-incrementalrelayfee`) have been +changed to 100 satoshis per kvB. They can still be changed using their respective configuration options, but it is +recommended to change both together if you decide to do so. + - Other minimum feerates (e.g. the dust feerate, the minimum returned by the fee estimator, and all feerates used by + the wallet) remain unchanged. The mempool minimum feerate still changes in response to high volume. + - Note that unless these lower defaults are widely adopted across the network, transactions created with lower fee + rates are not guaranteed to propagate or confirm. The wallet feerates remain unchanged; `-mintxfee` must be changed + before attempting to create transactions with lower feerates using the wallet. + +- #33106 policy: lower the default blockmintxfee, incrementalrelayfee, minrelaytxfee +- #33504 mempool: Do not enforce TRUC checks on reorg + +### P2P + +- #33395 net: do not apply whitelist permissions to onion inbounds + +### Test + +- #32765 test: Fix list index out of range error in feature_bip68_sequence.py +- #33001 test: Do not pass tests on unhandled exceptions +- #30125 test: improve BDB parser (handle internal/overflow pages, support all page sizes) +- #30948 test: Add missing sync_mempools() to fill_mempool() +- #30784 test: add BulkTransaction helper to unit test transaction utils + +### Build + +- #32678 guix: warn and abort when SOURCE_DATE_EPOCH is set +- #32943 depends: Force CMAKE_EXPORT_NO_PACKAGE_REGISTRY=TRUE +- #33073 guix: warn SOURCE_DATE_EPOCH set in guix-codesign +- #33563 build: fix depends Qt download link + +### Doc + +- #32776 doc: taproot became always active in v24.0 +- #32777 doc: fix Transifex 404s +- #33070 doc/zmq: fix unix socket path example +- #33133 rpc: fix getpeerinfo ping duration unit docs +- #33236 doc: Remove wrong and redundant doxygen tag + +### Misc + +- #33340 Fix benchmark CSV output +- #33482 contrib: fix macOS deployment with no translations +- #33581 ci: Properly include $FILE_ENV in DEPENDS_HASH + +Credits +======= + +Thanks to everyone who directly contributed to this release: +- 0xB10C +- amisha +- Ava Chow +- fanquake +- glozow +- Hennadii Stepanov +- MarcoFalke +- Martin Zumsande +- romanz +- Sjors Provoost +- theStack +- Vasil Dimov +- willcl-ark +- zaidmstrr + +As well as to everyone that helped with translations on +[Transifex](https://explore.transifex.com/bitcoin/bitcoin/). diff --git a/doc/release-notes/release-notes-29.2.md b/doc/release-notes/release-notes-29.2.md new file mode 100644 index 0000000..b981b8a --- /dev/null +++ b/doc/release-notes/release-notes-29.2.md @@ -0,0 +1,91 @@ +Bitcoin Core version 29.2 is now available from: + + + +This release includes various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + + +To receive security and update notifications, please subscribe to: + + + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes in some cases), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on macOS) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the data directory needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and tested on operating systems using the +Linux Kernel 3.17+, macOS 13+, and Windows 10+. Bitcoin +Core should also work on most other Unix-like systems but is not as +frequently tested on them. It is not recommended to use Bitcoin Core on +unsupported systems. + +Notable changes +=============== + +### P2P + +- #32646 p2p: Add witness mutation check inside FillBlock +- #33296 net: check for empty header before calling FillBlock +- #33395 net: do not apply whitelist permissions to onion inbounds + +### Mempool + +- #33504 mempool: Do not enforce TRUC checks on reorg + +### RPC + +- #33446 rpc: fix getblock(header) returns target for tip + +### CI + +- #32989 ci: Migrate CI to hosted Cirrus Runners +- #32999 ci: Use APT_LLVM_V in msan task +- #33099 ci: allow for any libc++ intrumentation & use it for TSAN +- #33258 ci: use LLVM 21 +- #33364 ci: always use tag for LLVM checkout + +### Doc + +- #33484 doc: rpc: fix case typo in `finalizepsbt` help + +### Misc + +- #33310 trace: Workaround GCC bug compiling with old systemtap +- #33340 Fix benchmark CSV output +- #33482 contrib: fix macOS deployment with no translations + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- Amisha Chhajed +- Eugene Siegel +- fanquake +- Greg Sanders +- Hennadii Stepanov +- Luke Dashjr +- MarcoFalke +- Martin Zumsande +- Sebastian Falbesoner +- Sjors Provoost +- Vasil Dimov +- Will Clark + +As well as to everyone that helped with translations on +[Transifex](https://explore.transifex.com/bitcoin/bitcoin/). diff --git a/doc/release-notes/release-notes-30.0.md b/doc/release-notes/release-notes-30.0.md new file mode 100644 index 0000000..a855d10 --- /dev/null +++ b/doc/release-notes/release-notes-30.0.md @@ -0,0 +1,438 @@ +v30.0 Release Notes +=================== + +Bitcoin Core version v30.0 is now available from: + + + +This release includes new features, various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + + +To receive security and update notifications, please subscribe to: + + + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes in some cases), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on macOS) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the data directory needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and tested on operating systems using the +Linux Kernel 3.17+, macOS 13+, and Windows 10+. Bitcoin +Core should also work on most other Unix-like systems but is not as +frequently tested on them. It is not recommended to use Bitcoin Core on +unsupported systems. + +Notable changes +=============== + +Policy +------ + +- The maximum number of potentially executed legacy signature operations in a + single standard transaction is now limited to 2500. Signature operations in all + previous output scripts, in all input scripts, as well as all P2SH redeem + scripts (if there are any) are counted toward the limit. The new limit is + assumed to not affect any known typically formed standard transactions. The + change was done to prepare for a possible BIP54 deployment in the future. (#32521) + +- `-datacarriersize` is increased to 100,000 by default, which effectively uncaps + the limit (as the maximum transaction size limit will be hit first). It can be + overridden with `-datacarriersize=83` to revert to the limit enforced in previous + versions. (#32406) + +- Multiple data carrier (OP_RETURN) outputs in a transaction are now permitted for + relay and mining. The `-datacarriersize` limit applies to the aggregate size of + the scriptPubKeys across all such outputs in a transaction, not including the + scriptPubKey size itself. (#32406) + +- The minimum block feerate (`-blockmintxfee`) has been changed to 0.001 satoshi per + vB. It can still be changed using the configuration option. This option can be used + by miners to set a minimum feerate on packages added to block templates. (#33106) + +- The default minimum relay feerate (`-minrelaytxfee`) and incremental relay feerate + (`-incrementalrelayfee`) have been changed to 0.1 satoshis per vB. They can still + be changed using their respective configuration options, but it is recommended to + change both together if you decide to do so. (#33106) + + Other minimum feerates (e.g. the dust feerate, the minimum returned by the fee + estimator, and all feerates used by the wallet) remain unchanged. The mempool minimum + feerate still changes in response to high volume. + + Note that unless these lower defaults are widely adopted across the network, transactions + created with lower fee rates are not guaranteed to propagate or confirm. The wallet + feerates remain unchanged; `-mintxfee` must be changed before attempting to create + transactions with lower feerates using the wallet. (#33106) + +P2P and network changes +----------------------- + +- Opportunistic 1-parent-1-child package relay has been improved to handle + situations when the child already has unconfirmed parent(s) in the mempool. + This means that 1p1c packages can be accepted and propagate, even if they are + connected to broader topologies: multi-parent-1-child (where only 1 parent + requires fee-bumping), grandparent-parent-child (where only parent requires + fee-bumping) etc. (#31385) + +- The transaction orphanage, which holds transactions with missing inputs temporarily + while the node attempts to fetch its parents, now has improved Denial of Service protections. + Previously, it enforced a maximum number of unique transactions (default 100, + configurable using `-maxorphantx`). Now, its limits are as follows: the number of + entries (unique by wtxid and peer), plus each unique transaction's input count divided + by 10, must not exceed 3,000. The total weight of unique transactions must not exceed + `404,000` Wu multiplied by the number of peers. (#31829) + +- The `-maxorphantx` option no longer has any effect, since the orphanage no longer + limits the number of unique transactions. Users should remove this configuration + option if they were using it, as the setting will cause an error in future versions + when it is no longer recognized. (#31829) + +New `bitcoin` command +--------------------- + +- A new `bitcoin` command line tool has been added to make features more discoverable + and convenient to use. The `bitcoin` tool just calls other executables and does not + implement any functionality on its own. Specifically `bitcoin node` is a synonym for + `bitcoind`, `bitcoin gui` is a synonym for `bitcoin-qt`, and `bitcoin rpc` is a synonym + for `bitcoin-cli -named`. Other commands and options can be listed with `bitcoin help`. + The new `bitcoin` command is an alternative to calling other commands directly, but it + doesn't replace them, and there are no plans to deprecate existing commands. (#31375) + +External Signing +---------------- + +- Support for external signing on Windows has been re-enabled. (#29868) + +IPC Mining Interface +-------------------- + +- The new `bitcoin` command does support one new feature: an (experimental) IPC Mining + Interface that allows the node to work with Stratum v2 or other mining client software, + see (#31098). When the node is started with `bitcoin -m node -ipcbind=unix` it will + listen on a unix socket for IPC client connections, allowing clients to request block + templates and submit mined blocks. The `-m` option launches a new internal binary + (`bitcoin-node` instead of `bitcoind`) and is currently required but will become optional + in the future (with [#33229](https://github.com/bitcoin/bitcoin/pull/33229)). + +- IPC connectivity introduces new dependencies (see [multiprocess.md](https://github.com/bitcoin/bitcoin/blob/master/doc/multiprocess.md)), + which can be turned off with the `-DENABLE_IPC=OFF` build option if you do not intend + to use IPC. (#31802) + +Install changes +--------------- + +- The `test_bitcoin` executable is now installed in `libexec/` instead of `bin/`. + It can still be executed directly, or accessed through the new `bitcoin` command + as `bitcoin test`. The `libexec/` directory also contains new `bitcoin-node` and + `bitcoin-gui` binaries which support IPC features and are called through the + `bitcoin` tool. In source builds only, `test_bitcoin-qt`, `bench_bitcoin`, and + `bitcoin-chainstate` are also now installed to `libexec/` instead of `bin/` and + can be accessed through the new `bitcoin` command. See `bitcoin help` output for + details. (#31679) + +- On Windows, the installer no longer adds a “(64-bit)” suffix to entries in the + Start Menu (#32132), and it now automatically removes obsolete artifacts during + upgrades (#33422). + +Indexes +------- + +- The implementation of coinstatsindex was changed to prevent an overflow bug that + could already be observed on the default Signet. The new version of the index will + need to be synced from scratch when starting the upgraded node for the first time. + + The new version is stored in `/indexes/coinstatsindex/` in contrast to the old version + which was stored at `/indexes/coinstats/`. The old version of the index is not deleted + by the upgraded node in case the user chooses to downgrade their node in the future. + If the user does not plan to downgrade it is safe for them to remove `/indexes/coinstats/` + from their datadir. A future release of Bitcoin Core may remove the old version of the + index automatically. (#30469) + +Logging +------- +- Unconditional logging to disk is now rate limited by giving each source location + a quota of 1MiB per hour. Unconditional logging is any logging with a log level + higher than debug, that is `info`, `warning`, and `error`. All logs will be + prefixed with `[*]` if there is at least one source location that is currently + being suppressed. (#32604) + +- When `-logsourcelocations` is enabled, the log output now contains the entire + function signature instead of just the function name. (#32604) + +Updated RPCs +------------ + +- The `-paytxfee` startup option and the `settxfee` RPC are now deprecated and + will be removed in Bitcoin Core 31.0. They allowed the user to set a static fee + rate for wallet transactions, which could potentially lead to overpaying or underpaying. + Users should instead rely on fee estimation or specify a fee rate per transaction + using the `fee_rate` argument in RPCs such as `fundrawtransaction`, `sendtoaddress`, + `send`, `sendall`, and `sendmany`. (#31278) + +- Any RPC in which one of the parameters is a descriptor will throw an error + if the provided descriptor contains a whitespace at the beginning or the end + of the public key within a fragment - e.g. `pk( KEY)` or `pk(KEY )`. (#31603) + +- The `submitpackage` RPC, which allows submissions of child-with-parents + packages, no longer requires that all unconfirmed parents be present. The + package may contain other in-mempool ancestors as well. (#31385) + +- The `waitfornewblock` RPC now takes an optional `current_tip` argument. It + is also no longer hidden. (#30635) + +- The `waitforblock` and `waitforblockheight` RPCs are no longer hidden. (#30635) + +- The `psbtbumpfee` and `bumpfee` RPCs allow a replacement under fullrbf and no + longer require BIP-125 signalling. (#31953) + +- Transaction Script validation errors used to return the reason for the error + prefixed by either `mandatory-script-verify-flag-failed` if it was a consensus + error, or `non-mandatory-script-verify-flag` (without "-failed") if it was a + standardness error. This has been changed to `block-script-verify-flag-failed` + and `mempool-script-verify-flag-failed` for all block and mempool errors + respectively. (#33183) + +- The `getmininginfo` RPC now returns "blockmintxfee" result specifying the value of + `-blockmintxfee` configuration. (#33189) + +- The `getmempoolinfo` RPC now returns an additional "permitbaremultisig" and + "maxdatacarriersize" field, reflecting the `-permitbaremultisig` and `-datacarriersize` + config values. (#29954) + +Changes to wallet-related RPCs can be found in the Wallet section below. + +New RPCs +-------- + +- A new REST API endpoint (`/rest/spenttxouts/BLOCKHASH`) has been introduced for + efficiently fetching spent transaction outputs using the block's undo data (#32540). + +Build System +------------ + +Updated settings +---------------- + +- The `-maxmempool` and `-dbcache` startup parameters are now capped on 32-bit systems + to 500MB and 1GiB respectively. (#32530) + +- The `-natpmp` option is now set to `1` by default. This means nodes with `-listen` + enabled (the default) but running behind a firewall, such as a local network router, + will be reachable if the firewall/router supports any of the `PCP` or `NAT-PMP` + protocols. (#33004) + +- The `-upnp` setting has now been fully removed. Use `-natpmp` instead. (#32500) + +- Previously, `-proxy` specified the proxy for all networks (except I2P which + uses `-i2psam`) and only the Tor proxy could have been specified separately + via `-onion`. Now, the syntax of `-proxy` has been extended and it is possible + to specify separately the proxy for IPv4, IPv6, Tor and CJDNS by appending `=` + followed by the network name, for example `-proxy=127.0.0.1:5555=ipv6` + configures a proxy only for IPv6. The `-proxy` option can be used multiple + times to define different proxies for different networks, such as + `-proxy=127.0.0.1:4444=ipv4 -proxy=10.0.0.1:6666=ipv6`. Later settings + override earlier ones for the same network; this can be used to remove an + earlier all-networks proxy and use direct connections only for a given + network, for example `-proxy=127.0.0.1:5555 -proxy=0=cjdns`. (#32425) + +- The `-blockmaxweight` startup option has been updated to be debug-only. + It is still available to users, but now hidden from the default `-help` text + and shown only in `-help-debug` (#32654). + +Changes to GUI or wallet related settings can be found in the GUI or Wallet section below. + +Wallet +------ + +- BDB legacy wallets can no longer be created or loaded. They can be migrated + to the new descriptor wallet format. Refer to the `migratewallet` RPC for more + details. + +- The legacy wallet removal drops redundant options in the bitcoin-wallet tool, + such as `-withinternalbdb`, `-legacy`, and `-descriptors`. Moreover, the + legacy-only RPCs `addmultisigaddress`, `dumpprivkey`, `dumpwallet`, + `importaddress`, `importmulti`, `importprivkey`, `importpubkey`, + `importwallet`, `newkeypool`, `sethdseed`, and `upgradewallet`, are removed. + (#32944, #28710, #32438, #31250) + +- Support has been added for spending TRUC transactions received by the + wallet, as well as creating TRUC transactions. The wallet ensures that + TRUC policy rules are being met. The wallet will throw an error if the + user is trying to spend TRUC utxos with utxos of other versions. + Additionally, the wallet will treat unconfirmed TRUC sibling + transactions as mempool conflicts. The wallet will also ensure that + transactions spending TRUC utxos meet the required size restrictions. (#32896) + +- Since descriptor wallets do not allow mixing watchonly and non-watchonly descriptors, + the `include_watchonly` option (and its variants in naming) are removed from all RPCs + that had it. (#32618) + +- The `iswatchonly` field is removed from any RPCs that returned it. (#32618) + +- `unloadwallet` - Return RPC_INVALID_PARAMETER when both the RPC wallet endpoint + and wallet_name parameters are unspecified. Previously the RPC failed with a JSON + parsing error. (#32845) + +- `getdescriptoractivity` - Mark blockhashes and scanobjects arguments as required, + so the user receives a clear help message when either is missing. As in `unloadwallet`, + previously the RPC failed with a JSON parsing error. (#32845) + +- `getwalletinfo` - Removes the fields `balance`, `immature_balance` and + `unconfirmed_balance`. (#32721) + +- `getunconfirmedbalance` - Removes this RPC command. You can query the `getbalances` + RPC and inspect the `["mine"]["untrusted_pending"]` entry within the JSON + response. (#32721) + +- The following RPCs now contain a `version` parameter that allows + the user to create transactions of any standard version number (1-3): + - `createrawtransaction` + - `createpsbt` + - `send` + - `sendall` + - `walletcreatefundedpsbt` + (#32896) + +GUI changes +----------- + +- The GUI has been migrated from Qt 5 to Qt 6. On Windows, dark mode is now supported. + On macOS, the Metal backend is now used. (#30997) + +- A transaction's fee bump is allowed under fullrbf and no longer requires + BIP-125 signalling. (#31953) + +- Custom column widths in the Transactions tab are reset as a side-effect of legacy + wallet removal. (#32459) + +Low-level changes +================= + +- Logs now include which peer sent us a header. Additionally there are fewer + redundant header log messages. A side-effect of this change is that for + some untypical cases new headers aren't logged anymore, e.g. a direct + `BLOCK` message with a previously unknown header and `submitheader` RPC. (#27826) + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- 0xb10c +- amisha +- Andrew Toth +- Anthony Towns +- Antoine Poinsot +- Ava Chow +- benthecarman +- Brandon Odiwuor +- brunoerg +- Bue-von-hon +- Bufo +- Chandra Pratap +- Chris Stewart +- Cory Fields +- Daniel Pfeifer +- Daniela Brozzoni +- David Gumberg +- deadmanoz +- dennsikl +- dergoegge +- enoch +- Ethan Heilman +- Eugene Siegel +- Eunovo +- Eval EXEC +- Fabian Jahr +- fanquake +- Florian Schmaus +- fuder.eth +- furszy +- glozow +- Greg Sanders +- Hao Xu +- Haoran Peng +- Haowen Liu +- Hennadii Stepanov +- Hodlinator +- hoffman +- ishaanam +- ismaelsadeeq +- Jameson Lopp +- janb84 +- Jiri Jakes +- John Bampton +- Jon Atack +- josibake +- jurraca +- kevkevin +- kevkevinpal +- kilavvy +- Kristaps Kaupe +- l0rinc +- laanwj +- leopardracer +- Lőrinc +- Luis Schwab +- Luke Dashjr +- MarcoFalke +- marcofleon +- Martin Zumsande +- Matt Corallo +- Matthew Zipkin +- Max Edwards +- monlovesmango +- Murch +- naiyoma +- nervana21 +- Nicola Leonardo Susca +- Novo +- pablomartin4btc +- Peter Todd +- Pieter Wuille +- Pol Espinasa +- Prabhat Verma +- rkrux +- Roman Zeyde +- Ryan Ofsky +- Saikiran +- Salvatore Ingala +- Sebastian Falbesoner +- Sergi Delgado Segura +- Shunsuke Shimizu +- Sjors Provoost +- stickies-v +- stratospher +- stringintech +- strmfos +- stutxo +- tdb3 +- TheCharlatan +- Tomás Andróil +- UdjinM6 +- Vasil Dimov +- VolodymyrBg +- w0xlt +- will +- willcl-ark +- William Casarin +- woltx +- yancy +- zaidmstrr + +As well as to everyone that helped with translations on +[Transifex](https://explore.transifex.com/bitcoin/bitcoin/). diff --git a/doc/release-process.md b/doc/release-process.md index 9159b8c..272f36e 100644 --- a/doc/release-process.md +++ b/doc/release-process.md @@ -53,7 +53,7 @@ Release Process - Set `MINCHAINWORK_HEADERS` to the height used for the `nMinimumChainWork` calculation above. - Check that the other variables still look reasonable. - Run the script. It works fine in CPython, but PyPy is much faster (seconds instead of minutes): `pypy3 contrib/devtools/headerssync-params.py`. - - Paste the output defining `HEADER_COMMITMENT_PERIOD` and `REDOWNLOAD_BUFFER_SIZE` into the top of [`src/headerssync.cpp`](/src/headerssync.cpp). + - Paste the output defining the header `commitment_period` and `redownload_buffer_size` into the mainnet section of [`src/kernel/chainparams.cpp`](/src/kernel/chainparams.cpp). - Clear the release notes and move them to the wiki (see "Write the release notes" below). - Translations on Transifex: - Pull translations from Transifex into the master branch. diff --git a/doc/tor.md b/doc/tor.md index 839c02e..e9db555 100644 --- a/doc/tor.md +++ b/doc/tor.md @@ -238,3 +238,10 @@ for normal IPv4/IPv6 communication, use: Otherwise it is trivial to link them, which may reduce privacy. Onion services created automatically (as in section 2) always have only one port open. +- Operating a node that listens on multiple networks (e.g. IPv4 and Tor) can help + strengthen the Bitcoin network, as nodes in this configuration (i.e. bridge nodes) increase + the cost and complexity of launching eclipse and partition attacks. However, under certain + conditions, an adversary that can connect to your node on multiple networks may be + able to correlate those identities by observing shared runtime characteristics. It + is not recommended to expose your node over multiple networks if you require + unlinkability across those identities. diff --git a/src/.clang-format b/src/.clang-format index f20e5ee..096c6f8 100644 --- a/src/.clang-format +++ b/src/.clang-format @@ -1,49 +1,222 @@ Language: Cpp AccessModifierOffset: -4 -AlignAfterOpenBracket: true -AlignEscapedNewlinesLeft: true -AlignTrailingComments: true +AlignAfterOpenBracket: Align +AlignArrayOfStructures: None +AlignConsecutiveAssignments: + Enabled: false + AcrossEmptyLines: false + AcrossComments: false + AlignCompound: false + PadOperators: true +AlignConsecutiveBitFields: + Enabled: false + AcrossEmptyLines: false + AcrossComments: false + AlignCompound: false + PadOperators: false +AlignConsecutiveDeclarations: + Enabled: false + AcrossEmptyLines: false + AcrossComments: false + AlignCompound: false + PadOperators: false +AlignConsecutiveMacros: + Enabled: false + AcrossEmptyLines: false + AcrossComments: false + AlignCompound: false + PadOperators: false +AlignEscapedNewlines: Left +AlignOperands: Align +AlignTrailingComments: + Kind: Always + OverEmptyLines: 0 +AllowAllArgumentsOnNextLine: true AllowAllParametersOfDeclarationOnNextLine: true -AllowShortBlocksOnASingleLine: false +AllowShortBlocksOnASingleLine: Never AllowShortCaseLabelsOnASingleLine: true +AllowShortEnumsOnASingleLine: true AllowShortFunctionsOnASingleLine: All -AllowShortIfStatementsOnASingleLine: true +AllowShortIfStatementsOnASingleLine: WithoutElse +AllowShortLambdasOnASingleLine: All AllowShortLoopsOnASingleLine: false +AlwaysBreakAfterDefinitionReturnType: None +AlwaysBreakAfterReturnType: None AlwaysBreakBeforeMultilineStrings: false -AlwaysBreakTemplateDeclarations: true +AlwaysBreakTemplateDeclarations: Yes +AttributeMacros: + - __capability BinPackArguments: true BinPackParameters: true -BreakBeforeBinaryOperators: false -BreakBeforeBraces: Custom +BitFieldColonSpacing: Both BraceWrapping: - AfterClass: true - AfterFunction: true + AfterCaseLabel: false + AfterClass: true + AfterControlStatement: Never + AfterEnum: false + AfterExternBlock: false + AfterFunction: true + AfterNamespace: false + AfterObjCDeclaration: false + AfterStruct: false + AfterUnion: false + BeforeCatch: false + BeforeElse: false + BeforeLambdaBody: false + BeforeWhile: false + IndentBraces: false + SplitEmptyFunction: true + SplitEmptyRecord: true + SplitEmptyNamespace: true +BreakAfterAttributes: Never +BreakAfterJavaFieldAnnotations: false +BreakArrays: true +BreakBeforeBinaryOperators: None +BreakBeforeConceptDeclarations: Always +BreakBeforeBraces: Custom +BreakBeforeInlineASMColon: OnlyMultiline BreakBeforeTernaryOperators: false -BreakConstructorInitializersBeforeComma: false +BreakConstructorInitializers: BeforeColon +BreakInheritanceList: BeforeColon +BreakStringLiterals: true ColumnLimit: 0 CommentPragmas: '^ IWYU pragma:' -ConstructorInitializerAllOnOneLineOrOnePerLine: false +CompactNamespaces: false ConstructorInitializerIndentWidth: 4 ContinuationIndentWidth: 4 Cpp11BracedListStyle: true DerivePointerAlignment: false DisableFormat: false +EmptyLineAfterAccessModifier: Never +EmptyLineBeforeAccessModifier: LogicalBlock +ExperimentalAutoDetectBinPacking: false +FixNamespaceComments: true +ForEachMacros: + - foreach + - Q_FOREACH + - BOOST_FOREACH +IfMacros: + - KJ_IF_MAYBE +IncludeBlocks: Preserve +IncludeCategories: + - Regex: '^"(llvm|llvm-c|clang|clang-c)/' + Priority: 2 + SortPriority: 0 + CaseSensitive: false + - Regex: '^(<|"(gtest|gmock|isl|json)/)' + Priority: 3 + SortPriority: 0 + CaseSensitive: false + - Regex: '.*' + Priority: 1 + SortPriority: 0 + CaseSensitive: false +IncludeIsMainRegex: '(Test)?$' +IncludeIsMainSourceRegex: '' +IndentAccessModifiers: false +IndentCaseBlocks: false IndentCaseLabels: false -IndentFunctionDeclarationAfterType: false +IndentExternBlock: AfterExternBlock +IndentGotoLabels: true +IndentPPDirectives: None +IndentRequiresClause: true IndentWidth: 4 +IndentWrappedFunctionNames: false +InsertBraces: false +InsertNewlineAtEOF: false +InsertTrailingCommas: None +IntegerLiteralSeparator: + Binary: 0 + BinaryMinDigits: 0 + Decimal: 0 + DecimalMinDigits: 0 + Hex: 0 + HexMinDigits: 0 +JavaScriptQuotes: Leave +JavaScriptWrapImports: true KeepEmptyLinesAtTheStartOfBlocks: false +LambdaBodyIndentation: Signature +LineEnding: DeriveLF +MacroBlockBegin: '' +MacroBlockEnd: '' MaxEmptyLinesToKeep: 2 NamespaceIndentation: None +ObjCBinPackProtocolList: Auto +ObjCBlockIndentWidth: 2 +ObjCBreakBeforeNestedBlockParam: true +ObjCSpaceAfterProperty: false +ObjCSpaceBeforeProtocolList: true +PackConstructorInitializers: BinPack +PenaltyBreakAssignment: 2 +PenaltyBreakBeforeFirstCallParameter: 19 +PenaltyBreakComment: 300 +PenaltyBreakFirstLessLess: 120 +PenaltyBreakOpenParenthesis: 0 +PenaltyBreakString: 1000 +PenaltyBreakTemplateDeclaration: 10 +PenaltyExcessCharacter: 1000000 +PenaltyIndentedWhitespace: 0 +PenaltyReturnTypeOnItsOwnLine: 60 PointerAlignment: Left +PPIndentWidth: -1 +QualifierAlignment: Leave +ReferenceAlignment: Pointer +ReflowComments: true +RemoveBracesLLVM: false +RemoveSemicolon: false +RequiresClausePosition: OwnLine +RequiresExpressionIndentation: OuterScope +SeparateDefinitionBlocks: Leave +ShortNamespaceLines: 1 +SortIncludes: CaseSensitive +SortJavaStaticImport: Before +SortUsingDeclarations: LexicographicNumeric +SpaceAfterCStyleCast: false +SpaceAfterLogicalNot: false +SpaceAfterTemplateKeyword: true +SpaceAroundPointerQualifiers: Default SpaceBeforeAssignmentOperators: true +SpaceBeforeCaseColon: false +SpaceBeforeCpp11BracedList: false +SpaceBeforeCtorInitializerColon: true +SpaceBeforeInheritanceColon: true SpaceBeforeParens: ControlStatements +SpaceBeforeParensOptions: + AfterControlStatements: true + AfterForeachMacros: true + AfterFunctionDefinitionName: false + AfterFunctionDeclarationName: false + AfterIfMacros: true + AfterOverloadedOperator: false + AfterRequiresInClause: false + AfterRequiresInExpression: false + BeforeNonEmptyParentheses: false +SpaceBeforeRangeBasedForLoopColon: true +SpaceBeforeSquareBrackets: false +SpaceInEmptyBlock: false SpaceInEmptyParentheses: false SpacesBeforeTrailingComments: 1 -SpacesInAngles: false +SpacesInAngles: Never +SpacesInConditionalStatement: false SpacesInContainerLiterals: true SpacesInCStyleCastParentheses: false +SpacesInLineCommentPrefix: + Minimum: 1 + Maximum: -1 SpacesInParentheses: false -BreakBeforeConceptDeclarations: Always -RequiresExpressionIndentation: OuterScope -Standard: c++20 +SpacesInSquareBrackets: false +Standard: c++20 +StatementAttributeLikeMacros: + - Q_EMIT +StatementMacros: + - Q_UNUSED + - QT_REQUIRE_VERSION +TabWidth: 8 UseTab: Never +WhitespaceSensitiveMacros: + - BOOST_PP_STRINGIZE + - CF_SWIFT_NAME + - NS_SWIFT_NAME + - PP_STRINGIZE + - STRINGIZE +... diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 81c8577..aed4061 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -238,8 +238,8 @@ add_library(bitcoin_node STATIC EXCLUDE_FROM_ALL node/warnings.cpp noui.cpp policy/ephemeral_policy.cpp - policy/fees.cpp - policy/fees_args.cpp + policy/fees/block_policy_estimator.cpp + policy/fees/block_policy_estimator_args.cpp policy/packages.cpp policy/rbf.cpp policy/settings.cpp @@ -411,6 +411,7 @@ if(BUILD_KERNEL_LIB) add_executable(bitcoin-chainstate bitcoin-chainstate.cpp ) + add_windows_application_manifest(bitcoin-chainstate) # TODO: The `SKIP_BUILD_RPATH` property setting can be deleted # in the future after reordering Guix script commands to # perform binary checks after the installation step. diff --git a/src/arith_uint256.h b/src/arith_uint256.h index 5cefff5..69e0aff 100644 --- a/src/arith_uint256.h +++ b/src/arith_uint256.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2022 The Bitcoin Core developers +// Copyright (c) 2009-present The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -21,7 +21,7 @@ class uint_error : public std::runtime_error { }; /** Template base class for unsigned big integers. */ -template +template class base_uint { protected: @@ -29,9 +29,9 @@ class base_uint static constexpr int WIDTH = BITS / 32; /** Big integer represented with 32-bit digits, least-significant first. */ uint32_t pn[WIDTH]; -public: - base_uint() +public: + constexpr base_uint() { for (int i = 0; i < WIDTH; i++) pn[i] = 0; @@ -40,7 +40,7 @@ class base_uint base_uint(const base_uint& b) = default; base_uint& operator=(const base_uint& b) = default; - base_uint(uint64_t b) + constexpr base_uint(uint64_t b) { pn[0] = (unsigned int)b; pn[1] = (unsigned int)(b >> 32); @@ -227,11 +227,12 @@ class base_uint }; /** 256-bit unsigned big integer. */ -class arith_uint256 : public base_uint<256> { +class arith_uint256 : public base_uint<256> +{ public: - arith_uint256() = default; - arith_uint256(const base_uint<256>& b) : base_uint<256>(b) {} - arith_uint256(uint64_t b) : base_uint<256>(b) {} + constexpr arith_uint256() = default; + constexpr arith_uint256(const base_uint& b) : base_uint(b) {} + constexpr arith_uint256(uint64_t b) : base_uint(b) {} /** * The "compact" format is a representation of a whole diff --git a/src/bench/CMakeLists.txt b/src/bench/CMakeLists.txt index 0bf469c..e0e03b1 100644 --- a/src/bench/CMakeLists.txt +++ b/src/bench/CMakeLists.txt @@ -56,6 +56,8 @@ add_executable(bench_bitcoin verify_script.cpp ) +add_windows_application_manifest(bench_bitcoin) + include(TargetDataSources) target_raw_data_sources(bench_bitcoin NAMESPACE benchmark::data data/block413567.raw diff --git a/src/bench/bench.cpp b/src/bench/bench.cpp index 26daff5..9cc1f30 100644 --- a/src/bench/bench.cpp +++ b/src/bench/bench.cpp @@ -56,7 +56,7 @@ void GenerateTemplateResults(const std::vector& bench // nothing to write, bail out return; } - std::ofstream fout{file}; + std::ofstream fout{file.std_path()}; if (fout.is_open()) { ankerl::nanobench::render(tpl, benchmarkResults, fout); std::cout << "Created " << file << std::endl; diff --git a/src/bench/descriptors.cpp b/src/bench/descriptors.cpp index c454566..ac5580a 100644 --- a/src/bench/descriptors.cpp +++ b/src/bench/descriptors.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -19,7 +20,7 @@ static void ExpandDescriptor(benchmark::Bench& bench) { ECC_Context ecc_context{}; - const auto desc_str = "sh(wsh(multi(16,03669b8afcec803a0d323e9a17f3ea8e68e8abe5a278020a929adbec52421adbd0,0260b2003c386519fc9eadf2b5cf124dd8eea4c4e68d5e154050a9346ea98ce600,0362a74e399c39ed5593852a30147f2959b56bb827dfa3e60e464b02ccf87dc5e8,0261345b53de74a4d721ef877c255429961b7e43714171ac06168d7e08c542a8b8,02da72e8b46901a65d4374fe6315538d8f368557dda3a1dcf9ea903f3afe7314c8,0318c82dd0b53fd3a932d16e0ba9e278fcc937c582d5781be626ff16e201f72286,0297ccef1ef99f9d73dec9ad37476ddb232f1238aff877af19e72ba04493361009,02e502cfd5c3f972fe9a3e2a18827820638f96b6f347e54d63deb839011fd5765d,03e687710f0e3ebe81c1037074da939d409c0025f17eb86adb9427d28f0f7ae0e9,02c04d3a5274952acdbc76987f3184b346a483d43be40874624b29e3692c1df5af,02ed06e0f418b5b43a7ec01d1d7d27290fa15f75771cb69b642a51471c29c84acd,036d46073cbb9ffee90473f3da429abc8de7f8751199da44485682a989a4bebb24,02f5d1ff7c9029a80a4e36b9a5497027ef7f3e73384a4a94fbfe7c4e9164eec8bc,02e41deffd1b7cce11cde209a781adcffdabd1b91c0ba0375857a2bfd9302419f3,02d76625f7956a7fc505ab02556c23ee72d832f1bac391bcd2d3abce5710a13d06,0399eb0a5487515802dc14544cf10b3666623762fbed2ec38a3975716e2c29c232)))"; + constexpr std::string_view desc_str{"sh(wsh(multi(16,03669b8afcec803a0d323e9a17f3ea8e68e8abe5a278020a929adbec52421adbd0,0260b2003c386519fc9eadf2b5cf124dd8eea4c4e68d5e154050a9346ea98ce600,0362a74e399c39ed5593852a30147f2959b56bb827dfa3e60e464b02ccf87dc5e8,0261345b53de74a4d721ef877c255429961b7e43714171ac06168d7e08c542a8b8,02da72e8b46901a65d4374fe6315538d8f368557dda3a1dcf9ea903f3afe7314c8,0318c82dd0b53fd3a932d16e0ba9e278fcc937c582d5781be626ff16e201f72286,0297ccef1ef99f9d73dec9ad37476ddb232f1238aff877af19e72ba04493361009,02e502cfd5c3f972fe9a3e2a18827820638f96b6f347e54d63deb839011fd5765d,03e687710f0e3ebe81c1037074da939d409c0025f17eb86adb9427d28f0f7ae0e9,02c04d3a5274952acdbc76987f3184b346a483d43be40874624b29e3692c1df5af,02ed06e0f418b5b43a7ec01d1d7d27290fa15f75771cb69b642a51471c29c84acd,036d46073cbb9ffee90473f3da429abc8de7f8751199da44485682a989a4bebb24,02f5d1ff7c9029a80a4e36b9a5497027ef7f3e73384a4a94fbfe7c4e9164eec8bc,02e41deffd1b7cce11cde209a781adcffdabd1b91c0ba0375857a2bfd9302419f3,02d76625f7956a7fc505ab02556c23ee72d832f1bac391bcd2d3abce5710a13d06,0399eb0a5487515802dc14544cf10b3666623762fbed2ec38a3975716e2c29c232)))"}; const std::pair range = {0, 1000}; FlatSigningProvider provider; std::string error; diff --git a/src/bitcoin-chainstate.cpp b/src/bitcoin-chainstate.cpp index 4a4bd87..310ca95 100644 --- a/src/bitcoin-chainstate.cpp +++ b/src/bitcoin-chainstate.cpp @@ -9,15 +9,6 @@ #include #include -#ifdef WIN32 -// clang-format off -#include -// clang-format on -#include -#include -#include -#endif - using namespace btck; std::vector hex_string_to_byte_vec(std::string_view hex) @@ -46,7 +37,7 @@ class KernelLog } }; -class TestValidationInterface : public ValidationInterface +class TestValidationInterface : public ValidationInterface { public: TestValidationInterface() = default; @@ -103,7 +94,7 @@ class TestValidationInterface : public ValidationInterface +class TestKernelNotifications : public KernelNotifications { public: void BlockTipHandler(SynchronizationState, const BlockTreeEntry, double) override @@ -149,22 +140,6 @@ int main(int argc, char* argv[]) << " BREAK IN FUTURE VERSIONS. DO NOT USE ON YOUR ACTUAL DATADIR." << std::endl; return 1; } - -#ifdef WIN32 - int win_argc; - wchar_t** wargv = CommandLineToArgvW(GetCommandLineW(), &win_argc); - std::vector utf8_args(win_argc); - std::vector win_argv(win_argc); - std::wstring_convert, wchar_t> utf8_cvt; - for (int i = 0; i < win_argc; i++) { - utf8_args[i] = utf8_cvt.to_bytes(wargv[i]); - win_argv[i] = &utf8_args[i][0]; - } - LocalFree(wargv); - argc = win_argc; - argv = win_argv.data(); -#endif - std::filesystem::path abs_datadir{std::filesystem::absolute(argv[1])}; std::filesystem::create_directories(abs_datadir); @@ -176,7 +151,9 @@ int main(int argc, char* argv[]) .always_print_category_levels = true, }; - Logger logger{std::make_unique(KernelLog{}), logging_options}; + logging_set_options(logging_options); + + Logger logger{std::make_unique()}; ContextOptions options{}; ChainParams params{ChainType::MAINNET}; diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index 56f2a90..279aa89 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -1328,10 +1328,6 @@ static int CommandLineRPC(int argc, char *argv[]) MAIN_FUNCTION { -#ifdef WIN32 - common::WinCmdLineArgs winArgs; - std::tie(argc, argv) = winArgs.get(); -#endif SetupEnvironment(); if (!SetupNetworking()) { tfm::format(std::cerr, "Error: Initializing networking failed\n"); diff --git a/src/bitcoin-wallet.cpp b/src/bitcoin-wallet.cpp index c871523..6811f8c 100644 --- a/src/bitcoin-wallet.cpp +++ b/src/bitcoin-wallet.cpp @@ -94,10 +94,6 @@ static std::optional WalletAppInit(ArgsManager& args, int argc, char* argv[ MAIN_FUNCTION { ArgsManager& args = gArgs; -#ifdef WIN32 - common::WinCmdLineArgs winArgs; - std::tie(argc, argv) = winArgs.get(); -#endif int exit_status; std::unique_ptr init = interfaces::MakeWalletInit(argc, argv, exit_status); diff --git a/src/bitcoin.cpp b/src/bitcoin.cpp index c1a5fce..a8ebab9 100644 --- a/src/bitcoin.cpp +++ b/src/bitcoin.cpp @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -61,6 +62,8 @@ static void ExecCommand(const std::vector& args, std::string_view a int main(int argc, char* argv[]) { + SetupEnvironment(); + try { CommandLine cmd{ParseCommandLine(argc, argv)}; if (cmd.show_version) { @@ -209,7 +212,7 @@ static void ExecCommand(const std::vector& args, std::string_view w // Try to resolve any symlinks and figure out the directory containing the wrapper executable. std::error_code ec; - fs::path wrapper_dir{fs::weakly_canonical(wrapper_path, ec)}; + auto wrapper_dir{fs::weakly_canonical(wrapper_path, ec)}; if (wrapper_dir.empty()) wrapper_dir = wrapper_path; // Restore previous path if weakly_canonical failed. wrapper_dir = wrapper_dir.parent_path(); @@ -225,7 +228,7 @@ static void ExecCommand(const std::vector& args, std::string_view w // If wrapper is installed in a bin/ directory, look for target executable // in libexec/ - (wrapper_dir.filename() == "bin" && try_exec(fs::path{wrapper_dir.parent_path()} / "libexec" / arg0.filename())) || + (wrapper_dir.filename() == "bin" && try_exec(wrapper_dir.parent_path() / "libexec" / arg0.filename())) || #ifdef WIN32 // Otherwise check the "daemon" subdirectory in a windows install. (!wrapper_dir.empty() && try_exec(wrapper_dir / "daemon" / arg0.filename())) || diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp index a4373da..37cacb2 100644 --- a/src/bitcoind.cpp +++ b/src/bitcoind.cpp @@ -259,11 +259,6 @@ static bool AppInit(NodeContext& node) MAIN_FUNCTION { -#ifdef WIN32 - common::WinCmdLineArgs winArgs; - std::tie(argc, argv) = winArgs.get(); -#endif - NodeContext node; int exit_status; std::unique_ptr init = interfaces::MakeNodeInit(node, argc, argv, exit_status); diff --git a/src/blockfilter.cpp b/src/blockfilter.cpp index 5e6702c..94bcc49 100644 --- a/src/blockfilter.cpp +++ b/src/blockfilter.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include @@ -151,7 +152,8 @@ const std::string& BlockFilterTypeName(BlockFilterType filter_type) return it != g_filter_types.end() ? it->second : unknown_retval; } -bool BlockFilterTypeByName(const std::string& name, BlockFilterType& filter_type) { +bool BlockFilterTypeByName(std::string_view name, BlockFilterType& filter_type) +{ for (const auto& entry : g_filter_types) { if (entry.second == name) { filter_type = entry.first; diff --git a/src/blockfilter.h b/src/blockfilter.h index 8eab4af..b70afb9 100644 --- a/src/blockfilter.h +++ b/src/blockfilter.h @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -99,7 +100,7 @@ enum class BlockFilterType : uint8_t const std::string& BlockFilterTypeName(BlockFilterType filter_type); /** Find a filter type by its human-readable name. */ -bool BlockFilterTypeByName(const std::string& name, BlockFilterType& filter_type); +bool BlockFilterTypeByName(std::string_view name, BlockFilterType& filter_type); /** Get a list of known filter types. */ const std::set& AllBlockFilterTypes(); diff --git a/src/chain.cpp b/src/chain.cpp index 3dd2263..94be718 100644 --- a/src/chain.cpp +++ b/src/chain.cpp @@ -6,12 +6,6 @@ #include #include #include -#include - -std::string CBlockFileInfo::ToString() const -{ - return strprintf("CBlockFileInfo(blocks=%u, size=%u, heights=%u...%u, time=%s...%s)", nBlocks, nSize, nHeightFirst, nHeightLast, FormatISO8601Date(nTimeFirst), FormatISO8601Date(nTimeLast)); -} std::string CBlockIndex::ToString() const { diff --git a/src/chain.h b/src/chain.h index 68aa612..64c8635 100644 --- a/src/chain.h +++ b/src/chain.h @@ -35,6 +35,9 @@ static constexpr int64_t MAX_FUTURE_BLOCK_TIME = 2 * 60 * 60; * MAX_FUTURE_BLOCK_TIME. */ static constexpr int64_t TIMESTAMP_WINDOW = MAX_FUTURE_BLOCK_TIME; +//! Init values for CBlockIndex nSequenceId when loaded from disk +static constexpr int32_t SEQ_ID_BEST_CHAIN_FROM_DISK = 0; +static constexpr int32_t SEQ_ID_INIT_FROM_DISK = 1; /** * Maximum gap between node time and block time used @@ -44,47 +47,6 @@ static constexpr int64_t TIMESTAMP_WINDOW = MAX_FUTURE_BLOCK_TIME; */ static constexpr int64_t MAX_BLOCK_TIME_GAP = 90 * 60; -class CBlockFileInfo -{ -public: - unsigned int nBlocks{}; //!< number of blocks stored in file - unsigned int nSize{}; //!< number of used bytes of block file - unsigned int nUndoSize{}; //!< number of used bytes in the undo file - unsigned int nHeightFirst{}; //!< lowest height of block in file - unsigned int nHeightLast{}; //!< highest height of block in file - uint64_t nTimeFirst{}; //!< earliest time of block in file - uint64_t nTimeLast{}; //!< latest time of block in file - - SERIALIZE_METHODS(CBlockFileInfo, obj) - { - READWRITE(VARINT(obj.nBlocks)); - READWRITE(VARINT(obj.nSize)); - READWRITE(VARINT(obj.nUndoSize)); - READWRITE(VARINT(obj.nHeightFirst)); - READWRITE(VARINT(obj.nHeightLast)); - READWRITE(VARINT(obj.nTimeFirst)); - READWRITE(VARINT(obj.nTimeLast)); - } - - CBlockFileInfo() = default; - - std::string ToString() const; - - /** update statistics (does not update nSize) */ - void AddBlock(unsigned int nHeightIn, uint64_t nTimeIn) - { - if (nBlocks == 0 || nHeightFirst > nHeightIn) - nHeightFirst = nHeightIn; - if (nBlocks == 0 || nTimeFirst > nTimeIn) - nTimeFirst = nTimeIn; - nBlocks++; - if (nHeightIn > nHeightLast) - nHeightLast = nHeightIn; - if (nTimeIn > nTimeLast) - nTimeLast = nTimeIn; - } -}; - enum BlockStatus : uint32_t { //! Unused. BLOCK_VALID_UNKNOWN = 0, @@ -191,7 +153,9 @@ class CBlockIndex uint32_t nNonce{0}; //! (memory only) Sequential id assigned to distinguish order in which blocks are received. - int32_t nSequenceId{0}; + //! Initialized to SEQ_ID_INIT_FROM_DISK{1} when loading blocks from disk, except for blocks + //! belonging to the best chain which overwrite it to SEQ_ID_BEST_CHAIN_FROM_DISK{0}. + int32_t nSequenceId{SEQ_ID_INIT_FROM_DISK}; //! (memory only) Maximum nTime in the chain up to and including this block. unsigned int nTimeMax{0}; diff --git a/src/cluster_linearize.h b/src/cluster_linearize.h index bec44d9..73c8a03 100644 --- a/src/cluster_linearize.h +++ b/src/cluster_linearize.h @@ -12,6 +12,7 @@ #include #include +#include #include #include #include @@ -332,6 +333,17 @@ class DepGraph } return true; } + + /** Reduce memory usage if possible. No observable effect. */ + void Compact() noexcept + { + entries.shrink_to_fit(); + } + + size_t DynamicMemoryUsage() const noexcept + { + return memusage::DynamicUsage(entries); + } }; /** A set of transactions together with their aggregate feerate. */ diff --git a/src/coins.cpp b/src/coins.cpp index 24a102b..554a3eb 100644 --- a/src/coins.cpp +++ b/src/coins.cpp @@ -76,9 +76,6 @@ void CCoinsViewCache::AddCoin(const COutPoint &outpoint, Coin&& coin, bool possi bool inserted; std::tie(it, inserted) = cacheCoins.emplace(std::piecewise_construct, std::forward_as_tuple(outpoint), std::tuple<>()); bool fresh = false; - if (!inserted) { - cachedCoinsUsage -= it->second.coin.DynamicMemoryUsage(); - } if (!possible_overwrite) { if (!it->second.coin.IsSpent()) { throw std::logic_error("Attempted to overwrite an unspent coin (when possible_overwrite is false)"); @@ -98,6 +95,9 @@ void CCoinsViewCache::AddCoin(const COutPoint &outpoint, Coin&& coin, bool possi // DIRTY, then it can be marked FRESH. fresh = !it->second.IsDirty(); } + if (!inserted) { + cachedCoinsUsage -= it->second.coin.DynamicMemoryUsage(); + } it->second.coin = std::move(coin); CCoinsCacheEntry::SetDirty(*it, m_sentinel); if (fresh) CCoinsCacheEntry::SetFresh(*it, m_sentinel); @@ -111,9 +111,12 @@ void CCoinsViewCache::AddCoin(const COutPoint &outpoint, Coin&& coin, bool possi } void CCoinsViewCache::EmplaceCoinInternalDANGER(COutPoint&& outpoint, Coin&& coin) { - cachedCoinsUsage += coin.DynamicMemoryUsage(); + const auto mem_usage{coin.DynamicMemoryUsage()}; auto [it, inserted] = cacheCoins.try_emplace(std::move(outpoint), std::move(coin)); - if (inserted) CCoinsCacheEntry::SetDirty(*it, m_sentinel); + if (inserted) { + CCoinsCacheEntry::SetDirty(*it, m_sentinel); + cachedCoinsUsage += mem_usage; + } } void AddCoins(CCoinsViewCache& cache, const CTransaction &tx, int nHeight, bool check_for_overwrite) { @@ -195,6 +198,7 @@ bool CCoinsViewCache::BatchWrite(CoinsViewCacheCursor& cursor, const uint256 &ha // and mark it as dirty. itUs = cacheCoins.try_emplace(it->first).first; CCoinsCacheEntry& entry{itUs->second}; + assert(entry.coin.DynamicMemoryUsage() == 0); if (cursor.WillErase(*it)) { // Since this entry will be erased, // we can move the coin into us instead of copying it @@ -248,19 +252,19 @@ bool CCoinsViewCache::BatchWrite(CoinsViewCacheCursor& cursor, const uint256 &ha } bool CCoinsViewCache::Flush() { - auto cursor{CoinsViewCacheCursor(cachedCoinsUsage, m_sentinel, cacheCoins, /*will_erase=*/true)}; + auto cursor{CoinsViewCacheCursor(m_sentinel, cacheCoins, /*will_erase=*/true)}; bool fOk = base->BatchWrite(cursor, hashBlock); if (fOk) { cacheCoins.clear(); ReallocateCache(); + cachedCoinsUsage = 0; } - cachedCoinsUsage = 0; return fOk; } bool CCoinsViewCache::Sync() { - auto cursor{CoinsViewCacheCursor(cachedCoinsUsage, m_sentinel, cacheCoins, /*will_erase=*/false)}; + auto cursor{CoinsViewCacheCursor(m_sentinel, cacheCoins, /*will_erase=*/false)}; bool fOk = base->BatchWrite(cursor, hashBlock); if (fOk) { if (m_sentinel.second.Next() != &m_sentinel) { @@ -345,8 +349,8 @@ void CCoinsViewCache::SanityCheck() const assert(recomputed_usage == cachedCoinsUsage); } -static const size_t MIN_TRANSACTION_OUTPUT_WEIGHT = WITNESS_SCALE_FACTOR * ::GetSerializeSize(CTxOut()); -static const size_t MAX_OUTPUTS_PER_BLOCK = MAX_BLOCK_WEIGHT / MIN_TRANSACTION_OUTPUT_WEIGHT; +static const uint64_t MIN_TRANSACTION_OUTPUT_WEIGHT{WITNESS_SCALE_FACTOR * ::GetSerializeSize(CTxOut())}; +static const uint64_t MAX_OUTPUTS_PER_BLOCK{MAX_BLOCK_WEIGHT / MIN_TRANSACTION_OUTPUT_WEIGHT}; const Coin& AccessByTxid(const CCoinsViewCache& view, const Txid& txid) { diff --git a/src/coins.h b/src/coins.h index 6725d5a..2fcc764 100644 --- a/src/coins.h +++ b/src/coins.h @@ -271,11 +271,10 @@ struct CoinsViewCacheCursor //! This is an optimization compared to erasing all entries as the cursor iterates them when will_erase is set. //! Calling CCoinsMap::clear() afterwards is faster because a CoinsCachePair cannot be coerced back into a //! CCoinsMap::iterator to be erased, and must therefore be looked up again by key in the CCoinsMap before being erased. - CoinsViewCacheCursor(size_t& usage LIFETIMEBOUND, - CoinsCachePair& sentinel LIFETIMEBOUND, - CCoinsMap& map LIFETIMEBOUND, - bool will_erase) noexcept - : m_usage(usage), m_sentinel(sentinel), m_map(map), m_will_erase(will_erase) {} + CoinsViewCacheCursor(CoinsCachePair& sentinel LIFETIMEBOUND, + CCoinsMap& map LIFETIMEBOUND, + bool will_erase) noexcept + : m_sentinel(sentinel), m_map(map), m_will_erase(will_erase) {} inline CoinsCachePair* Begin() const noexcept { return m_sentinel.second.Next(); } inline CoinsCachePair* End() const noexcept { return &m_sentinel; } @@ -288,7 +287,7 @@ struct CoinsViewCacheCursor // Otherwise, clear the state of the entry. if (!m_will_erase) { if (current.second.coin.IsSpent()) { - m_usage -= current.second.coin.DynamicMemoryUsage(); + assert(current.second.coin.DynamicMemoryUsage() == 0); // scriptPubKey was already cleared in SpendCoin m_map.erase(current.first); } else { current.second.SetClean(); @@ -299,7 +298,6 @@ struct CoinsViewCacheCursor inline bool WillErase(CoinsCachePair& current) const noexcept { return m_will_erase || current.second.coin.IsSpent(); } private: - size_t& m_usage; CoinsCachePair& m_sentinel; CCoinsMap& m_map; bool m_will_erase; diff --git a/src/common/args.cpp b/src/common/args.cpp index d44cd43..50b9902 100644 --- a/src/common/args.cpp +++ b/src/common/args.cpp @@ -19,8 +19,6 @@ #include #ifdef WIN32 -#include -#include #include #endif @@ -879,30 +877,3 @@ void ArgsManager::LogArgs() const } logArgsPrefix("Command-line arg:", "", m_settings.command_line_options); } - -namespace common { -#ifdef WIN32 -WinCmdLineArgs::WinCmdLineArgs() -{ - wchar_t** wargv = CommandLineToArgvW(GetCommandLineW(), &argc); - std::wstring_convert, wchar_t> utf8_cvt; - argv = new char*[argc]; - args.resize(argc); - for (int i = 0; i < argc; i++) { - args[i] = utf8_cvt.to_bytes(wargv[i]); - argv[i] = &*args[i].begin(); - } - LocalFree(wargv); -} - -WinCmdLineArgs::~WinCmdLineArgs() -{ - delete[] argv; -} - -std::pair WinCmdLineArgs::get() -{ - return std::make_pair(argc, argv); -} -#endif -} // namespace common diff --git a/src/common/args.h b/src/common/args.h index d907ad7..1b9233e 100644 --- a/src/common/args.h +++ b/src/common/args.h @@ -480,21 +480,4 @@ std::string HelpMessageGroup(const std::string& message); */ std::string HelpMessageOpt(const std::string& option, const std::string& message); -namespace common { -#ifdef WIN32 -class WinCmdLineArgs -{ -public: - WinCmdLineArgs(); - ~WinCmdLineArgs(); - std::pair get(); - -private: - int argc; - char** argv; - std::vector args; -}; -#endif -} // namespace common - #endif // BITCOIN_COMMON_ARGS_H diff --git a/src/common/config.cpp b/src/common/config.cpp index 42f5e28..7216022 100644 --- a/src/common/config.cpp +++ b/src/common/config.cpp @@ -135,7 +135,7 @@ bool ArgsManager::ReadConfigFiles(std::string& error, bool ignore_invalid_keys) error = strprintf("Config file \"%s\" is a directory.", fs::PathToString(conf_path)); return false; } - stream = std::ifstream{conf_path}; + stream = std::ifstream{conf_path.std_path()}; // If the file is explicitly specified, it must be readable if (IsArgSet("-conf") && !stream.good()) { error = strprintf("specified config file \"%s\" could not be opened.", fs::PathToString(conf_path)); @@ -187,7 +187,7 @@ bool ArgsManager::ReadConfigFiles(std::string& error, bool ignore_invalid_keys) error = strprintf("Included config file \"%s\" is a directory.", fs::PathToString(include_conf_path)); return false; } - std::ifstream conf_file_stream{include_conf_path}; + std::ifstream conf_file_stream{include_conf_path.std_path()}; if (conf_file_stream.good()) { if (!ReadConfigStream(conf_file_stream, conf_file_name, error, ignore_invalid_keys)) { return false; diff --git a/src/common/messages.cpp b/src/common/messages.cpp index fc0b4a8..983c982 100644 --- a/src/common/messages.cpp +++ b/src/common/messages.cpp @@ -6,7 +6,7 @@ #include #include -#include +#include #include #include #include @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -91,7 +92,7 @@ std::string InvalidEstimateModeErrorMessage() return "Invalid estimate_mode parameter, must be one of: \"" + FeeModes("\", \"") + "\""; } -bool FeeModeFromString(const std::string& mode_string, FeeEstimateMode& fee_estimate_mode) +bool FeeModeFromString(std::string_view mode_string, FeeEstimateMode& fee_estimate_mode) { auto searchkey = ToUpper(mode_string); for (const auto& pair : FeeModeMap()) { diff --git a/src/common/messages.h b/src/common/messages.h index 5827fcc..5d33eda 100644 --- a/src/common/messages.h +++ b/src/common/messages.h @@ -12,6 +12,7 @@ #define BITCOIN_COMMON_MESSAGES_H #include +#include struct bilingual_str; @@ -23,7 +24,7 @@ enum class TransactionError; namespace common { enum class PSBTError; -bool FeeModeFromString(const std::string& mode_string, FeeEstimateMode& fee_estimate_mode); +bool FeeModeFromString(std::string_view mode_string, FeeEstimateMode& fee_estimate_mode); std::string StringForFeeReason(FeeReason reason); std::string FeeModes(const std::string& delimiter); std::string FeeModeInfo(std::pair& mode); diff --git a/src/common/settings.cpp b/src/common/settings.cpp index 046afca..fbe531c 100644 --- a/src/common/settings.cpp +++ b/src/common/settings.cpp @@ -78,7 +78,7 @@ bool ReadSettings(const fs::path& path, std::map& va if (!fs::exists(path)) return true; std::ifstream file; - file.open(path); + file.open(path.std_path()); if (!file.is_open()) { errors.emplace_back(strprintf("%s. Please check permissions.", fs::PathToString(path))); return false; @@ -133,7 +133,7 @@ bool WriteSettings(const fs::path& path, out.pushKVEnd(value.first, value.second); } std::ofstream file; - file.open(path); + file.open(path.std_path()); if (file.fail()) { errors.emplace_back(strprintf("Error: Unable to open settings file %s for writing", fs::PathToString(path))); return false; diff --git a/src/common/system.cpp b/src/common/system.cpp index 35a6f44..3833763 100644 --- a/src/common/system.cpp +++ b/src/common/system.cpp @@ -12,6 +12,7 @@ #include #ifdef WIN32 +#include #include #include #include @@ -83,6 +84,7 @@ void SetupEnvironment() setenv("LC_ALL", "C.UTF-8", 1); } #elif defined(WIN32) + assert(GetACP() == CP_UTF8); // Set the default input/output charset is utf-8 SetConsoleCP(CP_UTF8); SetConsoleOutputCP(CP_UTF8); diff --git a/src/consensus/tx_verify.cpp b/src/consensus/tx_verify.cpp index 9d09872..c38b964 100644 --- a/src/consensus/tx_verify.cpp +++ b/src/consensus/tx_verify.cpp @@ -156,7 +156,7 @@ int64_t GetTransactionSigOpCost(const CTransaction& tx, const CCoinsViewCache& i const Coin& coin = inputs.AccessCoin(tx.vin[i].prevout); assert(!coin.IsSpent()); const CTxOut &prevout = coin.out; - nSigOps += CountWitnessSigOps(tx.vin[i].scriptSig, prevout.scriptPubKey, &tx.vin[i].scriptWitness, flags); + nSigOps += CountWitnessSigOps(tx.vin[i].scriptSig, prevout.scriptPubKey, tx.vin[i].scriptWitness, flags); } return nSigOps; } diff --git a/src/core_io.h b/src/core_io.h index ce2e8f6..1874c93 100644 --- a/src/core_io.h +++ b/src/core_io.h @@ -21,6 +21,7 @@ class SigningProvider; class uint256; class UniValue; class CTxUndo; +class CTxOut; /** * Verbose level for block's transaction @@ -46,6 +47,6 @@ std::string FormatScript(const CScript& script); std::string EncodeHexTx(const CTransaction& tx); std::string SighashToStr(unsigned char sighash_type); void ScriptToUniv(const CScript& script, UniValue& out, bool include_hex = true, bool include_address = false, const SigningProvider* provider = nullptr); -void TxToUniv(const CTransaction& tx, const uint256& block_hash, UniValue& entry, bool include_hex = true, const CTxUndo* txundo = nullptr, TxVerbosity verbosity = TxVerbosity::SHOW_DETAILS); +void TxToUniv(const CTransaction& tx, const uint256& block_hash, UniValue& entry, bool include_hex = true, const CTxUndo* txundo = nullptr, TxVerbosity verbosity = TxVerbosity::SHOW_DETAILS, std::function is_change_func = {}); #endif // BITCOIN_CORE_IO_H diff --git a/src/core_write.cpp b/src/core_write.cpp index 14836f5..d805cfd 100644 --- a/src/core_write.cpp +++ b/src/core_write.cpp @@ -168,7 +168,7 @@ void ScriptToUniv(const CScript& script, UniValue& out, bool include_hex, bool i out.pushKV("type", GetTxnOutputType(type)); } -void TxToUniv(const CTransaction& tx, const uint256& block_hash, UniValue& entry, bool include_hex, const CTxUndo* txundo, TxVerbosity verbosity) +void TxToUniv(const CTransaction& tx, const uint256& block_hash, UniValue& entry, bool include_hex, const CTxUndo* txundo, TxVerbosity verbosity, std::function is_change_func) { CHECK_NONFATAL(verbosity >= TxVerbosity::SHOW_DETAILS); @@ -246,6 +246,11 @@ void TxToUniv(const CTransaction& tx, const uint256& block_hash, UniValue& entry UniValue o(UniValue::VOBJ); ScriptToUniv(txout.scriptPubKey, /*out=*/o, /*include_hex=*/true, /*include_address=*/true); out.pushKV("scriptPubKey", std::move(o)); + + if (is_change_func && is_change_func(txout)) { + out.pushKV("ischange", true); + } + vout.push_back(std::move(out)); if (have_undo) { diff --git a/src/crypto/chacha20.cpp b/src/crypto/chacha20.cpp index 9106405..6bdffe6 100644 --- a/src/crypto/chacha20.cpp +++ b/src/crypto/chacha20.cpp @@ -8,11 +8,10 @@ #include #include #include -#include #include #include -#include +#include #define QUARTERROUND(a,b,c,d) \ a += b; d = std::rotl(d ^ a, 16); \ diff --git a/src/crypto/chacha20.h b/src/crypto/chacha20.h index 6653c2c..1503562 100644 --- a/src/crypto/chacha20.h +++ b/src/crypto/chacha20.h @@ -5,12 +5,11 @@ #ifndef BITCOIN_CRYPTO_CHACHA20_H #define BITCOIN_CRYPTO_CHACHA20_H -#include - #include #include #include -#include +#include +#include #include // classes for ChaCha20 256-bit stream cipher developed by Daniel J. Bernstein diff --git a/src/crypto/chacha20poly1305.h b/src/crypto/chacha20poly1305.h index a6c3a66..9a863dd 100644 --- a/src/crypto/chacha20poly1305.h +++ b/src/crypto/chacha20poly1305.h @@ -7,10 +7,10 @@ #include #include +#include #include #include -#include /** The AEAD_CHACHA20_POLY1305 authenticated encryption algorithm from RFC8439 section 2.8. */ class AEADChaCha20Poly1305 diff --git a/src/crypto/hex_base.cpp b/src/crypto/hex_base.cpp index e2eccf5..7098aba 100644 --- a/src/crypto/hex_base.cpp +++ b/src/crypto/hex_base.cpp @@ -5,8 +5,10 @@ #include #include +#include #include #include +#include namespace { diff --git a/src/crypto/hex_base.h b/src/crypto/hex_base.h index 704b19f..9975f7f 100644 --- a/src/crypto/hex_base.h +++ b/src/crypto/hex_base.h @@ -9,6 +9,7 @@ #include #include +#include #include /** diff --git a/src/crypto/hkdf_sha256_32.cpp b/src/crypto/hkdf_sha256_32.cpp index e622494..3cd2cf8 100644 --- a/src/crypto/hkdf_sha256_32.cpp +++ b/src/crypto/hkdf_sha256_32.cpp @@ -4,8 +4,9 @@ #include +#include + #include -#include CHKDF_HMAC_SHA256_L32::CHKDF_HMAC_SHA256_L32(const unsigned char* ikm, size_t ikmlen, const std::string& salt) { diff --git a/src/crypto/hkdf_sha256_32.h b/src/crypto/hkdf_sha256_32.h index f9f343b..7c5d5a7 100644 --- a/src/crypto/hkdf_sha256_32.h +++ b/src/crypto/hkdf_sha256_32.h @@ -5,10 +5,8 @@ #ifndef BITCOIN_CRYPTO_HKDF_SHA256_32_H #define BITCOIN_CRYPTO_HKDF_SHA256_32_H -#include - -#include -#include +#include +#include /** A rfc5869 HKDF implementation with HMAC_SHA256 and fixed key output length of 32 bytes (L=32) */ class CHKDF_HMAC_SHA256_L32 diff --git a/src/crypto/hmac_sha256.cpp b/src/crypto/hmac_sha256.cpp index 0346520..a95ef70 100644 --- a/src/crypto/hmac_sha256.cpp +++ b/src/crypto/hmac_sha256.cpp @@ -4,6 +4,8 @@ #include +#include + #include CHMAC_SHA256::CHMAC_SHA256(const unsigned char* key, size_t keylen) diff --git a/src/crypto/hmac_sha256.h b/src/crypto/hmac_sha256.h index ea8cee8..a26947d 100644 --- a/src/crypto/hmac_sha256.h +++ b/src/crypto/hmac_sha256.h @@ -7,8 +7,7 @@ #include -#include -#include +#include /** A hasher class for HMAC-SHA-256. */ class CHMAC_SHA256 diff --git a/src/crypto/hmac_sha512.cpp b/src/crypto/hmac_sha512.cpp index 245ed29..f37e709 100644 --- a/src/crypto/hmac_sha512.cpp +++ b/src/crypto/hmac_sha512.cpp @@ -4,6 +4,8 @@ #include +#include + #include CHMAC_SHA512::CHMAC_SHA512(const unsigned char* key, size_t keylen) diff --git a/src/crypto/hmac_sha512.h b/src/crypto/hmac_sha512.h index d6bebc0..dfae8d0 100644 --- a/src/crypto/hmac_sha512.h +++ b/src/crypto/hmac_sha512.h @@ -7,8 +7,7 @@ #include -#include -#include +#include /** A hasher class for HMAC-SHA-512. */ class CHMAC_SHA512 diff --git a/src/crypto/muhash.cpp b/src/crypto/muhash.cpp index 02c38c4..c04f58e 100644 --- a/src/crypto/muhash.cpp +++ b/src/crypto/muhash.cpp @@ -7,11 +7,12 @@ #include #include #include +#include +#include #include #include -#include -#include +#include #include namespace { diff --git a/src/crypto/muhash.h b/src/crypto/muhash.h index f340937..5e2be32 100644 --- a/src/crypto/muhash.h +++ b/src/crypto/muhash.h @@ -6,9 +6,12 @@ #define BITCOIN_CRYPTO_MUHASH_H #include -#include +#include #include +#include + +class uint256; class Num3072 { diff --git a/src/crypto/poly1305.cpp b/src/crypto/poly1305.cpp index c9c066b..d6762a1 100644 --- a/src/crypto/poly1305.cpp +++ b/src/crypto/poly1305.cpp @@ -5,8 +5,6 @@ #include #include -#include - namespace poly1305_donna { // Based on the public domain implementation by Andrew Moon diff --git a/src/crypto/poly1305.h b/src/crypto/poly1305.h index 494f7cd..af4aede 100644 --- a/src/crypto/poly1305.h +++ b/src/crypto/poly1305.h @@ -8,8 +8,9 @@ #include #include +#include #include -#include +#include #define POLY1305_BLOCK_SIZE 16 diff --git a/src/crypto/sha256.cpp b/src/crypto/sha256.cpp index d75d01b..8ad9a85 100644 --- a/src/crypto/sha256.cpp +++ b/src/crypto/sha256.cpp @@ -10,7 +10,7 @@ #include #if !defined(DISABLE_OPTIMIZED_SHA256) -#include +#include // IWYU pragma: keep #if defined(__linux__) && defined(ENABLE_ARM_SHANI) #include diff --git a/src/crypto/sha256_arm_shani.cpp b/src/crypto/sha256_arm_shani.cpp index 2ea1d9c..a778e87 100644 --- a/src/crypto/sha256_arm_shani.cpp +++ b/src/crypto/sha256_arm_shani.cpp @@ -13,7 +13,6 @@ #include #include #include -#include #include namespace { diff --git a/src/crypto/sha256_sse4.cpp b/src/crypto/sha256_sse4.cpp index d0a3493..2d37d12 100644 --- a/src/crypto/sha256_sse4.cpp +++ b/src/crypto/sha256_sse4.cpp @@ -5,11 +5,11 @@ // This is a translation to GCC extended asm syntax from YASM code by Intel // (available at the bottom of this file). +#if defined(__x86_64__) || defined(__amd64__) + #include #include -#if defined(__x86_64__) || defined(__amd64__) - namespace sha256_sse4 { void Transform(uint32_t* s, const unsigned char* chunk, size_t blocks) @@ -969,27 +969,27 @@ void Transform(uint32_t* s, const unsigned char* chunk, size_t blocks) /* ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Copyright (c) 2012, Intel Corporation -; -; All rights reserved. -; +; Copyright (c) 2012, Intel Corporation +; +; All rights reserved. +; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions are -; met: -; +; met: +; ; * Redistributions of source code must retain the above copyright -; notice, this list of conditions and the following disclaimer. -; +; notice, this list of conditions and the following disclaimer. +; ; * Redistributions in binary form must reproduce the above copyright ; notice, this list of conditions and the following disclaimer in the ; documentation and/or other materials provided with the -; distribution. -; +; distribution. +; ; * Neither the name of the Intel Corporation nor the names of its ; contributors may be used to endorse or promote products derived from -; this software without specific prior written permission. -; -; +; this software without specific prior written permission. +; +; ; THIS SOFTWARE IS PROVIDED BY INTEL CORPORATION "AS IS" AND ANY ; EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE ; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR @@ -1020,15 +1020,15 @@ void Transform(uint32_t* s, const unsigned char* chunk, size_t blocks) ; This code schedules 1 blocks at a time, with 4 lanes per block ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -%define MOVDQ movdqu ;; assume buffers not aligned +%define MOVDQ movdqu ;; assume buffers not aligned ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Define Macros ; addm [mem], reg ; Add reg to mem using reg-mem add and store %macro addm 2 - add %2, %1 - mov %1, %2 + add %2, %1 + mov %1, %2 %endm ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -1054,31 +1054,31 @@ void Transform(uint32_t* s, const unsigned char* chunk, size_t blocks) %define XTMP4 xmm8 %define XFER xmm9 -%define SHUF_00BA xmm10 ; shuffle xBxA -> 00BA -%define SHUF_DC00 xmm11 ; shuffle xDxC -> DC00 -%define BYTE_FLIP_MASK xmm12 - +%define SHUF_00BA xmm10 ; shuffle xBxA -> 00BA +%define SHUF_DC00 xmm11 ; shuffle xDxC -> DC00 +%define BYTE_FLIP_MASK xmm12 + %ifdef LINUX -%define NUM_BLKS rdx ; 3rd arg -%define CTX rsi ; 2nd arg -%define INP rdi ; 1st arg - -%define SRND rdi ; clobbers INP -%define c ecx -%define d r8d -%define e edx +%define NUM_BLKS rdx ; 3rd arg +%define CTX rsi ; 2nd arg +%define INP rdi ; 1st arg + +%define SRND rdi ; clobbers INP +%define c ecx +%define d r8d +%define e edx %else -%define NUM_BLKS r8 ; 3rd arg -%define CTX rdx ; 2nd arg -%define INP rcx ; 1st arg - -%define SRND rcx ; clobbers INP -%define c edi -%define d esi -%define e r8d - +%define NUM_BLKS r8 ; 3rd arg +%define CTX rdx ; 2nd arg +%define INP rcx ; 1st arg + +%define SRND rcx ; clobbers INP +%define c edi +%define d esi +%define e r8d + %endif -%define TBL rbp +%define TBL rbp %define a eax %define b ebx @@ -1092,22 +1092,22 @@ void Transform(uint32_t* s, const unsigned char* chunk, size_t blocks) -_INP_END_SIZE equ 8 -_INP_SIZE equ 8 -_XFER_SIZE equ 8 +_INP_END_SIZE equ 8 +_INP_SIZE equ 8 +_XFER_SIZE equ 8 %ifdef LINUX -_XMM_SAVE_SIZE equ 0 +_XMM_SAVE_SIZE equ 0 %else -_XMM_SAVE_SIZE equ 7*16 +_XMM_SAVE_SIZE equ 7*16 %endif ; STACK_SIZE plus pushes must be an odd multiple of 8 -_ALIGN_SIZE equ 8 +_ALIGN_SIZE equ 8 -_INP_END equ 0 -_INP equ _INP_END + _INP_END_SIZE -_XFER equ _INP + _INP_SIZE -_XMM_SAVE equ _XFER + _XFER_SIZE + _ALIGN_SIZE -STACK_SIZE equ _XMM_SAVE + _XMM_SAVE_SIZE +_INP_END equ 0 +_INP equ _INP_END + _INP_END_SIZE +_XFER equ _INP + _INP_SIZE +_XMM_SAVE equ _XFER + _XFER_SIZE + _ALIGN_SIZE +STACK_SIZE equ _XMM_SAVE + _XMM_SAVE_SIZE ; rotate_Xs ; Rotate values of symbols X0...X3 @@ -1134,168 +1134,168 @@ STACK_SIZE equ _XMM_SAVE + _XMM_SAVE_SIZE %endm %macro FOUR_ROUNDS_AND_SCHED 0 - ;; compute s0 four at a time and s1 two at a time - ;; compute W[-16] + W[-7] 4 at a time - movdqa XTMP0, X3 - mov y0, e ; y0 = e - ror y0, (25-11) ; y0 = e >> (25-11) - mov y1, a ; y1 = a - palignr XTMP0, X2, 4 ; XTMP0 = W[-7] - ror y1, (22-13) ; y1 = a >> (22-13) - xor y0, e ; y0 = e ^ (e >> (25-11)) - mov y2, f ; y2 = f - ror y0, (11-6) ; y0 = (e >> (11-6)) ^ (e >> (25-6)) - movdqa XTMP1, X1 - xor y1, a ; y1 = a ^ (a >> (22-13) - xor y2, g ; y2 = f^g - paddd XTMP0, X0 ; XTMP0 = W[-7] + W[-16] - xor y0, e ; y0 = e ^ (e >> (11-6)) ^ (e >> (25-6)) - and y2, e ; y2 = (f^g)&e - ror y1, (13-2) ; y1 = (a >> (13-2)) ^ (a >> (22-2)) - ;; compute s0 - palignr XTMP1, X0, 4 ; XTMP1 = W[-15] - xor y1, a ; y1 = a ^ (a >> (13-2)) ^ (a >> (22-2)) - ror y0, 6 ; y0 = S1 = (e>>6) & (e>>11) ^ (e>>25) - xor y2, g ; y2 = CH = ((f^g)&e)^g - movdqa XTMP2, XTMP1 ; XTMP2 = W[-15] - ror y1, 2 ; y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22) - add y2, y0 ; y2 = S1 + CH - add y2, [rsp + _XFER + 0*4] ; y2 = k + w + S1 + CH - movdqa XTMP3, XTMP1 ; XTMP3 = W[-15] - mov y0, a ; y0 = a - add h, y2 ; h = h + S1 + CH + k + w - mov y2, a ; y2 = a - pslld XTMP1, (32-7) - or y0, c ; y0 = a|c - add d, h ; d = d + h + S1 + CH + k + w - and y2, c ; y2 = a&c - psrld XTMP2, 7 - and y0, b ; y0 = (a|c)&b - add h, y1 ; h = h + S1 + CH + k + w + S0 - por XTMP1, XTMP2 ; XTMP1 = W[-15] ror 7 - or y0, y2 ; y0 = MAJ = (a|c)&b)|(a&c) - add h, y0 ; h = h + S1 + CH + k + w + S0 + MAJ + ;; compute s0 four at a time and s1 two at a time + ;; compute W[-16] + W[-7] 4 at a time + movdqa XTMP0, X3 + mov y0, e ; y0 = e + ror y0, (25-11) ; y0 = e >> (25-11) + mov y1, a ; y1 = a + palignr XTMP0, X2, 4 ; XTMP0 = W[-7] + ror y1, (22-13) ; y1 = a >> (22-13) + xor y0, e ; y0 = e ^ (e >> (25-11)) + mov y2, f ; y2 = f + ror y0, (11-6) ; y0 = (e >> (11-6)) ^ (e >> (25-6)) + movdqa XTMP1, X1 + xor y1, a ; y1 = a ^ (a >> (22-13) + xor y2, g ; y2 = f^g + paddd XTMP0, X0 ; XTMP0 = W[-7] + W[-16] + xor y0, e ; y0 = e ^ (e >> (11-6)) ^ (e >> (25-6)) + and y2, e ; y2 = (f^g)&e + ror y1, (13-2) ; y1 = (a >> (13-2)) ^ (a >> (22-2)) + ;; compute s0 + palignr XTMP1, X0, 4 ; XTMP1 = W[-15] + xor y1, a ; y1 = a ^ (a >> (13-2)) ^ (a >> (22-2)) + ror y0, 6 ; y0 = S1 = (e>>6) & (e>>11) ^ (e>>25) + xor y2, g ; y2 = CH = ((f^g)&e)^g + movdqa XTMP2, XTMP1 ; XTMP2 = W[-15] + ror y1, 2 ; y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22) + add y2, y0 ; y2 = S1 + CH + add y2, [rsp + _XFER + 0*4] ; y2 = k + w + S1 + CH + movdqa XTMP3, XTMP1 ; XTMP3 = W[-15] + mov y0, a ; y0 = a + add h, y2 ; h = h + S1 + CH + k + w + mov y2, a ; y2 = a + pslld XTMP1, (32-7) + or y0, c ; y0 = a|c + add d, h ; d = d + h + S1 + CH + k + w + and y2, c ; y2 = a&c + psrld XTMP2, 7 + and y0, b ; y0 = (a|c)&b + add h, y1 ; h = h + S1 + CH + k + w + S0 + por XTMP1, XTMP2 ; XTMP1 = W[-15] ror 7 + or y0, y2 ; y0 = MAJ = (a|c)&b)|(a&c) + add h, y0 ; h = h + S1 + CH + k + w + S0 + MAJ ROTATE_ARGS - movdqa XTMP2, XTMP3 ; XTMP2 = W[-15] - mov y0, e ; y0 = e - mov y1, a ; y1 = a - movdqa XTMP4, XTMP3 ; XTMP4 = W[-15] - ror y0, (25-11) ; y0 = e >> (25-11) - xor y0, e ; y0 = e ^ (e >> (25-11)) - mov y2, f ; y2 = f - ror y1, (22-13) ; y1 = a >> (22-13) - pslld XTMP3, (32-18) - xor y1, a ; y1 = a ^ (a >> (22-13) - ror y0, (11-6) ; y0 = (e >> (11-6)) ^ (e >> (25-6)) - xor y2, g ; y2 = f^g - psrld XTMP2, 18 - ror y1, (13-2) ; y1 = (a >> (13-2)) ^ (a >> (22-2)) - xor y0, e ; y0 = e ^ (e >> (11-6)) ^ (e >> (25-6)) - and y2, e ; y2 = (f^g)&e - ror y0, 6 ; y0 = S1 = (e>>6) & (e>>11) ^ (e>>25) - pxor XTMP1, XTMP3 - xor y1, a ; y1 = a ^ (a >> (13-2)) ^ (a >> (22-2)) - xor y2, g ; y2 = CH = ((f^g)&e)^g - psrld XTMP4, 3 ; XTMP4 = W[-15] >> 3 - add y2, y0 ; y2 = S1 + CH - add y2, [rsp + _XFER + 1*4] ; y2 = k + w + S1 + CH - ror y1, 2 ; y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22) - pxor XTMP1, XTMP2 ; XTMP1 = W[-15] ror 7 ^ W[-15] ror 18 - mov y0, a ; y0 = a - add h, y2 ; h = h + S1 + CH + k + w - mov y2, a ; y2 = a - pxor XTMP1, XTMP4 ; XTMP1 = s0 - or y0, c ; y0 = a|c - add d, h ; d = d + h + S1 + CH + k + w - and y2, c ; y2 = a&c - ;; compute low s1 - pshufd XTMP2, X3, 11111010b ; XTMP2 = W[-2] {BBAA} - and y0, b ; y0 = (a|c)&b - add h, y1 ; h = h + S1 + CH + k + w + S0 - paddd XTMP0, XTMP1 ; XTMP0 = W[-16] + W[-7] + s0 - or y0, y2 ; y0 = MAJ = (a|c)&b)|(a&c) - add h, y0 ; h = h + S1 + CH + k + w + S0 + MAJ + movdqa XTMP2, XTMP3 ; XTMP2 = W[-15] + mov y0, e ; y0 = e + mov y1, a ; y1 = a + movdqa XTMP4, XTMP3 ; XTMP4 = W[-15] + ror y0, (25-11) ; y0 = e >> (25-11) + xor y0, e ; y0 = e ^ (e >> (25-11)) + mov y2, f ; y2 = f + ror y1, (22-13) ; y1 = a >> (22-13) + pslld XTMP3, (32-18) + xor y1, a ; y1 = a ^ (a >> (22-13) + ror y0, (11-6) ; y0 = (e >> (11-6)) ^ (e >> (25-6)) + xor y2, g ; y2 = f^g + psrld XTMP2, 18 + ror y1, (13-2) ; y1 = (a >> (13-2)) ^ (a >> (22-2)) + xor y0, e ; y0 = e ^ (e >> (11-6)) ^ (e >> (25-6)) + and y2, e ; y2 = (f^g)&e + ror y0, 6 ; y0 = S1 = (e>>6) & (e>>11) ^ (e>>25) + pxor XTMP1, XTMP3 + xor y1, a ; y1 = a ^ (a >> (13-2)) ^ (a >> (22-2)) + xor y2, g ; y2 = CH = ((f^g)&e)^g + psrld XTMP4, 3 ; XTMP4 = W[-15] >> 3 + add y2, y0 ; y2 = S1 + CH + add y2, [rsp + _XFER + 1*4] ; y2 = k + w + S1 + CH + ror y1, 2 ; y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22) + pxor XTMP1, XTMP2 ; XTMP1 = W[-15] ror 7 ^ W[-15] ror 18 + mov y0, a ; y0 = a + add h, y2 ; h = h + S1 + CH + k + w + mov y2, a ; y2 = a + pxor XTMP1, XTMP4 ; XTMP1 = s0 + or y0, c ; y0 = a|c + add d, h ; d = d + h + S1 + CH + k + w + and y2, c ; y2 = a&c + ;; compute low s1 + pshufd XTMP2, X3, 11111010b ; XTMP2 = W[-2] {BBAA} + and y0, b ; y0 = (a|c)&b + add h, y1 ; h = h + S1 + CH + k + w + S0 + paddd XTMP0, XTMP1 ; XTMP0 = W[-16] + W[-7] + s0 + or y0, y2 ; y0 = MAJ = (a|c)&b)|(a&c) + add h, y0 ; h = h + S1 + CH + k + w + S0 + MAJ ROTATE_ARGS - movdqa XTMP3, XTMP2 ; XTMP3 = W[-2] {BBAA} - mov y0, e ; y0 = e - mov y1, a ; y1 = a - ror y0, (25-11) ; y0 = e >> (25-11) - movdqa XTMP4, XTMP2 ; XTMP4 = W[-2] {BBAA} - xor y0, e ; y0 = e ^ (e >> (25-11)) - ror y1, (22-13) ; y1 = a >> (22-13) - mov y2, f ; y2 = f - xor y1, a ; y1 = a ^ (a >> (22-13) - ror y0, (11-6) ; y0 = (e >> (11-6)) ^ (e >> (25-6)) - psrlq XTMP2, 17 ; XTMP2 = W[-2] ror 17 {xBxA} - xor y2, g ; y2 = f^g - psrlq XTMP3, 19 ; XTMP3 = W[-2] ror 19 {xBxA} - xor y0, e ; y0 = e ^ (e >> (11-6)) ^ (e >> (25-6)) - and y2, e ; y2 = (f^g)&e - psrld XTMP4, 10 ; XTMP4 = W[-2] >> 10 {BBAA} - ror y1, (13-2) ; y1 = (a >> (13-2)) ^ (a >> (22-2)) - xor y1, a ; y1 = a ^ (a >> (13-2)) ^ (a >> (22-2)) - xor y2, g ; y2 = CH = ((f^g)&e)^g - ror y0, 6 ; y0 = S1 = (e>>6) & (e>>11) ^ (e>>25) - pxor XTMP2, XTMP3 - add y2, y0 ; y2 = S1 + CH - ror y1, 2 ; y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22) - add y2, [rsp + _XFER + 2*4] ; y2 = k + w + S1 + CH - pxor XTMP4, XTMP2 ; XTMP4 = s1 {xBxA} - mov y0, a ; y0 = a - add h, y2 ; h = h + S1 + CH + k + w - mov y2, a ; y2 = a - pshufb XTMP4, SHUF_00BA ; XTMP4 = s1 {00BA} - or y0, c ; y0 = a|c - add d, h ; d = d + h + S1 + CH + k + w - and y2, c ; y2 = a&c - paddd XTMP0, XTMP4 ; XTMP0 = {..., ..., W[1], W[0]} - and y0, b ; y0 = (a|c)&b - add h, y1 ; h = h + S1 + CH + k + w + S0 - ;; compute high s1 - pshufd XTMP2, XTMP0, 01010000b ; XTMP2 = W[-2] {DDCC} - or y0, y2 ; y0 = MAJ = (a|c)&b)|(a&c) - add h, y0 ; h = h + S1 + CH + k + w + S0 + MAJ + movdqa XTMP3, XTMP2 ; XTMP3 = W[-2] {BBAA} + mov y0, e ; y0 = e + mov y1, a ; y1 = a + ror y0, (25-11) ; y0 = e >> (25-11) + movdqa XTMP4, XTMP2 ; XTMP4 = W[-2] {BBAA} + xor y0, e ; y0 = e ^ (e >> (25-11)) + ror y1, (22-13) ; y1 = a >> (22-13) + mov y2, f ; y2 = f + xor y1, a ; y1 = a ^ (a >> (22-13) + ror y0, (11-6) ; y0 = (e >> (11-6)) ^ (e >> (25-6)) + psrlq XTMP2, 17 ; XTMP2 = W[-2] ror 17 {xBxA} + xor y2, g ; y2 = f^g + psrlq XTMP3, 19 ; XTMP3 = W[-2] ror 19 {xBxA} + xor y0, e ; y0 = e ^ (e >> (11-6)) ^ (e >> (25-6)) + and y2, e ; y2 = (f^g)&e + psrld XTMP4, 10 ; XTMP4 = W[-2] >> 10 {BBAA} + ror y1, (13-2) ; y1 = (a >> (13-2)) ^ (a >> (22-2)) + xor y1, a ; y1 = a ^ (a >> (13-2)) ^ (a >> (22-2)) + xor y2, g ; y2 = CH = ((f^g)&e)^g + ror y0, 6 ; y0 = S1 = (e>>6) & (e>>11) ^ (e>>25) + pxor XTMP2, XTMP3 + add y2, y0 ; y2 = S1 + CH + ror y1, 2 ; y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22) + add y2, [rsp + _XFER + 2*4] ; y2 = k + w + S1 + CH + pxor XTMP4, XTMP2 ; XTMP4 = s1 {xBxA} + mov y0, a ; y0 = a + add h, y2 ; h = h + S1 + CH + k + w + mov y2, a ; y2 = a + pshufb XTMP4, SHUF_00BA ; XTMP4 = s1 {00BA} + or y0, c ; y0 = a|c + add d, h ; d = d + h + S1 + CH + k + w + and y2, c ; y2 = a&c + paddd XTMP0, XTMP4 ; XTMP0 = {..., ..., W[1], W[0]} + and y0, b ; y0 = (a|c)&b + add h, y1 ; h = h + S1 + CH + k + w + S0 + ;; compute high s1 + pshufd XTMP2, XTMP0, 01010000b ; XTMP2 = W[-2] {DDCC} + or y0, y2 ; y0 = MAJ = (a|c)&b)|(a&c) + add h, y0 ; h = h + S1 + CH + k + w + S0 + MAJ ROTATE_ARGS - movdqa XTMP3, XTMP2 ; XTMP3 = W[-2] {DDCC} - mov y0, e ; y0 = e - ror y0, (25-11) ; y0 = e >> (25-11) - mov y1, a ; y1 = a - movdqa X0, XTMP2 ; X0 = W[-2] {DDCC} - ror y1, (22-13) ; y1 = a >> (22-13) - xor y0, e ; y0 = e ^ (e >> (25-11)) - mov y2, f ; y2 = f - ror y0, (11-6) ; y0 = (e >> (11-6)) ^ (e >> (25-6)) - psrlq XTMP2, 17 ; XTMP2 = W[-2] ror 17 {xDxC} - xor y1, a ; y1 = a ^ (a >> (22-13) - xor y2, g ; y2 = f^g - psrlq XTMP3, 19 ; XTMP3 = W[-2] ror 19 {xDxC} - xor y0, e ; y0 = e ^ (e >> (11-6)) ^ (e >> (25-6)) - and y2, e ; y2 = (f^g)&e - ror y1, (13-2) ; y1 = (a >> (13-2)) ^ (a >> (22-2)) - psrld X0, 10 ; X0 = W[-2] >> 10 {DDCC} - xor y1, a ; y1 = a ^ (a >> (13-2)) ^ (a >> (22-2)) - ror y0, 6 ; y0 = S1 = (e>>6) & (e>>11) ^ (e>>25) - xor y2, g ; y2 = CH = ((f^g)&e)^g - pxor XTMP2, XTMP3 - ror y1, 2 ; y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22) - add y2, y0 ; y2 = S1 + CH - add y2, [rsp + _XFER + 3*4] ; y2 = k + w + S1 + CH - pxor X0, XTMP2 ; X0 = s1 {xDxC} - mov y0, a ; y0 = a - add h, y2 ; h = h + S1 + CH + k + w - mov y2, a ; y2 = a - pshufb X0, SHUF_DC00 ; X0 = s1 {DC00} - or y0, c ; y0 = a|c - add d, h ; d = d + h + S1 + CH + k + w - and y2, c ; y2 = a&c - paddd X0, XTMP0 ; X0 = {W[3], W[2], W[1], W[0]} - and y0, b ; y0 = (a|c)&b - add h, y1 ; h = h + S1 + CH + k + w + S0 - or y0, y2 ; y0 = MAJ = (a|c)&b)|(a&c) - add h, y0 ; h = h + S1 + CH + k + w + S0 + MAJ + movdqa XTMP3, XTMP2 ; XTMP3 = W[-2] {DDCC} + mov y0, e ; y0 = e + ror y0, (25-11) ; y0 = e >> (25-11) + mov y1, a ; y1 = a + movdqa X0, XTMP2 ; X0 = W[-2] {DDCC} + ror y1, (22-13) ; y1 = a >> (22-13) + xor y0, e ; y0 = e ^ (e >> (25-11)) + mov y2, f ; y2 = f + ror y0, (11-6) ; y0 = (e >> (11-6)) ^ (e >> (25-6)) + psrlq XTMP2, 17 ; XTMP2 = W[-2] ror 17 {xDxC} + xor y1, a ; y1 = a ^ (a >> (22-13) + xor y2, g ; y2 = f^g + psrlq XTMP3, 19 ; XTMP3 = W[-2] ror 19 {xDxC} + xor y0, e ; y0 = e ^ (e >> (11-6)) ^ (e >> (25-6)) + and y2, e ; y2 = (f^g)&e + ror y1, (13-2) ; y1 = (a >> (13-2)) ^ (a >> (22-2)) + psrld X0, 10 ; X0 = W[-2] >> 10 {DDCC} + xor y1, a ; y1 = a ^ (a >> (13-2)) ^ (a >> (22-2)) + ror y0, 6 ; y0 = S1 = (e>>6) & (e>>11) ^ (e>>25) + xor y2, g ; y2 = CH = ((f^g)&e)^g + pxor XTMP2, XTMP3 + ror y1, 2 ; y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22) + add y2, y0 ; y2 = S1 + CH + add y2, [rsp + _XFER + 3*4] ; y2 = k + w + S1 + CH + pxor X0, XTMP2 ; X0 = s1 {xDxC} + mov y0, a ; y0 = a + add h, y2 ; h = h + S1 + CH + k + w + mov y2, a ; y2 = a + pshufb X0, SHUF_DC00 ; X0 = s1 {DC00} + or y0, c ; y0 = a|c + add d, h ; d = d + h + S1 + CH + k + w + and y2, c ; y2 = a&c + paddd X0, XTMP0 ; X0 = {W[3], W[2], W[1], W[0]} + and y0, b ; y0 = (a|c)&b + add h, y1 ; h = h + S1 + CH + k + w + S0 + or y0, y2 ; y0 = MAJ = (a|c)&b)|(a&c) + add h, y0 ; h = h + S1 + CH + k + w + S0 + MAJ ROTATE_ARGS rotate_Xs @@ -1303,34 +1303,34 @@ rotate_Xs ;; input is [rsp + _XFER + %1 * 4] %macro DO_ROUND 1 - mov y0, e ; y0 = e - ror y0, (25-11) ; y0 = e >> (25-11) - mov y1, a ; y1 = a - xor y0, e ; y0 = e ^ (e >> (25-11)) - ror y1, (22-13) ; y1 = a >> (22-13) - mov y2, f ; y2 = f - xor y1, a ; y1 = a ^ (a >> (22-13) - ror y0, (11-6) ; y0 = (e >> (11-6)) ^ (e >> (25-6)) - xor y2, g ; y2 = f^g - xor y0, e ; y0 = e ^ (e >> (11-6)) ^ (e >> (25-6)) - ror y1, (13-2) ; y1 = (a >> (13-2)) ^ (a >> (22-2)) - and y2, e ; y2 = (f^g)&e - xor y1, a ; y1 = a ^ (a >> (13-2)) ^ (a >> (22-2)) - ror y0, 6 ; y0 = S1 = (e>>6) & (e>>11) ^ (e>>25) - xor y2, g ; y2 = CH = ((f^g)&e)^g - add y2, y0 ; y2 = S1 + CH - ror y1, 2 ; y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22) - add y2, [rsp + _XFER + %1 * 4] ; y2 = k + w + S1 + CH - mov y0, a ; y0 = a - add h, y2 ; h = h + S1 + CH + k + w - mov y2, a ; y2 = a - or y0, c ; y0 = a|c - add d, h ; d = d + h + S1 + CH + k + w - and y2, c ; y2 = a&c - and y0, b ; y0 = (a|c)&b - add h, y1 ; h = h + S1 + CH + k + w + S0 - or y0, y2 ; y0 = MAJ = (a|c)&b)|(a&c) - add h, y0 ; h = h + S1 + CH + k + w + S0 + MAJ + mov y0, e ; y0 = e + ror y0, (25-11) ; y0 = e >> (25-11) + mov y1, a ; y1 = a + xor y0, e ; y0 = e ^ (e >> (25-11)) + ror y1, (22-13) ; y1 = a >> (22-13) + mov y2, f ; y2 = f + xor y1, a ; y1 = a ^ (a >> (22-13) + ror y0, (11-6) ; y0 = (e >> (11-6)) ^ (e >> (25-6)) + xor y2, g ; y2 = f^g + xor y0, e ; y0 = e ^ (e >> (11-6)) ^ (e >> (25-6)) + ror y1, (13-2) ; y1 = (a >> (13-2)) ^ (a >> (22-2)) + and y2, e ; y2 = (f^g)&e + xor y1, a ; y1 = a ^ (a >> (13-2)) ^ (a >> (22-2)) + ror y0, 6 ; y0 = S1 = (e>>6) & (e>>11) ^ (e>>25) + xor y2, g ; y2 = CH = ((f^g)&e)^g + add y2, y0 ; y2 = S1 + CH + ror y1, 2 ; y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22) + add y2, [rsp + _XFER + %1 * 4] ; y2 = k + w + S1 + CH + mov y0, a ; y0 = a + add h, y2 ; h = h + S1 + CH + k + w + mov y2, a ; y2 = a + or y0, c ; y0 = a|c + add d, h ; d = d + h + S1 + CH + k + w + and y2, c ; y2 = a&c + and y0, b ; y0 = (a|c)&b + add h, y1 ; h = h + S1 + CH + k + w + S0 + or y0, y2 ; y0 = MAJ = (a|c)&b)|(a&c) + add h, y0 ; h = h + S1 + CH + k + w + S0 + MAJ ROTATE_ARGS %endm @@ -1344,166 +1344,166 @@ section .text global sha256_sse4 align 32 sha256_sse4: - push rbx + push rbx %ifndef LINUX - push rsi - push rdi + push rsi + push rdi %endif - push rbp - push r13 - push r14 - push r15 + push rbp + push r13 + push r14 + push r15 - sub rsp,STACK_SIZE + sub rsp,STACK_SIZE %ifndef LINUX - movdqa [rsp + _XMM_SAVE + 0*16],xmm6 - movdqa [rsp + _XMM_SAVE + 1*16],xmm7 - movdqa [rsp + _XMM_SAVE + 2*16],xmm8 - movdqa [rsp + _XMM_SAVE + 3*16],xmm9 - movdqa [rsp + _XMM_SAVE + 4*16],xmm10 - movdqa [rsp + _XMM_SAVE + 5*16],xmm11 - movdqa [rsp + _XMM_SAVE + 6*16],xmm12 + movdqa [rsp + _XMM_SAVE + 0*16],xmm6 + movdqa [rsp + _XMM_SAVE + 1*16],xmm7 + movdqa [rsp + _XMM_SAVE + 2*16],xmm8 + movdqa [rsp + _XMM_SAVE + 3*16],xmm9 + movdqa [rsp + _XMM_SAVE + 4*16],xmm10 + movdqa [rsp + _XMM_SAVE + 5*16],xmm11 + movdqa [rsp + _XMM_SAVE + 6*16],xmm12 %endif - shl NUM_BLKS, 6 ; convert to bytes - jz done_hash - add NUM_BLKS, INP ; pointer to end of data - mov [rsp + _INP_END], NUM_BLKS + shl NUM_BLKS, 6 ; convert to bytes + jz done_hash + add NUM_BLKS, INP ; pointer to end of data + mov [rsp + _INP_END], NUM_BLKS ;; load initial digest - mov a,[4*0 + CTX] - mov b,[4*1 + CTX] - mov c,[4*2 + CTX] - mov d,[4*3 + CTX] - mov e,[4*4 + CTX] - mov f,[4*5 + CTX] - mov g,[4*6 + CTX] - mov h,[4*7 + CTX] - - movdqa BYTE_FLIP_MASK, [PSHUFFLE_BYTE_FLIP_MASK wrt rip] - movdqa SHUF_00BA, [_SHUF_00BA wrt rip] - movdqa SHUF_DC00, [_SHUF_DC00 wrt rip] + mov a,[4*0 + CTX] + mov b,[4*1 + CTX] + mov c,[4*2 + CTX] + mov d,[4*3 + CTX] + mov e,[4*4 + CTX] + mov f,[4*5 + CTX] + mov g,[4*6 + CTX] + mov h,[4*7 + CTX] + + movdqa BYTE_FLIP_MASK, [PSHUFFLE_BYTE_FLIP_MASK wrt rip] + movdqa SHUF_00BA, [_SHUF_00BA wrt rip] + movdqa SHUF_DC00, [_SHUF_DC00 wrt rip] loop0: - lea TBL,[K256 wrt rip] + lea TBL,[K256 wrt rip] ;; byte swap first 16 dwords - COPY_XMM_AND_BSWAP X0, [INP + 0*16], BYTE_FLIP_MASK - COPY_XMM_AND_BSWAP X1, [INP + 1*16], BYTE_FLIP_MASK - COPY_XMM_AND_BSWAP X2, [INP + 2*16], BYTE_FLIP_MASK - COPY_XMM_AND_BSWAP X3, [INP + 3*16], BYTE_FLIP_MASK - - mov [rsp + _INP], INP + COPY_XMM_AND_BSWAP X0, [INP + 0*16], BYTE_FLIP_MASK + COPY_XMM_AND_BSWAP X1, [INP + 1*16], BYTE_FLIP_MASK + COPY_XMM_AND_BSWAP X2, [INP + 2*16], BYTE_FLIP_MASK + COPY_XMM_AND_BSWAP X3, [INP + 3*16], BYTE_FLIP_MASK + + mov [rsp + _INP], INP ;; schedule 48 input dwords, by doing 3 rounds of 16 each - mov SRND, 3 + mov SRND, 3 align 16 loop1: - movdqa XFER, [TBL + 0*16] - paddd XFER, X0 - movdqa [rsp + _XFER], XFER + movdqa XFER, [TBL + 0*16] + paddd XFER, X0 + movdqa [rsp + _XFER], XFER FOUR_ROUNDS_AND_SCHED - movdqa XFER, [TBL + 1*16] - paddd XFER, X0 - movdqa [rsp + _XFER], XFER + movdqa XFER, [TBL + 1*16] + paddd XFER, X0 + movdqa [rsp + _XFER], XFER FOUR_ROUNDS_AND_SCHED - movdqa XFER, [TBL + 2*16] - paddd XFER, X0 - movdqa [rsp + _XFER], XFER + movdqa XFER, [TBL + 2*16] + paddd XFER, X0 + movdqa [rsp + _XFER], XFER FOUR_ROUNDS_AND_SCHED - movdqa XFER, [TBL + 3*16] - paddd XFER, X0 - movdqa [rsp + _XFER], XFER - add TBL, 4*16 + movdqa XFER, [TBL + 3*16] + paddd XFER, X0 + movdqa [rsp + _XFER], XFER + add TBL, 4*16 FOUR_ROUNDS_AND_SCHED - sub SRND, 1 - jne loop1 + sub SRND, 1 + jne loop1 - mov SRND, 2 + mov SRND, 2 loop2: - paddd X0, [TBL + 0*16] - movdqa [rsp + _XFER], X0 - DO_ROUND 0 - DO_ROUND 1 - DO_ROUND 2 - DO_ROUND 3 - paddd X1, [TBL + 1*16] - movdqa [rsp + _XFER], X1 - add TBL, 2*16 - DO_ROUND 0 - DO_ROUND 1 - DO_ROUND 2 - DO_ROUND 3 - - movdqa X0, X2 - movdqa X1, X3 - - sub SRND, 1 - jne loop2 - - addm [4*0 + CTX],a - addm [4*1 + CTX],b - addm [4*2 + CTX],c - addm [4*3 + CTX],d - addm [4*4 + CTX],e - addm [4*5 + CTX],f - addm [4*6 + CTX],g - addm [4*7 + CTX],h - - mov INP, [rsp + _INP] - add INP, 64 - cmp INP, [rsp + _INP_END] - jne loop0 + paddd X0, [TBL + 0*16] + movdqa [rsp + _XFER], X0 + DO_ROUND 0 + DO_ROUND 1 + DO_ROUND 2 + DO_ROUND 3 + paddd X1, [TBL + 1*16] + movdqa [rsp + _XFER], X1 + add TBL, 2*16 + DO_ROUND 0 + DO_ROUND 1 + DO_ROUND 2 + DO_ROUND 3 + + movdqa X0, X2 + movdqa X1, X3 + + sub SRND, 1 + jne loop2 + + addm [4*0 + CTX],a + addm [4*1 + CTX],b + addm [4*2 + CTX],c + addm [4*3 + CTX],d + addm [4*4 + CTX],e + addm [4*5 + CTX],f + addm [4*6 + CTX],g + addm [4*7 + CTX],h + + mov INP, [rsp + _INP] + add INP, 64 + cmp INP, [rsp + _INP_END] + jne loop0 done_hash: %ifndef LINUX - movdqa xmm6,[rsp + _XMM_SAVE + 0*16] - movdqa xmm7,[rsp + _XMM_SAVE + 1*16] - movdqa xmm8,[rsp + _XMM_SAVE + 2*16] - movdqa xmm9,[rsp + _XMM_SAVE + 3*16] - movdqa xmm10,[rsp + _XMM_SAVE + 4*16] - movdqa xmm11,[rsp + _XMM_SAVE + 5*16] - movdqa xmm12,[rsp + _XMM_SAVE + 6*16] + movdqa xmm6,[rsp + _XMM_SAVE + 0*16] + movdqa xmm7,[rsp + _XMM_SAVE + 1*16] + movdqa xmm8,[rsp + _XMM_SAVE + 2*16] + movdqa xmm9,[rsp + _XMM_SAVE + 3*16] + movdqa xmm10,[rsp + _XMM_SAVE + 4*16] + movdqa xmm11,[rsp + _XMM_SAVE + 5*16] + movdqa xmm12,[rsp + _XMM_SAVE + 6*16] %endif - add rsp, STACK_SIZE + add rsp, STACK_SIZE - pop r15 - pop r14 - pop r13 - pop rbp + pop r15 + pop r14 + pop r13 + pop rbp %ifndef LINUX - pop rdi - pop rsi + pop rdi + pop rsi %endif - pop rbx + pop rbx + + ret - ret - section .data align 64 K256: - dd 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 - dd 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5 - dd 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3 - dd 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174 - dd 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc - dd 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da - dd 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7 - dd 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967 - dd 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13 - dd 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85 - dd 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3 - dd 0xd192e819,0xd6990624,0xf40e3585,0x106aa070 - dd 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5 - dd 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 - dd 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 - dd 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 + dd 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 + dd 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5 + dd 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3 + dd 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174 + dd 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc + dd 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da + dd 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7 + dd 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967 + dd 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13 + dd 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85 + dd 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3 + dd 0xd192e819,0xd6990624,0xf40e3585,0x106aa070 + dd 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5 + dd 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 + dd 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 + dd 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 PSHUFFLE_BYTE_FLIP_MASK: ddq 0x0c0d0e0f08090a0b0405060700010203 diff --git a/src/crypto/sha256_x86_shani.cpp b/src/crypto/sha256_x86_shani.cpp index 5ee957a..daccdc5 100644 --- a/src/crypto/sha256_x86_shani.cpp +++ b/src/crypto/sha256_x86_shani.cpp @@ -8,6 +8,7 @@ #if defined(ENABLE_SSE41) && defined(ENABLE_X86_SHANI) +#include #include #include diff --git a/src/crypto/sha3.cpp b/src/crypto/sha3.cpp index 4c61bce..f684447 100644 --- a/src/crypto/sha3.cpp +++ b/src/crypto/sha3.cpp @@ -9,9 +9,9 @@ #include #include -#include #include -#include +#include +#include #include void KeccakF(uint64_t (&st)[25]) diff --git a/src/crypto/sha3.h b/src/crypto/sha3.h index 7a538e4..91cdb5c 100644 --- a/src/crypto/sha3.h +++ b/src/crypto/sha3.h @@ -5,10 +5,9 @@ #ifndef BITCOIN_CRYPTO_SHA3_H #define BITCOIN_CRYPTO_SHA3_H -#include - #include #include +#include //! The Keccak-f[1600] transform. void KeccakF(uint64_t (&st)[25]); diff --git a/src/crypto/siphash.cpp b/src/crypto/siphash.cpp index dda28a6..1a9eb77 100644 --- a/src/crypto/siphash.cpp +++ b/src/crypto/siphash.cpp @@ -4,7 +4,11 @@ #include +#include + #include +#include +#include #define SIPROUND do { \ v0 += v1; v1 = std::rotl(v1, 13); v1 ^= v0; \ diff --git a/src/crypto/siphash.h b/src/crypto/siphash.h index 4c6a39e..8d41a08 100644 --- a/src/crypto/siphash.h +++ b/src/crypto/siphash.h @@ -6,9 +6,9 @@ #define BITCOIN_CRYPTO_SIPHASH_H #include +#include -#include -#include +class uint256; /** SipHash-2-4 */ class CSipHasher diff --git a/src/dbwrapper.cpp b/src/dbwrapper.cpp index fe5f9cb..cb9abee 100644 --- a/src/dbwrapper.cpp +++ b/src/dbwrapper.cpp @@ -274,7 +274,7 @@ CDBWrapper::~CDBWrapper() DBContext().options.env = nullptr; } -bool CDBWrapper::WriteBatch(CDBBatch& batch, bool fSync) +void CDBWrapper::WriteBatch(CDBBatch& batch, bool fSync) { const bool log_memory = LogAcceptCategory(BCLog::LEVELDB, BCLog::Level::Debug); double mem_before = 0; @@ -288,7 +288,6 @@ bool CDBWrapper::WriteBatch(CDBBatch& batch, bool fSync) LogDebug(BCLog::LEVELDB, "WriteBatch memory usage: db=%s, before=%.1fMiB, after=%.1fMiB\n", m_name, mem_before, mem_after); } - return true; } size_t CDBWrapper::DynamicMemoryUsage() const diff --git a/src/dbwrapper.h b/src/dbwrapper.h index b9b98bd..c770df8 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -230,11 +230,11 @@ class CDBWrapper } template - bool Write(const K& key, const V& value, bool fSync = false) + void Write(const K& key, const V& value, bool fSync = false) { CDBBatch batch(*this); batch.Write(key, value); - return WriteBatch(batch, fSync); + WriteBatch(batch, fSync); } //! @returns filesystem path to the on-disk data. @@ -255,14 +255,14 @@ class CDBWrapper } template - bool Erase(const K& key, bool fSync = false) + void Erase(const K& key, bool fSync = false) { CDBBatch batch(*this); batch.Erase(key); - return WriteBatch(batch, fSync); + WriteBatch(batch, fSync); } - bool WriteBatch(CDBBatch& batch, bool fSync = false); + void WriteBatch(CDBBatch& batch, bool fSync = false); // Get an estimate of LevelDB memory usage (in bytes). size_t DynamicMemoryUsage() const; diff --git a/src/flatfile.h b/src/flatfile.h index 3edb0b8..3ec4eaf 100644 --- a/src/flatfile.h +++ b/src/flatfile.h @@ -13,16 +13,16 @@ struct FlatFilePos { - int nFile{-1}; - unsigned int nPos{0}; + int32_t nFile{-1}; + uint32_t nPos{0}; SERIALIZE_METHODS(FlatFilePos, obj) { READWRITE(VARINT_MODE(obj.nFile, VarIntMode::NONNEGATIVE_SIGNED), VARINT(obj.nPos)); } FlatFilePos() = default; - FlatFilePos(int nFileIn, unsigned int nPosIn) : - nFile(nFileIn), - nPos(nPosIn) + FlatFilePos(int32_t nFileIn, uint32_t nPosIn) + : nFile{nFileIn}, + nPos{nPosIn} {} friend bool operator==(const FlatFilePos &a, const FlatFilePos &b) { diff --git a/src/headerssync.cpp b/src/headerssync.cpp index ae7187f..e52a8cd 100644 --- a/src/headerssync.cpp +++ b/src/headerssync.cpp @@ -1,32 +1,26 @@ -// Copyright (c) 2022 The Bitcoin Core developers +// Copyright (c) 2022-present The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include + #include #include #include #include #include -// The two constants below are computed using the simulation script in -// contrib/devtools/headerssync-params.py. - -//! Store one header commitment per HEADER_COMMITMENT_PERIOD blocks. -constexpr size_t HEADER_COMMITMENT_PERIOD{632}; - -//! Only feed headers to validation once this many headers on top have been -//! received and validated against commitments. -constexpr size_t REDOWNLOAD_BUFFER_SIZE{15009}; // 15009/632 = ~23.7 commitments - -// Our memory analysis assumes 48 bytes for a CompressedHeader (so we should -// re-calculate parameters if we compress further) +// Our memory analysis in headerssync-params.py assumes this many bytes for a +// CompressedHeader (we should re-calculate parameters if we compress further). static_assert(sizeof(CompressedHeader) == 48); HeadersSyncState::HeadersSyncState(NodeId id, const Consensus::Params& consensus_params, - const CBlockIndex* chain_start, const arith_uint256& minimum_required_work) : - m_commit_offset(FastRandomContext().randrange(HEADER_COMMITMENT_PERIOD)), + const HeadersSyncParams& params, const CBlockIndex* chain_start, + const arith_uint256& minimum_required_work) : + m_commit_offset((assert(params.commitment_period > 0), // HeadersSyncParams field must be initialized to non-zero. + FastRandomContext().randrange(params.commitment_period))), m_id(id), m_consensus_params(consensus_params), + m_params(params), m_chain_start(chain_start), m_minimum_required_work(minimum_required_work), m_current_chain_work(chain_start->nChainWork), @@ -41,7 +35,9 @@ HeadersSyncState::HeadersSyncState(NodeId id, const Consensus::Params& consensus // exceeds this bound, because it's not possible for a consensus-valid // chain to be longer than this (at the current time -- in the future we // could try again, if necessary, to sync a longer chain). - m_max_commitments = 6*(Ticks(NodeClock::now() - NodeSeconds{std::chrono::seconds{chain_start->GetMedianTimePast()}}) + MAX_FUTURE_BLOCK_TIME) / HEADER_COMMITMENT_PERIOD; + const auto max_seconds_since_start{(Ticks(NodeClock::now() - NodeSeconds{std::chrono::seconds{chain_start->GetMedianTimePast()}})) + + MAX_FUTURE_BLOCK_TIME}; + m_max_commitments = 6 * max_seconds_since_start / m_params.commitment_period; LogDebug(BCLog::NET, "Initial headers sync started with peer=%d: height=%i, max_commitments=%i, min_work=%s\n", m_id, m_current_height, m_max_commitments, m_minimum_required_work.ToString()); } @@ -66,8 +62,8 @@ void HeadersSyncState::Finalize() /** Process the next batch of headers received from our peer. * Validate and store commitments, and compare total chainwork to our target to * see if we can switch to REDOWNLOAD mode. */ -HeadersSyncState::ProcessingResult HeadersSyncState::ProcessNextHeaders(const - std::vector& received_headers, const bool full_headers_message) +HeadersSyncState::ProcessingResult HeadersSyncState::ProcessNextHeaders( + std::span received_headers, const bool full_headers_message) { ProcessingResult ret; @@ -137,7 +133,7 @@ HeadersSyncState::ProcessingResult HeadersSyncState::ProcessNextHeaders(const return ret; } -bool HeadersSyncState::ValidateAndStoreHeadersCommitments(const std::vector& headers) +bool HeadersSyncState::ValidateAndStoreHeadersCommitments(std::span headers) { // The caller should not give us an empty set of headers. Assume(headers.size() > 0); @@ -193,7 +189,7 @@ bool HeadersSyncState::ValidateAndProcessSingleHeader(const CBlockHeader& curren return false; } - if (next_height % HEADER_COMMITMENT_PERIOD == m_commit_offset) { + if (next_height % m_params.commitment_period == m_commit_offset) { // Add a commitment. m_header_commitments.push_back(m_hasher(current.GetHash()) & 1); if (m_header_commitments.size() > m_max_commitments) { @@ -254,7 +250,7 @@ bool HeadersSyncState::ValidateAndStoreRedownloadedHeader(const CBlockHeader& he // it's possible our peer has extended its chain between our first sync and // our second, and we don't want to return failure after we've seen our // target blockhash just because we ran out of commitments. - if (!m_process_all_remaining_headers && next_height % HEADER_COMMITMENT_PERIOD == m_commit_offset) { + if (!m_process_all_remaining_headers && next_height % m_params.commitment_period == m_commit_offset) { if (m_header_commitments.size() == 0) { LogDebug(BCLog::NET, "Initial headers sync aborted with peer=%d: commitment overrun at height=%i (redownload phase)\n", m_id, next_height); // Somehow our peer managed to feed us a different chain and @@ -285,7 +281,7 @@ std::vector HeadersSyncState::PopHeadersReadyForAcceptance() Assume(m_download_state == State::REDOWNLOAD); if (m_download_state != State::REDOWNLOAD) return ret; - while (m_redownloaded_headers.size() > REDOWNLOAD_BUFFER_SIZE || + while (m_redownloaded_headers.size() > m_params.redownload_buffer_size || (m_redownloaded_headers.size() > 0 && m_process_all_remaining_headers)) { ret.emplace_back(m_redownloaded_headers.front().GetFullHeader(m_redownload_buffer_first_prev_hash)); m_redownloaded_headers.pop_front(); diff --git a/src/headerssync.h b/src/headerssync.h index 56380c6..9e3af58 100644 --- a/src/headerssync.h +++ b/src/headerssync.h @@ -136,7 +136,8 @@ class HeadersSyncState { * minimum_required_work: amount of chain work required to accept the chain */ HeadersSyncState(NodeId id, const Consensus::Params& consensus_params, - const CBlockIndex* chain_start, const arith_uint256& minimum_required_work); + const HeadersSyncParams& params, const CBlockIndex* chain_start, + const arith_uint256& minimum_required_work); /** Result data structure for ProcessNextHeaders. */ struct ProcessingResult { @@ -165,7 +166,7 @@ class HeadersSyncState { * ProcessingResult.request_more: if true, the caller is suggested to call * NextHeadersRequestLocator and send a getheaders message using it. */ - ProcessingResult ProcessNextHeaders(const std::vector& + ProcessingResult ProcessNextHeaders(std::span received_headers, bool full_headers_message); /** Issue the next GETHEADERS message to our peer. @@ -179,8 +180,8 @@ class HeadersSyncState { /** The (secret) offset on the heights for which to create commitments. * * m_header_commitments entries are created at any height h for which - * (h % HEADER_COMMITMENT_PERIOD) == m_commit_offset. */ - const unsigned m_commit_offset; + * (h % m_params.commitment_period) == m_commit_offset. */ + const size_t m_commit_offset; private: /** Clear out all download state that might be in progress (freeing any used @@ -195,7 +196,7 @@ class HeadersSyncState { * processed headers. * On failure, this invokes Finalize() and returns false. */ - bool ValidateAndStoreHeadersCommitments(const std::vector& headers); + bool ValidateAndStoreHeadersCommitments(std::span headers); /** In PRESYNC, process and update state for a single header */ bool ValidateAndProcessSingleHeader(const CBlockHeader& current); @@ -214,6 +215,9 @@ class HeadersSyncState { /** We use the consensus params in our anti-DoS calculations */ const Consensus::Params& m_consensus_params; + /** Parameters that impact memory usage for a given chain, especially when attacked. */ + const HeadersSyncParams m_params; + /** Store the last block in our block index that the peer's chain builds from */ const CBlockIndex* m_chain_start{nullptr}; diff --git a/src/index/base.cpp b/src/index/base.cpp index 82259ac..d5c4044 100644 --- a/src/index/base.cpp +++ b/src/index/base.cpp @@ -2,10 +2,13 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include -#include #include + +#include +#include +#include #include +#include #include #include #include @@ -13,20 +16,31 @@ #include #include #include +#include +#include #include +#include #include +#include #include #include +#include +#include #include #include +#include -#include +#include +#include +#include #include #include +#include #include #include #include #include +#include constexpr uint8_t DB_BEST_BLOCK{'B'}; @@ -261,7 +275,7 @@ bool BaseIndex::Commit() ok = CustomCommit(batch); if (ok) { GetDB().WriteBestBlock(batch, GetLocator(*m_chain, m_best_block_index.load()->GetBlockHash())); - ok = GetDB().WriteBatch(batch); + GetDB().WriteBatch(batch); } } if (!ok) { diff --git a/src/index/base.h b/src/index/base.h index 4131b06..5a6f0a4 100644 --- a/src/index/base.h +++ b/src/index/base.h @@ -5,29 +5,41 @@ #ifndef BITCOIN_INDEX_BASE_H #define BITCOIN_INDEX_BASE_H +#include #include #include -#include -#include +#include +#include +#include +#include #include #include +#include +#include +#include +#include #include +#include class CBlock; class CBlockIndex; class Chainstate; -class ChainstateManager; -namespace interfaces { -class Chain; -} // namespace interfaces +struct CBlockLocator; struct IndexSummary { std::string name; bool synced{false}; int best_block_height{0}; uint256 best_block_hash; }; +namespace interfaces { +struct BlockRef; +} +namespace util { +template +struct ConstevalFormatString; +} /** * Base class for indices of blockchain data. This implements diff --git a/src/index/blockfilterindex.cpp b/src/index/blockfilterindex.cpp index 2ccae3a..4bf2f5e 100644 --- a/src/index/blockfilterindex.cpp +++ b/src/index/blockfilterindex.cpp @@ -2,19 +2,39 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include +#include -#include +#include +#include #include #include +#include #include -#include +#include +#include +#include #include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + /* The index database stores three items for each block: the disk location of the encoded filter, * its dSHA256 hash, and the header. Those belonging to blocks on the active chain are indexed by * height, and those belonging to blocks that have been reorganized out of the active chain are @@ -202,9 +222,9 @@ size_t BlockFilterIndex::WriteFilterToDisk(FlatFilePos& pos, const BlockFilter& { assert(filter.GetFilterType() == GetFilterType()); - size_t data_size = + uint64_t data_size{ GetSerializeSize(filter.GetBlockHash()) + - GetSerializeSize(filter.GetEncodedFilter()); + GetSerializeSize(filter.GetEncodedFilter())}; // If writing the filter would overflow the file, flush and move to the next one. if (pos.nPos + data_size > MAX_FLTR_FILE_SIZE) { @@ -291,9 +311,7 @@ bool BlockFilterIndex::Write(const BlockFilter& filter, uint32_t block_height, c value.second.header = filter_header; value.second.pos = m_next_filter_pos; - if (!m_db->Write(DBHeightKey(block_height), value)) { - return false; - } + m_db->Write(DBHeightKey(block_height), value); m_next_filter_pos.nPos += bytes_written; return true; @@ -338,7 +356,7 @@ bool BlockFilterIndex::CustomRemove(const interfaces::BlockInfo& block) // But since this creates new references to the filter, the position should get updated here // atomically as well in case Commit fails. batch.Write(DB_FILTER_POS, m_next_filter_pos); - if (!m_db->WriteBatch(batch)) return false; + m_db->WriteBatch(batch); // Update cached header to the previous block hash m_last_header = *Assert(ReadFilterHeader(block.height - 1, *Assert(block.prev_hash))); diff --git a/src/index/blockfilterindex.h b/src/index/blockfilterindex.h index 983cf32..b1ebd55 100644 --- a/src/index/blockfilterindex.h +++ b/src/index/blockfilterindex.h @@ -6,13 +6,24 @@ #define BITCOIN_INDEX_BLOCKFILTERINDEX_H #include -#include -#include #include #include +#include +#include +#include #include +#include +#include +#include +#include +#include #include +#include + +class BlockFilter; +class CBlockIndex; +enum class BlockFilterType : uint8_t; static const char* const DEFAULT_BLOCKFILTERINDEX = "0"; diff --git a/src/index/coinstatsindex.cpp b/src/index/coinstatsindex.cpp index af798e2..72bc09c 100644 --- a/src/index/coinstatsindex.cpp +++ b/src/index/coinstatsindex.cpp @@ -2,20 +2,39 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include + #include +#include #include #include #include +#include #include -#include +#include +#include +#include +#include #include #include -#include +#include +#include +#include