Skip to content

Commit

Permalink
Merge branch 'auxpow'
Browse files Browse the repository at this point in the history
Updated name RPCs for bitcoin/bitcoin#20012.

The rpc_help.py regtest now verifies that arguments with a certain name
(independent of the underlying method) are either converted or not by
the RPC client.  This causes a mismatch between the name RPCs'
"value" (which is a string and not converted) and the "setwalletflag"
RPC (whose value argument is a bool).  To fix this and keep compatibility
with name clients, we change the name of setwalletflag's argument to
"newvalue".
  • Loading branch information
domob1812 committed Feb 1, 2021
2 parents b2c49a6 + dd4f6f1 commit c4af2fd
Show file tree
Hide file tree
Showing 100 changed files with 1,412 additions and 908 deletions.
2 changes: 1 addition & 1 deletion .cirrus.yml
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ task:
FILE_ENV: "./ci/test/00_setup_env_native_asan.sh"

task:
name: '[no depends, sanitizers: fuzzer,address,undefined] [focal]'
name: '[no depends, sanitizers: fuzzer,address,undefined,integer] [focal]'
<< : *GLOBAL_TASK_TEMPLATE
container:
image: ubuntu:focal
Expand Down
3 changes: 3 additions & 0 deletions Makefile.am
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,9 @@ OSX_INSTALLER_ICONS=$(top_srcdir)/src/qt/res/icons/bitcoin.icns
OSX_PLIST=$(top_builddir)/share/qt/Info.plist #not installed

DIST_CONTRIB = \
$(top_srcdir)/test/sanitizer_suppressions/lsan \
$(top_srcdir)/test/sanitizer_suppressions/tsan \
$(top_srcdir)/test/sanitizer_suppressions/ubsan \
$(top_srcdir)/contrib/linearize/linearize-data.py \
$(top_srcdir)/contrib/linearize/linearize-hashes.py

Expand Down
1 change: 0 additions & 1 deletion ci/lint/06_script.sh
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ test/lint/git-subtree-check.sh src/univalue
test/lint/git-subtree-check.sh src/leveldb
test/lint/git-subtree-check.sh src/crc32c
test/lint/check-doc.py
test/lint/check-rpc-mappings.py .
test/lint/lint-all.sh

if [ "$CIRRUS_REPO_FULL_NAME" = "bitcoin/bitcoin" ] && [ -n "$CIRRUS_CRON" ]; then
Expand Down
2 changes: 1 addition & 1 deletion ci/test/00_setup_env_native_fuzz.sh
Original file line number Diff line number Diff line change
Expand Up @@ -14,5 +14,5 @@ export RUN_UNIT_TESTS=false
export RUN_FUNCTIONAL_TESTS=false
export RUN_FUZZ_TESTS=true
export GOAL="install"
export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,address,undefined CC=clang CXX=clang++ --with-boost-process"
export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,address,undefined,integer CC=clang CXX=clang++ --with-boost-process"
export CCACHE_SIZE=200M
5 changes: 5 additions & 0 deletions configure.ac
Original file line number Diff line number Diff line change
Expand Up @@ -813,6 +813,11 @@ AX_GCC_FUNC_ATTRIBUTE([dllimport])
if test x$use_glibc_compat != xno; then
AX_CHECK_LINK_FLAG([[-Wl,--wrap=__divmoddi4]], [COMPAT_LDFLAGS="$COMPAT_LDFLAGS -Wl,--wrap=__divmoddi4"])
AX_CHECK_LINK_FLAG([[-Wl,--wrap=log2f]], [COMPAT_LDFLAGS="$COMPAT_LDFLAGS -Wl,--wrap=log2f"])
case $host in
powerpc64* | ppc64*)
AX_CHECK_LINK_FLAG([[-Wl,--no-tls-get-addr-optimize]], [COMPAT_LDFLAGS="$COMPAT_LDFLAGS -Wl,--no-tls-get-addr-optimize"])
;;
esac
else
AC_SEARCH_LIBS([clock_gettime],[rt])
fi
Expand Down
19 changes: 10 additions & 9 deletions contrib/gitian-build.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,15 +13,16 @@ def setup():
programs = ['ruby', 'git', 'make', 'wget', 'curl']
if args.kvm:
programs += ['apt-cacher-ng', 'python-vm-builder', 'qemu-kvm', 'qemu-utils']
elif args.docker and not os.path.isfile('/lib/systemd/system/docker.service'):
dockers = ['docker.io', 'docker-ce']
for i in dockers:
return_code = subprocess.call(['sudo', 'apt-get', 'install', '-qq', i])
if return_code == 0:
break
if return_code != 0:
print('Cannot find any way to install Docker.', file=sys.stderr)
sys.exit(1)
elif args.docker:
if not os.path.isfile('/lib/systemd/system/docker.service'):
dockers = ['docker.io', 'docker-ce']
for i in dockers:
return_code = subprocess.call(['sudo', 'apt-get', 'install', '-qq', i])
if return_code == 0:
break
if return_code != 0:
print('Cannot find any way to install Docker.', file=sys.stderr)
sys.exit(1)
else:
programs += ['apt-cacher-ng', 'lxc', 'debootstrap']
subprocess.check_call(['sudo', 'apt-get', 'install', '-qq'] + programs)
Expand Down
20 changes: 16 additions & 4 deletions contrib/gitian-descriptors/gitian-linux.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,12 @@ packages:
# - aarch64-linux-gnu
- "binutils-aarch64-linux-gnu"
- "g++-8-aarch64-linux-gnu"
# - powerpc64-linux-gnu
- "binutils-powerpc64-linux-gnu"
- "g++-8-powerpc64-linux-gnu"
# - powerpc64le-linux-gnu
- "binutils-powerpc64le-linux-gnu"
- "g++-8-powerpc64le-linux-gnu"
# - riscv64-linux-gnu
- "binutils-riscv64-linux-gnu"
- "g++-8-riscv64-linux-gnu"
Expand All @@ -38,7 +44,7 @@ script: |
set -e -o pipefail
WRAP_DIR=$HOME/wrapped
HOSTS="x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv64-linux-gnu"
HOSTS="x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu powerpc64-linux-gnu powerpc64le-linux-gnu riscv64-linux-gnu"
CONFIGFLAGS="--enable-glibc-back-compat --enable-reduce-exports --disable-bench --disable-gui-tests"
FAKETIME_HOST_PROGS="gcc g++"
FAKETIME_PROGS="date ar ranlib nm"
Expand All @@ -64,7 +70,7 @@ script: |
echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog}
echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${prog}
echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog}
echo "\$REAL \$@" >> $WRAP_DIR/${prog}
echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${prog}
chmod +x ${WRAP_DIR}/${prog}
done
}
Expand All @@ -78,7 +84,13 @@ script: |
echo "REAL=\`which -a ${i}-${prog}-8 | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog}
echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${i}-${prog}
echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog}
echo "\$REAL \"\$@\"" >> $WRAP_DIR/${i}-${prog}
if [ "${i:0:11}" = "powerpc64le" ]; then
echo "exec \"\$REAL\" -mcpu=power8 -mtune=power9 \"\$@\"" >> $WRAP_DIR/${i}-${prog}
elif [ "${i:0:9}" = "powerpc64" ]; then
echo "exec \"\$REAL\" -mcpu=970 -mtune=power9 \"\$@\"" >> $WRAP_DIR/${i}-${prog}
else
echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${i}-${prog}
fi
chmod +x ${WRAP_DIR}/${i}-${prog}
fi
done
Expand Down Expand Up @@ -118,7 +130,7 @@ script: |
# Extract the git archive into a dir for each host and build
for i in ${HOSTS}; do
export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH}
if [ "${i}" = "riscv64-linux-gnu" ]; then
if [ "${i}" = "riscv64-linux-gnu" ] || [ "${i}" = "powerpc64-linux-gnu" ] || [ "${i}" = "powerpc64le-linux-gnu" ]; then
# Workaround for https://bugs.launchpad.net/ubuntu/+source/gcc-8-cross-ports/+bug/1853740
# TODO: remove this when no longer needed
HOST_LDFLAGS="${HOST_LDFLAGS_BASE} -Wl,-z,noexecstack"
Expand Down
2 changes: 1 addition & 1 deletion contrib/gitian-descriptors/gitian-osx-signer.yml
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ script: |
echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog}
echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${prog}
echo "export FAKETIME=\"${REFERENCE_DATETIME}\"" >> ${WRAP_DIR}/${prog}
echo "\$REAL \$@" >> $WRAP_DIR/${prog}
echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${prog}
chmod +x ${WRAP_DIR}/${prog}
done
Expand Down
4 changes: 2 additions & 2 deletions contrib/gitian-descriptors/gitian-osx.yml
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ script: |
echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog}
echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${prog}
echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog}
echo "\$REAL \$@" >> $WRAP_DIR/${prog}
echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${prog}
chmod +x ${WRAP_DIR}/${prog}
done
}
Expand All @@ -75,7 +75,7 @@ script: |
echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog}
echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${i}-${prog}
echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog}
echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog}
echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${i}-${prog}
chmod +x ${WRAP_DIR}/${i}-${prog}
done
done
Expand Down
6 changes: 3 additions & 3 deletions contrib/gitian-descriptors/gitian-win.yml
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ script: |
echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog}
echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${prog}
echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog}
echo "\$REAL \$@" >> $WRAP_DIR/${prog}
echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${prog}
chmod +x ${WRAP_DIR}/${prog}
done
}
Expand All @@ -67,7 +67,7 @@ script: |
echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog}
echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${i}-${prog}
echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog}
echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog}
echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${i}-${prog}
chmod +x ${WRAP_DIR}/${i}-${prog}
done
done
Expand All @@ -81,7 +81,7 @@ script: |
echo "REAL=\`which -a ${i}-${prog}-posix | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog}
echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${i}-${prog}
echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog}
echo "\$REAL \"\$@\"" >> $WRAP_DIR/${i}-${prog}
echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${i}-${prog}
chmod +x ${WRAP_DIR}/${i}-${prog}
done
done
Expand Down
14 changes: 7 additions & 7 deletions contrib/verifybinaries/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,21 +21,21 @@ The script returns 0 if everything passes the checks. It returns 1 if either the


```sh
./verify.sh bitcoin-core-0.11.2
./verify.sh bitcoin-core-0.12.0
./verify.sh bitcoin-core-0.13.0-rc3
./verify.py bitcoin-core-0.11.2
./verify.py bitcoin-core-0.12.0
./verify.py bitcoin-core-0.13.0-rc3
```

If you only want to download the binaries of certain platform, add the corresponding suffix, e.g.:

```sh
./verify.sh bitcoin-core-0.11.2-osx
./verify.sh 0.12.0-linux
./verify.sh bitcoin-core-0.13.0-rc3-win64
./verify.py bitcoin-core-0.11.2-osx
./verify.py 0.12.0-linux
./verify.py bitcoin-core-0.13.0-rc3-win64
```

If you do not want to keep the downloaded binaries, specify anything as the second parameter.

```sh
./verify.sh bitcoin-core-0.13.0 delete
./verify.py bitcoin-core-0.13.0 delete
```
183 changes: 183 additions & 0 deletions contrib/verifybinaries/verify.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,183 @@
#!/usr/bin/env python3
# Copyright (c) 2020 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
"""Script for verifying Bitoin Core release binaries
This script attempts to download the signature file SHA256SUMS.asc from
bitcoincore.org and bitcoin.org and compares them.
It first checks if the signature passes, and then downloads the files
specified in the file, and checks if the hashes of these files match those
that are specified in the signature file.
The script returns 0 if everything passes the checks. It returns 1 if either
the signature check or the hash check doesn't pass. If an error occurs the
return value is >= 2.
"""
from hashlib import sha256
import os
import subprocess
import sys
from textwrap import indent

WORKINGDIR = "/tmp/bitcoin_verify_binaries"
HASHFILE = "hashes.tmp"
HOST1 = "https://bitcoincore.org"
HOST2 = "https://bitcoin.org"
VERSIONPREFIX = "bitcoin-core-"
SIGNATUREFILENAME = "SHA256SUMS.asc"


def parse_version_string(version_str):
if version_str.startswith(VERSIONPREFIX): # remove version prefix
version_str = version_str[len(VERSIONPREFIX):]

parts = version_str.split('-')
version_base = parts[0]
version_rc = ""
version_os = ""
if len(parts) == 2: # "<version>-rcN" or "version-platform"
if "rc" in parts[1]:
version_rc = parts[1]
else:
version_os = parts[1]
elif len(parts) == 3: # "<version>-rcN-platform"
version_rc = parts[1]
version_os = parts[2]

return version_base, version_rc, version_os


def download_with_wget(remote_file, local_file=None):
if local_file:
wget_args = ['wget', '-O', local_file, remote_file]
else:
# use timestamping mechanism if local filename is not explicitely set
wget_args = ['wget', '-N', remote_file]

result = subprocess.run(wget_args,
stderr=subprocess.STDOUT, stdout=subprocess.PIPE)
return result.returncode == 0, result.stdout.decode().rstrip()


def files_are_equal(filename1, filename2):
with open(filename1, 'rb') as file1:
contents1 = file1.read()
with open(filename2, 'rb') as file2:
contents2 = file2.read()
return contents1 == contents2


def verify_with_gpg(signature_filename, output_filename):
result = subprocess.run(['gpg', '--yes', '--decrypt', '--output',
output_filename, signature_filename],
stderr=subprocess.STDOUT, stdout=subprocess.PIPE)
return result.returncode, result.stdout.decode().rstrip()


def remove_files(filenames):
for filename in filenames:
os.remove(filename)


def main(args):
# sanity check
if len(args) < 1:
print("Error: need to specify a version on the command line")
return 3

# determine remote dir dependend on provided version string
version_base, version_rc, os_filter = parse_version_string(args[0])
remote_dir = f"/bin/{VERSIONPREFIX}{version_base}/"
if version_rc:
remote_dir += f"test.{version_rc}/"
remote_sigfile = remote_dir + SIGNATUREFILENAME

# create working directory
os.makedirs(WORKINGDIR, exist_ok=True)
os.chdir(WORKINGDIR)

# fetch first signature file
sigfile1 = SIGNATUREFILENAME
success, output = download_with_wget(HOST1 + remote_sigfile, sigfile1)
if not success:
print("Error: couldn't fetch signature file. "
"Have you specified the version number in the following format?")
print(f"[{VERSIONPREFIX}]<version>[-rc[0-9]][-platform] "
f"(example: {VERSIONPREFIX}0.21.0-rc3-osx)")
print("wget output:")
print(indent(output, '\t'))
return 4

# fetch second signature file
sigfile2 = SIGNATUREFILENAME + ".2"
success, output = download_with_wget(HOST2 + remote_sigfile, sigfile2)
if not success:
print("bitcoin.org failed to provide signature file, "
"but bitcoincore.org did?")
print("wget output:")
print(indent(output, '\t'))
remove_files([sigfile1])
return 5

# ensure that both signature files are equal
if not files_are_equal(sigfile1, sigfile2):
print("bitcoin.org and bitcoincore.org signature files were not equal?")
print(f"See files {WORKINGDIR}/{sigfile1} and {WORKINGDIR}/{sigfile2}")
return 6

# check signature and extract data into file
retval, output = verify_with_gpg(sigfile1, HASHFILE)
if retval != 0:
if retval == 1:
print("Bad signature.")
elif retval == 2:
print("gpg error. Do you have the Bitcoin Core binary release "
"signing key installed?")
print("gpg output:")
print(indent(output, '\t'))
remove_files([sigfile1, sigfile2, HASHFILE])
return 1

# extract hashes/filenames of binaries to verify from hash file;
# each line has the following format: "<hash> <binary_filename>"
with open(HASHFILE, 'r', encoding='utf8') as hash_file:
hashes_to_verify = [
line.split()[:2] for line in hash_file if os_filter in line]
remove_files([HASHFILE])
if not hashes_to_verify:
print("error: no files matched the platform specified")
return 7

# download binaries
for _, binary_filename in hashes_to_verify:
print(f"Downloading {binary_filename}")
download_with_wget(HOST1 + remote_dir + binary_filename)

# verify hashes
offending_files = []
for hash_expected, binary_filename in hashes_to_verify:
with open(binary_filename, 'rb') as binary_file:
hash_calculated = sha256(binary_file.read()).hexdigest()
if hash_calculated != hash_expected:
offending_files.append(binary_filename)
if offending_files:
print("Hashes don't match.")
print("Offending files:")
print('\n'.join(offending_files))
return 1
verified_binaries = [entry[1] for entry in hashes_to_verify]

# clean up files if desired
if len(args) >= 2:
print("Clean up the binaries")
remove_files([sigfile1, sigfile2] + verified_binaries)
else:
print(f"Keep the binaries in {WORKINGDIR}")

print("Verified hashes of")
print('\n'.join(verified_binaries))
return 0


if __name__ == '__main__':
sys.exit(main(sys.argv[1:]))
Loading

0 comments on commit c4af2fd

Please sign in to comment.