From 306cb1b47a6d12b06988c373c5d9a2a33ecf854d Mon Sep 17 00:00:00 2001 From: fanquake Date: Tue, 28 Apr 2020 16:28:34 +0800 Subject: [PATCH] Merge #18556: build: Drop make dist in gitian builds 2aa48edec0101f8a77a2189244fc62722ff7a123 refactor: Drop unused ${WRAP_DIR}/${HOST} directory (Hennadii Stepanov) 1362be044724bb49d785ca2e296a3b43343c1690 build: Drop make dist in gitian builds (Hennadii Stepanov) Pull request description: After the merge of #18331, the packaged source tarball is created by `git archive`, but the binaries are built from another one which is made by `make dist`. With this PR the only source tarball, created by `git archive`, is used both for binaries building and for packaging to users. Close #16588. Close #18547. As a good side-effect, #18349 becomes redundant. **Change in behavior** The following variables https://github.com/bitcoin/bitcoin/blob/1b151e3ffce7c1a2ee46bf280cc1d96775d1f91e/configure.ac#L2-L6 are no longer used for naming of directories and tarballs. Instead of them the gitian descriptors use a git tag (if available) or a commit hash. --- Also a small refactor commit picked from #18404. ACKs for top commit: dongcarl: ACK 2aa48edec0101f8a77a2189244fc62722ff7a123 MarcoFalke: ACK 2aa48edec0101f8a77a2189244fc62722ff7a123 fanquake: ACK 2aa48edec0101f8a77a2189244fc62722ff7a123 - I've had a quick look over this, and don't want to block merging if this actually gets as closer to finally having this all sorted out. Obviously we've still got #18741, and after speaking to Carl this morning, there will likely be even more changes after that (not Guix specific). Tree-SHA512: d3b16f87e48d1790a3264940c28acd5d881bfd10f3ce94fb0c8a6af76d8039289d01e0cd4972adac49ae24362857251f6c1e5e09e3e9fbf636c10708b4015a7c --- contrib/gitian-descriptors/gitian-linux.yml | 26 +++++------------ contrib/gitian-descriptors/gitian-osx.yml | 25 ++++------------- contrib/gitian-descriptors/gitian-win.yml | 31 ++++++--------------- contrib/gitian-descriptors/make_git_archive | 20 +++++++++++++ share/genbuild.sh | 6 +--- test/lint/lint-shell.sh | 10 ++++--- 6 files changed, 48 insertions(+), 70 deletions(-) create mode 100755 contrib/gitian-descriptors/make_git_archive diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml index 93394cbdf898f1..97e98a2866aa9e 100755 --- a/contrib/gitian-descriptors/gitian-linux.yml +++ b/contrib/gitian-descriptors/gitian-linux.yml @@ -167,18 +167,12 @@ script: | create_per-host_faketime_wrappers "${REFERENCE_DATETIME}" export PATH=${WRAP_DIR}:${PATH} - # Create the release tarball using (arbitrarily) the first host - ./autogen.sh - CONFIG_SITE=${BASEPREFIX}/$(echo "${HOSTS}" | awk '{print $1;}')/share/config.site ./configure --prefix=/ - make dist - SOURCEDIST=$(echo dashcore-*.tar.gz) - DISTNAME=${SOURCEDIST/%.tar.gz} - - # Workaround for tarball not building with the bare tag version (prep) - make -C src obj/build.h + # Create the git archive, and define DISTNAME and GIT_ARCHIVE variables. + # shellcheck source=contrib/gitian-descriptors/make_git_archive + source contrib/gitian-descriptors/make_git_archive ORIGPATH="$PATH" - # Extract the release tarball into a dir for each host and build + # Extract the git archive into a dir for each host and build for i in ${HOSTS}; do export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} if [ "${i}" = "riscv64-linux-gnu" ]; then @@ -192,13 +186,9 @@ script: | cd distsrc-${i} INSTALLPATH="${PWD}/installed/${DISTNAME}" mkdir -p ${INSTALLPATH} - tar --strip-components=1 -xf ../$SOURCEDIST - - # Workaround for tarball not building with the bare tag version - echo '#!/bin/true' >share/genbuild.sh - mkdir src/obj - cp ../src/obj/build.h src/obj/ + tar -xf $GIT_ARCHIVE + ./autogen.sh CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} CFLAGS="${HOST_CFLAGS}" CXXFLAGS="${HOST_CXXFLAGS}" LDFLAGS="${HOST_LDFLAGS}" make ${MAKEOPTS} make ${MAKEOPTS} -C src check-security @@ -210,14 +200,12 @@ script: | rm -rf ${DISTNAME}/lib/pkgconfig find ${DISTNAME}/bin -type f -executable -print0 | xargs -0 -n1 -I{} ../contrib/devtools/split-debug.sh {} {} {}.dbg find ${DISTNAME}/lib -type f -print0 | xargs -0 -n1 -I{} ../contrib/devtools/split-debug.sh {} {} {}.dbg - cp ../../README.md ${DISTNAME}/ + cp ../README.md ${DISTNAME}/ find ${DISTNAME} -not -name "*.dbg" | sort | tar --mtime="$REFERENCE_DATETIME" --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}.tar.gz find ${DISTNAME} -name "*.dbg" | sort | tar --mtime="$REFERENCE_DATETIME" --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}-debug.tar.gz cd ../../ rm -rf distsrc-${i} done - mkdir -p ${OUTDIR}/src - git archive --output=${OUTDIR}/src/${DISTNAME}.tar.gz HEAD # Compress ccache (otherwise the assert file will get too huge) if [ "$CCACHE_DIR" != "" ]; then diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml index 6d7a492c8f6a3b..b73cd703b39bed 100644 --- a/contrib/gitian-descriptors/gitian-osx.yml +++ b/contrib/gitian-descriptors/gitian-osx.yml @@ -122,31 +122,21 @@ script: | create_per-host_faketime_wrappers "${REFERENCE_DATETIME}" export PATH=${WRAP_DIR}:${PATH} - # Create the release tarball using (arbitrarily) the first host - ./autogen.sh - CONFIG_SITE=${BASEPREFIX}/$(echo "${HOSTS}" | awk '{print $1;}')/share/config.site ./configure --prefix=/ - make dist - SOURCEDIST=$(echo dashcore-*.tar.gz) - DISTNAME=${SOURCEDIST/%.tar.gz} - - # Workaround for tarball not building with the bare tag version (prep) - make -C src obj/build.h + # Create the git archive, and define DISTNAME and GIT_ARCHIVE variables. + # shellcheck source=contrib/gitian-descriptors/make_git_archive + source contrib/gitian-descriptors/make_git_archive ORIGPATH="$PATH" - # Extract the release tarball into a dir for each host and build + # Extract the git archive into a dir for each host and build for i in ${HOSTS}; do export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} mkdir -p distsrc-${i} cd distsrc-${i} INSTALLPATH="${PWD}/installed/${DISTNAME}" mkdir -p ${INSTALLPATH} - tar --strip-components=1 -xf ../$SOURCEDIST - - # Workaround for tarball not building with the bare tag version - echo '#!/bin/true' >share/genbuild.sh - mkdir src/obj - cp ../src/obj/build.h src/obj/ + tar -xf $GIT_ARCHIVE + ./autogen.sh CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} make ${MAKEOPTS} make -C src osx_debug @@ -177,9 +167,6 @@ script: | find ${DISTNAME} -path '*.dSYM*' | sort | tar --mtime="$REFERENCE_DATETIME" --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}-debug.tar.gz cd ../../ done - mkdir -p ${OUTDIR}/src - git archive --output=${OUTDIR}/src/${DISTNAME}.tar.gz HEAD - mv ${OUTDIR}/${DISTNAME}-x86_64-apple-darwin19.tar.gz ${OUTDIR}/${DISTNAME}-osx64.tar.gz mv ${OUTDIR}/${DISTNAME}-x86_64-apple-darwin19-debug.tar.gz ${OUTDIR}/${DISTNAME}-osx64-debug.tar.gz diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml index 8c17304aee30a9..658badbcdee709 100755 --- a/contrib/gitian-descriptors/gitian-win.yml +++ b/contrib/gitian-descriptors/gitian-win.yml @@ -101,7 +101,6 @@ script: | function create_per-host_compiler_wrapper { # -posix variant is required for c++11 threading. for i in $HOSTS; do - mkdir -p ${WRAP_DIR}/${i} for prog in gcc g++; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} echo "# GCCVERSION=${GCCVERSION}" >> ${WRAP_DIR}/${i}-${prog} @@ -110,7 +109,6 @@ script: | echo "# $(${prog} --version | head -1)" >> ${WRAP_DIR}/${i}-${prog} echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${i}-${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} - echo "export COMPILER_PATH=${WRAP_DIR}/${i}" >> ${WRAP_DIR}/${i}-${prog} echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} chmod +x ${WRAP_DIR}/${i}-${prog} touch -d "${REFERENCE_DATETIME}" ${WRAP_DIR}/${i}-${prog} @@ -139,37 +137,27 @@ script: | create_per-host_compiler_wrapper "${REFERENCE_DATETIME}" export PATH=${WRAP_DIR}:${PATH} - # Create the release tarball using (arbitrarily) the first host - ./autogen.sh - CONFIG_SITE=${BASEPREFIX}/$(echo "${HOSTS}" | awk '{print $1;}')/share/config.site ./configure --prefix=/ - make dist - SOURCEDIST=$(echo dashcore-*.tar.gz) - DISTNAME=${SOURCEDIST/%.tar.gz} - - # Workaround for tarball not building with the bare tag version (prep) - make -C src obj/build.h + # Create the git archive, and define DISTNAME and GIT_ARCHIVE variables. + # shellcheck source=contrib/gitian-descriptors/make_git_archive + source contrib/gitian-descriptors/make_git_archive ORIGPATH="$PATH" - # Extract the release tarball into a dir for each host and build + # Extract the git archive into a dir for each host and build for i in ${HOSTS}; do export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} mkdir -p distsrc-${i} cd distsrc-${i} INSTALLPATH="${PWD}/installed/${DISTNAME}" mkdir -p ${INSTALLPATH} - tar --strip-components=1 -xf ../$SOURCEDIST - - # Workaround for tarball not building with the bare tag version - echo '#!/bin/true' >share/genbuild.sh - mkdir src/obj - cp ../src/obj/build.h src/obj/ + tar -xf $GIT_ARCHIVE + ./autogen.sh CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} CFLAGS="${HOST_CFLAGS}" CXXFLAGS="${HOST_CXXFLAGS}" make ${MAKEOPTS} make ${MAKEOPTS} -C src check-security make deploy make install DESTDIR=${INSTALLPATH} - cp -f --target-directory="${OUTDIR}" ./bitcoin-*-setup-unsigned.exe + cp -f ./bitcoin-*-win64-setup-unsigned.exe ${OUTDIR}/${DISTNAME}-win64-setup-unsigned.exe cd installed mv ${DISTNAME}/bin/*.dll ${DISTNAME}/lib/ find . -name "lib*.la" -delete @@ -184,13 +172,10 @@ script: | rm -rf distsrc-${i} done - mkdir -p ${OUTDIR}/src - git archive --output=${OUTDIR}/src/${DISTNAME}.tar.gz HEAD - cp -rf contrib/windeploy $BUILD_DIR cd $BUILD_DIR/windeploy mkdir unsigned - cp $OUTDIR/dashcore-*setup-unsigned.exe unsigned/ + cp ${OUTDIR}/${DISTNAME}-win64-setup-unsigned.exe unsigned/ find . | sort | tar --mtime="$REFERENCE_DATETIME" --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-win-unsigned.tar.gz # Compress ccache (otherwise the assert file will get too huge) diff --git a/contrib/gitian-descriptors/make_git_archive b/contrib/gitian-descriptors/make_git_archive new file mode 100755 index 00000000000000..d922c94c60bf24 --- /dev/null +++ b/contrib/gitian-descriptors/make_git_archive @@ -0,0 +1,20 @@ +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# A helper script to be sourced into the gitian descriptors + +mkdir -p ${OUTDIR}/src +RECENT_TAG=$(git describe --abbrev=0 HEAD) +if [ $RECENT_TAG = $(git describe HEAD) ]; then + if [[ $RECENT_TAG == v* ]]; then + VERSION=${RECENT_TAG:1} + else + VERSION=$RECENT_TAG + fi +else + VERSION=$(git rev-parse --short HEAD) +fi +DISTNAME=bitcoin-${VERSION} +GIT_ARCHIVE="${OUTDIR}/src/${DISTNAME}.tar.gz" +git archive --output=$GIT_ARCHIVE HEAD diff --git a/share/genbuild.sh b/share/genbuild.sh index cd6214d5fab259..a5a561878a4d2e 100755 --- a/share/genbuild.sh +++ b/share/genbuild.sh @@ -18,13 +18,9 @@ else exit 1 fi -git_check_in_repo() { - ! { git status --porcelain -uall --ignored "$@" 2>/dev/null || echo '??'; } | grep -q '?' -} - DESC="" SUFFIX="" -if [ "${BITCOIN_GENBUILD_NO_GIT}" != "1" ] && [ -e "$(command -v git)" ] && [ "$(git rev-parse --is-inside-work-tree 2>/dev/null)" = "true" ] && git_check_in_repo share/genbuild.sh; then +if [ "${BITCOIN_GENBUILD_NO_GIT}" != "1" ] && [ -e "$(command -v git)" ] && [ "$(git rev-parse --is-inside-work-tree 2>/dev/null)" = "true" ]; then # clean 'dirty' status of touched files that haven't been modified git diff >/dev/null 2>/dev/null diff --git a/test/lint/lint-shell.sh b/test/lint/lint-shell.sh index def78bcb922b10..262ac27379b87d 100755 --- a/test/lint/lint-shell.sh +++ b/test/lint/lint-shell.sh @@ -56,15 +56,17 @@ if ! command -v jq > /dev/null; then fi EXCLUDE_GITIAN=${EXCLUDE}",$(IFS=','; echo "${disabled_gitian[*]}")" +SHELLCHECK_CMD="shellcheck --external-sources --check-sourced $EXCLUDE_GITIAN" for descriptor in $(git ls-files -- 'contrib/gitian-descriptors/*.yml') do - echo - echo "$descriptor" + script=$(basename "$descriptor") # Use #!/bin/bash as gitian-builder/bin/gbuild does to complete a script. - SCRIPT=$'#!/bin/bash\n'$(yq -r .script "$descriptor") - if ! echo "$SCRIPT" | shellcheck "$EXCLUDE_GITIAN" -; then + echo "#!/bin/bash" > $script + yq -r .script "$descriptor" >> $script + if ! $SHELLCHECK_CMD $script; then EXIT_CODE=1 fi + rm $script done exit $EXIT_CODE