diff --git a/contrib/gitian-descriptors/assign_DISTNAME b/contrib/gitian-descriptors/assign_DISTNAME new file mode 100755 index 0000000000000..a2ca768aaa70b --- /dev/null +++ b/contrib/gitian-descriptors/assign_DISTNAME @@ -0,0 +1,12 @@ +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# A helper script to be sourced into the gitian descriptors + +if RECENT_TAG="$(git describe --exact-match HEAD)"; then + VERSION="${RECENT_TAG#v}" +else + VERSION="$(git rev-parse --short=12 HEAD)" +fi +DISTNAME="bitcoin-${VERSION}" diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml index 6e55459f9c785..f63844ed9f91b 100644 --- a/contrib/gitian-descriptors/gitian-linux.yml +++ b/contrib/gitian-descriptors/gitian-linux.yml @@ -140,18 +140,19 @@ script: | create_per-host_faketime_wrappers "${REFERENCE_DATETIME}" export PATH=${WRAP_DIR}:${PATH} - # Create the release tarball using (arbitrarily) the first host - ./autogen.sh - CONFIG_SITE=${BASEPREFIX}/$(echo "${HOSTS}" | awk '{print $1;}')/share/config.site ./configure --prefix=/ - make dist - SOURCEDIST=$(echo bitcoin-*.tar.gz) - DISTNAME=${SOURCEDIST/%.tar.gz} + # Define DISTNAME variable. + # shellcheck source=contrib/gitian-descriptors/assign_DISTNAME + source contrib/gitian-descriptors/assign_DISTNAME + + GIT_ARCHIVE="${OUTDIR}/src/${DISTNAME}.tar.gz" - # Workaround for tarball not building with the bare tag version (prep) - make -C src obj/build.h + # Create the release tarball using (arbitrarily) the first host + mkdir -p "$(dirname "$GIT_ARCHIVE")" + CONFIG_SITE="${BASEPREFIX}/${HOSTS/ */}/share/config.site" \ + contrib/gitian-descriptors/make_release_tarball "${GIT_ARCHIVE}" ORIGPATH="$PATH" - # Extract the release tarball into a dir for each host and build + # Extract the git archive into a dir for each host and build for i in ${HOSTS}; do export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} if [ "${i}" = "riscv64-linux-gnu" ]; then @@ -165,12 +166,7 @@ script: | cd distsrc-${i} INSTALLPATH="${PWD}/installed/${DISTNAME}" mkdir -p ${INSTALLPATH} - tar --strip-components=1 -xf ../$SOURCEDIST - - # Workaround for tarball not building with the bare tag version - echo '#!/bin/true' >share/genbuild.sh - mkdir src/obj - cp ../src/obj/build.h src/obj/ + tar --strip-components=1 -xf $GIT_ARCHIVE CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} CFLAGS="${HOST_CFLAGS}" CXXFLAGS="${HOST_CXXFLAGS}" LDFLAGS="${HOST_LDFLAGS}" make ${MAKEOPTS} @@ -183,12 +179,9 @@ script: | rm -rf ${DISTNAME}/lib/pkgconfig find ${DISTNAME}/bin -type f -executable -print0 | xargs -0 -n1 -I{} ../contrib/devtools/split-debug.sh {} {} {}.dbg find ${DISTNAME}/lib -type f -print0 | xargs -0 -n1 -I{} ../contrib/devtools/split-debug.sh {} {} {}.dbg - cp ../../README.md ${DISTNAME}/ + cp ../README.md ${DISTNAME}/ find ${DISTNAME} -not -name "*.dbg" | sort | tar --mtime="$REFERENCE_DATETIME" --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}.tar.gz find ${DISTNAME} -name "*.dbg" | sort | tar --mtime="$REFERENCE_DATETIME" --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}-debug.tar.gz cd ../../ rm -rf distsrc-${i} done - - mkdir -p ${OUTDIR}/src - git archive --prefix="${DISTNAME}/" --output=${OUTDIR}/src/${DISTNAME}.tar.gz HEAD diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml index 61fc5d2e6a5e9..98f85167958aa 100644 --- a/contrib/gitian-descriptors/gitian-osx.yml +++ b/contrib/gitian-descriptors/gitian-osx.yml @@ -103,30 +103,26 @@ script: | create_per-host_faketime_wrappers "${REFERENCE_DATETIME}" export PATH=${WRAP_DIR}:${PATH} - # Create the release tarball using (arbitrarily) the first host - ./autogen.sh - CONFIG_SITE=${BASEPREFIX}/$(echo "${HOSTS}" | awk '{print $1;}')/share/config.site ./configure --prefix=/ - make dist - SOURCEDIST=$(echo bitcoin-*.tar.gz) - DISTNAME=${SOURCEDIST/%.tar.gz} + # Define DISTNAME variable. + # shellcheck source=contrib/gitian-descriptors/assign_DISTNAME + source contrib/gitian-descriptors/assign_DISTNAME + + GIT_ARCHIVE="${OUTDIR}/src/${DISTNAME}.tar.gz" - # Workaround for tarball not building with the bare tag version (prep) - make -C src obj/build.h + # Create the release tarball using (arbitrarily) the first host + mkdir -p "$(dirname "$GIT_ARCHIVE")" + CONFIG_SITE="${BASEPREFIX}/${HOSTS/ */}/share/config.site" \ + contrib/gitian-descriptors/make_release_tarball "${GIT_ARCHIVE}" ORIGPATH="$PATH" - # Extract the release tarball into a dir for each host and build + # Extract the git archive into a dir for each host and build for i in ${HOSTS}; do export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} mkdir -p distsrc-${i} cd distsrc-${i} INSTALLPATH="${PWD}/installed/${DISTNAME}" mkdir -p ${INSTALLPATH} - tar --strip-components=1 -xf ../$SOURCEDIST - - # Workaround for tarball not building with the bare tag version - echo '#!/bin/true' >share/genbuild.sh - mkdir src/obj - cp ../src/obj/build.h src/obj/ + tar --strip-components=1 -xf $GIT_ARCHIVE CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} make ${MAKEOPTS} @@ -160,7 +156,4 @@ script: | cd ../../ done - mkdir -p ${OUTDIR}/src - git archive --prefix="${DISTNAME}/" --output=${OUTDIR}/src/${DISTNAME}.tar.gz HEAD - mv ${OUTDIR}/${DISTNAME}-x86_64-*.tar.gz ${OUTDIR}/${DISTNAME}-osx64.tar.gz diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml index 1f9ffd13893ae..c60a072a928ba 100644 --- a/contrib/gitian-descriptors/gitian-win.yml +++ b/contrib/gitian-descriptors/gitian-win.yml @@ -76,13 +76,11 @@ script: | function create_per-host_compiler_wrapper { # -posix variant is required for c++11 threading. for i in $HOSTS; do - mkdir -p ${WRAP_DIR}/${i} for prog in gcc g++; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} echo "REAL=\`which -a ${i}-${prog}-posix | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${i}-${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} - echo "export COMPILER_PATH=${WRAP_DIR}/${i}" >> ${WRAP_DIR}/${i}-${prog} echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} chmod +x ${WRAP_DIR}/${i}-${prog} done @@ -110,30 +108,26 @@ script: | create_per-host_compiler_wrapper "${REFERENCE_DATETIME}" export PATH=${WRAP_DIR}:${PATH} - # Create the release tarball using (arbitrarily) the first host - ./autogen.sh - CONFIG_SITE=${BASEPREFIX}/$(echo "${HOSTS}" | awk '{print $1;}')/share/config.site ./configure --prefix=/ - make dist - SOURCEDIST=$(echo bitcoin-*.tar.gz) - DISTNAME=${SOURCEDIST/%.tar.gz} + # Define DISTNAME variable. + # shellcheck source=contrib/gitian-descriptors/assign_DISTNAME + source contrib/gitian-descriptors/assign_DISTNAME + + GIT_ARCHIVE="${OUTDIR}/src/${DISTNAME}.tar.gz" - # Workaround for tarball not building with the bare tag version (prep) - make -C src obj/build.h + # Create the release tarball using (arbitrarily) the first host + mkdir -p "$(dirname "$GIT_ARCHIVE")" + CONFIG_SITE="${BASEPREFIX}/${HOSTS/ */}/share/config.site" \ + contrib/gitian-descriptors/make_release_tarball "${GIT_ARCHIVE}" ORIGPATH="$PATH" - # Extract the release tarball into a dir for each host and build + # Extract the git archive into a dir for each host and build for i in ${HOSTS}; do export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} mkdir -p distsrc-${i} cd distsrc-${i} INSTALLPATH="${PWD}/installed/${DISTNAME}" mkdir -p ${INSTALLPATH} - tar --strip-components=1 -xf ../$SOURCEDIST - - # Workaround for tarball not building with the bare tag version - echo '#!/bin/true' >share/genbuild.sh - mkdir src/obj - cp ../src/obj/build.h src/obj/ + tar --strip-components=1 -xf $GIT_ARCHIVE CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} CFLAGS="${HOST_CFLAGS}" CXXFLAGS="${HOST_CXXFLAGS}" make ${MAKEOPTS} @@ -141,7 +135,7 @@ script: | make ${MAKEOPTS} -C src check-symbols make deploy make install DESTDIR=${INSTALLPATH} - cp -f --target-directory="${OUTDIR}" ./bitcoin-*-setup-unsigned.exe + cp -f ./bitcoin-*-win64-setup-unsigned.exe ${OUTDIR}/${DISTNAME}-win64-setup-unsigned.exe cd installed mv ${DISTNAME}/bin/*.dll ${DISTNAME}/lib/ find . -name "lib*.la" -delete @@ -156,11 +150,8 @@ script: | rm -rf distsrc-${i} done - mkdir -p ${OUTDIR}/src - git archive --prefix="${DISTNAME}/" --output=${OUTDIR}/src/${DISTNAME}.tar.gz HEAD - cp -rf contrib/windeploy $BUILD_DIR cd $BUILD_DIR/windeploy mkdir unsigned - cp $OUTDIR/bitcoin-*setup-unsigned.exe unsigned/ + cp ${OUTDIR}/${DISTNAME}-win64-setup-unsigned.exe unsigned/ find . | sort | tar --mtime="$REFERENCE_DATETIME" --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-win-unsigned.tar.gz diff --git a/contrib/gitian-descriptors/make_release_tarball b/contrib/gitian-descriptors/make_release_tarball new file mode 100755 index 0000000000000..9c348c270dc94 --- /dev/null +++ b/contrib/gitian-descriptors/make_release_tarball @@ -0,0 +1,38 @@ +#!/bin/sh +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# A helper script to generate source release tarball + +set -e + +GIT_ARCHIVE="$1" +. "`dirname "$0"`/assign_DISTNAME" + +git archive --prefix="${DISTNAME}/" HEAD | tar -xp + +# Generate correct build info file from git, before we lose git +GIT_BUILD_INFO="$(share/genbuild.sh /dev/stdout)" +sed 's/\/\/ No build information available/'"${GIT_BUILD_INFO}"'/' -i "${DISTNAME}/share/genbuild.sh" + +cd "${DISTNAME}" + +./autogen.sh +./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking +make distclean + +cd .. +tar \ + --format=ustar \ + --exclude autom4te.cache \ + --exclude .deps \ + --exclude .git \ + --sort=name \ + --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 \ + --mtime="${REFERENCE_DATETIME}" \ + -c "${DISTNAME}" | \ + gzip -9n \ + >"${GIT_ARCHIVE}" + +rm -r "${DISTNAME}" diff --git a/share/genbuild.sh b/share/genbuild.sh index 197787d5e0917..e68da2cdf774b 100755 --- a/share/genbuild.sh +++ b/share/genbuild.sh @@ -18,6 +18,7 @@ else exit 1 fi +# This checks that we are actually part of the intended git repository, and not just getting info about some unrelated git repository that the code happens to be in a directory under git_check_in_repo() { ! { git status --porcelain -uall --ignored "$@" 2>/dev/null || echo '??'; } | grep -q '?' } @@ -44,7 +45,9 @@ if [ -n "$DESC" ]; then elif [ -n "$SUFFIX" ]; then NEWINFO="#define BUILD_SUFFIX $SUFFIX" else - NEWINFO="// No build information available" + # NOTE: The NEWINFO line below this comment gets replaced by a string-match in contrib/gitian-descriptors/make_release_tarball + # If changing it, update the script too! + NEWINFO='// No build information available' fi # only update build.h if necessary diff --git a/test/lint/lint-shell.sh b/test/lint/lint-shell.sh index f59b2c994559d..d31ea14ea86af 100755 --- a/test/lint/lint-shell.sh +++ b/test/lint/lint-shell.sh @@ -46,15 +46,17 @@ if ! command -v yq > /dev/null; then fi EXCLUDE_GITIAN=${EXCLUDE}",$(IFS=','; echo "${disabled_gitian[*]}")" +SHELLCHECK_CMD="shellcheck --external-sources --check-sourced $EXCLUDE_GITIAN" for descriptor in $(git ls-files -- 'contrib/gitian-descriptors/*.yml') do - echo - echo "$descriptor" + script=$(basename "$descriptor") # Use #!/bin/bash as gitian-builder/bin/gbuild does to complete a script. - SCRIPT=$'#!/bin/bash\n'$(yq -r .script "$descriptor") - if ! echo "$SCRIPT" | shellcheck "$EXCLUDE_GITIAN" -; then + echo "#!/bin/bash" > $script + yq -r .script "$descriptor" >> $script + if ! $SHELLCHECK_CMD $script; then EXIT_CODE=1 fi + rm $script done exit $EXIT_CODE