[0.20] Fix release tarball

contrib/gitian-descriptors/assign_DISTNAME

@@ -0,0 +1,12 @@
+# Copyright (c) 2020 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+#
+# A helper script to be sourced into the gitian descriptors
+
+if RECENT_TAG="$(git describe --exact-match HEAD)"; then
+    VERSION="${RECENT_TAG#v}"
+else
+    VERSION="$(git rev-parse --short=12 HEAD)"
+fi
+DISTNAME="bitcoin-${VERSION}"

contrib/gitian-descriptors/gitian-linux.yml

@@ -140,18 +140,19 @@ script: |
   create_per-host_faketime_wrappers "${REFERENCE_DATETIME}"
   export PATH=${WRAP_DIR}:${PATH}
 
-  # Create the release tarball using (arbitrarily) the first host
-  ./autogen.sh
-  CONFIG_SITE=${BASEPREFIX}/$(echo "${HOSTS}" | awk '{print $1;}')/share/config.site ./configure --prefix=/
-  make dist
-  SOURCEDIST=$(echo bitcoin-*.tar.gz)
-  DISTNAME=${SOURCEDIST/%.tar.gz}
+  # Define DISTNAME variable.
+  # shellcheck source=contrib/gitian-descriptors/assign_DISTNAME
+  source contrib/gitian-descriptors/assign_DISTNAME
+
+  GIT_ARCHIVE="${OUTDIR}/src/${DISTNAME}.tar.gz"
 
-  # Workaround for tarball not building with the bare tag version (prep)
-  make -C src obj/build.h
+  # Create the release tarball using (arbitrarily) the first host
+  mkdir -p "$(dirname "$GIT_ARCHIVE")"
+  CONFIG_SITE="${BASEPREFIX}/${HOSTS/ */}/share/config.site" \
+  contrib/gitian-descriptors/make_release_tarball "${GIT_ARCHIVE}"
 
   ORIGPATH="$PATH"
-  # Extract the release tarball into a dir for each host and build
+  # Extract the git archive into a dir for each host and build
   for i in ${HOSTS}; do
     export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH}
     if [ "${i}" = "riscv64-linux-gnu" ]; then
@@ -165,12 +166,7 @@ script: |
     cd distsrc-${i}
     INSTALLPATH="${PWD}/installed/${DISTNAME}"
     mkdir -p ${INSTALLPATH}
-    tar --strip-components=1 -xf ../$SOURCEDIST
-
-    # Workaround for tarball not building with the bare tag version
-    echo '#!/bin/true' >share/genbuild.sh
-    mkdir src/obj
-    cp ../src/obj/build.h src/obj/
+    tar --strip-components=1 -xf $GIT_ARCHIVE
 
     CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} CFLAGS="${HOST_CFLAGS}" CXXFLAGS="${HOST_CXXFLAGS}" LDFLAGS="${HOST_LDFLAGS}"
     make ${MAKEOPTS}
@@ -183,12 +179,9 @@ script: |
     rm -rf ${DISTNAME}/lib/pkgconfig
     find ${DISTNAME}/bin -type f -executable -print0 | xargs -0 -n1 -I{} ../contrib/devtools/split-debug.sh {} {} {}.dbg
     find ${DISTNAME}/lib -type f -print0 | xargs -0 -n1 -I{} ../contrib/devtools/split-debug.sh {} {} {}.dbg
-    cp ../../README.md ${DISTNAME}/
+    cp ../README.md ${DISTNAME}/
     find ${DISTNAME} -not -name "*.dbg" | sort | tar --mtime="$REFERENCE_DATETIME" --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}.tar.gz
     find ${DISTNAME} -name "*.dbg" | sort | tar --mtime="$REFERENCE_DATETIME" --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}-debug.tar.gz
     cd ../../
     rm -rf distsrc-${i}
   done
-
-  mkdir -p ${OUTDIR}/src
-  git archive --prefix="${DISTNAME}/" --output=${OUTDIR}/src/${DISTNAME}.tar.gz HEAD

contrib/gitian-descriptors/gitian-osx.yml

@@ -103,30 +103,26 @@ script: |
   create_per-host_faketime_wrappers "${REFERENCE_DATETIME}"
   export PATH=${WRAP_DIR}:${PATH}
 
-  # Create the release tarball using (arbitrarily) the first host
-  ./autogen.sh
-  CONFIG_SITE=${BASEPREFIX}/$(echo "${HOSTS}" | awk '{print $1;}')/share/config.site ./configure --prefix=/
-  make dist
-  SOURCEDIST=$(echo bitcoin-*.tar.gz)
-  DISTNAME=${SOURCEDIST/%.tar.gz}
+  # Define DISTNAME variable.
+  # shellcheck source=contrib/gitian-descriptors/assign_DISTNAME
+  source contrib/gitian-descriptors/assign_DISTNAME
+
+  GIT_ARCHIVE="${OUTDIR}/src/${DISTNAME}.tar.gz"
 
-  # Workaround for tarball not building with the bare tag version (prep)
-  make -C src obj/build.h
+  # Create the release tarball using (arbitrarily) the first host
+  mkdir -p "$(dirname "$GIT_ARCHIVE")"
+  CONFIG_SITE="${BASEPREFIX}/${HOSTS/ */}/share/config.site" \
+  contrib/gitian-descriptors/make_release_tarball "${GIT_ARCHIVE}"
 
   ORIGPATH="$PATH"
-  # Extract the release tarball into a dir for each host and build
+  # Extract the git archive into a dir for each host and build
   for i in ${HOSTS}; do
     export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH}
     mkdir -p distsrc-${i}
     cd distsrc-${i}
     INSTALLPATH="${PWD}/installed/${DISTNAME}"
     mkdir -p ${INSTALLPATH}
-    tar --strip-components=1 -xf ../$SOURCEDIST
-
-    # Workaround for tarball not building with the bare tag version
-    echo '#!/bin/true' >share/genbuild.sh
-    mkdir src/obj
-    cp ../src/obj/build.h src/obj/
+    tar --strip-components=1 -xf $GIT_ARCHIVE
 
     CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS}
     make ${MAKEOPTS}
@@ -160,7 +156,4 @@ script: |
     cd ../../
   done
 
-  mkdir -p ${OUTDIR}/src
-  git archive --prefix="${DISTNAME}/" --output=${OUTDIR}/src/${DISTNAME}.tar.gz HEAD
-
   mv ${OUTDIR}/${DISTNAME}-x86_64-*.tar.gz ${OUTDIR}/${DISTNAME}-osx64.tar.gz

contrib/gitian-descriptors/gitian-win.yml

@@ -76,13 +76,11 @@ script: |
   function create_per-host_compiler_wrapper {
   # -posix variant is required for c++11 threading.
   for i in $HOSTS; do
-    mkdir -p ${WRAP_DIR}/${i}
     for prog in gcc g++; do
         echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog}
         echo "REAL=\`which -a ${i}-${prog}-posix | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog}
         echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${i}-${prog}
         echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog}
-        echo "export COMPILER_PATH=${WRAP_DIR}/${i}" >> ${WRAP_DIR}/${i}-${prog}
         echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog}
         chmod +x ${WRAP_DIR}/${i}-${prog}
     done
@@ -110,38 +108,34 @@ script: |
   create_per-host_compiler_wrapper "${REFERENCE_DATETIME}"
   export PATH=${WRAP_DIR}:${PATH}
 
-  # Create the release tarball using (arbitrarily) the first host
-  ./autogen.sh
-  CONFIG_SITE=${BASEPREFIX}/$(echo "${HOSTS}" | awk '{print $1;}')/share/config.site ./configure --prefix=/
-  make dist
-  SOURCEDIST=$(echo bitcoin-*.tar.gz)
-  DISTNAME=${SOURCEDIST/%.tar.gz}
+  # Define DISTNAME variable.
+  # shellcheck source=contrib/gitian-descriptors/assign_DISTNAME
+  source contrib/gitian-descriptors/assign_DISTNAME
+
+  GIT_ARCHIVE="${OUTDIR}/src/${DISTNAME}.tar.gz"
 
-  # Workaround for tarball not building with the bare tag version (prep)
-  make -C src obj/build.h
+  # Create the release tarball using (arbitrarily) the first host
+  mkdir -p "$(dirname "$GIT_ARCHIVE")"
+  CONFIG_SITE="${BASEPREFIX}/${HOSTS/ */}/share/config.site" \
+  contrib/gitian-descriptors/make_release_tarball "${GIT_ARCHIVE}"
 
   ORIGPATH="$PATH"
-  # Extract the release tarball into a dir for each host and build
+  # Extract the git archive into a dir for each host and build
   for i in ${HOSTS}; do
     export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH}
     mkdir -p distsrc-${i}
     cd distsrc-${i}
     INSTALLPATH="${PWD}/installed/${DISTNAME}"
     mkdir -p ${INSTALLPATH}
-    tar --strip-components=1 -xf ../$SOURCEDIST
-
-    # Workaround for tarball not building with the bare tag version
-    echo '#!/bin/true' >share/genbuild.sh
-    mkdir src/obj
-    cp ../src/obj/build.h src/obj/
+    tar --strip-components=1 -xf $GIT_ARCHIVE
 
     CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} CFLAGS="${HOST_CFLAGS}" CXXFLAGS="${HOST_CXXFLAGS}"
     make ${MAKEOPTS}
     make ${MAKEOPTS} -C src check-security
     make ${MAKEOPTS} -C src check-symbols
     make deploy
     make install DESTDIR=${INSTALLPATH}
-    cp -f --target-directory="${OUTDIR}" ./bitcoin-*-setup-unsigned.exe
+    cp -f ./bitcoin-*-win64-setup-unsigned.exe ${OUTDIR}/${DISTNAME}-win64-setup-unsigned.exe
     cd installed
     mv ${DISTNAME}/bin/*.dll ${DISTNAME}/lib/
     find . -name "lib*.la" -delete
@@ -156,11 +150,8 @@ script: |
     rm -rf distsrc-${i}
   done
 
-  mkdir -p ${OUTDIR}/src
-  git archive --prefix="${DISTNAME}/" --output=${OUTDIR}/src/${DISTNAME}.tar.gz HEAD
-
   cp -rf contrib/windeploy $BUILD_DIR
   cd $BUILD_DIR/windeploy
   mkdir unsigned
-  cp $OUTDIR/bitcoin-*setup-unsigned.exe unsigned/
+  cp ${OUTDIR}/${DISTNAME}-win64-setup-unsigned.exe unsigned/
   find . | sort | tar --mtime="$REFERENCE_DATETIME" --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-win-unsigned.tar.gz

contrib/gitian-descriptors/make_release_tarball

@@ -0,0 +1,38 @@
+#!/bin/sh
+# Copyright (c) 2020 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+#
+# A helper script to generate source release tarball
+
+set -e
+
+GIT_ARCHIVE="$1"
+. "`dirname "$0"`/assign_DISTNAME"
+
+git archive --prefix="${DISTNAME}/" HEAD | tar -xp
+
+# Generate correct build info file from git, before we lose git
+GIT_BUILD_INFO="$(share/genbuild.sh /dev/stdout)"
+sed 's/\/\/ No build information available/'"${GIT_BUILD_INFO}"'/' -i "${DISTNAME}/share/genbuild.sh"
+
+cd "${DISTNAME}"
+
+./autogen.sh
+./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking
+make distclean
+
+cd ..
+tar \
+  --format=ustar \
+  --exclude autom4te.cache \
+  --exclude .deps \
+  --exclude .git \
+  --sort=name \
+  --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 \
+  --mtime="${REFERENCE_DATETIME}" \
+  -c "${DISTNAME}" | \
+  gzip -9n \
+  >"${GIT_ARCHIVE}"
+
+rm -r "${DISTNAME}"

share/genbuild.sh

@@ -18,6 +18,7 @@ else
     exit 1
 fi
 
+# This checks that we are actually part of the intended git repository, and not just getting info about some unrelated git repository that the code happens to be in a directory under
 git_check_in_repo() {
     ! { git status --porcelain -uall --ignored "$@" 2>/dev/null || echo '??'; } | grep -q '?'
 }
@@ -44,7 +45,9 @@ if [ -n "$DESC" ]; then
 elif [ -n "$SUFFIX" ]; then
     NEWINFO="#define BUILD_SUFFIX $SUFFIX"
 else
-    NEWINFO="// No build information available"
+    # NOTE: The NEWINFO line below this comment gets replaced by a string-match in contrib/gitian-descriptors/make_release_tarball
+    # If changing it, update the script too!
+    NEWINFO='// No build information available'
 fi
 
 # only update build.h if necessary

test/lint/lint-shell.sh

@@ -46,15 +46,17 @@ if ! command -v yq > /dev/null; then
 fi
 
 EXCLUDE_GITIAN=${EXCLUDE}",$(IFS=','; echo "${disabled_gitian[*]}")"
+SHELLCHECK_CMD="shellcheck --external-sources --check-sourced $EXCLUDE_GITIAN"
 for descriptor in $(git ls-files -- 'contrib/gitian-descriptors/*.yml')
 do
-    echo
-    echo "$descriptor"
+    script=$(basename "$descriptor")
     # Use #!/bin/bash as gitian-builder/bin/gbuild does to complete a script.
-    SCRIPT=$'#!/bin/bash\n'$(yq -r .script "$descriptor")
-    if ! echo "$SCRIPT" | shellcheck "$EXCLUDE_GITIAN" -; then
+    echo "#!/bin/bash" > $script
+    yq -r .script "$descriptor" >> $script
+    if ! $SHELLCHECK_CMD $script; then
         EXIT_CODE=1
     fi
+    rm $script
 done
 
 exit $EXIT_CODE