Update gitian build to most recent version (dashpay#15)

* Update gitian build to most recent version * Remove installation of cmake Co-authored-by: BytzCK <bytzck@protonmail.com>
ckti-dash-current · Apr 10, 2022 · ed279fe · ed279fe
1 parent 9bcefd7
commit ed279fe
Show file tree

Hide file tree

Showing 7 changed files with 274 additions and 231 deletions.
diff --git a/contrib/gitian-build.py b/contrib/gitian-build.py
diff --git a/contrib/gitian-descriptors/README.md b/contrib/gitian-descriptors/README.md
@@ -1,4 +1,4 @@
-### Gavin's notes on getting Gitian builds up and running using KVM
+### Notes on getting Gitian builds up and running
 
 These instructions distilled from
 [https://help.ubuntu.com/community/KVM/Installation](https://help.ubuntu.com/community/KVM/Installation).
@@ -45,11 +45,11 @@ If your main machine is a 64-bit Mac or PC with a few gigabytes of memory
 and at least 10 gigabytes of free disk space, you can `gitian-build` using
 LXC running inside a virtual machine.
 
-Here's a description of Gavin's setup on OSX 10.6:
+Here's a description of a setup on OSX 10.6:
 
 1. Download and install VirtualBox from [https://www.virtualbox.org/](https://www.virtualbox.org/)
 
-2. Download the 64-bit Ubuntu Desktop 12.04 LTS .iso CD image from
+2. Download the 64-bit Ubuntu Desktop 20.04 LTS .iso CD image from
    [http://www.ubuntu.com/](http://www.ubuntu.com/)
 
 3. Run VirtualBox and create a new virtual machine, using the Ubuntu .iso (see the [VirtualBox documentation](https://www.virtualbox.org/wiki/Documentation) for details). Create it with at least 2 gigabytes of memory and a disk that is at least 20 gigabytes big.

diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml
@@ -1,22 +1,35 @@
 ---
-name: "wagerr-linux-4.0"
+name: "wagerr-linux-4.0.99"
 enable_cache: true
+distro: "ubuntu"
 suites:
 - "bionic"
 architectures:
 - "amd64"
 packages:
 - "curl"
 - "g++-aarch64-linux-gnu"
-- "g++-7-aarch64-linux-gnu"
-- "gcc-7-aarch64-linux-gnu"
+- "g++-8-aarch64-linux-gnu"
+- "gcc-8-aarch64-linux-gnu"
 - "binutils-aarch64-linux-gnu"
 - "g++-arm-linux-gnueabihf"
-- "g++-7-arm-linux-gnueabihf"
-- "gcc-7-arm-linux-gnueabihf"
+- "g++-8-arm-linux-gnueabihf"
+- "gcc-8-arm-linux-gnueabihf"
 - "binutils-arm-linux-gnueabihf"
-- "g++-7-multilib"
-- "gcc-7-multilib"
+- "g++-riscv64-linux-gnu"
+- "g++-8-riscv64-linux-gnu"
+- "gcc-8-riscv64-linux-gnu"
+- "binutils-riscv64-linux-gnu"
+- "g++-powerpc64le-linux-gnu"
+- "g++-8-powerpc64le-linux-gnu"
+- "gcc-8-powerpc64le-linux-gnu"
+- "binutils-powerpc64le-linux-gnu"
+- "g++-i686-linux-gnu"
+- "g++-8-i686-linux-gnu"
+- "gcc-8-i686-linux-gnu"
+- "binutils-i686-linux-gnu"
+- "g++-8-multilib"
+- "gcc-8-multilib"
 - "binutils-gold"
 - "git"
 - "pkg-config"
@@ -26,20 +39,21 @@ packages:
 - "faketime"
 - "bsdmainutils"
 - "ca-certificates"
-- "python"
 - "python3"
 - "libxkbcommon0"
-- "ccache"
+- "linux-libc-dev"
+- "doxygen"
 remotes:
 - "url": "https://github.com/wagerr/wagerr.git"
   "dir": "wagerr"
 files: []
 script: |
+  set -e -o pipefail
 
   WRAP_DIR=$HOME/wrapped
-  HOSTS="i686-pc-linux-gnu x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu"
+  HOSTS="i686-pc-linux-gnu x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu" # powerpc64le-linux-gnu riscv64-linux-gnu s390x-linux-gnu powerpc64-linux-gnu mipsel-linux-gnu mips-linux-gnu
   CONFIGFLAGS="--enable-glibc-back-compat --enable-reduce-exports --disable-bench --disable-gui-tests --enable-crash-hooks"
-  FAKETIME_HOST_PROGS=""
+  FAKETIME_HOST_PROGS="gcc g++"
   FAKETIME_PROGS="date ar ranlib nm"
   HOST_CFLAGS="-O2 -g"
   HOST_CXXFLAGS="-O2 -g"
@@ -54,54 +68,33 @@ script: |
   mkdir -p ${WRAP_DIR}
   if test -n "$GBUILD_CACHE_ENABLED"; then
     export SOURCES_PATH=${GBUILD_COMMON_CACHE}
-    export BASE_CACHE=${GBUILD_PACKAGE_CACHE}/depends
+    export BASE_CACHE=${GBUILD_PACKAGE_CACHE}
     mkdir -p ${BASE_CACHE} ${SOURCES_PATH}
-
-    # Setup ccache to use correct cache directories and fix the compiler check of ccache
-    CONFIGFLAGS="${CONFIGFLAGS} --enable-ccache"
-    export CCACHE_DIR=${GBUILD_PACKAGE_CACHE}/ccache
-    # As we later wrap the gcc binaries, this is fast
-    export CCACHE_COMPILERCHECK="content"
-    if [ -f ${GBUILD_PACKAGE_CACHE}/ccache.tar ]; then
-      pushd ${GBUILD_PACKAGE_CACHE}
-      tar xf ccache.tar
-      rm ccache.tar
-      popd
-    fi
-    # instead of compressing ccache.tar, we let ccache handle it by itself
-    # Otherwise we end up uncompressing/compressing a lot of cache files which we actually never use
-    export CCACHE_COMPRESS=1
-  else
-    CONFIGFLAGS="${CONFIGFLAGS} --disable-ccache"
   fi
 
-  # We include the GCC version in all wrappers so that ccache can detect compiler upgrades when hashing the wrappers
-  GCCVERSION=`gcc --version | head -1`
-
   function create_global_faketime_wrappers {
   for prog in ${FAKETIME_PROGS}; do
     echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog}
-    echo "# GCCVERSION=${GCCVERSION}" >> ${WRAP_DIR}/${prog}
     echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog}
     echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog}
     echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog}
     echo "\$REAL \$@" >> $WRAP_DIR/${prog}
     chmod +x ${WRAP_DIR}/${prog}
-    touch -d "${REFERENCE_DATETIME}" ${WRAP_DIR}/${prog}
   done
   }
 
   function create_per-host_faketime_wrappers {
   for i in $HOSTS; do
     for prog in ${FAKETIME_HOST_PROGS}; do
-        echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog}
-        echo "# GCCVERSION=${GCCVERSION}" >> ${WRAP_DIR}/${i}-${prog}
-        echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog}
-        echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog}
-        echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog}
-        echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog}
-        chmod +x ${WRAP_DIR}/${i}-${prog}
-        touch -d "${REFERENCE_DATETIME}" ${WRAP_DIR}/${i}-${prog}
+        if which ${i}-${prog}-8
+        then
+            echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog}
+            echo "REAL=\`which -a ${i}-${prog}-8 | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog}
+            echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog}
+            echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog}
+            echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog}
+            chmod +x ${WRAP_DIR}/${i}-${prog}
+        fi
     done
   done
   }
@@ -127,8 +120,7 @@ script: |
   rm -f ${WRAP_DIR}/${prog}
   cat << EOF > ${WRAP_DIR}/${prog}
   #!/usr/bin/env bash
-  # GCCVERSION=${GCCVERSION}
-  REAL="`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1`"
+  REAL="`which -a ${prog}-8 | grep -v ${WRAP_DIR}/${prog} | head -1`"
   for var in "\$@"
   do
     if [ "\$var" = "-m32" ]; then
@@ -140,7 +132,6 @@ script: |
   \$REAL \$@
   EOF
   chmod +x ${WRAP_DIR}/${prog}
-  touch -d "${REFERENCE_DATETIME}" ${WRAP_DIR}/${prog}
   done
 
   cd wagerr
@@ -192,39 +183,26 @@ script: |
     mkdir src/obj
     cp ../src/obj/build.h src/obj/
 
-    CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} CFLAGS="${HOST_CFLAGS}" CXXFLAGS="${HOST_CXXFLAGS}" LDFLAGS="${HOST_LDFLAGS}"
+    CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} CFLAGS="${HOST_CFLAGS}" CXXFLAGS="${HOST_CXXFLAGS}" LDFLAGS="${HOST_LDFLAGS}"
     make ${MAKEOPTS}
     make ${MAKEOPTS} -C src check-security
-
-    #TODO: This is a quick hack that disables symbol checking for arm.
-    #      Instead, we should investigate why these are popping up.
-    #      For aarch64, we'll need to bump up the min GLIBC version, as the abi
-    #      support wasn't introduced until 2.17.
     case $i in
        aarch64-*) : ;;
        arm-*) : ;;
        *) make ${MAKEOPTS} -C src check-symbols ;;
     esac
-
     make install DESTDIR=${INSTALLPATH}
     cd installed
     find . -name "lib*.la" -delete
     find . -name "lib*.a" -delete
     rm -rf ${DISTNAME}/lib/pkgconfig
-    find ${DISTNAME}/bin -type f -executable -exec ../contrib/devtools/split-debug.sh {} {} {}.dbg \;
-    find ${DISTNAME}/lib -type f -exec ../contrib/devtools/split-debug.sh {} {} {}.dbg \;
+    find ${DISTNAME}/bin -type f -executable -print0 | xargs -0 -n1 -I{} ../contrib/devtools/split-debug.sh {} {} {}.dbg
+    #find ${DISTNAME}/lib -type f -print0 | xargs -0 -n1 -I{} ../contrib/devtools/split-debug.sh {} {} {}.dbg
+    cp ../doc/README.md ${DISTNAME}/
     find ${DISTNAME} -not -name "*.dbg" | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}.tar.gz
     find ${DISTNAME} -name "*.dbg" | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}-debug.tar.gz
     cd ../../
     rm -rf distsrc-${i}
   done
   mkdir -p $OUTDIR/src
   mv $SOURCEDIST $OUTDIR/src
-
-  # Compress ccache (otherwise the assert file will get too huge)
-  if [ "$CCACHE_DIR" != "" ]; then
-    pushd ${GBUILD_PACKAGE_CACHE}
-    tar cf ccache.tar ccache
-    rm -rf ccache
-    popd
-  fi
diff --git a/contrib/gitian-descriptors/gitian-osx-signer.yml b/contrib/gitian-descriptors/gitian-osx-signer.yml
@@ -1,5 +1,6 @@
 ---
 name: "wagerr-dmg-signer"
+distro: "ubuntu"
 suites:
 - "bionic"
 architectures:
@@ -12,24 +13,22 @@ remotes:
 files:
 - "wagerr-osx-unsigned.tar.gz"
 script: |
+  set -e -o pipefail
   WRAP_DIR=$HOME/wrapped
   mkdir -p ${WRAP_DIR}
-  export PATH=`pwd`:$PATH
+  export PATH="$PWD":$PATH
   FAKETIME_PROGS="dmg genisoimage"
-
   # Create global faketime wrappers
   for prog in ${FAKETIME_PROGS}; do
     echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog}
     echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog}
-    echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog}
+    echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${prog}
     echo "export FAKETIME=\"${REFERENCE_DATETIME}\"" >> ${WRAP_DIR}/${prog}
     echo "\$REAL \$@" >> $WRAP_DIR/${prog}
     chmod +x ${WRAP_DIR}/${prog}
   done
-
   UNSIGNED=wagerr-osx-unsigned.tar.gz
   SIGNED=wagerr-osx-signed.dmg
-
   tar -xf ${UNSIGNED}
   OSX_VOLNAME="$(cat osx_volname)"
   ./detached-sig-apply.sh ${UNSIGNED} signature/osx

diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml
@@ -1,6 +1,7 @@
 ---
-name: "wagerr-osx-4.0"
+name: "wagerr-osx-4.0.99"
 enable_cache: true
+distro: "ubuntu"
 suites:
 - "bionic"
 architectures:
@@ -22,22 +23,22 @@ packages:
 - "libcap-dev"
 - "libz-dev"
 - "libbz2-dev"
-- "python"
-- "python-dev"
-- "python3"
 - "python3-dev"
 - "python3-setuptools"
+- "python3-biplist"
 - "fonts-tuffy"
-- "ccache"
+- "doxygen"
 remotes:
 - "url": "https://github.com/wagerr/wagerr.git"
   "dir": "wagerr"
 files:
-- "MacOSX10.11.sdk.tar.gz"
+- "MacOSX10.11.sdk.tar.xz"
 script: |
+  set -e -o pipefail
+
   WRAP_DIR=$HOME/wrapped
-  HOSTS="x86_64-apple-darwin14"
-  CONFIGFLAGS="--enable-reduce-exports --disable-miner --disable-bench --disable-gui-tests GENISOIMAGE=$WRAP_DIR/genisoimage --enable-crash-hooks"
+  HOSTS="x86_64-apple-darwin16"
+  CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests GENISOIMAGE=$WRAP_DIR/genisoimage --enable-crash-hooks"
   FAKETIME_HOST_PROGS=""
   FAKETIME_PROGS="ar ranlib date dmg genisoimage"
 
@@ -50,23 +51,8 @@ script: |
   mkdir -p ${WRAP_DIR}
   if test -n "$GBUILD_CACHE_ENABLED"; then
     export SOURCES_PATH=${GBUILD_COMMON_CACHE}
-    export BASE_CACHE=${GBUILD_PACKAGE_CACHE}/depends
+    export BASE_CACHE=${GBUILD_PACKAGE_CACHE}
     mkdir -p ${BASE_CACHE} ${SOURCES_PATH}
-
-    # Setup ccache to use correct cache directories
-    CONFIGFLAGS="${CONFIGFLAGS} --enable-ccache"
-    export CCACHE_DIR=${GBUILD_PACKAGE_CACHE}/ccache
-    if [ -f ${GBUILD_PACKAGE_CACHE}/ccache.tar ]; then
-      pushd ${GBUILD_PACKAGE_CACHE}
-      tar xf ccache.tar
-      rm ccache.tar
-      popd
-    fi
-    # instead of compressing ccache.tar, we let ccache handle it by itself
-    # Otherwise we end up uncompressing/compressing a lot of cache files which we actually never use
-    export CCACHE_COMPRESS=1
-  else
-    CONFIGFLAGS="${CONFIGFLAGS} --disable-ccache"
   fi
 
   export ZERO_AR_DATE=1
@@ -79,7 +65,6 @@ script: |
     echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog}
     echo "\$REAL \$@" >> $WRAP_DIR/${prog}
     chmod +x ${WRAP_DIR}/${prog}
-    touch -d "${REFERENCE_DATETIME}" ${WRAP_DIR}/${prog}
   done
   }
 
@@ -92,7 +77,6 @@ script: |
         echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog}
         echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog}
         chmod +x ${WRAP_DIR}/${i}-${prog}
-        touch -d "${REFERENCE_DATETIME}" ${WRAP_DIR}/${i}-${prog}
     done
   done
   }
@@ -107,7 +91,7 @@ script: |
   BASEPREFIX=`pwd`/depends
 
   mkdir -p ${BASEPREFIX}/SDKs
-  tar -C ${BASEPREFIX}/SDKs -xf ${BUILD_DIR}/MacOSX10.11.sdk.tar.gz
+  tar -C ${BASEPREFIX}/SDKs -xJvf ${BUILD_DIR}/MacOSX10.11.sdk.tar.xz
 
   # Build dependencies for each host
   for i in $HOSTS; do
@@ -152,9 +136,8 @@ script: |
     mkdir src/obj
     cp ../src/obj/build.h src/obj/
 
-    CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS}
+    CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS}
     make ${MAKEOPTS}
-    make -C src osx_debug
     make install-strip DESTDIR=${INSTALLPATH}
 
     make osx_volname
@@ -179,20 +162,9 @@ script: |
     find . -name "lib*.la" -delete
     find . -name "lib*.a" -delete
     rm -rf ${DISTNAME}/lib/pkgconfig
-    find .. -name *.dSYM -exec cp -ra {} ${DISTNAME}/bin \;
-    find ${DISTNAME} -not -path '*.dSYM*' | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}.tar.gz
-    find ${DISTNAME} -path '*.dSYM*' | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}-debug.tar.gz
+    find ${DISTNAME} | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}.tar.gz
     cd ../../
   done
   mkdir -p $OUTDIR/src
   mv $SOURCEDIST $OUTDIR/src
-  mv ${OUTDIR}/${DISTNAME}-x86_64-apple-darwin14.tar.gz ${OUTDIR}/${DISTNAME}-osx64.tar.gz
-  mv ${OUTDIR}/${DISTNAME}-x86_64-apple-darwin14-debug.tar.gz ${OUTDIR}/${DISTNAME}-osx64-debug.tar.gz
-
-  # Compress ccache (otherwise the assert file will get too huge)
-  if [ "$CCACHE_DIR" != "" ]; then
-    pushd ${GBUILD_PACKAGE_CACHE}
-    tar cf ccache.tar ccache
-    rm -rf ccache
-    popd
-  fi
+  mv ${OUTDIR}/${DISTNAME}-x86_64-*.tar.gz ${OUTDIR}/${DISTNAME}-osx64.tar.gz
diff --git a/contrib/gitian-descriptors/gitian-win-signer.yml b/contrib/gitian-descriptors/gitian-win-signer.yml
@@ -1,5 +1,6 @@
 ---
 name: "wagerr-win-signer"
+distro: "ubuntu"
 suites:
 - "bionic"
 architectures:
@@ -17,7 +18,9 @@ files:
 - "osslsigncode-2.0.tar.gz"
 - "wagerr-win-unsigned.tar.gz"
 script: |
-  BUILD_DIR=`pwd`
+  set -e -o pipefail
+
+  BUILD_DIR="$PWD"
   SIGDIR=${BUILD_DIR}/signature/win
   UNSIGNED_DIR=${BUILD_DIR}/unsigned
 
@@ -34,6 +37,6 @@ script: |
   make
   find ${UNSIGNED_DIR} -name "*-unsigned.exe" | while read i; do
     INFILE="`basename "${i}"`"
-    OUTFILE="`echo "${INFILE}" | sed s/-unsigned//`"
+    OUTFILE="${INFILE/-unsigned}"
     ./osslsigncode attach-signature -in "${i}" -out "${OUTDIR}/${OUTFILE}" -sigin "${SIGDIR}/${INFILE}.pem"
   done