CBD-4127: v8 for aarch64 (and gcc10)

Change-Id: I3d7167198d65740c84db994ac9daf41b406c4fe8
Reviewed-on: http://review.couchbase.org/c/tlm/+/157273
Tested-by: Build Bot <build@couchbase.com>
Reviewed-by: Blair Watt <blair.watt@couchbase.com>

deps/manifest.cmake

@@ -55,7 +55,8 @@ DECLARE_DEP (rocksdb VERSION 5.18.3-cb6 PLATFORMS amzn2 centos7 debian9 debian10
 ENDIF (NOT CB_PRODUCTION_BUILD)
 DECLARE_DEP (snappy VERSION 1.1.1 PLATFORMS windows_msvc2017)
 DECLARE_DEP (snappy VERSION 1.1.1-cb5 PLATFORMS amzn2 centos7 debian9 debian10 macosx suse12 suse15 ubuntu18.04 ubuntu20.04)
-DECLARE_DEP (v8 VERSION 8.3-cb4 PLATFORMS amzn2 centos7 debian9 debian10 macosx suse12 suse15 ubuntu18.04 ubuntu20.04 windows_msvc2017)
+DECLARE_DEP (v8 VERSION 8.3-cb4 PLATFORMS suse12)
+DECLARE_DEP (v8 VERSION 8.3-cb5 PLATFORMS amzn2 centos7 debian9 debian10 macosx suse15 ubuntu18.04 ubuntu20.04 windows_msvc2017)
 DECLARE_DEP (zlib V2 VERSION 1.2.11 BUILD 7 PLATFORMS amzn2 centos7 debian9 debian10 macosx suse12 suse15 ubuntu18.04 ubuntu20.04 windows_msvc2017)
 
 #

deps/packages/CMakeLists.txt

@@ -161,7 +161,7 @@ _ADD_DEP_PACKAGE(snappy 1.1.1 1.1.1 5)
 _ADD_DEP_PACKAGE(python-snappy c97d633 c97d633 2)
 _ADD_DEP_PACKAGE(icu4c 59.1 r59.1 4)
 _ADD_DEP_PACKAGE(libevent 2.1.11 release-2.1.11-stable 7)
-_ADD_DEP_PACKAGE(v8 8.3 8.3.110.13 4)
+_ADD_DEP_PACKAGE(v8 8.3 8.3.110.13 5)
 _ADD_DEP_PACKAGE(jemalloc 5.2.1 5.2.1 6)
 
 # -cb11 was built and published but never used in builds

deps/packages/v8/linux_patches/iosfwd_gcc10.patch

@@ -0,0 +1,12 @@
+diff --git a/src/base/template-utils.h b/src/base/template-utils.h
+index 8f89672..d6d4ca3 100644
+--- a/src/base/template-utils.h
++++ b/src/base/template-utils.h
+@@ -7,6 +7,7 @@
+
+ #include <array>
+ #include <functional>
++#include <iosfwd>
+ #include <type_traits>
+ #include <utility>
+

deps/packages/v8/linux_patches/no_werror_gcc10.patch

@@ -0,0 +1,14 @@
+diff --git a/config/compiler/BUILD.gn b/config/compiler/BUILD.gn
+index 5fa34150a..f83ca12fc 100644
+--- a/config/compiler/BUILD.gn
++++ b/config/compiler/BUILD.gn
+@@ -1547,7 +1547,7 @@ config("chromium_code") {
+   } else {
+     cflags = [ "-Wall" ]
+     if (treat_warnings_as_errors) {
+-      cflags += [ "-Werror" ]
++      cflags += [ ]
+
+       # The compiler driver can sometimes (rarely) emit warnings before calling
+       # the actual linker.  Make sure these warnings are treated as errors as
+

deps/packages/v8/linux_patches/v8_linux_runpath.patch

@@ -0,0 +1,15 @@
+diff --git a/toolchain/linux/BUILD.gn b/toolchain/linux/BUILD.gn
+index fa8b17e9d..11554193a 100644
+--- a/toolchain/linux/BUILD.gn
++++ b/toolchain/linux/BUILD.gn
+@@ -153,6 +153,10 @@ clang_toolchain("clang_x64_v8_mips64") {
+ }
+
+ gcc_toolchain("x64") {
++  # necessary for Centos 7 due to old binutils
++  # https://sourceware.org/bugzilla/show_bug.cgi?id=20535
++  extra_ldflags = "-Wl,-rpath=\\\$ORIGIN -Wl,-rpath-link ."
++
+   cc = "gcc"
+   cxx = "g++"
+

deps/packages/v8/linux_patches/zlib_aarch64.patch

@@ -0,0 +1,32 @@
+diff --git a/contrib/optimizations/insert_string.h b/contrib/optimizations/insert_string.h
+index 1826601..d123305 100644
+--- a/contrib/optimizations/insert_string.h
++++ b/contrib/optimizations/insert_string.h
+@@ -26,15 +26,23 @@
+   #define _cpu_crc32_u32 _mm_crc32_u32
+
+ #elif defined(CRC32_ARMV8_CRC32)
+-  #if defined(__clang__)
++  #if defined(__GNUC__) || defined(__clang__)
+     #undef TARGET_CPU_WITH_CRC
+-    #define __crc32cw __builtin_arm_crc32cw
++    #if defined(__clang__)
++      #define __crc32cw __builtin_arm_crc32cw
++    #elif defined(__GNUC__)
++      #define __crc32cw __builtin_aarch64_crc32cw
++    #endif
+   #endif
+
+   #define _cpu_crc32_u32 __crc32cw
+
+   #if defined(__aarch64__)
+-    #define TARGET_CPU_WITH_CRC __attribute__((target("crc")))
++    #if defined(__clang__)
++      #define TARGET_CPU_WITH_CRC __attribute__((target("crc")))
++    #elif defined(__GNUC__)
++      #define TARGET_CPU_WITH_CRC __attribute__((target("+crc")))
++    #endif
+   #else  // !defined(__aarch64__)
+     #define TARGET_CPU_WITH_CRC __attribute__((target("armv8-a,crc")))
+   #endif  // defined(__aarch64__)
+

deps/packages/v8/v8_unix.sh

@@ -3,6 +3,8 @@
 INSTALL_DIR=$1
 PLATFORM=$2
 
+ARCH=`uname -m`
+
 pushd $(dirname $0) > /dev/null
 SCRIPTPATH=$(pwd -P)
 popd > /dev/null
@@ -11,48 +13,111 @@ DEPS=/tmp/deps
 rm -rf ${DEPS}
 mkdir -p ${DEPS}
 
-# Download cbdep to get python2 (still required for this build sadly).
-CBDEP_TOOL_VER=0.9.15
-MINICONDA_VER=4.7.12.1
-
-# Download cbdep, unless it's already available in the local .cbdepscache
-OPSYS=$(uname -s | tr "[:upper:]" "[:lower:]")
-CBDEP_BIN_CACHE=/home/couchbase/.cbdepscache/cbdep/${CBDEP_TOOL_VER}/cbdep-${CBDEP_TOOL_VER}-${OPSYS}
-
-if [[ -f ${CBDEP_BIN_CACHE} ]]; then
-    cp ${CBDEP_BIN_CACHE} /tmp/cbdep
-else
-    CBDEP_URL=https://packages.couchbase.com/cbdep/${CBDEP_TOOL_VER}/cbdep-${CBDEP_TOOL_VER}-${OPSYS}
-    curl -o /tmp/cbdep ${CBDEP_URL}
+# This is horrible and I hate it, but it's required for building v8
+# using gcc, and I really don't like the idea of baking this into
+# our build agent images *just* to build v8. I suggest destroying
+# the build agents after compiling v8 ("docker stack rm" then
+# re-deploy).
+if [[ "${PLATFORM}" =~ amzn*|centos*|rhel* ]]; then
+    sudo yum install -y glib2-devel
+elif [[ "${PLATFORM}" =~ suse* ]]; then
+    # suse12 can't do this because the agents aren't hosted on
+    # suse12 VMs, so we have to bake it in there :(
+    [ "${PLATFORM}" = suse12 ] || sudo zypper install -y glib2-devel
+elif [[ "${PLATFORM}" =~ debian*|ubuntu* ]]; then
+    # This env var helps work around a problem with libstdc++ from
+    # /usr/local breaking apt-get on ubuntu20; should be harmless
+    # on other platforms
+    sudo LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu apt-get update
+    sudo LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu apt-get install -y libglib2.0-dev
 fi
 
-chmod +x /tmp/cbdep
-
-# Use cbdep to install miniconda2. Add to PATH *last* (so it only adds
-# python2, not overriding anything). Also add to LD_LIBRARY_PATH as the
-# "vpython" script the build uses creates a copy of "python2" but doesn't
-# copy libpython2 as well.
-# On macosx, we need to use python from miniconda; otherwise, gclient will
-# fail due to SIP.  Creating an alias doesn't seem to work; have to add
-# miniconda2 in the front of PATH.  ubuntu20 comes with python3, v8 doesn't
-# work well with python3, put miniconda2 in the front as well.
-
-/tmp/cbdep install -d ${DEPS} miniconda2 ${MINICONDA_VER}
-export PATH_ORG=${PATH}
-export LD_LIBRARY_PATH=${DEPS}/miniconda2-${MINICONDA_VER}/lib
+# If a newer GCC exists in /opt, use it
+if [ -d /opt/gcc/bin ]; then
+    export PATH=/opt/gcc/bin:${PATH}
+fi
 
-if [[ $PLATFORM != "macosx" && $PLATFORM != "ubuntu20.04" && $PLATFORM != "ubuntu18.04" ]]; then
-    export PATH=${PATH_ORG}:${DEPS}/miniconda2-${MINICONDA_VER}/bin
-else
-    export PATH=${DEPS}/miniconda2-${MINICONDA_VER}/bin:${PATH_ORG}
+# We assume a reasonable python2 is available for aarch64 builds.
+# Can't fall back to miniconda there because miniconda doesn't have
+# aarch64 versions of python2.
+if [ "${ARCH}" = "x86_64" ]; then
+    # Download cbdep to get python2 (still required for this build sadly).
+    CBDEP_TOOL_VER=0.9.15
+    MINICONDA_VER=4.7.12.1
+
+    # Download cbdep, unless it's already available in the local .cbdepscache
+    OPSYS=$(uname -s | tr "[:upper:]" "[:lower:]")
+    CBDEP_BIN_CACHE=/home/couchbase/.cbdepscache/cbdep/${CBDEP_TOOL_VER}/cbdep-${CBDEP_TOOL_VER}-${OPSYS}
+
+    if [[ -f ${CBDEP_BIN_CACHE} ]]; then
+        cp ${CBDEP_BIN_CACHE} /tmp/cbdep
+    else
+        CBDEP_URL=https://packages.couchbase.com/cbdep/${CBDEP_TOOL_VER}/cbdep-${CBDEP_TOOL_VER}-${OPSYS}
+        curl -o /tmp/cbdep ${CBDEP_URL}
+    fi
+
+    chmod +x /tmp/cbdep
+
+    # Use cbdep to install miniconda2. Add to PATH *last* (so it only adds
+    # python2, not overriding anything). Also add to LD_LIBRARY_PATH as the
+    # "vpython" script the build uses creates a copy of "python2" but doesn't
+    # copy libpython2 as well.
+    # On macosx, we need to use python from miniconda; otherwise, gclient will
+    # fail due to SIP.  Creating an alias doesn't seem to work; have to add
+    # miniconda2 in the front of PATH.  ubuntu20 comes with python3, v8 doesn't
+    # work well with python3, put miniconda2 in the front as well.
+
+    /tmp/cbdep install -d ${DEPS} miniconda2 ${MINICONDA_VER}
+    export PATH_ORG=${PATH}
+    export LD_LIBRARY_PATH=${DEPS}/miniconda2-${MINICONDA_VER}/lib
+
+    if [[ $PLATFORM != "macosx" && $PLATFORM != "ubuntu20.04" && $PLATFORM != "ubuntu18.04" ]]; then
+        export PATH=${PATH_ORG}:${DEPS}/miniconda2-${MINICONDA_VER}/bin
+    else
+        export PATH=${DEPS}/miniconda2-${MINICONDA_VER}/bin:${PATH_ORG}
+    fi
 fi
 
-# Get Google's depot_tools; checkout from October 18th, 2018,
-# which worked for the SuSE platforms on the last build.
+# Get Google's depot_tools; lock to moderately recent build (as of June 2021)
 git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git
-pushd depot_tools && git checkout 93277a7 && popd
+pushd depot_tools && git checkout e319aba2 && popd
 export PATH=$(pwd)/depot_tools:$PATH
 
+# Do NOT use their bundled python binary
+export VPYTHON_BYPASS="manually managed python not supported by chrome operations"
+
+# Build Ninja
+pushd ${DEPS}
+if [ ! -d ninja ]; then
+    mkdir ninja
+    pushd ninja
+    git clone https://github.com/ninja-build/ninja.git -b v1.8.2
+    cd ninja
+    ./configure.py --bootstrap
+    popd
+fi
+export PATH=$(pwd)/ninja/ninja:$PATH
+popd
+
+# Build an older version of GN that's still able to build v8 8.3
+# (later versions cannot). We have to build our own on at least
+# some platforms either due to Google's depot_tools' gn using too
+# new a glibc, or else not working on aarch64. We also have to
+# force it to use our compiler.
+pushd ${DEPS}
+if [ ! -d gn ]; then
+    mkdir gn
+    pushd gn
+    git clone https://gn.googlesource.com/gn
+    cd gn
+    git checkout 5ed3c
+    CC=$(which gcc) CXX=$(which g++) python build/gen.py
+    ninja -C out
+    popd
+fi
+export PATH=$(pwd)/gn/gn/out:$PATH
+popd
+
 # Set up gclient config for tag to pull for v8, then do sync
 # (this handles the 'fetch v8' done by the usual process)
 cat > .gclient <<EOF
@@ -68,36 +133,63 @@ EOF
 
 gclient sync
 
-# Apply change to enable RPATH (runpath) for libraries/binaries on Linux
+# Several patches and tweaks for Linux builds
 if [[ $PLATFORM != "macosx" ]]; then
+    # Apply change to enable RPATH (runpath) for libraries/binaries on Linux
     pushd v8/build
-    git apply $SCRIPTPATH/v8_linux_runpath.patch
+    git apply $SCRIPTPATH/linux_patches/v8_linux_runpath.patch
+    popd
+
+    # Necessary for building on aarch64.
+    # https://gist.github.com/Adenilson/de40a2de85d60c9d30bd79fbcb4ad4a2
+    pushd v8/third_party/zlib
+    git apply $SCRIPTPATH/linux_patches/zlib_aarch64.patch
     popd
-fi
 
-#download older version of gn for centos7
-#latest requires glibc 2.18, centos7 comes with 2.17
-if [[ $PLATFORM == "centos7" ]]; then
-    pushd v8/buildtools/linux64
-    mv gn gn.org
-    curl -L -o gn.zip https://chrome-infra-packages.appspot.com/dl/gn/gn/linux-amd64/+/jCF636-ci00rA64wJkdAPZx3NrIvafSvZ_bajkcCMbUC
-    unzip gn.zip
+    # Necessary for building with gcc10.
+    # https://chromium-review.googlesource.com/c/v8/v8/+/2248196/2/src/base/template-utils.h
+    pushd v8
+    git apply $SCRIPTPATH/linux_patches/iosfwd_gcc10.patch
+    popd
+
+    # Also necessary with gcc10 because it has a number of new warnings.
+    pushd v8/build
+    git apply $SCRIPTPATH/linux_patches/no_werror_gcc10.patch
+    popd
+
+    # Set up all the weird tool aliases that gn expects
+    pushd ${DEPS}
+    for tool in gcc g++ ar readelf nm; do
+	ln -s $(which ${tool}) $(uname -m)-linux-gnu-${tool}
+    done
+    export PATH=$(pwd):$PATH
     popd
 fi
 
 # Actual v8 configure and build steps - we build debug and release.
+if [ "${ARCH}" = "aarch64" ]; then
+    TARGET_CPU=arm64
+else
+    TARGET_CPU=x64
+fi
+if [ "${PLATFORM}" = "macosx" ]; then
+    IS_CLANG=true
+else
+    IS_CLANG=false
+fi
+
 cd v8
-V8_ARGS='target_cpu="x64" is_component_build=true v8_enable_backtrace=true v8_use_external_startup_data=false use_custom_libcxx=false v8_enable_pointer_compression=false'
+V8_ARGS="target_cpu=\"${TARGET_CPU}\" is_clang=${IS_CLANG} use_sysroot=false use_gold=false linux_use_bundled_binutils=false is_component_build=true v8_enable_backtrace=true v8_use_external_startup_data=false use_custom_libcxx=false v8_enable_pointer_compression=false"
 
 gn gen out.gn/x64.release --args="$V8_ARGS is_debug=false"
-ninja -C out.gn/x64.release
+ninja -j4 -C out.gn/x64.release
 V8_ARGS="$V8_ARGS v8_enable_slow_dchecks=true"
 #only enable v8_optimized_debug=false if it is not macosx as of V8 8.3 as it causes unitest failures
 if [[ $PLATFORM != "macosx" ]]; then
     V8_ARGS="$V8_ARGS v8_optimized_debug=false"
 fi
 gn gen out.gn/x64.debug --args="$V8_ARGS is_debug=true"
-ninja -C out.gn/x64.debug
+ninja -j4 -C out.gn/x64.debug
 
 # Copy right stuff to output directory.
 mkdir -p \