Support aarch64 as a builder

Images can now be compiled either by an x86 or aarch64 builder.

When the builder arch matches the target arch, the image is produced
natively (rather than running through emulation). When the builder arch
and the target arch differ, the appropriate toolchain is installed and
used for image compilation, thus avoiding emulation and slow compile
times for all cases.

Until this change, the builder was always assumed to be x86-based.

Author: aasmith

Dockerfile.multiarch

@@ -2,6 +2,7 @@ ARG OS=
 
 ARG ARCH=
 ARG TOOLCHAIN=
+ARG TOOLCHAIN_PREFIX=
 ARG OPENSSL_TARGET=
 
 ARG OPENSSL_VERSION=
@@ -17,9 +18,9 @@ ARG HAPROXY_SHA256=
 ARG LUA_VERSION=
 ARG LUA_MD5=
 
-# Build using cross-compilation, with amd64 as the build arch and $ARCH as the host
+# Build using cross-compilation, using $TOOLCHAIN to target $ARCH
 
-FROM --platform=linux/amd64 $OS as builder
+FROM $OS as builder
 
 ARG ARCH
 ARG TOOLCHAIN
@@ -28,14 +29,16 @@ ARG ARCH_FLAGS
 RUN dpkg --add-architecture "${ARCH}" && \
     apt-get update && \
     apt-get install --no-install-recommends -y \
-            gcc-10-${TOOLCHAIN} libc6-dev-${ARCH}-cross make file libc6-dev perl libtext-template-perl libreadline-dev curl ca-certificates libcrypt-dev:${ARCH}
+            gcc-10-${TOOLCHAIN} libc6-dev-${ARCH}-cross make file libc6-dev perl libtext-template-perl \
+            libreadline-dev curl ca-certificates libcrypt-dev:${ARCH} gcc-10 binutils-${TOOLCHAIN} binutils
 
 ### OpenSSL
 
-FROM --platform=linux/amd64 builder as ssl
+FROM builder as ssl
 
 ARG ARCH
 ARG TOOLCHAIN
+ARG TOOLCHAIN_PREFIX
 ARG ARCH_FLAGS
 
 ARG OPENSSL_VERSION
@@ -46,18 +49,19 @@ RUN curl -OJL https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz &
     echo ${OPENSSL_SHA256} openssl-${OPENSSL_VERSION}.tar.gz | sha256sum -c && \
     tar zxvf openssl-${OPENSSL_VERSION}.tar.gz && \
     cd openssl-${OPENSSL_VERSION} && \
-    ./Configure ${OPENSSL_TARGET} --cross-compile-prefix=/usr/bin/${TOOLCHAIN}- CC=gcc-10 \
+    ./Configure ${OPENSSL_TARGET} --cross-compile-prefix=/usr/bin/${TOOLCHAIN_PREFIX}- CC=gcc-10 \
              -march=${ARCH_FLAGS} enable-ec_nistp_64_gcc_128 \
-             no-shared --prefix=/tmp/openssl --openssldir=/tmp/openssl && \
+             no-shared --prefix=/tmp/openssl --openssldir=/tmp/openssl --libdir=lib && \
     make && \
     make install_sw
 
 ### PCRE2
 
-FROM --platform=linux/amd64 builder as pcre2
+FROM builder as pcre2
 
 ARG ARCH
 ARG TOOLCHAIN
+ARG TOOLCHAIN_PREFIX
 ARG ARCH_FLAGS
 
 ARG PCRE2_VERSION
@@ -67,7 +71,7 @@ RUN curl -OJL "https://github.com/PhilipHazel/pcre2/releases/download/pcre2-${PC
     echo ${PCRE2_SHA256} pcre2-${PCRE2_VERSION}.tar.gz | sha256sum -c && \
     tar zxvf pcre2-${PCRE2_VERSION}.tar.gz && \
     cd pcre2-${PCRE2_VERSION} && \
-    CC=/usr/bin/${TOOLCHAIN}-gcc-10 CFLAGS="-O3 -march=${ARCH_FLAGS} -g" \
+    CC=/usr/bin/${TOOLCHAIN_PREFIX}-gcc-10 CFLAGS="-O3 -march=${ARCH_FLAGS} -g" \
     ./configure --prefix=/tmp/pcre2 --disable-shared --enable-jit --host=${TOOLCHAIN} && \
     make install
 
@@ -77,6 +81,7 @@ FROM builder as lua
 
 ARG ARCH
 ARG TOOLCHAIN
+ARG TOOLCHAIN_PREFIX
 ARG ARCH_FLAGS
 
 ARG LUA_VERSION
@@ -86,22 +91,23 @@ RUN curl -OJL "http://www.lua.org/ftp/lua-${LUA_VERSION}.tar.gz" && \
     echo "${LUA_MD5} lua-${LUA_VERSION}.tar.gz" | md5sum -c && \
     tar zxf lua-${LUA_VERSION}.tar.gz && \
     cd lua-${LUA_VERSION} && \
-    make CC="/usr/bin/${TOOLCHAIN}-gcc-10" \
+    make CC="/usr/bin/${TOOLCHAIN_PREFIX}-gcc-10" \
          MYCFLAGS="-march=${ARCH_FLAGS} -g" \
-         AR="/usr/bin/${TOOLCHAIN}-ar rcu" \
-         RANLIB=/usr/bin/${TOOLCHAIN}-ranlib && \
+         AR="/usr/bin/${TOOLCHAIN_PREFIX}-ar rcu" \
+         RANLIB=/usr/bin/${TOOLCHAIN_PREFIX}-ranlib && \
     make install INSTALL_TOP=/tmp/lua
 
 ### HAProxy
 
-FROM --platform=linux/amd64 builder as haproxy
+FROM builder as haproxy
 
 COPY --from=ssl   /tmp/openssl /tmp/openssl
 COPY --from=pcre2 /tmp/pcre2   /tmp/pcre2
 COPY --from=lua   /tmp/lua     /tmp/lua
 
 ARG ARCH
 ARG TOOLCHAIN
+ARG TOOLCHAIN_PREFIX
 ARG ARCH_FLAGS
 
 ARG HAPROXY_MAJOR
@@ -114,7 +120,7 @@ RUN curl -OJL "http://www.haproxy.org/download/${HAPROXY_MAJOR}/src/haproxy-${HA
     make -C haproxy-${HAPROXY_VERSION} \
       TARGET=linux-glibc \
       ARCH_FLAGS="-march=${ARCH_FLAGS}" \
-      CC=/usr/bin/${TOOLCHAIN}-gcc-10 \
+      CC=/usr/bin/${TOOLCHAIN_PREFIX}-gcc-10 \
       USE_SLZ=1 \
       USE_STATIC_PCRE2=1 USE_PCRE2_JIT=1 PCRE2DIR=/tmp/pcre2 \
       USE_OPENSSL=1 SSL_INC=/tmp/openssl/include SSL_LIB=/tmp/openssl/lib \

Dockerfile renamed to Dockerfile.native

@@ -32,7 +32,7 @@ RUN curl -OJL https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz &
     echo ${OPENSSL_SHA256} openssl-${OPENSSL_VERSION}.tar.gz | sha256sum -c && \
     tar zxvf openssl-${OPENSSL_VERSION}.tar.gz && \
     cd openssl-${OPENSSL_VERSION} && \
-    ./config no-shared --prefix=/tmp/openssl --openssldir=/tmp/openssl && \
+    ./config no-shared --prefix=/tmp/openssl --openssldir=/tmp/openssl --libdir=lib && \
     make && \
     make test && \
     make install_sw
@@ -89,10 +89,10 @@ RUN curl -OJL "http://www.haproxy.org/download/${HAPROXY_MAJOR}/src/haproxy-${HA
     echo "${HAPROXY_SHA256} haproxy-${HAPROXY_VERSION}.tar.gz" | sha256sum -c && \
     tar zxvf haproxy-${HAPROXY_VERSION}.tar.gz && \
     make -C haproxy-${HAPROXY_VERSION} \
-      TARGET=linux-glibc ARCH=x86_64 \
+      TARGET=linux-glibc \
       USE_SLZ=1 \
       USE_STATIC_PCRE2=1 USE_PCRE2_JIT=1 PCRE2DIR=/tmp/pcre2 \
-      USE_OPENSSL=1 SSL_INC=/tmp/openssl/include SSL_LIB=/tmp/openssl/lib64 \
+      USE_OPENSSL=1 SSL_INC=/tmp/openssl/include SSL_LIB=/tmp/openssl/lib \
       USE_LUA=1 \
       USE_PROMEX=1 \
       DESTDIR=/tmp/haproxy PREFIX= \
@@ -104,7 +104,7 @@ RUN curl -OJL "http://www.haproxy.org/download/${HAPROXY_MAJOR}/src/haproxy-${HA
 
 ### HAProxy runtime image
 
-FROM --platform=linux/amd64 $OS as runtime
+FROM $OS as runtime
 
 RUN apt-get update && \
     apt-get install --no-install-recommends -y curl ca-certificates

build.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 
 set -eu
+shopt -s extglob
 
 export OS=debian:bullseye-slim
 export OPENSSL_VERSION=3.0.8
@@ -39,7 +40,24 @@ echo "Preparing manifest '$MANIFEST_NAME'"
 
 # Build images that require cross-compliation
 
-for buildspec in buildspec.*; do
+echo "Builder is an $(uname -m)"
+
+case "$(uname -m)" in
+	arm64)
+		CROSS_SPECS=buildspec.!(aarch64)
+		NATIVE_DOCKER_ARCH=arm64
+		;;
+	x86_64)
+		CROSS_SPECS=buildspec.!(x86_64)
+		NATIVE_DOCKER_ARCH=amd64
+		;;
+	*)
+		echo "unknown arch"
+		exit 1
+		;;
+esac
+
+for buildspec in $CROSS_SPECS; do
 
   set -o allexport
   source "$buildspec"
@@ -63,6 +81,7 @@ for buildspec in buildspec.*; do
     --build-arg ARCH \
     --build-arg ARCH_FLAGS \
     --build-arg TOOLCHAIN \
+    --build-arg TOOLCHAIN_PREFIX \
     --build-arg OPENSSL_TARGET \
     --provenance=false \
     --$ACTION \
@@ -78,14 +97,11 @@ done
 
 echo "building native"
 
-# Build "native" amd64 image
-
-ARCH=amd64
-IMAGE_NAME=$BASE:$IMAGE_TAG-$ARCH
+IMAGE_NAME=$BASE:$IMAGE_TAG-$NATIVE_DOCKER_ARCH
 
 echo "Building '$IMAGE_NAME'..."
 
-docker buildx build -f Dockerfile -t "$IMAGE_NAME" \
+docker buildx build -f Dockerfile.native -t "$IMAGE_NAME" \
   --build-arg OS \
   --build-arg OPENSSL_VERSION \
   --build-arg OPENSSL_SHA256 \
@@ -102,7 +118,7 @@ docker buildx build -f Dockerfile -t "$IMAGE_NAME" \
 
 if [ -n "$REALBUILD" ]; then
   docker manifest create "$MANIFEST_NAME" --amend "$IMAGE_NAME"
-  docker manifest annotate --arch=$ARCH "$MANIFEST_NAME" "$IMAGE_NAME"
+  docker manifest annotate --arch=$NATIVE_DOCKER_ARCH "$MANIFEST_NAME" "$IMAGE_NAME"
   docker manifest inspect "$MANIFEST_NAME"
 
   # Push the complete manifest

buildspec.aarch64

@@ -22,6 +22,7 @@ export ARCH_FLAGS=armv8.2-a+fp16+rcpc+dotprod+crypto
 # The GNU triplet toolchain for cross-compiling to the specified architecture.
 # These are specifically "debian multiarch tuples" per https://wiki.debian.org/Multiarch/Tuples
 export TOOLCHAIN=aarch64-linux-gnu
+export TOOLCHAIN_PREFIX=aarch64-linux-gnu
 
 # Valid openssl targets are defined in openssl/Configurations/10-main.conf.
 export OPENSSL_TARGET=linux-aarch64

buildspec.x86_64

@@ -0,0 +1,23 @@
+# Target architecture in docker parlance. This is essentially $GOARCH, per the
+# manifest spec at https://docs.docker.com/registry/spec/manifest-v2-2/
+#
+# The "linux/" OS prefix is assumed.
+#
+export ARCH=amd64
+
+# The advertised variant that will be used in the manifest.
+export VARIANT=
+
+# Generic x86-64
+#
+# https://gcc.gnu.org/onlinedocs/gcc/x86-Options.html
+export ARCH_FLAGS=x86-64
+
+# The GNU triplet toolchain for cross-compiling to the specified architecture.
+# These are specifically "debian multiarch tuples" per https://wiki.debian.org/Multiarch/Tuples
+export TOOLCHAIN=x86-64-linux-gnu
+export TOOLCHAIN_PREFIX=x86_64-linux-gnu
+
+# Valid openssl targets are defined in openssl/Configurations/10-main.conf.
+export OPENSSL_TARGET=linux-x86_64
+