From 8dec244a9ddfa91944cface00f01a45ea8f09beb Mon Sep 17 00:00:00 2001 From: Jiawei Huang Date: Thu, 9 May 2024 08:36:16 -0700 Subject: [PATCH 1/2] Add qemu emulated arm64 node image build Qemu emulated arm64 node image build was remove in [1] in favor of the native builds. However, our release process isn't update to date with native runners. This changeset adds qemu back to arm64 node image build and updates qemu static binary copy and clean up for ppc64le and s390x. [1] https://github.com/projectcalico/calico/pull/8558/files#diff-c6e82bd404a904c10b9a0756d78d913a8a5e2ff833fefd384c80af1d7c93c3bb --- node/Dockerfile.arm64 | 15 +++++++++++++++ node/Dockerfile.ppc64le | 6 ++---- node/Dockerfile.s390x | 4 +++- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/node/Dockerfile.arm64 b/node/Dockerfile.arm64 index 99e5777ed66..2df7d9e4f82 100644 --- a/node/Dockerfile.arm64 +++ b/node/Dockerfile.arm64 @@ -17,9 +17,11 @@ ARG LIBNFTNL_VER=1.2.2-1 ARG IPSET_VER=7.11-6 ARG RUNIT_VER=2.1.2 ARG BIRD_IMAGE=calico/bird:latest +ARG QEMU_IMAGE ARG UBI_IMAGE FROM calico/bpftool:v5.3-arm64 as bpftool +FROM ${QEMU_IMAGE} as qemu FROM ${BIRD_IMAGE} as bird # Use this build stage to build iptables rpm and runit binaries. @@ -27,6 +29,10 @@ FROM ${BIRD_IMAGE} as bird # We need to build runit because there aren't any rpms for it in CentOS or ubi repositories. FROM quay.io/centos/centos:stream8 as centos +# Enable non-native builds of this image on an amd64 hosts. +# This must be the first RUN command in this file! +COPY --from=qemu /usr/bin/qemu-aarch64-static /usr/bin/qemu-aarch64-static + ARG IPTABLES_VER ARG LIBNFTNL_VER ARG IPSET_VER @@ -89,6 +95,10 @@ ARG LIBNFTNL_VER ARG IPSET_VER ARG RUNIT_VER +# Enable non-native builds of this image on an amd64 hosts. +# This must be the first RUN command in this file! +COPY --from=qemu /usr/bin/qemu-aarch64-static /usr/bin/qemu-aarch64-static + # Update base packages to pick up security updates. Must do this before adding the centos repo. RUN microdnf upgrade @@ -203,8 +213,13 @@ RUN chmod u+s /bin/mountns # Clean out as many files as we can from the filesystem. We no longer need dnf or the platform python install # or any of its dependencies. COPY clean-up-filesystem.sh /clean-up-filesystem.sh +# Allowing qemu binaries to persist. +RUN sed -i 's#zmore#zmore\n\tqemu\n#m' /clean-up-filesystem.sh RUN /clean-up-filesystem.sh +# Delete qemu binaries +RUN rm /usr/bin/qemu-aarch64-static + # Add in top-level license file COPY LICENSE /licenses/LICENSE diff --git a/node/Dockerfile.ppc64le b/node/Dockerfile.ppc64le index b864f6f69ed..f09376cb9fd 100644 --- a/node/Dockerfile.ppc64le +++ b/node/Dockerfile.ppc64le @@ -42,9 +42,7 @@ LABEL version=${GIT_VERSION} # Enable non-native builds of this image on an amd64 hosts. # This must be the first RUN command in this file! -# we only need this for the intermediate "base" image, so we can run all the apk and other commands -# when running on a kernel >= 4.8, this will become less relevant -COPY --from=qemu /usr/bin/qemu-${ARCH}-static /usr/bin/ +COPY --from=qemu /usr/bin/qemu-ppc64le-static /usr/bin/qemu-ppc64le-static # Install remaining runtime deps required for felix from the global repository RUN apk add --no-cache bash ip6tables ipset iputils iproute2 conntrack-tools runit file ca-certificates @@ -69,6 +67,6 @@ RUN chmod u+s /bin/mountns COPY --from=bpftool /bpftool /bin -RUN rm /usr/bin/qemu-${ARCH}-static +RUN rm /usr/bin/qemu-ppc64le-static CMD ["start_runit"] diff --git a/node/Dockerfile.s390x b/node/Dockerfile.s390x index 15f25ea540c..0db5743da57 100644 --- a/node/Dockerfile.s390x +++ b/node/Dockerfile.s390x @@ -39,7 +39,7 @@ LABEL version=${GIT_VERSION} # Enable non-native builds of this image on an amd64 hosts. # This must be the first RUN command in this file! -COPY --from=qemu /usr/bin/qemu-*-static /usr/bin/ +COPY --from=qemu /usr/bin/qemu-s390x-static /usr/bin/qemu-s390x-static # Install remaining runtime deps required for felix from the global repository RUN apk add --no-cache bash ip6tables ipset iputils iproute2 conntrack-tools runit file ca-certificates @@ -67,4 +67,6 @@ COPY --from=bpftool /bpftool /bin # Add in top-level license file COPY LICENSE /licenses +RUN rm /usr/bin/qemu-s390x-static + CMD ["start_runit"] From 407509ad8c6e2cb637642f7ea79023076ff44908 Mon Sep 17 00:00:00 2001 From: Jiawei Huang Date: Thu, 9 May 2024 09:57:51 -0700 Subject: [PATCH 2/2] Streamline runit build for amd64 and arm64 `package/check` (part of `package/install`) fails on qemu emulated arm64 build without the hacks we made in [1]. However, this hack will break native arm64 build [2]. This change workaround this issue by calling `pacpackage/compile` as the runit binaries are important to us and copied in the next stage. [1] https://github.com/projectcalico/node/pull/1044/files#diff-02ae637382f3cca949bd439c8b796acdb08abfbbe4b712eafae1bb80dd08f866R99 [2] https://github.com/projectcalico/calico/issues/8285 --- node/Dockerfile.amd64 | 10 ++++------ node/Dockerfile.arm64 | 10 ++++------ 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/node/Dockerfile.amd64 b/node/Dockerfile.amd64 index aae34ba3661..c5f80b437fb 100644 --- a/node/Dockerfile.amd64 +++ b/node/Dockerfile.amd64 @@ -75,11 +75,9 @@ RUN rpm -i ${IPSET_SOURCERPM_URL} && \ # runit is not available in ubi or CentOS repos so build it. # get it from the debian repos as the official website doesn't support https -RUN wget -P /tmp https://ftp.debian.org/debian/pool/main/r/runit/runit_${RUNIT_VER}.orig.tar.gz && \ - gunzip /tmp/runit_${RUNIT_VER}.orig.tar.gz && \ - tar -xpf /tmp/runit_${RUNIT_VER}.orig.tar -C /tmp && \ - cd /tmp/admin/runit-${RUNIT_VER}/ && \ - package/install +RUN curl -sfL https://ftp.debian.org/debian/pool/main/r/runit/runit_${RUNIT_VER}.orig.tar.gz | tar xz -C /root && \ + cd /root/admin/runit-${RUNIT_VER} && \ + package/compile FROM ${UBI_IMAGE} as ubi @@ -93,7 +91,7 @@ ARG RUNIT_VER RUN microdnf upgrade # Copy in runit binaries -COPY --from=centos /tmp/admin/runit-${RUNIT_VER}/command/* /usr/local/bin/ +COPY --from=centos /root/admin/runit-${RUNIT_VER}/command/* /usr/local/bin/ # Copy in our rpms COPY --from=centos /root/rpmbuild/RPMS/x86_64/* /tmp/rpms/ diff --git a/node/Dockerfile.arm64 b/node/Dockerfile.arm64 index 2df7d9e4f82..877efea9b17 100644 --- a/node/Dockerfile.arm64 +++ b/node/Dockerfile.arm64 @@ -81,11 +81,9 @@ RUN rpm -i ${IPSET_SOURCERPM_URL} && \ # runit is not available in ubi or CentOS repos so build it. # get it from the debian repos as the official website doesn't support https -RUN wget -P /tmp https://ftp.debian.org/debian/pool/main/r/runit/runit_${RUNIT_VER}.orig.tar.gz && \ - gunzip /tmp/runit_${RUNIT_VER}.orig.tar.gz && \ - tar -xpf /tmp/runit_${RUNIT_VER}.orig.tar -C /tmp && \ - cd /tmp/admin/runit-${RUNIT_VER}/ && \ - package/install +RUN curl -sfL https://ftp.debian.org/debian/pool/main/r/runit/runit_${RUNIT_VER}.orig.tar.gz | tar xz -C /root && \ + cd /root/admin/runit-${RUNIT_VER} && \ + package/compile FROM ${UBI_IMAGE} as ubi @@ -103,7 +101,7 @@ COPY --from=qemu /usr/bin/qemu-aarch64-static /usr/bin/qemu-aarch64-static RUN microdnf upgrade # Copy in runit binaries -COPY --from=centos /tmp/admin/runit-${RUNIT_VER}/command/* /usr/local/bin/ +COPY --from=centos /root/admin/runit-${RUNIT_VER}/command/* /usr/local/bin/ # Copy in our rpms COPY --from=centos /root/rpmbuild/RPMS/aarch64/* /tmp/rpms/