From a297364665b8308297382acbd9d31dfd9b283845 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Wed, 11 Mar 2020 10:29:04 +0100 Subject: [PATCH] docker, runtime: upgrade to recent clang/llvm image in runtime Do not rely on clang-7/llvm-7 shipped by Ubuntu base image and instead upgrade to clang-11/llvm-11 with a BPF-only backend. This would help overcoming the blockade of [0] where we're hitting the 1 mio instruction complexity limit which was bisected by Paul to the kernel commit f7cf25b20 ("bpf: track spill/fill of constants"). As it is stated there: Newer clang generates better code by spilling less to the stack. Instead it keeps more constants in the registers which hurts state pruning since the verifier already tracks constants in the registers [...]. Tracking constants in the registers hurts state pruning already. Adding tracking of constants through stack hurts pruning even more. The later patch address this general constant tracking issue with coarse/precise logic. Side-effect of going with a custom clang-11/llvm-11 build with a BPF-only backend is that i) we can also shrink the image since x86 is not needed anymore, and ii) avoid shipping a generic compiler in our image that can generate x86 executable code. This depends on [1] where we first need to get rid of our custom runtime probes in Cilium and instead rely on bpftool to take over that job. Size shrinkage around 36.6M: clang-7 (90,777,392) -> clang-11/stripped (75,617,520) llc-7 (51,453,072) -> llc-11/bpf/stripped (29,997,384) [0] https://github.com/cilium/cilium/issues/10517 [1] https://github.com/cilium/cilium/pull/10019 Complexity comparison: https://user-images.githubusercontent.com/677393/76440789-aae08b80-63be-11ea-863f-37ab12106ad9.png Signed-off-by: Daniel Borkmann Cc: Paul Chaignon --- contrib/packaging/docker/Dockerfile.runtime | 38 ++++++++++++--------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/contrib/packaging/docker/Dockerfile.runtime b/contrib/packaging/docker/Dockerfile.runtime index 2cd40f37c3c7..99abdd798075 100644 --- a/contrib/packaging/docker/Dockerfile.runtime +++ b/contrib/packaging/docker/Dockerfile.runtime @@ -9,23 +9,9 @@ apt-get upgrade -y && \ # apt-get install -y --no-install-recommends \ gpg gpg-agent libelf-dev libmnl-dev libc6-dev-i386 iptables libgcc-5-dev \ - bash-completion binutils binutils-dev ca-certificates clang-7 llvm-7 kmod && \ + bash-completion binutils binutils-dev ca-certificates kmod && \ apt-get purge --auto-remove && \ -apt-get clean && \ -# -# clang-7 -# -rm -fr /usr/lib/llvm-7/include/llvm-c && \ -rm -fr /usr/lib/llvm-7/include/clang-c && \ -rm -fr /usr/lib/llvm-7/include/c++ && \ -rm -fr /usr/lib/llvm-7/include/polly && \ -rm -fr /usr/lib/llvm-7/share && \ -ls -d /usr/lib/llvm-7/lib/* | grep -vE clang$ | xargs rm -r && \ -ls -d /usr/lib/llvm-7/bin/* | grep -vE "clang$|clang-7$|llc$" | xargs basename -a | awk '{ print "/usr/bin/"$1"-7" }' | xargs rm -r && \ -ls -d /usr/lib/llvm-7/bin/* | grep -vE "clang$|clang-7$|llc$" | xargs rm -r && \ -strip /usr/lib/llvm-7/bin/* && \ -update-alternatives --install /usr/bin/clang clang /usr/lib/llvm-7/bin/clang 1000 && \ -update-alternatives --install /usr/bin/llc llc /usr/lib/llvm-7/bin/llc 1000 +apt-get clean # # Build Cilium runtime dependencies. @@ -42,6 +28,8 @@ apt-get install -y --no-install-recommends make git curl ca-certificates xz-util gcc git pkg-config bison flex build-essential \ # Additional bpftool dependencies python3 && \ +# Additional clang/llvm dependencies + cmake ninja-build && \ # # iproute2 # @@ -53,6 +41,21 @@ strip tc/tc && \ strip ip/ip && \ cd .. && \ # +# clang/llvm image with only BPF backend +# +git clone -b master https://github.com/llvm/llvm-project.git llvm && \ +mkdir -p llvm/llvm/build/install && \ +cd llvm/ && \ +git checkout -b d941df363d1cb621a3836b909c37d79f2a3e27e2 d941df363d1cb621a3836b909c37d79f2a3e27e2 && \ +cd llvm/build && \ +cmake .. -G "Ninja" -DLLVM_TARGETS_TO_BUILD="BPF" -DLLVM_ENABLE_PROJECTS="clang" -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=Release -DLLVM_BUILD_RUNTIME=OFF && \ +ninja && \ +strip bin/clang && \ +strip bin/llc && \ +cp bin/clang /usr/bin/clang && \ +cp bin/llc /usr/bin/llc && \ +cd ../../../ && \ +# # bpftool # git clone --depth 1 -b master git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git linux && \ @@ -69,7 +72,7 @@ strip -s ./loopback && \ # # Cleanup # -apt-get purge --auto-remove -y gpg gpg-agent gcc make bison flex git curl xz-utils ca-certificates && \ +apt-get purge --auto-remove -y gpg gpg-agent gcc make bison flex git curl xz-utils ca-certificates cmake ninja-build && \ apt-get clean # @@ -91,6 +94,7 @@ LABEL maintainer="maintainer@cilium.io" WORKDIR /bin COPY --from=runtime-build /tmp/iproute2/tc/tc /tmp/iproute2/ip/ip ./ COPY --from=runtime-build /tmp/linux/tools/bpf/bpftool/bpftool ./ +COPY --from=runtime-build /tmp/llvm/llvm/build/bin/clang /tmp/llvm/llvm/build/bin/llc ./ COPY --from=runtime-gobuild /go/bin/gops ./ WORKDIR /cni COPY --from=runtime-build /tmp/loopback ./