From 1e91b25ef57e0546530a6d46d6988e0ff9b1ba30 Mon Sep 17 00:00:00 2001 From: Benjamin Elder Date: Thu, 15 Jul 2021 20:18:49 -0700 Subject: [PATCH 1/3] stop building for ppc64le because qemu coredumps and it's not clear that we have users anyhow --- images/Makefile.common.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/Makefile.common.in b/images/Makefile.common.in index ee335d1e60..973727777e 100644 --- a/images/Makefile.common.in +++ b/images/Makefile.common.in @@ -13,7 +13,7 @@ IMAGE?=$(REGISTRY)/$(IMAGE_NAME):$(TAG) export DOCKER_CLI_EXPERIMENTAL=enabled # build with buildx -PLATFORMS?=linux/amd64,linux/arm64,linux/ppc64le +PLATFORMS?=linux/amd64,linux/arm64 OUTPUT= PROGRESS=auto build: ensure-buildx From a6da3463feb6dedce18f1ad36c5456d8e3e862e5 Mon Sep 17 00:00:00 2001 From: Benjamin Elder Date: Thu, 15 Jul 2021 20:02:23 -0700 Subject: [PATCH 2/3] build a distroless haproxy image --- images/haproxy/Dockerfile | 56 ++++++++++++++- images/haproxy/stage-binary-and-deps.sh | 90 +++++++++++++++++++++++++ 2 files changed, 143 insertions(+), 3 deletions(-) create mode 100755 images/haproxy/stage-binary-and-deps.sh diff --git a/images/haproxy/Dockerfile b/images/haproxy/Dockerfile index db0089d5bc..5df6cc2985 100644 --- a/images/haproxy/Dockerfile +++ b/images/haproxy/Dockerfile @@ -12,7 +12,57 @@ # See the License for the specific language governing permissions and # limitations under the License. -# standard haproxy image + minimal config so the container will not exit -ARG BASE="haproxy:2.2.0-alpine" -FROM ${BASE} +# This image is a haproxy image + minimal config so the container will not exit +# while we rewrite the config at runtime and signal haproxy to reload. + +ARG BASE="k8s.gcr.io/build-image/debian-base:buster-v1.8.0" +FROM ${BASE} as build + +# NOTE: copyrights.tar.gz is a quirk of Kubernetes's debian-base image +# We extract these here so we can grab the relevant files are easily +# staged for copying into our final image. +RUN [ ! -f /usr/share/copyrights.tar.gz ] || tar -C / -xzvf /usr/share/copyrights.tar.gz + +# install: +# - haproxy (see: https://haproxy.debian.net/) +# - bash (ldd is a bash script and debian-base removes bash) +# - procps (for `kill` which kind needs) +RUN echo deb http://deb.debian.org/debian buster-backports main \ + >/etc/apt/sources.list.d/backports.list && \ + apt update && \ + apt install -y --no-install-recommends haproxy=2.2.\* \ + procps bash + +# copy in script for staging distro provided binary to distroless +COPY --chmod=0755 stage-binary-and-deps.sh /usr/local/bin/ + +# stage everything for copying into the final image +# NOTE: kind currently also uses "mkdir" and "cp" to write files within the container +# TODO: mkdir especially should be unnecessary, with a little refactoring +# NOTE: kill is used to signal haproxy to reload +ARG STAGE_DIR="/opt/stage" +RUN mkdir -p "${STAGE_DIR}" && \ + stage-binary-and-deps.sh haproxy "${STAGE_DIR}" && \ + stage-binary-and-deps.sh cp "${STAGE_DIR}" && \ + stage-binary-and-deps.sh mkdir "${STAGE_DIR}" && \ + stage-binary-and-deps.sh kill "${STAGE_DIR}" + +################################################################################ + +# haproxy is a c++ binary, so we will use the c++ distroless image +# See: https://github.com/GoogleContainerTools/distroless/tree/main/base +# See: https://github.com/GoogleContainerTools/distroless/tree/main/cc +# This has /etc/passwd, tzdata, cacerts, glibc, libssl, openssl, and libgcc1 +FROM "gcr.io/distroless/cc" + +ARG STAGE_DIR="/opt/stage" + +# copy staged binary + deps + copyright +COPY --from=build "${STAGE_DIR}/" / + +# add our minimal config COPY haproxy.cfg /usr/local/etc/haproxy/haproxy.cfg + +# below roughly matches the standard haproxy image +STOPSIGNAL SIGUSR1 +ENTRYPOINT ["haproxy", "-sf", "7", "-W", "-db", "-f", "/usr/local/etc/haproxy/haproxy.cfg"] diff --git a/images/haproxy/stage-binary-and-deps.sh b/images/haproxy/stage-binary-and-deps.sh new file mode 100755 index 0000000000..b2381a1fde --- /dev/null +++ b/images/haproxy/stage-binary-and-deps.sh @@ -0,0 +1,90 @@ +#!/bin/bash + +# Copyright 2021 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# USAGE: stage-binary-and-deps.sh haproxy /opt/stage +# +# Stages $1 and it's dependencies + their copyright files to $2 +# +# This is intended to be used in a multi-stage docker build with a distroless/base +# or distroless/cc image. + +set -o errexit +set -o nounset +set -o pipefail + +# file_to_package identifies the debian package that provided the file $1 +file_to_package() { + # `dpkg-query --search $file-pattern` outputs lines with the format: "$package: $file-path" + # where $file-path belongs to $package + # https://manpages.debian.org/jessie/dpkg/dpkg-query.1.en.html + dpkg-query --search "${1}" | cut -d':' -f1 +} + +# package_to_copyright gives the path to the copyright file for the package $1 +package_to_copyright() { + echo "/usr/share/doc/${1}/copyright" +} + +# stage_file stages the filepath $1 to $2, following symlinks +# and staging copyrights +stage_file() { + cp -a --parents "${1}" "${2}" + # recursively follow symlinks + if [[ -L "${1}" ]]; then + stage_file "$(cd "$(dirname "${1}")"; realpath -s "$(readlink "${1}")")" "${2}" + fi + # stage the copyright for the file + cp -a --parents "$(package_to_copyright "$(file_to_package "${1}")")" "${2}" +} + +# binary_to_libraries identifies the library files needed by the binary $1 with ldd +binary_to_libraries() { + # see: https://man7.org/linux/man-pages/man1/ldd.1.html + ldd "${1}" \ + `# strip the leading '${name} => ' if any so only '/lib-foo.so (0xf00)' remains` \ + | sed -E 's#.* => /#/#' \ + `# we want only the path remaining, not the (0x${LOCATION})` \ + | awk '{print $1}' \ + `# linux-vdso.so.1 is a special virtual shared object from the kernel` \ + `# see: http://man7.org/linux/man-pages/man7/vdso.7.html` \ + | grep -v 'linux-vdso.so.1' +} + +# main script logic +main(){ + local BINARY=$1 + local STAGE_DIR="${2}/" + + # locate the path to the binary + local binary_path + binary_path="$(which "${BINARY}")" + + # stage the binary itself + stage_file "${binary_path}" "${STAGE_DIR}" + + # stage the dependencies of the binary + while IFS= read -r c_dep; do + # skip libc, libgcc1 we already have this in the distroless images + pkg="$(file_to_package "${c_dep}")" + if [[ "${pkg}" == "libc6" || "${pkg}" == "libgcc1" ]]; then + continue + fi + # otherwise stage dependency + stage_file "${c_dep}" "${STAGE_DIR}" + done < <(binary_to_libraries "${binary_path}") +} + +main "$@" From 8f293e11855e6545789ed81dd3507fc6c8359ce8 Mon Sep 17 00:00:00 2001 From: Benjamin Elder Date: Thu, 15 Jul 2021 22:40:37 -0700 Subject: [PATCH 3/3] bump haproxy image --- pkg/cluster/internal/loadbalancer/const.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cluster/internal/loadbalancer/const.go b/pkg/cluster/internal/loadbalancer/const.go index 14df1565ce..260e029100 100644 --- a/pkg/cluster/internal/loadbalancer/const.go +++ b/pkg/cluster/internal/loadbalancer/const.go @@ -17,7 +17,7 @@ limitations under the License. package loadbalancer // Image defines the loadbalancer image:tag -const Image = "kindest/haproxy:v20200708-548e36db" +const Image = "kindest/haproxy:v20210715-a6da3463" // ConfigPath defines the path to the config file in the image const ConfigPath = "/usr/local/etc/haproxy/haproxy.cfg"