Add instructions to build multi-arch container image (#2019)

* add instructions to build multi-arch container image for skipper and tools

Signed-off-by: Martin Linkhorst <martin.linkhorst@zalando.de>

* add custom multi-arch builder for CDP pipeline

Signed-off-by: Martin Linkhorst <martin.linkhorst@zalando.de>

* inline multi-arch build with current build commands

Signed-off-by: Martin Linkhorst <martin.linkhorst@zalando.de>

Makefile

@@ -38,7 +38,7 @@ endif
 
 build: $(SOURCES) lib skipper eskip webhook routesrv
 
-build.linux.armv8:
+build.linux.arm64:
 	GOOS=linux GOARCH=arm64 CGO_ENABLED=0 go build -o bin/skipper -ldflags "-X main.version=$(VERSION) -X main.commit=$(COMMIT_HASH)" ./cmd/skipper
 
 build.linux.armv7:
@@ -47,7 +47,10 @@ build.linux.armv7:
 build.linux:
 	GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o bin/skipper -ldflags "-X main.version=$(VERSION) -X main.commit=$(COMMIT_HASH)" ./cmd/skipper
 
-build.osx:
+build.darwin.arm64:
+	GOOS=darwin GOARCH=arm64 CGO_ENABLED=0 go build -o bin/skipper -ldflags "-X main.version=$(VERSION) -X main.commit=$(COMMIT_HASH)" ./cmd/skipper
+
+build.darwin:
 	GOOS=darwin GOARCH=amd64 CGO_ENABLED=0 go build -o bin/skipper -ldflags "-X main.version=$(VERSION) -X main.commit=$(COMMIT_HASH)" ./cmd/skipper
 
 build.windows:

delivery.yaml

@@ -9,6 +9,7 @@ pipeline:
   - desc: build-push
     cmd: |
       IMAGE_REGISTRY="registry-write.opensource.zalan.do"
+      MULTIARCH_REGISTRY="container-registry-test.zalando.net"
       if [[ $CDP_TARGET_BRANCH == master && ! $CDP_PULL_REQUEST_NUMBER ]]; then
         LATEST_VERSION=$(git describe --tags --always | awk -F \- '{print $1}')
         CUR_PART=$(echo $LATEST_VERSION | awk -F . '{print $1"."$2}')
@@ -21,19 +22,23 @@ pipeline:
         IMAGE="${IMAGE_REGISTRY}/teapot/skipper:${RELEASE_VERSION}"
         ARM_IMAGE="${IMAGE_REGISTRY}/teapot/skipper-armv7:${RELEASE_VERSION}"
         ARM64_IMAGE="${IMAGE_REGISTRY}/teapot/skipper-arm64:${RELEASE_VERSION}"
+        MULTIARCH_IMAGE="${MULTIARCH_REGISTRY}/teapot/skipper:${RELEASE_VERSION}"
       else
         IMAGE="${IMAGE_REGISTRY}/teapot/skipper-test:${CDP_BUILD_VERSION}"
         ARM_IMAGE="${IMAGE_REGISTRY}/teapot/skipper-armv7-test:${CDP_BUILD_VERSION}"
         ARM64_IMAGE="${IMAGE_REGISTRY}/teapot/skipper-arm64-test:${CDP_BUILD_VERSION}"
+        MULTIARCH_IMAGE="${MULTIARCH_REGISTRY}/teapot/skipper-test:${CDP_BUILD_VERSION}"
       fi
-      export IMAGE ARM_IMAGE ARM64_IMAGE
+      export IMAGE ARM_IMAGE ARM64_IMAGE MULTIARCH_IMAGE
 
       make deps cicheck staticcheck gosec
       git status
       git diff
-      cd packaging && make docker.build.amd64 && git status && git diff && make docker.push.amd64
+      cd packaging && make docker.build.amd64 && git status && git diff && make docker.push.amd64 && make docker.push.multiarch
       if [[ $CDP_TARGET_BRANCH == master && ! $CDP_PULL_REQUEST_NUMBER ]]; then
         echo "Created docker image registry.opensource.zalan.do/teapot/skipper:${RELEASE_VERSION}"
+        cdp-promote-image "${MULTIARCH_IMAGE}"
+        echo "Created multi-arch docker image container-registry.zalando.net/teapot/skipper:${RELEASE_VERSION}"
         echo "Creating docker image registry.opensource.zalan.do/teapot/skipper-arm64:${RELEASE_VERSION}"
         make docker.build.arm64 && git status && git diff && make docker.push.arm64
         echo "Creating docker image registry.opensource.zalan.do/teapot/skipper-armv7:${RELEASE_VERSION}"
@@ -62,6 +67,7 @@ pipeline:
         git gh-release --message-from-file "${tf}" "${files[@]}" "$RELEASE_VERSION"
       else
         echo "Created docker image registry.opensource.zalan.do/teapot/skipper-test:${CDP_BUILD_VERSION}"
+        echo "Created multi-arch docker image container-registry-test.zalando.net/teapot/skipper-test:${CDP_BUILD_VERSION}"
         echo "Not creating a release. No release version defined."
       fi
 - id: docs

docs/tutorials/basics.md

@@ -244,7 +244,7 @@ We use Go modules to build skipper, therefore you need [Go](https://golang.org/d
 To get a local build of skipper for your CPU architecture, you can run
 `make skipper`. To cross compile to non Linux platforms you can use:
 
-- `make build.osx` for Mac OS X (amd64)
+- `make build.darwin` for Mac OS X (amd64)
 - `make build.windows` for Windows (amd64)
 
 The local build will write into `./bin/` directory.

packaging/Dockerfile

@@ -1,8 +1,14 @@
-FROM registry.opensource.zalan.do/library/alpine-3.13:latest
+ARG BASE_IMAGE=registry.opensource.zalan.do/library/alpine-3.13:latest
+FROM ${BASE_IMAGE}
 LABEL maintainer="Team Teapot @ Zalando SE <team-teapot@zalando.de>"
 RUN apk --no-cache add ca-certificates && update-ca-certificates
 RUN mkdir -p /usr/bin
-ADD skipper eskip webhook routesrv /usr/bin/
+ARG BUILD_FOLDER=build
+ARG TARGETPLATFORM
+ADD ${BUILD_FOLDER}/${TARGETPLATFORM}/skipper \
+    ${BUILD_FOLDER}/${TARGETPLATFORM}/eskip \
+    ${BUILD_FOLDER}/${TARGETPLATFORM}/webhook \
+    ${BUILD_FOLDER}/${TARGETPLATFORM}/routesrv /usr/bin/
 ENV PATH $PATH:/usr/bin
 
 EXPOSE 9090 9911

packaging/Dockerfile.arm64

@@ -1,10 +1,10 @@
-FROM --platform=linux/amd64 alpine:3.13 AS build
+FROM --platform=linux/arm64 alpine:3.13
 LABEL maintainer="Team Teapot @ Zalando SE <team-teapot@zalando.de>"
 RUN apk --no-cache add ca-certificates && update-ca-certificates
-
-FROM --platform=linux/arm64 alpine:3.13
-COPY --from=build /etc/ssl/certs/ /etc/ssl/certs/
-ADD build/linux_arm8/skipper /usr/bin/
+ADD build/linux/arm64/skipper \
+    build/linux/arm64/eskip \
+    build/linux/arm64/webhook \
+    build/linux/arm64/routesrv /usr/bin/
 ENV PATH $PATH:/usr/bin
 
 EXPOSE 9090 9911

packaging/Dockerfile.arm → packaging/Dockerfile.armv7

@@ -1,10 +1,10 @@
-FROM --platform=linux/amd64 alpine:3.13 AS build
+FROM --platform=linux/arm/v7 alpine:3.13
 LABEL maintainer="Team Teapot @ Zalando SE <team-teapot@zalando.de>"
 RUN apk --no-cache add ca-certificates && update-ca-certificates
-
-FROM --platform=linux/arm/v7 alpine:3.13
-COPY --from=build /etc/ssl/certs/ /etc/ssl/certs/
-ADD build/linux_arm7/skipper /usr/bin/
+ADD build/linux/arm/v7/skipper \
+    build/linux/arm/v7/eskip \
+    build/linux/arm/v7/webhook \
+    build/linux/arm/v7/routesrv /usr/bin/
 ENV PATH $PATH:/usr/bin
 
 EXPOSE 9090 9911

packaging/Makefile

@@ -2,6 +2,7 @@
 
 VERSION            ?= $(shell git rev-parse HEAD)
 REGISTRY           ?= registry-write.opensource.zalan.do/teapot
+BINARIES           ?= skipper webhook eskip routesrv
 IMAGE              ?= $(REGISTRY)/skipper:$(VERSION)
 ARM64_IMAGE        ?= $(REGISTRY)/skipper-arm64:$(VERSION)
 ARM_IMAGE          ?= $(REGISTRY)/skipper-armv7:$(VERSION)
@@ -34,15 +35,18 @@ routesrv:
 	GO111MODULE=$(GO111) GOOS=$(GOOS) GOARCH=$(GOARCH) $(GOARM) CGO_ENABLED=$(CGO_ENABLED) go build -o routesrv ../cmd/routesrv/*.go
 
 clean:
-	rm -rf skipper eskip webhook routesrv build/
+	rm -rf $(BINARIES) build/
+
+docker.build: clean $(BINARIES)
+	docker build -t $(IMAGE) --build-arg BUILD_FOLDER=. --build-arg TARGETPLATFORM= .
 
 docker-build: docker.build.amd64 docker.build.arm64 docker.build.armv7
-docker.build.amd64: clean skipper eskip webhook routesrv
-	docker build -t $(IMAGE) .
+docker.build.amd64: clean build.linux.amd64 docker.build.enable
+	docker buildx build -t $(IMAGE) --platform linux/amd64 -f Dockerfile .
 docker.build.arm64: clean build.linux.arm64 docker.build.enable
 	docker buildx build -t $(ARM64_IMAGE) --platform linux/arm64 -f Dockerfile.arm64 .
 docker.build.armv7: clean build.linux.armv7 docker.build.enable
-	docker buildx build -t $(ARM_IMAGE) --platform linux/arm/v7 -f Dockerfile.arm .
+	docker buildx build -t $(ARM_IMAGE) --platform linux/arm/v7 -f Dockerfile.armv7 .
 
 docker-push: docker.push.amd64 docker.push.arm64 docker.push.armv7
 docker.push.amd64:
@@ -52,6 +56,17 @@ docker.push.arm64: docker.build.arm64
 docker.push.armv7: docker.build.armv7
 	docker push $(ARM_IMAGE)
 
+docker.push.multiarch: clean build.linux docker.build.enable
+	# Install qemu interpreter for arm64 (https://docs.docker.com/buildx/working-with-buildx/#build-multi-platform-images)
+	docker run --rm --privileged container-registry.zalando.net/teapot/tonistiigi-binfmt:qemu-v6.2.0-main-2 --install arm64
+
+	# create a Buildkit builder with CDP specific configuration (https://cloud.docs.zalando.net/howtos/cdp-multiarch/)
+	docker buildx create --config /etc/cdp-buildkitd.toml --driver-opt network=host --bootstrap --use
+
+	# build multi-arch container image using a trusted multi-arch base image
+	docker buildx build --rm -t $(MULTIARCH_IMAGE) --platform linux/amd64,linux/arm64 --push \
+	  --build-arg BASE_IMAGE=container-registry.zalando.net/library/alpine-3.13:latest .
+
 # https://docs.docker.com/build/working-with-build/
 # ~/.docker/config.json add: "experimental": "enabled",
 docker.build.enable:
@@ -60,48 +75,65 @@ docker.build.enable:
 	[ -f $$HOME/.docker/config.json ] || touch $$HOME/.docker/config.json
 	if [ -s $$HOME/.docker/config.json ]; then jq -r '. += {experimental: "enabled"}' $$HOME/.docker/config.json > $$HOME/.docker/config.json.new; mv $$HOME/.docker/config.json.new $$HOME/.docker/config.json; else echo '{"experimental": "enabled"}' >$$HOME/.docker/config.json; fi
 
-build.linux: $(SOURCES) build.linux.amd64 build.linux.arm64 build.linux.armv7
+build.linux: build.linux.amd64 build.linux.arm64 build.linux.armv7
+build.linux.amd64: $(addprefix build/linux/amd64/,$(BINARIES))
+build.linux.arm64: $(addprefix build/linux/arm64/,$(BINARIES))
+build.linux.armv7: $(addprefix build/linux/arm/v7/,$(BINARIES))
+
+build.darwin: build.darwin.amd64 build.darwin.arm64
+build.darwin.amd64: $(addprefix build/darwin/amd64/,$(BINARIES))
+build.darwin.arm64: $(addprefix build/darwin/arm64/,$(BINARIES))
+
+build.windows: $(addprefix build/windows/amd64/,$(BINARIES))
 
-build.linux.amd64:
+build/linux/amd64/%:
 	GO111MODULE=$(GO111) \
 	GOOS=linux \
 	GOARCH=amd64 \
 	CGO_ENABLED=$(CGO_ENABLED) \
-	go build -ldflags "-X main.version=$(VERSION) -X main.commit=$(COMMIT_HASH)" -o build/linux/skipper ../cmd/skipper/*.go
+	go build -ldflags "-X main.version=$(VERSION) -X main.commit=$(COMMIT_HASH)" -o build/linux/amd64/$(notdir $@) ../cmd/$(notdir $@)/*.go
 
-build.linux.arm64:
+build/linux/arm64/%:
 	GO111MODULE=$(GO111) \
 	GOOS=linux \
 	GOARCH=arm64 \
 	CGO_ENABLED=$(CGO_ENABLED) \
-	go build -ldflags "-X main.version=$(VERSION) -X main.commit=$(COMMIT_HASH)" -o build/linux_arm8/skipper ../cmd/skipper/*.go
+	go build -ldflags "-X main.version=$(VERSION) -X main.commit=$(COMMIT_HASH)" -o build/linux/arm64/$(notdir $@) ../cmd/$(notdir $@)/*.go
 
-build.linux.armv7:
+build/linux/arm/v7/%:
 	GO111MODULE=$(GO111) \
 	GOOS=linux \
 	GOARCH=arm \
 	GOARM=7 \
 	CGO_ENABLED=$(CGO_ENABLED) \
-	go build -ldflags "-X main.version=$(VERSION) -X main.commit=$(COMMIT_HASH)" -o build/linux_arm7/skipper ../cmd/skipper/*.go
+	go build -ldflags "-X main.version=$(VERSION) -X main.commit=$(COMMIT_HASH)" -o build/linux/arm/v7/$(notdir $@) ../cmd/$(notdir $@)/*.go
 
-build.osx:
+build/darwin/amd64/%:
 	GO111MODULE=$(GO111) \
 	GOOS=darwin \
 	GOARCH=amd64 \
 	CGO_ENABLED=$(CGO_ENABLED) \
-	go build -o build/osx/skipper -ldflags "-X main.version=$(VERSION) -X main.commit=$(COMMIT_HASH)" ../cmd/skipper
+	go build -ldflags "-X main.version=$(VERSION) -X main.commit=$(COMMIT_HASH)" -o build/darwin/amd64/$(notdir $@) ../cmd/$(notdir $@)/*.go
+
+build/darwin/arm64/%:
+	GO111MODULE=$(GO111) \
+	GOOS=darwin \
+	GOARCH=arm64 \
+	CGO_ENABLED=$(CGO_ENABLED) \
+	go build -ldflags "-X main.version=$(VERSION) -X main.commit=$(COMMIT_HASH)" -o build/darwin/arm64/$(notdir $@) ../cmd/$(notdir $@)/*.go
 
-build.windows:
+build/windows/amd64/%:
 	GO111MODULE=$(GO111) \
 	GOOS=windows \
 	GOARCH=amd64 \
 	CGO_ENABLED=$(CGO_ENABLED) \
-	go build -o build/windows/skipper.exe -ldflags "-X main.version=$(VERSION) -X main.commit=$(COMMIT_HASH)" ../cmd/skipper
-
-build.package: build.linux build.osx build.windows
-	tar --transform 's,^\.,skipper-$(VERSION)-linux-amd64,' -C build/linux -czvf skipper-$(VERSION)-linux-amd64.tar.gz .
-	tar --transform 's,^\.,skipper-$(VERSION)-linux-arm8,' -C build/linux_arm8 -czvf skipper-$(VERSION)-linux-arm8.tar.gz .
-	tar --transform 's,^\.,skipper-$(VERSION)-linux-arm7,' -C build/linux_arm7 -czvf skipper-$(VERSION)-linux-arm7.tar.gz .
-	tar --transform 's,^\.,skipper-$(VERSION)-osx-amd64,' -C build/osx -czvf skipper-$(VERSION)-osx-amd64.tar.gz .
-	tar --transform 's,^\.,skipper-$(VERSION)-windows-amd64,' -C build/windows -czvf skipper-$(VERSION)-windows-amd64.tar.gz .
+	go build -ldflags "-X main.version=$(VERSION) -X main.commit=$(COMMIT_HASH)" -o build/windows/amd64/$(notdir $@) ../cmd/$(notdir $@)/*.go
+
+build.package: build.linux build.darwin build.windows
+	tar --transform 's,^\.,skipper-$(VERSION)-linux-amd64,' -C build/linux/amd64 -czvf skipper-$(VERSION)-linux-amd64.tar.gz .
+	tar --transform 's,^\.,skipper-$(VERSION)-linux-arm64,' -C build/linux/arm64 -czvf skipper-$(VERSION)-linux-arm64.tar.gz .
+	tar --transform 's,^\.,skipper-$(VERSION)-linux-armv7,' -C build/linux/arm/v7 -czvf skipper-$(VERSION)-linux-armv7.tar.gz .
+	tar --transform 's,^\.,skipper-$(VERSION)-darwin-amd64,' -C build/darwin/amd64 -czvf skipper-$(VERSION)-darwin-amd64.tar.gz .
+	tar --transform 's,^\.,skipper-$(VERSION)-darwin-arm64,' -C build/darwin/arm64 -czvf skipper-$(VERSION)-darwin-arm64.tar.gz .
+	tar --transform 's,^\.,skipper-$(VERSION)-windows-amd64,' -C build/windows/amd64 -czvf skipper-$(VERSION)-windows-amd64.tar.gz .
 	for f in *.tar.gz; do sha256sum $$f >> sha256sum.txt; done