packaging: refactor docker image build

Makefile changes
==

Build local architecture binaries into build/ folder and nest other
architecture-dependent binaries into build/<arch> subfolders (e.g. build/linux/amd64).

Add help and make it a default target.

Cleanup tar packages.

Remove unused variables.

Dockerfile changes
==

Remove BUILD_FOLDER build arg, see Makefile changes.

Use COPY instead of ADD following https://docs.docker.com/develop/develop-images/dockerfile_best-practices/#add-or-copy

Remove redundant commands.

Use one Docker file for all architectures.

Use alpine:latest in Dockerfile and configure trusted base images for amd64 and multiarch builds in Makefile.

Testing
==

Using https://github.com/multiarch/qemu-user-static it is possible to
test that all make targets produce images and binaries for proper architectures,
see `make test`

Links
==

* https://www.docker.com/blog/multi-arch-images/
* https://www.thorsten-hans.com/how-to-build-multi-arch-docker-images-with-ease/
* https://www.stereolabs.com/docs/docker/building-arm-container-on-x86/
* https://github.com/multiarch/qemu-user-static
* https://stackoverflow.com/questions/72444103/what-does-running-the-multiarch-qemu-user-static-does-before-building-a-containe

Signed-off-by: Alexander Yastrebov <alexander.yastrebov@zalando.de>

packaging/Dockerfile

@@ -1,16 +1,18 @@
 ARG BASE_IMAGE=default
-FROM registry.opensource.zalan.do/library/alpine-3:latest AS default
+FROM alpine:latest AS default
 FROM ${BASE_IMAGE}
 LABEL maintainer="Team Gateway&Proxy @ Zalando SE <team-gwproxy@zalando.de>"
-RUN apk --no-cache add ca-certificates && update-ca-certificates
-RUN mkdir -p /usr/bin
-ARG BUILD_FOLDER=build
+
+# https://docs.docker.com/engine/reference/builder/#automatic-platform-args-in-the-global-scope
 ARG TARGETPLATFORM
-ADD ${BUILD_FOLDER}/${TARGETPLATFORM}/skipper \
-    ${BUILD_FOLDER}/${TARGETPLATFORM}/eskip \
-    ${BUILD_FOLDER}/${TARGETPLATFORM}/webhook \
-    ${BUILD_FOLDER}/${TARGETPLATFORM}/routesrv /usr/bin/
-ENV PATH $PATH:/usr/bin
+
+RUN apk --no-cache add ca-certificates && update-ca-certificates
+
+COPY build/${TARGETPLATFORM}/skipper \
+    build/${TARGETPLATFORM}/eskip \
+    build/${TARGETPLATFORM}/webhook \
+    build/${TARGETPLATFORM}/routesrv \
+    /usr/bin/
 
 EXPOSE 9090 9911
 

packaging/Makefile

@@ -1,64 +1,60 @@
-.PHONY: docker-push
-
 VERSION            ?= $(shell git rev-parse HEAD)
 REGISTRY           ?= registry-write.opensource.zalan.do/teapot
 BINARIES           ?= skipper webhook eskip routesrv
 IMAGE              ?= $(REGISTRY)/skipper:$(VERSION)
 ARM64_IMAGE        ?= $(REGISTRY)/skipper-arm64:$(VERSION)
 ARM_IMAGE          ?= $(REGISTRY)/skipper-armv7:$(VERSION)
-PACKAGE            ?= github.com/zalando/skipper
+MULTIARCH_IMAGE    ?= container-registry-test.zalando.net/teapot/skipper:$(VERSION)
 CGO_ENABLED        ?= 0
-GOOS               ?= linux
-GOARCH             ?= amd64
-GOARM              ?=
-GO111              ?= on
 COMMIT_HASH        = $(shell git rev-parse --short HEAD)
 
-default: docker-build
-
-skipper:
-	GO111MODULE=$(GO111) \
-	GOOS=$(GOOS) \
-	GOARCH=$(GOARCH) \
-	$(GOARM) \
-	CGO_ENABLED=$(CGO_ENABLED) \
-	go build -ldflags "-X main.version=$(VERSION) -X main.commit=$(COMMIT_HASH)" -o skipper ../cmd/skipper/*.go
+.PHONY: help
+help: ## display this help
+	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-.]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
 
-eskip:
-	GO111MODULE=$(GO111) GOOS=$(GOOS) GOARCH=$(GOARCH) $(GOARM) CGO_ENABLED=$(CGO_ENABLED) go build -o eskip ../cmd/eskip/*.go
+.PHONY: clean
+clean: ## clean binaries
+	rm -rf sha256sum.txt *.tar.gz build/
 
-webhook:
-	GO111MODULE=$(GO111) GOOS=$(GOOS) GOARCH=$(GOARCH) $(GOARM) CGO_ENABLED=$(CGO_ENABLED) go build -o webhook ../cmd/webhook/*.go
+.PHONY: docker.build.local
+docker.build.local: clean $(addprefix build/,$(BINARIES)) ## build local architecture image
+	docker build -t $(IMAGE) --build-arg TARGETPLATFORM= .
 
-routesrv:
-	GO111MODULE=$(GO111) GOOS=$(GOOS) GOARCH=$(GOARCH) $(GOARM) CGO_ENABLED=$(CGO_ENABLED) go build -o routesrv ../cmd/routesrv/*.go
+.PHONY: docker.build.amd64
+docker.build.amd64: clean build.linux.amd64 docker.build.enable ## build linux/amd64 image using a trusted amd64 alpine base image
+	docker buildx build -t $(IMAGE) --platform linux/amd64 --load . \
+		--build-arg BASE_IMAGE=registry.opensource.zalan.do/library/alpine-3:latest
 
-clean:
-	rm -rf $(BINARIES) build/
+.PHONY: docker.build.arm64
+docker.build.arm64: clean build.linux.arm64 docker.build.enable ## build linux/arm64 image
+	docker buildx build -t $(ARM64_IMAGE) --platform linux/arm64 --load .
 
-docker.build: clean $(BINARIES)
-	docker build -t $(IMAGE) --build-arg BUILD_FOLDER=. --build-arg TARGETPLATFORM= .
+.PHONY: docker.build.armv7
+docker.build.armv7: clean build.linux.armv7 docker.build.enable ## build linux/arm/v7 image
+	docker buildx build -t $(ARM_IMAGE) --platform linux/arm/v7 --load .
 
-docker-build: docker.build.amd64 docker.build.arm64 docker.build.armv7
-docker.build.amd64: clean build.linux.amd64 docker.build.enable
-	docker buildx build -t $(IMAGE) --platform linux/amd64 -f Dockerfile --load .
-docker.build.arm64: clean build.linux.arm64 docker.build.enable
-	docker buildx build -t $(ARM64_IMAGE) --platform linux/arm64 -f Dockerfile.arm64 --load .
-docker.build.armv7: clean build.linux.armv7 docker.build.enable
-	docker buildx build -t $(ARM_IMAGE) --platform linux/arm/v7 -f Dockerfile.armv7 --load .
+.PHONY: docker.push.local
+docker.push.local: ## push local image
+	docker push $(IMAGE)
 
-docker-push: docker.push.amd64 docker.push.arm64 docker.push.armv7
-docker.push.amd64:
+.PHONY: docker.push.amd64
+docker.push.amd64: ## push linux/amd64 image
 	docker push $(IMAGE)
-docker.push.arm64: docker.build.arm64
+
+.PHONY: docker.push.arm64
+docker.push.arm64: ## push linux/arm64 image
 	docker push $(ARM64_IMAGE)
-docker.push.armv7: docker.build.armv7
+
+.PHONY: docker.push.armv7
+docker.push.armv7: ## push linux/arm/v7 image
 	docker push $(ARM_IMAGE)
 
-# build multi-arch container image using a trusted multi-arch base image
-docker.push.multiarch: clean build.linux docker.build.enable
+# Currently it is not possible to build and push multiarch image separately,
+# see https://github.com/docker/buildx/issues/59
+.PHONY: docker.push.multiarch
+docker.push.multiarch: clean build.linux docker.build.enable ## build and push multi-arch image using a trusted multi-arch alpine base image
 	docker buildx build --rm -t $(MULTIARCH_IMAGE) --platform linux/amd64,linux/arm64 --push \
-	  --build-arg BASE_IMAGE=container-registry.zalando.net/library/alpine-3:latest .
+		--build-arg BASE_IMAGE=container-registry.zalando.net/library/alpine-3:latest .
 
 # https://docs.docker.com/build/working-with-build/
 # ~/.docker/config.json add: "experimental": "enabled",
@@ -68,65 +64,88 @@ docker.build.enable:
 	[ -f $$HOME/.docker/config.json ] || touch $$HOME/.docker/config.json
 	if [ -s $$HOME/.docker/config.json ]; then jq -r '. += {experimental: "enabled"}' $$HOME/.docker/config.json > $$HOME/.docker/config.json.new; mv $$HOME/.docker/config.json.new $$HOME/.docker/config.json; else echo '{"experimental": "enabled"}' >$$HOME/.docker/config.json; fi
 
+.PHONY: build.linux build.linux.amd64 build.linux.arm64 build.linux.armv7
 build.linux: build.linux.amd64 build.linux.arm64 build.linux.armv7
 build.linux.amd64: $(addprefix build/linux/amd64/,$(BINARIES))
 build.linux.arm64: $(addprefix build/linux/arm64/,$(BINARIES))
 build.linux.armv7: $(addprefix build/linux/arm/v7/,$(BINARIES))
 
+.PHONY: build.darwin build.darwin.amd64 build.darwin.arm64
 build.darwin: build.darwin.amd64 build.darwin.arm64
 build.darwin.amd64: $(addprefix build/darwin/amd64/,$(BINARIES))
 build.darwin.arm64: $(addprefix build/darwin/arm64/,$(BINARIES))
 
+.PHONY: build.windows
 build.windows: $(addprefix build/windows/amd64/,$(BINARIES))
 
+# local build
+build/%:
+	$(GOARM) \
+	CGO_ENABLED=$(CGO_ENABLED) \
+	go build -ldflags "-X main.version=$(VERSION) -X main.commit=$(COMMIT_HASH)" -o $@ ../cmd/$(notdir $@)
+
 build/linux/amd64/%:
-	GO111MODULE=$(GO111) \
 	GOOS=linux \
 	GOARCH=amd64 \
 	CGO_ENABLED=$(CGO_ENABLED) \
-	go build -ldflags "-X main.version=$(VERSION) -X main.commit=$(COMMIT_HASH)" -o build/linux/amd64/$(notdir $@) ../cmd/$(notdir $@)/*.go
+	go build -ldflags "-X main.version=$(VERSION) -X main.commit=$(COMMIT_HASH)" -o $@ ../cmd/$(notdir $@)
 
 build/linux/arm64/%:
-	GO111MODULE=$(GO111) \
 	GOOS=linux \
 	GOARCH=arm64 \
 	CGO_ENABLED=$(CGO_ENABLED) \
-	go build -ldflags "-X main.version=$(VERSION) -X main.commit=$(COMMIT_HASH)" -o build/linux/arm64/$(notdir $@) ../cmd/$(notdir $@)/*.go
+	go build -ldflags "-X main.version=$(VERSION) -X main.commit=$(COMMIT_HASH)" -o $@ ../cmd/$(notdir $@)
 
 build/linux/arm/v7/%:
-	GO111MODULE=$(GO111) \
 	GOOS=linux \
 	GOARCH=arm \
 	GOARM=7 \
 	CGO_ENABLED=$(CGO_ENABLED) \
-	go build -ldflags "-X main.version=$(VERSION) -X main.commit=$(COMMIT_HASH)" -o build/linux/arm/v7/$(notdir $@) ../cmd/$(notdir $@)/*.go
+	go build -ldflags "-X main.version=$(VERSION) -X main.commit=$(COMMIT_HASH)" -o $@ ../cmd/$(notdir $@)
 
 build/darwin/amd64/%:
-	GO111MODULE=$(GO111) \
 	GOOS=darwin \
 	GOARCH=amd64 \
 	CGO_ENABLED=$(CGO_ENABLED) \
-	go build -ldflags "-X main.version=$(VERSION) -X main.commit=$(COMMIT_HASH)" -o build/darwin/amd64/$(notdir $@) ../cmd/$(notdir $@)/*.go
+	go build -ldflags "-X main.version=$(VERSION) -X main.commit=$(COMMIT_HASH)" -o $@ ../cmd/$(notdir $@)
 
 build/darwin/arm64/%:
-	GO111MODULE=$(GO111) \
 	GOOS=darwin \
 	GOARCH=arm64 \
 	CGO_ENABLED=$(CGO_ENABLED) \
-	go build -ldflags "-X main.version=$(VERSION) -X main.commit=$(COMMIT_HASH)" -o build/darwin/arm64/$(notdir $@) ../cmd/$(notdir $@)/*.go
+	go build -ldflags "-X main.version=$(VERSION) -X main.commit=$(COMMIT_HASH)" -o $@ ../cmd/$(notdir $@)
 
 build/windows/amd64/%:
-	GO111MODULE=$(GO111) \
 	GOOS=windows \
 	GOARCH=amd64 \
 	CGO_ENABLED=$(CGO_ENABLED) \
-	go build -ldflags "-X main.version=$(VERSION) -X main.commit=$(COMMIT_HASH)" -o build/windows/amd64/$(notdir $@) ../cmd/$(notdir $@)/*.go
+	go build -ldflags "-X main.version=$(VERSION) -X main.commit=$(COMMIT_HASH)" -o $@ ../cmd/$(notdir $@)
 
-build.package: build.linux build.darwin build.windows
+.PHONY: build.package
+build.package: build.linux build.darwin build.windows ## package binaries
 	tar --transform 's,^\.,skipper-$(VERSION)-linux-amd64,' -C build/linux/amd64 -czvf skipper-$(VERSION)-linux-amd64.tar.gz .
 	tar --transform 's,^\.,skipper-$(VERSION)-linux-arm64,' -C build/linux/arm64 -czvf skipper-$(VERSION)-linux-arm64.tar.gz .
 	tar --transform 's,^\.,skipper-$(VERSION)-linux-armv7,' -C build/linux/arm/v7 -czvf skipper-$(VERSION)-linux-armv7.tar.gz .
 	tar --transform 's,^\.,skipper-$(VERSION)-darwin-amd64,' -C build/darwin/amd64 -czvf skipper-$(VERSION)-darwin-amd64.tar.gz .
 	tar --transform 's,^\.,skipper-$(VERSION)-darwin-arm64,' -C build/darwin/arm64 -czvf skipper-$(VERSION)-darwin-arm64.tar.gz .
 	tar --transform 's,^\.,skipper-$(VERSION)-windows-amd64,' -C build/windows/amd64 -czvf skipper-$(VERSION)-windows-amd64.tar.gz .
 	for f in *.tar.gz; do sha256sum $$f >> sha256sum.txt; done
+
+.PHONY: test
+test: ## test build targets using https://github.com/multiarch/qemu-user-static
+	IMAGE=skipper-packaging-test:local make docker.build.local
+	IMAGE=skipper-packaging-test:local make test-run
+
+	IMAGE=skipper-packaging-test:amd64 make docker.build.amd64
+	IMAGE=skipper-packaging-test:amd64 TARGETPLATFORM=linux/amd64 make test-run
+
+	ARM64_IMAGE=skipper-packaging-test:arm64 make docker.build.arm64
+	IMAGE=skipper-packaging-test:arm64 TARGETPLATFORM=linux/arm64 make test-run
+
+	ARM_IMAGE=skipper-packaging-test:armv7 make docker.build.armv7
+	IMAGE=skipper-packaging-test:armv7 TARGETPLATFORM=linux/arm/v7 make test-run
+
+.PHONY: test-run
+test-run:
+	docker run --platform=$(TARGETPLATFORM) --rm -it $(IMAGE) uname -m
+	docker run --platform=$(TARGETPLATFORM) --rm -it $(IMAGE) skipper -version

packaging/readme.md

@@ -5,17 +5,16 @@ package managers. (Currently, only docker.)
 
 ## Docker
 
-Use the provided Dockerfile to build a docker image with Alpine Linux and the latest version of skipper and
-eskip.
+Use the provided Dockerfile to build a docker image with Alpine Linux and the latest version of skipper and tools.
 
 Use the `VERSION` and `REGISTRY` or `IMAGE` environment variables to set the docker tag.
 `VERSION` defaults to the hash of the current git head, which revision is also used to build the packaged
-binary of skipper and eskip.
+binary of skipper and tools.
 
 **Example:**
 
 ```
-REGISTRY=my-repo VERSION=latest-SNAPSHOT make docker-build docker-push
+REGISTRY=my-repo VERSION=latest-SNAPSHOT make docker.build.local docker.push.local
 ```
 
 The above command will build a docker image with a tag 'my-repo/skipper:latest-SNAPSHOT' and push it to