New go-runner image for distroless scenarios

build/BUILD

@@ -20,6 +20,7 @@ filegroup(
     name = "all-srcs",
     srcs = [
         ":package-srcs",
+        "//build/go-runner:all-srcs",
         "//build/release-tars:all-srcs",
         "//build/visible_to:all-srcs",
     ],

build/go-runner/BUILD

@@ -0,0 +1,29 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["go-runner.go"],
+    importpath = "k8s.io/kubernetes/build/go-runner",
+    visibility = ["//visibility:private"],
+    deps = ["//vendor/github.com/pkg/errors:go_default_library"],
+)
+
+go_binary(
+    name = "go-runner",
+    embed = [":go_default_library"],
+    visibility = ["//visibility:public"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

build/go-runner/Dockerfile

@@ -0,0 +1,45 @@
+# Copyright 2020 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Build the manager binary
+FROM golang:1.13 as builder
+WORKDIR /workspace
+
+# Run this with docker build --build_arg goproxy=$(go env GOPROXY) to override the goproxy
+ARG goproxy=https://proxy.golang.org
+# Run this with docker build --build_arg package=./controlplane/kubeadm or --build_arg package=./bootstrap/kubeadm
+ENV GOPROXY=$goproxy
+
+# Copy the sources
+COPY ./ ./
+
+# Cache the go build
+RUN go build .
+
+# Build
+ARG package=.
+ARG ARCH
+
+# Do not force rebuild of up-to-date packages (do not use -a)
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} \
+    go build -ldflags '-s -w -buildid= -extldflags "-static"' \
+    -o go-runner ${package}
+
+# Production image
+FROM gcr.io/distroless/static:latest
+LABEL maintainers="Kubernetes Authors"
+LABEL description="go based runner for distroless scenarios"
+WORKDIR /
+COPY --from=builder /workspace/go-runner .
+ENTRYPOINT ["/go-runner"]

build/go-runner/Makefile

@@ -0,0 +1,71 @@
+# Copyright 2020 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# set default shell
+SHELL=/bin/bash -o pipefail
+
+TAG ?= 0.1.0
+REGISTRY ?= k8s.gcr.io
+
+IMGNAME = go-runner
+IMAGE = $(REGISTRY)/$(IMGNAME)
+
+PLATFORMS = linux/amd64 linux/arm64 linux/arm linux/ppc64le linux/s390x
+
+HOST_GOOS ?= $(shell go env GOOS)
+HOST_GOARCH ?= $(shell go env GOARCH)
+GO_BUILD ?= go build
+
+.PHONY: all build clean
+
+.PHONY: all
+all: build
+
+.PHONY: build
+build:
+	$(GO_BUILD)
+
+.PHONY: clean
+clean:
+	rm go-runner
+
+.PHONY: container
+container: init-docker-buildx
+	# https://github.com/docker/buildx/issues/59
+	$(foreach PLATFORM,$(PLATFORMS), \
+		DOCKER_CLI_EXPERIMENTAL=enabled docker buildx build \
+		--load \
+		--progress plain \
+		--platform $(PLATFORM) \
+		--tag $(IMAGE)-$(PLATFORM):$(TAG) .;)
+
+.PHONY: push
+push: container
+	$(foreach PLATFORM,$(PLATFORMS), \
+		docker push $(IMAGE)-$(PLATFORM):$(TAG);)
+
+.PHONY: manifest
+manifest: push
+	docker manifest create --amend $(IMAGE):$(TAG) $(shell echo $(PLATFORMS) | sed -e "s~[^ ]*~$(IMAGE)\-&:$(TAG)~g")
+	@for arch in $(PLATFORMS); do docker manifest annotate --arch "$${arch##*/}" ${IMAGE}:${TAG} ${IMAGE}-$${arch}:${TAG}; done
+	docker manifest push --purge $(IMAGE):$(TAG)
+
+.PHONY: init-docker-buildx
+init-docker-buildx:
+ifneq ($(shell docker buildx 2>&1 >/dev/null; echo $?),)
+	$(error "buildx not vailable. Docker 19.03 or higher is required")
+endif
+	docker run --rm --privileged linuxkit/binfmt:4ea3b9b0938cbd19834c096aa31ff475cc75d281
+	docker buildx create --name multiarch-go-runner --use || true
+	docker buildx inspect --bootstrap

build/go-runner/OWNERS

@@ -0,0 +1,6 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+approvers:
+  - build-image-approvers
+reviewers:
+  - build-image-reviewers

build/go-runner/README.md

@@ -0,0 +1,30 @@
+# Kubernetes go-runner image
+
+The Kubernetes go-runner image wraps the gcr.io/distroless/static image and provides a go based
+binary that can run commands and wrap stdout/stderr etc. 
+
+Why do we need this? Some of our images like kube-apiserver currently use bash for collecting
+logs, so we are not able to switch to distroless images directly for these images. The klog's
+`--log-file` was supposed to fix this problem, but we ran into trouble in scalability CI jobs
+around log rotation and picked this option instead. we essentially publish a multi-arch 
+manifest with support for various platforms. This can be used as a base for other kubernetes
+components.
+
+For example instead of running kube-apiserver like this:
+```bash
+"/bin/sh",
+  "-c",
+  "exec /usr/local/bin/kube-apiserver {{params}} --allow-privileged={{pillar['allow_privileged']}} 1>>/var/log/kube-apiserver.log 2>&1"
+```  
+
+we would use go-runner like so:
+```bash
+"/go-runner", "--log-file=/var/log/kube-apiserver.log", "--also-stdout=false", "--redirect-stderr=true",
+  "/usr/local/bin/kube-apiserver",
+  "--allow-privileged={{pillar['allow_privileged']}}",
+  {{params}}
+```
+
+The go-runner would then ensure that we run the `/usr/local/bin/kube-apiserver` with the 
+specified parameters and redirect stdout ONLY to the log file specified and ensure anything 
+logged to stderr also ends up in the log file.

build/go-runner/cloudbuild.yaml

@@ -0,0 +1,22 @@
+# See https://github.com/kubernetes/test-infra/blob/master/config/jobs/image-pushing/README.md for more details on image pushing process
+
+# this must be specified in seconds. If omitted, defaults to 600s (10 mins)
+timeout: 1200s
+# this prevents errors if you don't use both _GIT_TAG and _PULL_BASE_REF,
+# or any new substitutions added in the future.
+options:
+  substitution_option: ALLOW_LOOSE
+  machineType: 'N1_HIGHCPU_8'
+steps:
+  - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20200422-b25d964'
+    entrypoint: 'bash'
+    dir: ./build/go-runner
+    env:
+    - DOCKER_CLI_EXPERIMENTAL=enabled
+    - REGISTRY=gcr.io/$PROJECT_ID
+    - HOME=/root
+    args:
+    - '-c'
+    - |
+      gcloud auth configure-docker \
+      && make manifest

build/go-runner/go-runner.go

@@ -0,0 +1,122 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"os/exec"
+	"os/signal"
+	"strings"
+	"syscall"
+
+	"github.com/pkg/errors"
+)
+
+var (
+	logFilePath    = flag.String("log-file", "", "If non-empty, save stdout to this file")
+	alsoToStdOut   = flag.Bool("also-stdout", false, "useful with log-file, log to standard output as well as the log file")
+	redirectStderr = flag.Bool("redirect-stderr", true, "treat stderr same as stdout")
+)
+
+func main() {
+	flag.Parse()
+
+	if err := configureAndRun(); err != nil {
+		log.Fatal(err)
+	}
+}
+
+func configureAndRun() error {
+	var (
+		outputStream io.Writer = os.Stdout
+		errStream    io.Writer = os.Stderr
+	)
+
+	args := flag.Args()
+	if len(args) == 0 {
+		return errors.Errorf("not enough arguments to run")
+	}
+
+	if logFilePath != nil && *logFilePath != "" {
+		logFile, err := os.OpenFile(*logFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+		if err != nil {
+			return errors.Wrapf(err, "failed to create log file %v", *logFilePath)
+		}
+		if *alsoToStdOut {
+			outputStream = io.MultiWriter(os.Stdout, logFile)
+		} else {
+			outputStream = logFile
+		}
+	}
+
+	if *redirectStderr {
+		errStream = outputStream
+	}
+
+	exe := args[0]
+	var exeArgs []string
+	if len(args) > 1 {
+		exeArgs = args[1:]
+	}
+	cmd := exec.Command(exe, exeArgs...)
+	cmd.Stdout = outputStream
+	cmd.Stderr = errStream
+
+	log.Printf("Running command:\n%v", cmdInfo(cmd))
+	err := cmd.Start()
+	if err != nil {
+		return errors.Wrap(err, "starting command")
+	}
+
+	// Handle signals and shutdown process gracefully.
+	go setupSigHandler(cmd.Process)
+	return errors.Wrap(cmd.Wait(), "running command")
+}
+
+// cmdInfo generates a useful look at what the command is for printing/debug.
+func cmdInfo(cmd *exec.Cmd) string {
+	return fmt.Sprintf(
+		`Command env: (log-file=%v, also-stdout=%v, redirect-stderr=%v)
+Run from directory: %v
+Executable path: %v
+Args (comma-delimited): %v`, *logFilePath, *alsoToStdOut, *redirectStderr,
+		cmd.Dir, cmd.Path, strings.Join(cmd.Args, ","),
+	)
+}
+
+// setupSigHandler will forward any termination signals to the process
+func setupSigHandler(process *os.Process) {
+	// terminationSignals are signals that cause the program to exit in the
+	// supported platforms (linux, darwin, windows).
+	terminationSignals := []os.Signal{syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT}
+
+	c := make(chan os.Signal, 1)
+	signal.Notify(c, terminationSignals...)
+
+	// Block until a signal is received.
+	log.Println("Now listening for interrupts")
+	s := <-c
+	log.Printf("Got signal: %v. Sending down to process (PID: %v)", s, process.Pid)
+	if err := process.Signal(s); err != nil {
+		log.Fatalf("Failed to signal process: %v", err)
+	}
+	log.Printf("Signalled process %v successfully.", process.Pid)
+}

build/go-runner/go.mod

@@ -0,0 +1,5 @@
+module k8s.io/kubernetes/build/go-runner
+
+go 1.13
+
+require github.com/pkg/errors v0.9.1

build/go-runner/go.sum

@@ -0,0 +1,2 @@
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=