Skip to content

Commit

Permalink
Adds support for building Windows pause image
Browse files Browse the repository at this point in the history
We can use docker buildx in order to build and push Windows images from the same Linux
node, as long as the Dockerfile does not have any RUN commands in the Windows step.

We also need to create a non-default builder instance in order to be able to
build and push Windows images.

The Windows images have to be built and pushed directly to the registry. Because of
this, the make target "push" has been removed (the target "all" will build and push
the images).

We need wincat for a few kubectl proxy scenarios.

For Windows containers without Hyper-V isolation, the host OS Version and the
Container OS Version need to match, which is why we added multiple Windows OS Versions
to the building process.

Adds support for Windows OS Versions: 1809, 1903, 1909, 2004.

Bumps pause image version to 3.4.

Co-Authored-By: Claudiu Belu <cbelu@cloudbasesolutions.com>
Co-Authored-By: Ben Moss <bmoss@pivotal.io>

Signed-off-by: Leah Hanson <lhanson@pivotal.io>
  • Loading branch information
Ben Moss authored and claudiubelu committed Sep 26, 2020
1 parent 14a1106 commit 82ac28c
Show file tree
Hide file tree
Showing 11 changed files with 281 additions and 57 deletions.
1 change: 1 addition & 0 deletions build/BUILD
Expand Up @@ -20,6 +20,7 @@ filegroup(
name = "all-srcs",
srcs = [
":package-srcs",
"//build/pause/windows/wincat:all-srcs",
"//build/release-tars:all-srcs",
"//build/visible_to:all-srcs",
],
Expand Down
2 changes: 1 addition & 1 deletion build/dependencies.yaml
Expand Up @@ -152,7 +152,7 @@ dependencies:
match: tag =

- name: "k8s.gcr.io/pause"
version: 3.3
version: 3.4
refPaths:
- path: build/pause/Makefile
match: TAG =
Expand Down
5 changes: 3 additions & 2 deletions build/pause/Dockerfile
Expand Up @@ -12,7 +12,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.

FROM scratch
ARG BASE
FROM ${BASE}
ARG ARCH
ADD bin/pause-${ARCH} /pause
ADD bin/pause-linux-${ARCH} /pause
ENTRYPOINT ["/pause"]
25 changes: 25 additions & 0 deletions build/pause/Dockerfile_windows
@@ -0,0 +1,25 @@
# Copyright 2020 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

ARG BASE
FROM ${BASE}
ARG ARCH
ADD bin/pause-windows-${ARCH}.exe /pause.exe
ADD windows/wincat.exe /Windows/System32/wincat.exe

# NOTE(claudiub): docker buildx sets the PATH env variable to a Linux-like PATH,
# which is not desirable. See: https://github.com/moby/buildkit/issues/1560
# TODO(claudiub): remove this once the issue has been resolved.
ENV PATH="C:\Windows\system32;C:\Windows;"
ENTRYPOINT ["/pause.exe"]
149 changes: 95 additions & 54 deletions build/pause/Makefile
Expand Up @@ -12,91 +12,132 @@
# See the License for the specific language governing permissions and
# limitations under the License.

.PHONY: all push container clean orphan all-push push-manifest
.PHONY: all container clean orphan all-push push-manifest

REGISTRY ?= staging-k8s.gcr.io
IMAGE = $(REGISTRY)/pause
IMAGE_WITH_ARCH = $(IMAGE)-$(ARCH)
IMAGE_WITH_OS_ARCH = $(IMAGE)-$(OS)-$(ARCH)

TAG = 3.3
TAG = 3.4
REV = $(shell git describe --contains --always --match='v*')

# Architectures supported: amd64, arm, arm64, ppc64le and s390x
ARCH ?= amd64

ALL_ARCH = amd64 arm arm64 ppc64le s390x
# Operating systems supported: linux, windows
OS ?= linux
# OS Version for the Windows images: 1809, 1903, 1909 2004
OSVERSION ?= 1809 1903 1909 2004

# The output type could either be docker (local), or registry.
# If it is registry, it will also allow us to push the Windows images.
OUTPUT_TYPE ?= docker

ALL_OS = linux windows
ALL_ARCH.linux = amd64 arm arm64 ppc64le s390x
ALL_OS_ARCH.linux = $(foreach arch, ${ALL_ARCH.linux}, linux-$(arch))
ALL_ARCH.windows = amd64
ALL_OSVERSIONS.windows := 1809 1903 1909 2004
ALL_OS_ARCH.windows = $(foreach arch, $(ALL_ARCH.windows), $(foreach osversion, ${ALL_OSVERSIONS.windows}, windows-$(arch)-${osversion}))
ALL_OS_ARCH = $(foreach os, $(ALL_OS), ${ALL_OS_ARCH.${os}})

CFLAGS = -Os -Wall -Werror -static -DVERSION=v$(TAG)-$(REV)
KUBE_CROSS_IMAGE ?= k8s.gcr.io/build-image/kube-cross
KUBE_CROSS_VERSION ?= $(shell cat ../build-image/cross/VERSION)

BIN = pause
SRCS = pause.c

ifeq ($(ARCH),amd64)
TRIPLE ?= x86_64-linux-gnu
endif

ifeq ($(ARCH),arm)
TRIPLE ?= arm-linux-gnueabihf
endif

ifeq ($(ARCH),arm64)
TRIPLE ?= aarch64-linux-gnu
endif

ifeq ($(ARCH),ppc64le)
TRIPLE ?= powerpc64le-linux-gnu
endif

ifeq ($(ARCH),s390x)
TRIPLE ?= s390x-linux-gnu
endif
KUBE_CROSS_IMAGE.linux ?= k8s.gcr.io/build-image/kube-cross
KUBE_CROSS_VERSION.linux ?= $(shell cat ../build-image/cross/VERSION)
KUBE_CROSS_IMAGE.windows ?= dockcross/windows-static-x64
KUBE_CROSS_VERSION.windows ?= latest
KUBE_CROSS_IMAGE := ${KUBE_CROSS_IMAGE.${OS}}
KUBE_CROSS_VERSION := ${KUBE_CROSS_VERSION.${OS}}

# NOTE(claudiub): The Windows pause image also requires the wincat binary we're compiling for the
# port-forwarding scenarios. If it's no longer necessary, it can be removed.
# For more information, see: https://github.com/kubernetes/kubernetes/pull/91452
BIN.linux = pause
BIN.windows = pause wincat
BIN := ${BIN.${OS}}
SRCS.linux = linux/pause.c
SRCS.windows = windows/pause.c
SRCS := ${SRCS.${OS}}

EXTENSION.linux =
EXTENSION.windows = .exe
EXTENSION := ${EXTENSION.${OS}}

# The manifest command is still experimental as of Docker 18.09.3
export DOCKER_CLI_EXPERIMENTAL=enabled

TRIPLE.windows-amd64 := x86_64-w64-mingw32.static
TRIPLE.linux-amd64 := x86_64-linux-gnu
TRIPLE.linux-arm := arm-linux-gnueabihf
TRIPLE.linux-arm64 := aarch64-linux-gnu
TRIPLE.linux-ppc64le := powerpc64le-linux-gnu
TRIPLE.linux-s390x := s390x-linux-gnu
TRIPLE := ${TRIPLE.${OS}-${ARCH}}
BASE.linux := scratch
BASE.windows := mcr.microsoft.com/windows/nanoserver
BASE := ${BASE.${OS}}

# If you want to build AND push all containers, see the 'all-push' rule.
all: all-container
all: all-container-docker

all-push: all-push-images push-manifest
# NOTE(claudiub): A non-default builder instance is needed in order to build Windows images.
all-push: all-container-registry push-manifest

push-manifest:
docker manifest create --amend $(IMAGE):$(TAG) $(shell echo $(ALL_ARCH) | sed -e "s~[^ ]*~$(IMAGE)\-&:$(TAG)~g")
set -x; for arch in $(ALL_ARCH); do docker manifest annotate --arch $${arch} ${IMAGE}:${TAG} ${IMAGE}-$${arch}:${TAG}; done
docker manifest create --amend $(IMAGE):$(TAG) $(shell echo $(ALL_OS_ARCH) | sed -e "s~[^ ]*~$(IMAGE)\-&:$(TAG)~g")
set -x; for arch in $(ALL_ARCH.linux); do docker manifest annotate --os linux --arch $${arch} ${IMAGE}:${TAG} ${IMAGE}-linux-$${arch}:${TAG}; done
# For Windows images, we also need to include the "os.version" in the manifest list, so the Windows node can pull the proper image it needs.
# At the moment, docker manifest annotate doesn't allow us to set the os.version, so we'll have to it ourselves. The manifest list can be found locally as JSONs.
# See: https://github.com/moby/moby/issues/41417
# TODO(claudiub): Clean this up once the above issue has been fixed.
set -x; \
registry_prefix=$(shell (echo ${REGISTRY} | grep -Eq "[a-z]*") && echo "docker.io/" || echo ""); \
manifest_image_folder=`echo "$${registry_prefix}${IMAGE}" | sed "s|/|_|g" | sed "s/:/-/"`; \
for arch in $(ALL_ARCH.windows); do \
for osversion in ${ALL_OSVERSIONS.windows}; do \
docker manifest annotate --os windows --arch $${arch} ${IMAGE}:${TAG} ${IMAGE}-windows-$${arch}-$${osversion}:${TAG}; \
BASEIMAGE=${BASE.windows}:$${osversion}; \
full_version=`docker manifest inspect ${BASE.windows}:$${osversion} | grep "os.version" | head -n 1 | awk '{print $$2}'` || true; \
sed -i -r "s/(\"os\"\:\"windows\")/\0,\"os.version\":$${full_version}/" "${HOME}/.docker/manifests/$${manifest_image_folder}-${TAG}/$${manifest_image_folder}-windows-$${arch}-$${osversion}-${TAG}"; \
done; \
done
docker manifest push --purge ${IMAGE}:${TAG}

sub-container-%:
$(MAKE) ARCH=$* container

sub-push-%:
$(MAKE) ARCH=$* push

all-container: $(addprefix sub-container-,$(ALL_ARCH))
all-container-docker: $(addprefix sub-container-docker-,$(ALL_OS_ARCH.linux))
all-container-registry: $(addprefix sub-container-registry-,$(ALL_OS_ARCH))

all-push-images: $(addprefix sub-push-,$(ALL_ARCH))
# split words on hyphen, access by 1-index
word-hyphen = $(word $2,$(subst -, ,$1))
sub-container-%:
$(MAKE) OUTPUT_TYPE=$(call word-hyphen,$*,1) OS=$(call word-hyphen,$*,2) ARCH=$(call word-hyphen,$*,3) OSVERSION=$(call word-hyphen,$*,4) container

build: bin/$(BIN)-$(ARCH)
build: $(foreach binary, ${BIN}, bin/${binary}-${OS}-${ARCH})

bin/$(BIN)-$(ARCH): $(SRCS)
bin/${BIN.linux}-$(OS)-$(ARCH): $(SRCS)
mkdir -p bin
docker run --rm -u $$(id -u):$$(id -g) -v $$(pwd):/build \
$(KUBE_CROSS_IMAGE):$(KUBE_CROSS_VERSION) \
/bin/bash -c "\
cd /build && \
$(TRIPLE)-gcc $(CFLAGS) -o $@ $^ && \
$(TRIPLE)-strip $@"
$(TRIPLE)-strip $(foreach binary, $@, ${binary}${EXTENSION})"

bin/wincat-windows-${ARCH}:
CGO_ENABLED=0 GOOS=windows GOARCH=${ARCH} go build -o windows/wincat.exe windows/wincat/wincat.go

container: .container-$(ARCH)
.container-$(ARCH): bin/$(BIN)-$(ARCH)
DOCKER_CLI_EXPERIMENTAL=enabled docker buildx build --load --pull --platform linux/$(ARCH) -t $(IMAGE_WITH_ARCH):$(TAG) --build-arg ARCH=$(ARCH) .
container: .container-${OS}-$(ARCH)
.container-linux-$(ARCH): bin/$(BIN)-$(OS)-$(ARCH)
docker buildx build --pull --output=type=${OUTPUT_TYPE} --platform ${OS}/$(ARCH) \
-t $(IMAGE_WITH_OS_ARCH):$(TAG) --build-arg BASE=${BASE} --build-arg ARCH=$(ARCH) .
touch $@

push: .push-$(ARCH)
.push-$(ARCH): .container-$(ARCH)
docker push $(IMAGE_WITH_ARCH):$(TAG)
.container-windows-$(ARCH): $(foreach binary, ${BIN}, bin/${binary}-${OS}-${ARCH})
docker buildx build --pull --output=type=${OUTPUT_TYPE} --platform ${OS}/$(ARCH) \
-t $(IMAGE_WITH_OS_ARCH)-${OSVERSION}:$(TAG) --build-arg BASE=${BASE}:${OSVERSION} --build-arg ARCH=$(ARCH) -f Dockerfile_windows .
touch $@

# Useful for testing, not automatically included in container image
orphan: bin/orphan-$(ARCH)
bin/orphan-$(ARCH): orphan.c
orphan: bin/orphan-linux-$(ARCH)
bin/orphan-linux-$(ARCH): linux/orphan.c
mkdir -p bin
docker run -u $$(id -u):$$(id -g) -v $$(pwd):/build \
$(KUBE_CROSS_IMAGE):$(KUBE_CROSS_VERSION) \
Expand All @@ -106,4 +147,4 @@ bin/orphan-$(ARCH): orphan.c
$(TRIPLE)-strip $@"

clean:
rm -rf .container-* .push-* bin/
rm -rf .*-container-* .push-* bin/
1 change: 1 addition & 0 deletions build/pause/cloudbuild.yaml
Expand Up @@ -16,4 +16,5 @@ steps:
- '-c'
- |
gcloud auth configure-docker \
&& docker buildx create --name img-builder --use \
&& make all-push
File renamed without changes.
File renamed without changes.
49 changes: 49 additions & 0 deletions build/pause/windows/pause.c
@@ -0,0 +1,49 @@
/*
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

#include <windows.h>
#include <stdio.h>

BOOL WINAPI CtrlHandler(DWORD fdwCtrlType)
{
switch (fdwCtrlType)
{
case CTRL_C_EVENT:
fprintf(stderr, "Shutting down, got signal\n");
exit(0);

case CTRL_BREAK_EVENT:
fprintf(stderr, "Shutting down, got signal\n");
exit(0);

default:
return FALSE;
}
}

int main(void)
{
if (SetConsoleCtrlHandler(CtrlHandler, TRUE))
{
Sleep(INFINITE);
}
else
{
printf("\nERROR: Could not set control handler\n");
return 1;
}
return 0;
}
28 changes: 28 additions & 0 deletions build/pause/windows/wincat/BUILD
@@ -0,0 +1,28 @@
load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")

go_library(
name = "go_default_library",
srcs = ["wincat.go"],
importpath = "k8s.io/kubernetes/build/pause/windows/wincat",
visibility = ["//visibility:private"],
)

go_binary(
name = "windows",
embed = [":go_default_library"],
visibility = ["//visibility:public"],
)

filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)

filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

0 comments on commit 82ac28c

Please sign in to comment.