Make addon-manager cross-platform and use it with hyperkube

cluster/addons/addon-manager/Dockerfile

@@ -12,13 +12,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM python:2.7-slim
+FROM BASEIMAGE
+
+# If we're building for another architecture than amd64, the CROSS_BUILD_ placeholder is removed so e.g. CROSS_BUILD_COPY turns into COPY
+# If we're building normally, for amd64, CROSS_BUILD lines are removed
+CROSS_BUILD_COPY qemu-ARCH-static /usr/bin/
 
 RUN pip install pyyaml
 
 ADD kube-addons.sh /opt/
 ADD kube-addon-update.sh /opt/
 ADD namespace.yaml /opt/
-ADD kubectl /usr/local/bin/kubectl
+ADD kubectl /usr/local/bin/
 
-CMD /opt/kube-addons.sh
+CMD ["/opt/kube-addons.sh"]

cluster/addons/addon-manager/Makefile

@@ -13,22 +13,62 @@
 # limitations under the License.
 
 IMAGE=gcr.io/google-containers/kube-addon-manager
-VERSION=v1
-KUBECTL_VERSION=v1.2.3
+ARCH?=amd64
+TEMP_DIR:=$(shell mktemp -d)
+VERSION=v2
 
-.PHONY: build push container
+# amd64 and arm has "stable" binaries pushed for v1.2, arm64 and ppc64le hasn't so they have to fetch the latest alpha
+# however, arm64 and ppc64le are very experimental right now, so it's okay
+ifeq ($(ARCH),amd64)
+	KUBECTL_VERSION?=v1.2.4
+	BASEIMAGE?=python:2.7-slim
+endif
+ifeq ($(ARCH),arm)
+	KUBECTL_VERSION?=v1.2.4
+	BASEIMAGE?=hypriot/rpi-python:2.7
+	QEMUARCH=arm
+endif
+ifeq ($(ARCH),arm64)
+	KUBECTL_VERSION?=v1.3.0-alpha.3
+	BASEIMAGE?=aarch64/python:2.7-slim
+	QEMUARCH=aarch64
+endif
+ifeq ($(ARCH),ppc64le)
+	KUBECTL_VERSION?=v1.3.0-alpha.3
+	BASEIMAGE?=ppc64le/python:2.7-slim
+	QEMUARCH=ppc64le
+endif
 
-build: kubectl
-	docker build -t "$(IMAGE):$(VERSION)" .
+.PHONY: build push
 
-kubectl:
-	curl "https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" \
-		-o kubectl
-	chmod +x kubectl
+all: build
+build:
+	cp ./* $(TEMP_DIR)
+	curl -sSL --retry 5 https://storage.googleapis.com/kubernetes-release/release/$(KUBECTL_VERSION)/bin/linux/$(ARCH)/kubectl > $(TEMP_DIR)/kubectl
+	chmod +x $(TEMP_DIR)/kubectl
+	cd ${TEMP_DIR} && sed -i.back "s|ARCH|$(QEMUARCH)|g" Dockerfile
+	cd $(TEMP_DIR) && sed -i.back "s|BASEIMAGE|$(BASEIMAGE)|g" Dockerfile
+
+ifeq ($(ARCH),amd64)
+	# When building "normally" for amd64, remove the whole line, it has no part in the amd64 image
+	cd $(TEMP_DIR) && sed -i "/CROSS_BUILD_/d" Dockerfile
+else
+	# When cross-building, only the placeholder "CROSS_BUILD_" should be removed
+	# Register /usr/bin/qemu-ARCH-static as the handler for other-arch binaries in the kernel
+	docker run --rm --privileged multiarch/qemu-user-static:register --reset
+	curl -sSL --retry 5 https://github.com/multiarch/qemu-user-static/releases/download/v2.5.0/x86_64_qemu-$(QEMUARCH)-static.tar.xz | tar -xJ -C $(TEMP_DIR)
+	cd $(TEMP_DIR) && sed -i "s/CROSS_BUILD_//g" Dockerfile
+endif
+
+	docker build -t $(IMAGE)-$(ARCH):$(VERSION) $(TEMP_DIR)
 
 push: build
-	gcloud docker push "$(IMAGE):$(VERSION)"
+	gcloud docker push $(IMAGE)-$(ARCH):$(VERSION)
+ifeq ($(ARCH),amd64)
+	# Backward compatibility. TODO: deprecate this image tag
+	docker tag -f $(IMAGE)-$(ARCH):$(VERSION) $(IMAGE):$(VERSION)
+	gcloud docker push $(IMAGE):$(VERSION)
+endif
 
 clean:
-	rm kubectl
-	docker rmi -f "$(IMAGE):$(VERSION)"
+	docker rmi -f $(IMAGE)-$(ARCH):$(VERSION)

cluster/addons/addon-manager/README.md

@@ -0,0 +1,37 @@
+### addon-manager
+
+The `addon-manager` periodically checks for Kubernetes manifest changes in the `/etc/kubernetes/addons` directory,
+and when there's a new or changed addon, the `addon-manager` automatically `kubectl create`s it.
+
+It supports `ReplicationControllers`, `Deployments`, `DaemonSets`, `Services`, `PersistentVolumes` and `PersistentVolumeClaims`.
+
+The `addon-manager` is built for multiple architectures.
+
+#### How to release
+
+1. Change something in the source
+2. Bump `VERSION` in the `Makefile`
+3. Bump `KUBECTL_VERSION` in the `Makefile` if required
+4. Build the `amd64` image and test it on a cluster
+5. Push all images
+
+```console
+# Build for linux/amd64 (default)
+$ make push ARCH=amd64
+# ---> gcr.io/google-containers/kube-addon-manager-amd64:VERSION
+# ---> gcr.io/google-containers/kube-addon-manager:VERSION (image with backwards-compatible naming)
+
+$ make push ARCH=arm
+# ---> gcr.io/google-containers/kube-addon-manager-arm:VERSION
+
+$ make push ARCH=arm64
+# ---> gcr.io/google-containers/kube-addon-manager-arm64:VERSION
+
+$ make push ARCH=ppc64le
+# ---> gcr.io/google-containers/kube-addon-manager-ppc64le:VERSION
+```
+
+If you don't want to push the images, run `make` or `make build` instead
+
+
+[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/cluster/addons/addon-manager/README.md?pixel)]()

cluster/addons/addon-manager/kube-addon-update.sh

@@ -198,7 +198,7 @@ function run-until-success() {
 # returns a list of <namespace>/<name> pairs (nsnames)
 function get-addon-nsnames-from-server() {
     local -r obj_type=$1
-    "${KUBECTL}" get "${obj_type}" --all-namespaces -o go-template="{{range.items}}{{.metadata.namespace}}/{{.metadata.name}} {{end}}" --api-version=v1 -l kubernetes.io/cluster-service=true
+    "${KUBECTL}" get "${obj_type}" --all-namespaces -o go-template="{{range.items}}{{.metadata.namespace}}/{{.metadata.name}} {{end}}" -l kubernetes.io/cluster-service=true
 }
 
 # returns the characters after the last separator (including)
@@ -476,6 +476,7 @@ function update-addons() {
     # be careful, reconcile-objects uses global variables
     reconcile-objects ${addon_path} ReplicationController "-" &
     reconcile-objects ${addon_path} Deployment "-" &
+    reconcile-objects ${addon_path} DaemonSet "-" &
 
     # We don't expect names to be versioned for the following kinds, so
     # we match the entire name, ignoring version suffix.

cluster/addons/dashboard/dashboard-controller.yaml

@@ -1,8 +1,8 @@
+# This file should be kept in sync with cluster/images/hyperkube/dashboard-rc.yaml
+# and cluster/gce/coreos/kube-manifests/addons/dashboard/dashboard-controller.yaml
 apiVersion: v1
 kind: ReplicationController
 metadata:
-  # Keep the name in sync with image version and
-  # gce/coreos/kube-manifests/addons/dashboard counterparts
   name: kubernetes-dashboard-v1.0.1
   namespace: kube-system
   labels:

cluster/addons/dashboard/dashboard-service.yaml

@@ -1,3 +1,5 @@
+# This file should be kept in sync with cluster/images/hyperkube/dashboard-svc.yaml
+# and cluster/gce/coreos/kube-manifests/addons/dashboard/dashboard-service.yaml
 apiVersion: v1
 kind: Service
 metadata:

cluster/addons/dns/skydns-rc.yaml.in

@@ -1,3 +1,4 @@
+# This file should be kept in sync with cluster/images/hyperkube/dns-rc.yaml
 apiVersion: v1
 kind: ReplicationController
 metadata:

cluster/addons/dns/skydns-svc.yaml.in

@@ -1,3 +1,4 @@
+# This file should be kept in sync with cluster/images/hyperkube/dns-svc.yaml
 apiVersion: v1
 kind: Service
 metadata:

cluster/images/hyperkube/Dockerfile

@@ -38,25 +38,28 @@ RUN cp /usr/bin/nsenter /nsenter
 COPY hyperkube /hyperkube
 
 # Manifests for the docker guide
-COPY master.json /etc/kubernetes/manifests/master.json
-COPY etcd.json /etc/kubernetes/manifests/etcd.json
-COPY kube-proxy.json /etc/kubernetes/manifests/kube-proxy.json
+COPY static-pods/master.json /etc/kubernetes/manifests/
+COPY static-pods/etcd.json /etc/kubernetes/manifests/
+COPY static-pods/addon-manager.json /etc/kubernetes/manifests/
 
 # Manifests for the docker-multinode guide
-COPY master-multi.json /etc/kubernetes/manifests-multi/master.json
-COPY kube-proxy.json /etc/kubernetes/manifests-multi/kube-proxy.json
+COPY static-pods/master-multi.json /etc/kubernetes/manifests-multi/
+COPY static-pods/addon-manager.json /etc/kubernetes/manifests-multi/
+
+# Copy over all addons
+COPY addons /etc/kubernetes/addons
 
 # Other required scripts for the setup
 COPY safe_format_and_mount /usr/share/google/safe_format_and_mount
 COPY setup-files.sh /setup-files.sh
 COPY make-ca-cert.sh /make-ca-cert.sh
+COPY copy-addons.sh /copy-addons.sh
 
 # easy-rsa package required by make-ca-cert
 ADD https://storage.googleapis.com/kubernetes-release/easy-rsa/easy-rsa.tar.gz /root/kube/
 
-RUN mkdir -p /opt/cni
-RUN curl https://storage.googleapis.com/kubernetes-release/network-plugins/cni-c864f0e1ea73719b8f4582402b0847064f9883b0.tar.gz \
-  | tar xzv -C /opt/cni
+# Copy the cni folder into /opt/
+COPY cni /opt/cni
 
 # Create symlinks for each hyperkube server
 # TODO: this is unreliable for now (e.g. running "/kubelet" panics)

cluster/images/hyperkube/Makefile

@@ -20,6 +20,7 @@
 REGISTRY?="gcr.io/google_containers"
 ARCH?=amd64
 TEMP_DIR:=$(shell mktemp -d)
+CNI_RELEASE=c864f0e1ea73719b8f4582402b0847064f9883b0
 
 UNAME_S:=$(shell uname -s)
 ifeq ($(UNAME_S),Darwin)
@@ -28,6 +29,7 @@ endif
 ifeq ($(UNAME_S),Linux)
 	SED_CMD?=sed -i
 endif
+
 ifeq ($(ARCH),amd64)
 	BASEIMAGE?=debian:jessie
 endif
@@ -51,27 +53,32 @@ build:
 ifndef VERSION
     $(error VERSION is undefined)
 endif
-	cp ./* ${TEMP_DIR}
+	cp -r ./* ${TEMP_DIR}
+	mkdir -p ${TEMP_DIR}/cni
 	cp ../../saltbase/salt/helpers/safe_format_and_mount ${TEMP_DIR}
 	cp ../../saltbase/salt/generate-cert/make-ca-cert.sh ${TEMP_DIR}
 	cp ../../../_output/dockerized/bin/linux/${ARCH}/hyperkube ${TEMP_DIR}
-	cd ${TEMP_DIR} && sed -i.back "s|VERSION|${VERSION}|g" master-multi.json master.json kube-proxy.json
-	cd ${TEMP_DIR} && sed -i.back "s|ARCH|${ARCH}|g" master-multi.json master.json kube-proxy.json etcd.json
+
+	cd ${TEMP_DIR} && sed -i.back "s|VERSION|${VERSION}|g" addons/*.yaml static-pods/*.json
+	cd ${TEMP_DIR} && sed -i.back "s|ARCH|${ARCH}|g" addons/*.yaml static-pods/*.json
 	cd ${TEMP_DIR} && sed -i.back "s|ARCH|${QEMUARCH}|g" Dockerfile
 	cd ${TEMP_DIR} && sed -i.back "s|BASEIMAGE|${BASEIMAGE}|g" Dockerfile
-	rm ${TEMP_DIR}/*.back
+	rm ${TEMP_DIR}/addons/*.back
 
 	# Make scripts executable before they are copied into the Docker image. If we make them executable later, in another layer
 	# they'll take up twice the space because the new executable binary differs from the old one, but everything is cached in layers.
 	cd ${TEMP_DIR} && chmod a+rx \
-        hyperkube \
-        safe_format_and_mount \
-        setup-files.sh \
-        make-ca-cert.sh
+		hyperkube \
+		safe_format_and_mount \
+		setup-files.sh \
+		make-ca-cert.sh \
+		copy-addons.sh
 
 ifeq ($(ARCH),amd64)
 	# When building "normally" for amd64, remove the whole line, it has no part in the amd64 image
 	cd ${TEMP_DIR} && ${SED_CMD} "/CROSS_BUILD_/d" Dockerfile
+	# Download CNI
+	curl -sSL --retry 5 https://storage.googleapis.com/kubernetes-release/network-plugins/cni-${CNI_RELEASE}.tar.gz | tar -xz -C ${TEMP_DIR}/cni
 else
 	# When cross-building, only the placeholder "CROSS_BUILD_" should be removed
 	# Register /usr/bin/qemu-ARCH-static as the handler for ARM binaries in the kernel

cluster/images/hyperkube/addons/dashboard-rc.yaml

@@ -0,0 +1,51 @@
+# Copyright 2016 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file should be kept in sync with cluster/addons/dashboard/dashboard-controller.yaml
+apiVersion: v1
+kind: ReplicationController
+metadata:
+  name: kubernetes-dashboard
+  namespace: kube-system
+  labels:
+    app: kubernetes-dashboard
+    version: v1.0.1
+    kubernetes.io/cluster-service: "true"
+spec:
+  replicas: 1
+  selector:
+    app: kubernetes-dashboard
+    version: v1.0.1
+    kubernetes.io/cluster-service: "true"
+  template:
+    metadata:
+      labels:
+        app: kubernetes-dashboard
+        version: v1.0.1
+        kubernetes.io/cluster-service: "true"
+    spec:
+      containers:
+      - name: kubernetes-dashboard
+        # ARCH will be replaced with the architecture it's built for. Check out the Makefile for more details
+        image: gcr.io/google_containers/kubernetes-dashboard-ARCH:v1.0.1
+        imagePullPolicy: Always
+        ports:
+        - containerPort: 9090
+          protocol: TCP
+        livenessProbe:
+          httpGet:
+            path: /
+            port: 9090
+          initialDelaySeconds: 30
+          timeoutSeconds: 30

cluster/images/hyperkube/teardown.sh → cluster/images/hyperkube/addons/dashboard-svc.yaml

@@ -1,6 +1,4 @@
-#!/bin/bash
-
-# Copyright 2015 The Kubernetes Authors All rights reserved.
+# Copyright 2016 The Kubernetes Authors All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,18 +12,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Tears down an existing cluster.  Warning destroys _all_ docker containers on the machine
-
-set -o errexit
-set -o nounset
-set -o pipefail
-
-echo "Warning, this will delete all Docker containers on this machine."
-echo "Proceed? [Y/n]"
-
-read resp
-if [[ $resp == "n" || $resp == "N" ]]; then
-  exit 0
-fi
-
-docker ps -aq | xargs docker rm -f
+# This file should be kept in sync with cluster/addons/dashboard/dashboard-service.yaml
+kind: Service
+apiVersion: v1
+metadata:
+  name: kubernetes-dashboard
+  namespace: kube-system
+  labels:
+    app: kubernetes-dashboard
+    kubernetes.io/cluster-service: "true"
+spec:
+  ports:
+  - port: 80
+    targetPort: 9090
+  selector:
+    app: kubernetes-dashboard