Switch from upstream hyperkube image to individual images

* Kubernetes plans to stop releasing the hyperkube container image
* Upstream will continue to publish `kube-apiserver`, `kube-controller-manager`,
`kube-scheduler`, and `kube-proxy` container images to `k8s.gcr.io`
* Upstream will publish Kubelet only as a binary for distros to package,
either as a DEB/RPM on traditional distros or a container image on
container-optimized operating systems
* Typhoon will package the upstream Kubelet (checksummed) and its
dependencies as a container image for use on CoreOS Container Linux,
Flatcar Linux, and Fedora CoreOS
* Update the Typhoon container image security policy to list
`quay.io/poseidon/kubelet`as an official distributed artifact

Hyperkube: kubernetes/kubernetes#88676
Kubelet Container Image: https://github.com/poseidon/kubelet
Kubelet Quay Repo: https://quay.io/repository/poseidon/kubelet

CHANGES.md

@@ -5,6 +5,15 @@ Notable changes between versions.
 ## Latest
 
 * Update etcd from v3.4.4 to [v3.4.5](https://github.com/etcd-io/etcd/releases/tag/v3.4.5)
+* Switch from upstream hyperkube image to individual images ([#669](https://github.com/poseidon/typhoon/pull/669))
+  * Use upstream `k8s.gcr.io` `kube-apiserver`, `kube-controller-manager`, `kube-scheduler`, and `kube-proxy` container images
+  * Use [poseidon/kubelet](https://github.com/poseidon/kubelet) to package the upstream Kubelet binary (checksummed) and
+  its dependencies as an automated build container image [quay.io/poseidon/kubelet](https://quay.io/repository/poseidon/kubelet)
+  * Update base images for control plane and Kubelet images used by Typhoon from upstream's debian 9 (stretch) to debian 10
+  (buster) base
+  * Update Typhoon container image security policy to list `quay.io/poseidon/kubelet`as an official distributed artifact
+  * Background: Kubernetes will [stop releasing](https://github.com/kubernetes/kubernetes/pull/88676) the hyperkube container
+  image and provide the Kubelet as a binary for distros to package
 
 #### Addons
 

aws/container-linux/kubernetes/bootstrap.tf

@@ -1,6 +1,6 @@
 # Kubernetes assets (kubeconfig, manifests)
 module "bootstrap" {
-  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=73784c1b2c791d9ba586a1478979ac34dd324dad"
+  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e76f0a09fa9e6421a9cf697ee03714c6224e2581"
 
   cluster_name          = var.cluster_name
   api_servers           = [format("%s.%s", var.cluster_name, var.dns_zone)]

aws/container-linux/kubernetes/cl/controller.yaml

@@ -91,8 +91,7 @@ systemd:
           --mount volume=var-log,target=/var/log \
           --volume opt-cni-bin,kind=host,source=/opt/cni/bin \
           --mount volume=opt-cni-bin,target=/opt/cni/bin \
-          docker://k8s.gcr.io/hyperkube:v1.17.4 \
-          --exec=/usr/local/bin/kubelet -- \
+          docker://quay.io/poseidon/kubelet:v1.17.4 -- \
           --anonymous-auth=false \
           --authentication-token-webhook \
           --authorization-mode=Webhook \
@@ -136,7 +135,7 @@ systemd:
             --volume script,kind=host,source=/opt/bootstrap/apply \
             --mount volume=script,target=/apply \
             --insecure-options=image \
-            docker://k8s.gcr.io/hyperkube:v1.17.4 \
+            docker://quay.io/poseidon/kubelet:v1.17.4 \
             --net=host \
             --dns=host \
             --exec=/apply

aws/container-linux/kubernetes/workers/cl/worker.yaml

@@ -64,8 +64,7 @@ systemd:
           --mount volume=var-log,target=/var/log \
           --volume opt-cni-bin,kind=host,source=/opt/cni/bin \
           --mount volume=opt-cni-bin,target=/opt/cni/bin \
-          docker://k8s.gcr.io/hyperkube:v1.17.4 \
-          --exec=/usr/local/bin/kubelet -- \
+          docker://quay.io/poseidon/kubelet:v1.17.4 -- \
           --anonymous-auth=false \
           --authentication-token-webhook \
           --authorization-mode=Webhook \
@@ -80,9 +79,9 @@ systemd:
           --lock-file=/var/run/lock/kubelet.lock \
           --network-plugin=cni \
           --node-labels=node.kubernetes.io/node \
-          %{ for label in split(",", node_labels) }
+          %{~ for label in split(",", node_labels) ~}
           --node-labels=${label} \
-          %{ endfor ~}
+          %{~ endfor ~}
           --pod-manifest-path=/etc/kubernetes/manifests \
           --read-only-port=0 \
           --volume-plugin-dir=/var/lib/kubelet/volumeplugins
@@ -128,11 +127,11 @@ storage:
             --volume config,kind=host,source=/etc/kubernetes \
             --mount volume=config,target=/etc/kubernetes \
             --insecure-options=image \
-            docker://k8s.gcr.io/hyperkube:v1.17.4 \
+            docker://quay.io/poseidon/kubelet:612b947 \
             --net=host \
             --dns=host \
             -- \
-            kubectl --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname)
+            /usr/local/bin/kubectl --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname)
 passwd:
   users:
     - name: core

aws/fedora-coreos/kubernetes/bootstrap.tf

@@ -1,6 +1,6 @@
 # Kubernetes assets (kubeconfig, manifests)
 module "bootstrap" {
-  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=73784c1b2c791d9ba586a1478979ac34dd324dad"
+  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e76f0a09fa9e6421a9cf697ee03714c6224e2581"
 
   cluster_name          = var.cluster_name
   api_servers           = [format("%s.%s", var.cluster_name, var.dns_zone)]

aws/fedora-coreos/kubernetes/fcc/controller.yaml

@@ -79,7 +79,7 @@ systemd:
           --volume /var/log:/var/log \
           --volume /var/run/lock:/var/run/lock:z \
           --volume /opt/cni/bin:/opt/cni/bin:z \
-          k8s.gcr.io/hyperkube:v1.17.4 kubelet \
+          quay.io/poseidon/kubelet:v1.17.4 \
           --anonymous-auth=false \
           --authentication-token-webhook \
           --authorization-mode=Webhook \
@@ -123,7 +123,7 @@ systemd:
             --volume /opt/bootstrap/assets:/assets:ro,Z \
             --volume /opt/bootstrap/apply:/apply:ro,Z \
             --entrypoint=/apply \
-            k8s.gcr.io/hyperkube:v1.17.4
+            quay.io/poseidon/kubelet:v1.17.4
         ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done
         ExecStartPost=-/usr/bin/podman stop bootstrap
 storage:

aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml

@@ -49,7 +49,7 @@ systemd:
           --volume /var/log:/var/log \
           --volume /var/run/lock:/var/run/lock:z \
           --volume /opt/cni/bin:/opt/cni/bin:z \
-          k8s.gcr.io/hyperkube:v1.17.4 kubelet \
+          quay.io/poseidon/kubelet:v1.17.4 \
           --anonymous-auth=false \
           --authentication-token-webhook \
           --authorization-mode=Webhook \
@@ -87,7 +87,7 @@ systemd:
         Type=oneshot
         RemainAfterExit=true
         ExecStart=/bin/true
-        ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z k8s.gcr.io/hyperkube:v1.17.4 kubectl --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME'
+        ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.17.4 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME'
         [Install]
         WantedBy=multi-user.target
 storage:

azure/container-linux/kubernetes/bootstrap.tf

@@ -1,6 +1,6 @@
 # Kubernetes assets (kubeconfig, manifests)
 module "bootstrap" {
-  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=73784c1b2c791d9ba586a1478979ac34dd324dad"
+  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e76f0a09fa9e6421a9cf697ee03714c6224e2581"
 
   cluster_name = var.cluster_name
   api_servers  = [format("%s.%s", var.cluster_name, var.dns_zone)]

azure/container-linux/kubernetes/cl/controller.yaml

@@ -90,8 +90,7 @@ systemd:
           --mount volume=var-log,target=/var/log \
           --volume opt-cni-bin,kind=host,source=/opt/cni/bin \
           --mount volume=opt-cni-bin,target=/opt/cni/bin \
-          docker://k8s.gcr.io/hyperkube:v1.17.4 \
-          --exec=/usr/local/bin/kubelet -- \
+          docker://quay.io/poseidon/kubelet:v1.17.4 -- \
           --anonymous-auth=false \
           --authentication-token-webhook \
           --authorization-mode=Webhook \
@@ -134,7 +133,7 @@ systemd:
             --volume script,kind=host,source=/opt/bootstrap/apply \
             --mount volume=script,target=/apply \
             --insecure-options=image \
-            docker://k8s.gcr.io/hyperkube:v1.17.4 \
+            docker://quay.io/poseidon/kubelet:v1.17.4 \
             --net=host \
             --dns=host \
             --exec=/apply

azure/container-linux/kubernetes/workers/cl/worker.yaml

@@ -63,8 +63,7 @@ systemd:
           --mount volume=var-log,target=/var/log \
           --volume opt-cni-bin,kind=host,source=/opt/cni/bin \
           --mount volume=opt-cni-bin,target=/opt/cni/bin \
-          docker://k8s.gcr.io/hyperkube:v1.17.4 \
-          --exec=/usr/local/bin/kubelet -- \
+          docker://quay.io/poseidon/kubelet:v1.17.4 -- \
           --anonymous-auth=false \
           --authentication-token-webhook \
           --authorization-mode=Webhook \
@@ -78,9 +77,9 @@ systemd:
           --lock-file=/var/run/lock/kubelet.lock \
           --network-plugin=cni \
           --node-labels=node.kubernetes.io/node \
-          %{ for label in split(",", node_labels) }
+          %{~ for label in split(",", node_labels) ~}
           --node-labels=${label} \
-          %{ endfor ~}
+          %{~ endfor ~}
           --pod-manifest-path=/etc/kubernetes/manifests \
           --read-only-port=0 \
           --volume-plugin-dir=/var/lib/kubelet/volumeplugins
@@ -126,11 +125,11 @@ storage:
             --volume config,kind=host,source=/etc/kubernetes \
             --mount volume=config,target=/etc/kubernetes \
             --insecure-options=image \
-            docker://k8s.gcr.io/hyperkube:v1.17.4 \
+            docker://quay.io/poseidon/kubelet:v1.17.4 \
             --net=host \
             --dns=host \
             -- \
-            kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname | tr '[:upper:]' '[:lower:]')
+            /usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname | tr '[:upper:]' '[:lower:]')
 passwd:
   users:
     - name: core

bare-metal/container-linux/kubernetes/bootstrap.tf

@@ -1,6 +1,6 @@
 # Kubernetes assets (kubeconfig, manifests)
 module "bootstrap" {
-  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=73784c1b2c791d9ba586a1478979ac34dd324dad"
+  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e76f0a09fa9e6421a9cf697ee03714c6224e2581"
 
   cluster_name                    = var.cluster_name
   api_servers                     = [var.k8s_domain_name]

bare-metal/container-linux/kubernetes/cl/controller.yaml

@@ -103,8 +103,7 @@ systemd:
           --mount volume=etc-iscsi,target=/etc/iscsi \
           --volume usr-sbin-iscsiadm,kind=host,source=/usr/sbin/iscsiadm \
           --mount volume=usr-sbin-iscsiadm,target=/sbin/iscsiadm \
-          docker://k8s.gcr.io/hyperkube:v1.17.4 \
-          --exec=/usr/local/bin/kubelet -- \
+          docker://quay.io/poseidon/kubelet:v1.17.4 -- \
           --anonymous-auth=false \
           --authentication-token-webhook \
           --authorization-mode=Webhook \
@@ -149,7 +148,7 @@ systemd:
             --volume script,kind=host,source=/opt/bootstrap/apply \
             --mount volume=script,target=/apply \
             --insecure-options=image \
-            docker://k8s.gcr.io/hyperkube:v1.17.4 \
+            docker://quay.io/poseidon/kubelet:v1.17.4 \
             --net=host \
             --dns=host \
             --exec=/apply

bare-metal/container-linux/kubernetes/cl/worker.yaml

@@ -76,8 +76,7 @@ systemd:
           --mount volume=etc-iscsi,target=/etc/iscsi \
           --volume usr-sbin-iscsiadm,kind=host,source=/usr/sbin/iscsiadm \
           --mount volume=usr-sbin-iscsiadm,target=/sbin/iscsiadm \
-          docker://k8s.gcr.io/hyperkube:v1.17.4 \
-          --exec=/usr/local/bin/kubelet -- \
+          docker://quay.io/poseidon/kubelet:v1.17.4 -- \
           --anonymous-auth=false \
           --authentication-token-webhook \
           --authorization-mode=Webhook \

bare-metal/fedora-coreos/kubernetes/bootstrap.tf

@@ -1,6 +1,6 @@
 # Kubernetes assets (kubeconfig, manifests)
 module "bootstrap" {
-  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=73784c1b2c791d9ba586a1478979ac34dd324dad"
+  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e76f0a09fa9e6421a9cf697ee03714c6224e2581"
 
   cluster_name                    = var.cluster_name
   api_servers                     = [var.k8s_domain_name]

bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml

@@ -80,7 +80,7 @@ systemd:
           --volume /opt/cni/bin:/opt/cni/bin:z \
           --volume /etc/iscsi:/etc/iscsi \
           --volume /sbin/iscsiadm:/sbin/iscsiadm \
-          k8s.gcr.io/hyperkube:v1.17.4 kubelet \
+          quay.io/poseidon/kubelet:v1.17.4 \
           --anonymous-auth=false \
           --authentication-token-webhook \
           --authorization-mode=Webhook \
@@ -134,7 +134,7 @@ systemd:
             --volume /opt/bootstrap/assets:/assets:ro,Z \
             --volume /opt/bootstrap/apply:/apply:ro,Z \
             --entrypoint=/apply \
-            k8s.gcr.io/hyperkube:v1.17.4
+            quay.io/poseidon/kubelet:v1.17.4
         ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done
         ExecStartPost=-/usr/bin/podman stop bootstrap
 storage:

bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml

@@ -50,7 +50,7 @@ systemd:
           --volume /opt/cni/bin:/opt/cni/bin:z \
           --volume /etc/iscsi:/etc/iscsi \
           --volume /sbin/iscsiadm:/sbin/iscsiadm \
-          k8s.gcr.io/hyperkube:v1.17.4 kubelet \
+          quay.io/poseidon/kubelet:v1.17.4 \
           --anonymous-auth=false \
           --authentication-token-webhook \
           --authorization-mode=Webhook \

digital-ocean/container-linux/kubernetes/bootstrap.tf

@@ -1,6 +1,6 @@
 # Kubernetes assets (kubeconfig, manifests)
 module "bootstrap" {
-  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=73784c1b2c791d9ba586a1478979ac34dd324dad"
+  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e76f0a09fa9e6421a9cf697ee03714c6224e2581"
 
   cluster_name = var.cluster_name
   api_servers  = [format("%s.%s", var.cluster_name, var.dns_zone)]

digital-ocean/container-linux/kubernetes/cl/controller.yaml

@@ -101,8 +101,7 @@ systemd:
           --mount volume=var-log,target=/var/log \
           --volume opt-cni-bin,kind=host,source=/opt/cni/bin \
           --mount volume=opt-cni-bin,target=/opt/cni/bin \
-          docker://k8s.gcr.io/hyperkube:v1.17.4 \
-          --exec=/usr/local/bin/kubelet -- \
+          docker://quay.io/poseidon/kubelet:v1.17.4 -- \
           --anonymous-auth=false \
           --authentication-token-webhook \
           --authorization-mode=Webhook \
@@ -146,7 +145,7 @@ systemd:
             --volume script,kind=host,source=/opt/bootstrap/apply \
             --mount volume=script,target=/apply \
             --insecure-options=image \
-            docker://k8s.gcr.io/hyperkube:v1.17.4 \
+            docker://quay.io/poseidon/kubelet:v1.17.4 \
             --net=host \
             --dns=host \
             --exec=/apply

digital-ocean/container-linux/kubernetes/cl/worker.yaml

@@ -74,8 +74,7 @@ systemd:
           --mount volume=var-log,target=/var/log \
           --volume opt-cni-bin,kind=host,source=/opt/cni/bin \
           --mount volume=opt-cni-bin,target=/opt/cni/bin \
-          docker://k8s.gcr.io/hyperkube:v1.17.4 \
-          --exec=/usr/local/bin/kubelet -- \
+          docker://quay.io/poseidon/kubelet:v1.17.4 -- \
           --anonymous-auth=false \
           --authentication-token-webhook \
           --authorization-mode=Webhook \
@@ -132,8 +131,8 @@ storage:
             --volume config,kind=host,source=/etc/kubernetes \
             --mount volume=config,target=/etc/kubernetes \
             --insecure-options=image \
-            docker://k8s.gcr.io/hyperkube:v1.17.4 \
+            docker://quay.io/poseidon/kubelet:v1.17.4 \
             --net=host \
             --dns=host \
             -- \
-            kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname)
+            /usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname)

docs/architecture/operating-systems.md

@@ -31,8 +31,8 @@ Together, they diversify Typhoon to support a range of container technologies.
 | single-master     | all platforms | all platforms |
 | multi-master      | all platforms | all platforms |
 | control plane     | static pods   | static pods   |
-| kubelet image     | upstream hyperkube | upstream hyperkube |
-| control plane images | upstream hyperkube | upstream hyperkube |
+| kubelet image     | kubelet [image](https://github.com/poseidon/kubelet) with upstream binary | kubelet [image](https://github.com/poseidon/kubelet) with upstream binary |
+| control plane images | upstream images | upstream images |
 | on-host etcd      | rkt-fly   | podman |
 | on-host kubelet   | rkt-fly   | podman |
 | CNI plugins       | calico or flannel | calico or flannel |

docs/topics/security.md

@@ -40,9 +40,14 @@ Typhoon limits exposure to many security threats, but it is not a silver bullet.
 * Do not give untrusted users a shell behind your firewall
 * Define network policies for your namespaces
 
-## OpenPGP Signing
+## Container Images
 
-Typhoon uses upstream container images and binaries. We do not distribute artifacts of our own. If you find artifacts claiming to be from Typhoon, please send a note.
+Typhoon uses upstream container images (where possible) and upstream binaries.
+
+!!! note
+    Kubernetes releases `kubelet` as a binary for distros to package, either as a DEB/RPM on traditional distros or as a container image for container-optimized operating systems.
+
+Typhoon [packages](https://github.com/poseidon/kubelet) the upstream Kubelet and its dependencies as a [container image](https://quay.io/repository/poseidon/kubelet) for use in Typhoon. The upstream Kubelet binary is checksummed and packaged directly. Quay automated builds provide verifiability and confidence in image contents.
 
 ## Disclosures
 

google-cloud/container-linux/kubernetes/bootstrap.tf

@@ -1,6 +1,6 @@
 # Kubernetes assets (kubeconfig, manifests)
 module "bootstrap" {
-  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=73784c1b2c791d9ba586a1478979ac34dd324dad"
+  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e76f0a09fa9e6421a9cf697ee03714c6224e2581"
 
   cluster_name          = var.cluster_name
   api_servers           = [format("%s.%s", var.cluster_name, var.dns_zone)]

google-cloud/container-linux/kubernetes/cl/controller.yaml

@@ -90,8 +90,7 @@ systemd:
           --mount volume=var-log,target=/var/log \
           --volume opt-cni-bin,kind=host,source=/opt/cni/bin \
           --mount volume=opt-cni-bin,target=/opt/cni/bin \
-          docker://k8s.gcr.io/hyperkube:v1.17.4 \
-          --exec=/usr/local/bin/kubelet -- \
+          docker://quay.io/poseidon/kubelet:v1.17.4 -- \
           --anonymous-auth=false \
           --authentication-token-webhook \
           --authorization-mode=Webhook \
@@ -134,7 +133,7 @@ systemd:
             --volume script,kind=host,source=/opt/bootstrap/apply \
             --mount volume=script,target=/apply \
             --insecure-options=image \
-            docker://k8s.gcr.io/hyperkube:v1.17.4 \
+            docker://quay.io/poseidon/kubelet:v1.17.4 \
             --net=host \
             --dns=host \
             --exec=/apply

google-cloud/container-linux/kubernetes/workers/cl/worker.yaml

@@ -63,8 +63,7 @@ systemd:
           --mount volume=var-log,target=/var/log \
           --volume opt-cni-bin,kind=host,source=/opt/cni/bin \
           --mount volume=opt-cni-bin,target=/opt/cni/bin \
-          docker://k8s.gcr.io/hyperkube:v1.17.4 \
-          --exec=/usr/local/bin/kubelet -- \
+          docker://quay.io/poseidon/kubelet:v1.17.4 -- \
           --anonymous-auth=false \
           --authentication-token-webhook \
           --authorization-mode=Webhook \
@@ -78,9 +77,9 @@ systemd:
           --lock-file=/var/run/lock/kubelet.lock \
           --network-plugin=cni \
           --node-labels=node.kubernetes.io/node \
-          %{ for label in split(",", node_labels) }
+          %{~ for label in split(",", node_labels) ~}
           --node-labels=${label} \
-          %{ endfor ~}
+          %{~ endfor ~}
           --pod-manifest-path=/etc/kubernetes/manifests \
           --read-only-port=0 \
           --volume-plugin-dir=/var/lib/kubelet/volumeplugins
@@ -126,11 +125,11 @@ storage:
             --volume config,kind=host,source=/etc/kubernetes \
             --mount volume=config,target=/etc/kubernetes \
             --insecure-options=image \
-            docker://k8s.gcr.io/hyperkube:v1.17.4 \
+            docker://quay.io/poseidon/kubelet:v1.17.4 \
             --net=host \
             --dns=host \
             -- \
-            kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname)
+            /usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname)
 passwd:
   users:
     - name: core