Provider for k8s 1.15.1 (kubevirt#127)

* Add 1.15.1 provider

* Fix nodes startup script

* Remove outdated comments

* Add shasum

cluster-provision/k8s/1.15.1/provision.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -ex
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+export version=1.15.1
+../provision.sh

cluster-provision/k8s/1.15.1/publish.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -ex
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+export version=1.15.1
+../publish.sh

cluster-provision/k8s/scripts/nodes.sh

@@ -8,19 +8,17 @@ do
     sleep 2
 done
 
-# enable CPU manager
-# kubeadm 1.11 uses a new config method for the kubelet
 if [ -f /etc/sysconfig/kubelet ]; then
     # TODO use config file! this is deprecated
     cat <<EOT >>/etc/sysconfig/kubelet
-KUBELET_CPUMANAGER_ARGS=--feature-gates=CPUManager=true --cpu-manager-policy=static --kube-reserved=cpu=500m --system-reserved=cpu=500m
+KUBELET_EXTRA_ARGS=${KUBELET_EXTRA_ARGS} --feature-gates=CPUManager=true --cpu-manager-policy=static --kube-reserved=cpu=500m --system-reserved=cpu=500m
 EOT
 else
     cat <<EOT >>/etc/systemd/system/kubelet.service.d/09-kubeadm.conf
 Environment="KUBELET_CPUMANAGER_ARGS=--feature-gates=CPUManager=true --cpu-manager-policy=static --kube-reserved=cpu=500m --system-reserved=cpu=500m"
 EOT
-fi
 sed -i 's/$KUBELET_EXTRA_ARGS/$KUBELET_EXTRA_ARGS $KUBELET_CPUMANAGER_ARGS/' /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
+fi
 
 systemctl daemon-reload
 service kubelet restart

cluster-provision/k8s/scripts/provision.sh

@@ -59,9 +59,12 @@ yum install --nogpgcheck -y \
     kubectl-${version} \
     kubernetes-cni-0.6.0
 
-# Latest docker on CentOS uses systemd for cgroup management
-# kubeadm 1.11 uses a new config method for the kubelet
-if [[ $version =~ \.([0-9]+) ]] && [[ ${BASH_REMATCH[1]} -ge "12" ]]; then
+if [[ $version =~ \.([0-9]+) ]] && [[ ${BASH_REMATCH[1]} -ge "15" ]]; then
+    # TODO use config file! this is deprecated
+    cat <<EOT >/etc/sysconfig/kubelet
+KUBELET_EXTRA_ARGS=--cgroup-driver=systemd --runtime-cgroups=/systemd/system.slice --kubelet-cgroups=/systemd/system.slice --feature-gates="BlockVolume=true,CSIBlockVolume=true,VolumeSnapshotDataSource=true"
+EOT
+elif [[ $version =~ \.([0-9]+) ]] && [[ ${BASH_REMATCH[1]} -ge "12" ]]; then
     # TODO use config file! this is deprecated
     cat <<EOT >/etc/sysconfig/kubelet
 KUBELET_EXTRA_ARGS=--cgroup-driver=systemd --runtime-cgroups=/systemd/system.slice --kubelet-cgroups=/systemd/system.slice --allow-privileged=true --feature-gates="BlockVolume=true,CSIBlockVolume=true,VolumeSnapshotDataSource=true"
@@ -144,8 +147,78 @@ $reset_command
 # audit log configuration
 mkdir /etc/kubernetes/audit
 
-# New configuration for kubernetes >= 1.12
-if [[ ${BASH_REMATCH[1]} -ge "12" ]]; then
+if [[ ${BASH_REMATCH[1]} -ge "15" ]]; then
+cat > /etc/kubernetes/audit/adv-audit.yaml <<EOF
+apiVersion: audit.k8s.io/v1
+kind: Policy
+rules:
+- level: Request
+  users: ["kubernetes-admin"]
+  resources:
+  - group: kubevirt.io
+    resources:
+    - virtualmachines
+    - virtualmachineinstances
+    - virtualmachineinstancereplicasets
+    - virtualmachineinstancepresets
+    - virtualmachineinstancemigrations
+  omitStages:
+  - RequestReceived
+  - ResponseStarted
+  - Panic
+EOF
+
+cat > /etc/kubernetes/kubeadm.conf <<EOF
+apiVersion: kubeadm.k8s.io/v1beta1
+bootstrapTokens:
+- groups:
+  - system:bootstrappers:kubeadm:default-node-token
+  token: abcdef.1234567890123456
+  ttl: 24h0m0s
+  usages:
+  - signing
+  - authentication
+kind: InitConfiguration
+---
+apiServer:
+  extraArgs:
+    allow-privileged: "true"
+    audit-log-format: json
+    audit-log-path: /var/log/k8s-audit/k8s-audit.log
+    audit-policy-file: /etc/kubernetes/audit/adv-audit.yaml
+    enable-admission-plugins: NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota
+    feature-gates: BlockVolume=true,CSIBlockVolume=true,VolumeSnapshotDataSource=true,AdvancedAuditing=true
+  extraVolumes:
+  - hostPath: /etc/kubernetes/audit
+    mountPath: /etc/kubernetes/audit
+    name: audit-conf
+    readOnly: true
+  - hostPath: /var/log/k8s-audit
+    mountPath: /var/log/k8s-audit
+    name: audit-log
+  timeoutForControlPlane: 4m0s
+apiVersion: kubeadm.k8s.io/v1beta1
+certificatesDir: /etc/kubernetes/pki
+clusterName: kubernetes
+controlPlaneEndpoint: ""
+controllerManager:
+  extraArgs:
+    feature-gates: BlockVolume=true,CSIBlockVolume=true,VolumeSnapshotDataSource=true
+dns:
+  type: CoreDNS
+etcd:
+  local:
+    dataDir: /var/lib/etcd
+imageRepository: k8s.gcr.io
+kind: ClusterConfiguration
+kubernetesVersion: ${version}
+networking:
+  dnsDomain: cluster.local
+  podSubnet: 10.244.0.0/16
+  serviceSubnet: 10.96.0.0/12
+EOF
+
+elif [[ ${BASH_REMATCH[1]} -ge "12" ]]; then
 cat > /etc/kubernetes/audit/adv-audit.yaml <<EOF
 apiVersion: audit.k8s.io/v1
 kind: Policy

cluster-up/cluster/k8s-1.15.1/README.md

@@ -0,0 +1,45 @@
+# Kubernetes 1.15.1 in ephemeral containers
+
+Provides a pre-deployed Kubernetes with version 1.15.1 purely in docker
+containers with qemu. The provided VMs are completely ephemeral and are
+recreated on every cluster restart. The KubeVirt containers are built on the
+local machine and are the pushed to a registry which is exposed at
+`localhost:5000`.
+
+## Bringing the cluster up
+
+```bash
+export KUBEVIRT_PROVIDER=k8s-1.15.1
+export KUBEVIRT_NUM_NODES=2 # master + one node
+make cluster-up
+```
+
+The cluster can be accessed as usual:
+
+```bash
+$ cluster/kubectl.sh get nodes
+NAME      STATUS     ROLES     AGE       VERSION
+node01    NotReady   master    31s       v1.15.1
+node02    NotReady   <none>    5s        v1.15.1
+```
+
+## Bringing the cluster down
+
+```bash
+export KUBEVIRT_PROVIDER=k8s-1.15.1
+make cluster-down
+```
+
+This destroys the whole cluster. Recreating the cluster is fast, since k8s is
+already pre-deployed. The only state which is kept is the state of the local
+docker registry.
+
+## Destroying the docker registry state
+
+The docker registry survives a `make cluster-down`. It's state is stored in a
+docker volume called `kubevirt_registry`. If the volume gets too big or the
+volume contains corrupt data, it can be deleted with
+
+```bash
+docker volume rm kubevirt_registry
+```

cluster-up/cluster/k8s-1.15.1/provider.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+set -e
+
+image="k8s-1.15.1@sha256:14d7b1806f24e527167d2913deafd910ea46e69b830bf0b094dde35ba961b159"
+
+source ${KUBEVIRTCI_PATH}/cluster/ephemeral-provider-common.sh
+
+function up() {
+    ${_cli} run $(_add_common_params)
+
+    # Copy k8s config and kubectl
+    ${_cli} scp --prefix $provider_prefix /usr/bin/kubectl - >${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/.kubectl
+    chmod u+x ${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/.kubectl
+    ${_cli} scp --prefix $provider_prefix /etc/kubernetes/admin.conf - >${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/.kubeconfig
+
+    # Set server and disable tls check
+    export KUBECONFIG=${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/.kubeconfig
+    ${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/.kubectl config set-cluster kubernetes --server=https://$(_main_ip):$(_port k8s)
+    ${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/.kubectl config set-cluster kubernetes --insecure-skip-tls-verify=true
+
+    # Make sure that local config is correct
+    prepare_config
+}