From 1955c7d7cfb99e8f1ce84c21c746c7311779e9e8 Mon Sep 17 00:00:00 2001
From: Patrick Ohly <patrick.ohly@intel.com>
Date: Thu, 28 Mar 2019 11:02:08 +0100
Subject: [PATCH] deploy: preliminary example for Kubernetes 1.14

The only difference is in the image versions. We still need two
examples, because some CSI driver developers may need the older
example for Kubernetes 1.13 if they depend on the alpha features.

It's preliminary because the actual images haven't been released yet.
---
 deploy/kubernetes-1.13/README.md              |   8 +-
 deploy/kubernetes-1.14/README.md              |  10 ++
 deploy/kubernetes-1.14/deploy-hostpath.sh     | 138 ++++++++++++++++++
 .../hostpath/csi-hostpath-attacher.yaml       |  55 +++++++
 .../hostpath/csi-hostpath-plugin.yaml         | 136 +++++++++++++++++
 .../hostpath/csi-hostpath-provisioner.yaml    |  55 +++++++
 .../hostpath/csi-hostpath-service.yaml        |  56 +++++++
 .../hostpath/csi-hostpath-snapshotter.yaml    |  55 +++++++
 deploy/kubernetes-latest                      |   2 +-
 9 files changed, 509 insertions(+), 6 deletions(-)
 create mode 100644 deploy/kubernetes-1.14/README.md
 create mode 100755 deploy/kubernetes-1.14/deploy-hostpath.sh
 create mode 100644 deploy/kubernetes-1.14/hostpath/csi-hostpath-attacher.yaml
 create mode 100644 deploy/kubernetes-1.14/hostpath/csi-hostpath-plugin.yaml
 create mode 100644 deploy/kubernetes-1.14/hostpath/csi-hostpath-provisioner.yaml
 create mode 100644 deploy/kubernetes-1.14/hostpath/csi-hostpath-service.yaml
 create mode 100644 deploy/kubernetes-1.14/hostpath/csi-hostpath-snapshotter.yaml

diff --git a/deploy/kubernetes-1.13/README.md b/deploy/kubernetes-1.13/README.md
index c0b85d61a..5f59381fd 100644
--- a/deploy/kubernetes-1.13/README.md
+++ b/deploy/kubernetes-1.13/README.md
@@ -1,9 +1,7 @@
-The deployment for Kubernetes 1.13 uses CSI 1.0 and this is
+The deployment for Kubernetes 1.13 uses CSI 1.0 and thus is
 incompatible with older Kubernetes releases.
 
-It relies on the CRDs for CSIDriverInfo and CSINodeInfo, which are
-about to be replaced with builtin APIs in Kubernetes 1.14. It can be
+The sidecar images rely on the CRDs for CSIDriverInfo and CSINodeInfo,
+which were replaced with builtin APIs in Kubernetes 1.14. They can be
 deployed on Kubernetes 1.14 if the CRDs are installed, but features
 relying on these CRDs (like topology) are unlikely to work.
-
-Kubernetes 1.14 will need a different deployment.
diff --git a/deploy/kubernetes-1.14/README.md b/deploy/kubernetes-1.14/README.md
new file mode 100644
index 000000000..3f41e8464
--- /dev/null
+++ b/deploy/kubernetes-1.14/README.md
@@ -0,0 +1,10 @@
+The deployment for Kubernetes 1.14 uses CSI 1.0 and thus is incompatible with
+Kubernetes < 1.13.
+
+It uses the builtin APIs for CSIDriverInfo and CSINodeInfo that were
+introduced in Kubernetes 1.14, so features depending on those (like
+topology) will not work on Kubernetes 1.13. But because this example
+deployment does not enable those features, it can run on Kubernetes 1.13.
+
+WARNING: the images for Kubernetes 1.14 have not been released yet, so this
+example uses the "canary" images instead.
diff --git a/deploy/kubernetes-1.14/deploy-hostpath.sh b/deploy/kubernetes-1.14/deploy-hostpath.sh
new file mode 100755
index 000000000..a84d25a43
--- /dev/null
+++ b/deploy/kubernetes-1.14/deploy-hostpath.sh
@@ -0,0 +1,138 @@
+#!/usr/bin/env bash
+
+# This script captures the steps required to successfully
+# deploy the hostpath plugin driver.  This should be considered
+# authoritative and all updates for this process should be
+# done here and referenced elsewhere.
+
+# The script assumes that kubectl is available on the OS path 
+# where it is executed.
+
+set -e
+set -o pipefail
+
+BASE_DIR=$(dirname "$0")
+K8S_RELEASE=${K8S_RELEASE:-"release-1.13"}
+
+# If set, the following env variables override image registry and/or tag for each of the images.
+# They are named after the image name, with hyphen replaced by underscore and in upper case.
+#
+# - CSI_ATTACHER_REGISTRY
+# - CSI_ATTACHER_TAG
+# - CSI_NODE_DRIVER_REGISTRAR_REGISTRY
+# - CSI_NODE_DRIVER_REGISTRAR_TAG
+# - CSI_PROVISIONER_REGISTRY
+# - CSI_PROVISIONER_TAG
+# - CSI_SNAPSHOTTER_REGISTRY
+# - CSI_SNAPSHOTTER_TAG
+# - HOSTPATHPLUGIN_REGISTRY
+# - HOSTPATHPLUGIN_TAG
+#
+# Alternatively, it is possible to override all registries or tags with:
+# - IMAGE_REGISTRY
+# - IMAGE_TAG
+# These are used as fallback when the more specific variables are unset or empty.
+#
+# Beware that the .yaml files do not have "imagePullPolicy: Always". That means that
+# also the "canary" images will only be pulled once. This is good for testing
+# (starting a pod multiple times will always run with the same canary image), but
+# implies that refreshing that image has to be done manually.
+#
+# As a special case, 'none' as registry removes the registry name.
+
+function image_version () {
+    yaml="$1"
+    image="$2"
+
+    # get version from `image: quay.io/k8scsi/csi-attacher:v1.0.1`
+    version="$(grep "image:.*$image" "$yaml" | sed -e 's/.*:v/v/')"
+
+    # apply overrides
+    varname=$(echo $image | tr - _ | tr a-z A-Z)
+    eval version=\${${varname}_TAG:-\${IMAGE_TAG:-\$version}}
+
+    # When using canary images, we have to assume that the
+    # canary images were built from the corresponding branch.
+    case "$version" in canary) version=master;;
+                        *-canary) version="$(echo "$version" | sed -e 's/\(.*\)-canary/release-\1/')";;
+    esac
+    echo "$version"
+}
+
+# In addition, the RBAC rules can be overridden for provisioner and attacher.
+CSI_PROVISIONER_RBAC=${PROVISIONER_RBAC:-https://raw.githubusercontent.com/kubernetes-csi/external-provisioner/$(image_version "${BASE_DIR}/hostpath/csi-hostpath-provisioner.yaml" csi-provisioner)/deploy/kubernetes/rbac.yaml}
+CSI_ATTACHER_RBAC=${ATTACHER_RBAC:-https://raw.githubusercontent.com/kubernetes-csi/external-attacher/$(image_version "${BASE_DIR}/hostpath/csi-hostpath-attacher.yaml" csi-attacher)/deploy/kubernetes/rbac.yaml}
+CSI_SNAPSHOTTER_RBAC=${CSI_SNAPSHOTTER_RBAC:-https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/$(image_version "${BASE_DIR}/hostpath/csi-hostpath-snapshotter.yaml" csi-snapshotter)/deploy/kubernetes/rbac.yaml}
+
+INSTALL_CRD=${INSTALL_CRD:-"false"}
+
+run () {
+    echo "$@" >&2
+    "$@"
+}
+
+# apply CSIDriver and CSINodeInfo API objects
+if [[ "${INSTALL_CRD}" =~ ^(y|Y|yes|true)$ ]] ; then
+    echo "installing CRDs"
+    run kubectl apply -f https://raw.githubusercontent.com/kubernetes/csi-api/${K8S_RELEASE}/pkg/crd/manifests/csidriver.yaml --validate=false
+    run kubectl apply -f https://raw.githubusercontent.com/kubernetes/csi-api/${K8S_RELEASE}/pkg/crd/manifests/csinodeinfo.yaml --validate=false
+fi
+
+# rbac rules
+echo "applying RBAC rules"
+run kubectl apply -f "${CSI_PROVISIONER_RBAC}"
+run kubectl apply -f "${CSI_ATTACHER_RBAC}"
+run kubectl apply -f "${CSI_SNAPSHOTTER_RBAC}"
+
+# deploy hostpath plugin and registrar sidecar
+echo "deploying hostpath components"
+for i in ${BASE_DIR}/hostpath/*.yaml; do
+    echo "   $i"
+    modified="$(cat "$i" | while IFS= read -r line; do
+        if echo "$line" | grep -q '^\s*image:\s*'; then
+            # Split 'image: quay.io/k8scsi/csi-attacher:v1.0.1'
+            # into image (quay.io/k8scsi/csi-attacher:v1.0.1),
+            # registry (quay.io/k8scsi),
+            # name (csi-attacher),
+            # tag (v1.0.1).
+            image=$(echo "$line" | sed -e 's;.*image:\s*;;')
+            registry=$(echo "$image" | sed -e 's;\(.*\)/.*;\1;')
+            name=$(echo "$image" | sed -e 's;.*/\([^:]*\).*;\1;')
+            tag=$(echo "$image" | sed -e 's;.*:;;')
+
+            # Variables are with underscores and upper case.
+            varname=$(echo $name | tr - _ | tr a-z A-Z)
+
+            # Now replace registry and/or tag, if set as env variables.
+            # If not set, the replacement is the same as the original value.
+            prefix=$(eval echo \${${varname}_REGISTRY:-${IMAGE_REGISTRY:-${registry}}}/ | sed -e 's;none/;;')
+            suffix=$(eval echo :\${${varname}_TAG:-${IMAGE_TAG:-${tag}}})
+            line="$(echo "$line" | sed -e "s;$image;${prefix}${name}${suffix};")"
+            echo "        using $line" >&2
+        fi
+        echo "$line"
+    done)"
+    if ! echo "$modified" | kubectl apply -f -; then
+        echo "modified version of $i:"
+        echo "$modified"
+        exit 1
+    fi
+done
+
+# Wait until all pods are running. We have to make some assumptions
+# about the deployment here, otherwise we wouldn't know what to wait
+# for: the expectation is that we run attacher, provisioner,
+# snapshotter, socat and hostpath plugin in the default namespace.
+cnt=0
+while [ $(kubectl get pods 2>/dev/null | grep '^csi-hostpath.* Running ' | wc -l) -lt 5 ]; do
+    if [ $cnt -gt 30 ]; then
+        echo "Running pods:"
+        kubectl describe pods
+
+        echo >&2 "ERROR: hostpath deployment not ready after over 5min"
+        exit 1
+    fi
+    echo $(date +%H:%M:%S) "waiting for hostpath deployment to complete, attempt #$cnt"
+    cnt=$(($cnt + 1))
+    sleep 10
+done
diff --git a/deploy/kubernetes-1.14/hostpath/csi-hostpath-attacher.yaml b/deploy/kubernetes-1.14/hostpath/csi-hostpath-attacher.yaml
new file mode 100644
index 000000000..00d913bf8
--- /dev/null
+++ b/deploy/kubernetes-1.14/hostpath/csi-hostpath-attacher.yaml
@@ -0,0 +1,55 @@
+kind: Service
+apiVersion: v1
+metadata:
+  name: csi-hostpath-attacher
+  labels:
+    app: csi-hostpath-attacher
+spec:
+  selector:
+    app: csi-hostpath-attacher
+  ports:
+    - name: dummy
+      port: 12345
+
+---
+kind: StatefulSet
+apiVersion: apps/v1
+metadata:
+  name: csi-hostpath-attacher
+spec:
+  serviceName: "csi-hostpath-attacher"
+  replicas: 1
+  selector:
+    matchLabels:
+      app: csi-hostpath-attacher
+  template:
+    metadata:
+      labels:
+        app: csi-hostpath-attacher
+    spec:
+      affinity:
+        podAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+          - labelSelector:
+              matchExpressions:
+              - key: app
+                operator: In
+                values:
+                - csi-hostpathplugin
+            topologyKey: kubernetes.io/hostname
+      serviceAccountName: csi-attacher
+      containers:
+        - name: csi-attacher
+          image: quay.io/k8scsi/csi-attacher:canary # TODO: replace with released version
+          args:
+            - --v=5
+            - --csi-address=/csi/csi.sock
+          volumeMounts:
+          - mountPath: /csi
+            name: socket-dir
+
+      volumes:
+        - hostPath:
+            path: /var/lib/kubelet/plugins/csi-hostpath
+            type: DirectoryOrCreate
+          name: socket-dir
diff --git a/deploy/kubernetes-1.14/hostpath/csi-hostpath-plugin.yaml b/deploy/kubernetes-1.14/hostpath/csi-hostpath-plugin.yaml
new file mode 100644
index 000000000..8a7f72425
--- /dev/null
+++ b/deploy/kubernetes-1.14/hostpath/csi-hostpath-plugin.yaml
@@ -0,0 +1,136 @@
+# Service defined here, plus serviceName below in StatefulSet,
+# are needed only because of condition explained in
+# https://github.com/kubernetes/kubernetes/issues/69608
+
+kind: Service
+apiVersion: v1
+metadata:
+  name: csi-hostpathplugin
+  labels:
+    app: csi-hostpathplugin
+spec:
+  selector:
+    app: csi-hostpathplugin
+  ports:
+    - name: dummy
+      port: 12345
+---
+kind: StatefulSet
+apiVersion: apps/v1
+metadata:
+  name: csi-hostpathplugin
+spec:
+  serviceName: "csi-hostpathplugin"
+  # One replica only:
+  # Host path driver only works when everything runs
+  # on a single node. We achieve that by starting it once and then
+  # co-locate all other pods via inter-pod affinity
+  replicas: 1
+  selector:
+    matchLabels:
+      app: csi-hostpathplugin
+  template:
+    metadata:
+      labels:
+        app: csi-hostpathplugin
+    spec:
+      hostNetwork: true
+      containers:
+        - name: node-driver-registrar
+          image: quay.io/k8scsi/csi-node-driver-registrar:canary # TODO: replace with released version
+          lifecycle:
+            preStop:
+              exec:
+                command: ["/bin/sh", "-c", "rm -rf /registration/csi-hostpath /registration/csi-hostpath-reg.sock"]
+          args:
+            - --v=5
+            - --csi-address=/csi/csi.sock
+            - --kubelet-registration-path=/var/lib/kubelet/plugins/csi-hostpath/csi.sock
+          securityContext:
+            privileged: true
+          env:
+            - name: KUBE_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: spec.nodeName
+          volumeMounts:
+          - mountPath: /csi
+            name: socket-dir
+          - mountPath: /registration
+            name: registration-dir
+          - mountPath: /csi-data-dir
+            name: csi-data-dir
+
+        - name: hostpath
+          image: quay.io/k8scsi/hostpathplugin:canary # TODO: replace with released version
+          args:
+            - "--v=5"
+            - "--endpoint=$(CSI_ENDPOINT)"
+            - "--nodeid=$(KUBE_NODE_NAME)"
+          env:
+            - name: CSI_ENDPOINT
+              value: unix:///csi/csi.sock
+            - name: KUBE_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: spec.nodeName
+          securityContext:
+            privileged: true
+          ports:
+          - containerPort: 9898
+            name: healthz
+            protocol: TCP
+          livenessProbe:
+            failureThreshold: 5
+            httpGet:
+              path: /healthz
+              port: healthz
+            initialDelaySeconds: 10
+            timeoutSeconds: 3
+            periodSeconds: 2
+          volumeMounts:
+            - mountPath: /csi
+              name: socket-dir
+            - mountPath: /var/lib/kubelet/pods
+              mountPropagation: Bidirectional
+              name: mountpoint-dir
+            - mountPath: /var/lib/kubelet/plugins
+              mountPropagation: Bidirectional
+              name: plugins-dir
+
+        - name: liveness-probe
+          imagePullPolicy: Always
+          volumeMounts:
+          - mountPath: /csi
+            name: socket-dir
+          image: quay.io/k8scsi/livenessprobe:v1.0.2
+          args:
+          - --csi-address=/csi/csi.sock
+          - --connection-timeout=3s
+          - --health-port=9898
+
+      volumes:
+        - hostPath:
+            path: /var/lib/kubelet/plugins/csi-hostpath
+            type: DirectoryOrCreate
+          name: socket-dir
+        - hostPath:
+            path: /var/lib/kubelet/pods
+            type: DirectoryOrCreate
+          name: mountpoint-dir
+        - hostPath:
+            path: /var/lib/kubelet/plugins_registry
+            type: Directory
+          name: registration-dir
+        - hostPath:
+            path: /var/lib/kubelet/plugins
+            type: Directory
+          name: plugins-dir
+        - hostPath:
+            # 'path' is where PV data is persisted on host.
+            # using /tmp is also possible while the PVs will not available after plugin container recreation or host reboot
+            path: /var/lib/csi-hostpath-data/
+            type: DirectoryOrCreate
+          name: csi-data-dir
diff --git a/deploy/kubernetes-1.14/hostpath/csi-hostpath-provisioner.yaml b/deploy/kubernetes-1.14/hostpath/csi-hostpath-provisioner.yaml
new file mode 100644
index 000000000..f5673e264
--- /dev/null
+++ b/deploy/kubernetes-1.14/hostpath/csi-hostpath-provisioner.yaml
@@ -0,0 +1,55 @@
+kind: Service
+apiVersion: v1
+metadata:
+  name: csi-hostpath-provisioner 
+  labels:
+    app: csi-hostpath-provisioner 
+spec:
+  selector:
+    app: csi-hostpath-provisioner 
+  ports:
+    - name: dummy
+      port: 12345
+
+---
+kind: StatefulSet
+apiVersion: apps/v1
+metadata:
+  name: csi-hostpath-provisioner
+spec:
+  serviceName: "csi-hostpath-provisioner"
+  replicas: 1
+  selector:
+    matchLabels:
+      app: csi-hostpath-provisioner
+  template:
+    metadata:
+      labels:
+        app: csi-hostpath-provisioner
+    spec:
+      affinity:
+        podAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+          - labelSelector:
+              matchExpressions:
+              - key: app
+                operator: In
+                values:
+                - csi-hostpathplugin
+            topologyKey: kubernetes.io/hostname
+      serviceAccountName: csi-provisioner
+      containers:
+        - name: csi-provisioner
+          image: quay.io/k8scsi/csi-provisioner:canary # TODO: replace with released version
+          args:
+            - -v=5
+            - --csi-address=/csi/csi.sock
+            - --connection-timeout=15s
+          volumeMounts:
+            - mountPath: /csi
+              name: socket-dir
+      volumes:
+        - hostPath:
+            path: /var/lib/kubelet/plugins/csi-hostpath
+            type: DirectoryOrCreate
+          name: socket-dir
diff --git a/deploy/kubernetes-1.14/hostpath/csi-hostpath-service.yaml b/deploy/kubernetes-1.14/hostpath/csi-hostpath-service.yaml
new file mode 100644
index 000000000..86d515aac
--- /dev/null
+++ b/deploy/kubernetes-1.14/hostpath/csi-hostpath-service.yaml
@@ -0,0 +1,56 @@
+# This exposes the hostpath's Unix domain csi.sock as a TCP port to the
+# outside world. The mapping from Unix domain socket to TCP is done
+# by socat.
+#
+# This is useful for testing with csi-sanity or csc.
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: hostpath-service
+spec:
+  type: NodePort
+  selector:
+    app: csi-hostpath-socat
+  ports:
+  - port: 10000 # fixed port inside the pod, dynamically allocated port outside
+---
+kind: StatefulSet
+apiVersion: apps/v1
+metadata:
+  name: csi-hostpath-socat
+spec:
+  serviceName: "csi-hostpath-socat"
+  replicas: 1
+  selector:
+    matchLabels:
+      app: csi-hostpath-socat
+  template:
+    metadata:
+      labels:
+        app: csi-hostpath-socat
+    spec:
+      affinity:
+        podAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+          - labelSelector:
+              matchExpressions:
+              - key: app
+                operator: In
+                values:
+                - csi-hostpathplugin
+            topologyKey: kubernetes.io/hostname
+      containers:
+        - name: socat
+          image: alpine/socat:1.0.3
+          args:
+            - tcp-listen:10000,fork,reuseaddr
+            - unix-connect:/csi/csi.sock
+          volumeMounts:
+          - mountPath: /csi
+            name: socket-dir
+      volumes:
+        - hostPath:
+            path: /var/lib/kubelet/plugins/csi-hostpath
+            type: DirectoryOrCreate
+          name: socket-dir
diff --git a/deploy/kubernetes-1.14/hostpath/csi-hostpath-snapshotter.yaml b/deploy/kubernetes-1.14/hostpath/csi-hostpath-snapshotter.yaml
new file mode 100644
index 000000000..b9a11171c
--- /dev/null
+++ b/deploy/kubernetes-1.14/hostpath/csi-hostpath-snapshotter.yaml
@@ -0,0 +1,55 @@
+kind: Service
+apiVersion: v1
+metadata:
+  name: csi-hostpath-snapshotter
+  labels:
+    app: csi-hostpath-snapshotter
+spec:
+  selector:
+    app: csi-hostpath-snapshotter
+  ports:
+    - name: dummy
+      port: 12345
+
+---
+kind: StatefulSet
+apiVersion: apps/v1
+metadata:
+  name: csi-hostpath-snapshotter
+spec:
+  serviceName: "csi-hostpath-snapshotter"
+  replicas: 1
+  selector:
+    matchLabels:
+      app: csi-hostpath-snapshotter
+  template:
+    metadata:
+      labels:
+        app: csi-hostpath-snapshotter
+    spec:
+      affinity:
+        podAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+          - labelSelector:
+              matchExpressions:
+              - key: app
+                operator: In
+                values:
+                - csi-hostpathplugin
+            topologyKey: kubernetes.io/hostname
+      serviceAccount: csi-snapshotter
+      containers:
+        - name: csi-snapshotter
+          image: quay.io/k8scsi/csi-snapshotter:canary # TODO: replace with released version
+          args:
+            - -v=5
+            - --csi-address=/csi/csi.sock
+            - --connection-timeout=15s
+          volumeMounts:
+            - mountPath: /csi
+              name: socket-dir
+      volumes:
+        - hostPath:
+            path: /var/lib/kubelet/plugins/csi-hostpath
+            type: DirectoryOrCreate
+          name: socket-dir
diff --git a/deploy/kubernetes-latest b/deploy/kubernetes-latest
index c7e193148..dcca3cd83 120000
--- a/deploy/kubernetes-latest
+++ b/deploy/kubernetes-latest
@@ -1 +1 @@
-kubernetes-1.13
\ No newline at end of file
+kubernetes-1.14
\ No newline at end of file