deploy: preliminary example for Kubernetes 1.14

The only difference is in the image versions. We still need two
examples, because some CSI driver developers may need the older
example for Kubernetes 1.13 if they depend on the alpha features.

It's preliminary because the actual images haven't been released yet.

deploy/kubernetes-1.13/README.md

@@ -1,9 +1,7 @@
-The deployment for Kubernetes 1.13 uses CSI 1.0 and this is
+The deployment for Kubernetes 1.13 uses CSI 1.0 and thus is
 incompatible with older Kubernetes releases.
 
-It relies on the CRDs for CSIDriverInfo and CSINodeInfo, which are
-about to be replaced with builtin APIs in Kubernetes 1.14. It can be
+The sidecar images rely on the CRDs for CSIDriverInfo and CSINodeInfo,
+which were replaced with builtin APIs in Kubernetes 1.14. They can be
 deployed on Kubernetes 1.14 if the CRDs are installed, but features
 relying on these CRDs (like topology) are unlikely to work.
-
-Kubernetes 1.14 will need a different deployment.

deploy/kubernetes-1.14/README.md

@@ -0,0 +1,10 @@
+The deployment for Kubernetes 1.14 uses CSI 1.0 and thus is incompatible with
+Kubernetes < 1.13.
+
+It uses the builtin APIs for CSIDriverInfo and CSINodeInfo that were
+introduced in Kubernetes 1.14, so features depending on those (like
+topology) will not work on Kubernetes 1.13. But because this example
+deployment does not enable those features, it can run on Kubernetes 1.13.
+
+WARNING: the images for Kubernetes 1.14 have not been released yet, so this
+example uses the "canary" images instead.

deploy/kubernetes-1.14/deploy-hostpath.sh

@@ -0,0 +1,139 @@
+#!/usr/bin/env bash
+
+# This script captures the steps required to successfully
+# deploy the hostpath plugin driver.  This should be considered
+# authoritative and all updates for this process should be
+# done here and referenced elsewhere.
+
+# The script assumes that kubectl is available on the OS path 
+# where it is executed.
+
+set -e
+set -o pipefail
+
+BASE_DIR=$(dirname "$0")
+K8S_RELEASE=${K8S_RELEASE:-"release-1.13"}
+
+# If set, the following env variables override image registry and/or tag for each of the images.
+# They are named after the image name, with hyphen replaced by underscore and in upper case.
+#
+# - CSI_ATTACHER_REGISTRY
+# - CSI_ATTACHER_TAG
+# - CSI_NODE_DRIVER_REGISTRAR_REGISTRY
+# - CSI_NODE_DRIVER_REGISTRAR_TAG
+# - CSI_PROVISIONER_REGISTRY
+# - CSI_PROVISIONER_TAG
+# - CSI_SNAPSHOTTER_REGISTRY
+# - CSI_SNAPSHOTTER_TAG
+# - HOSTPATHPLUGIN_REGISTRY
+# - HOSTPATHPLUGIN_TAG
+#
+# Alternatively, it is possible to override all registries or tags with:
+# - IMAGE_REGISTRY
+# - IMAGE_TAG
+# These are used as fallback when the more specific variables are unset or empty.
+#
+# Beware that the .yaml files do not have "imagePullPolicy: Always". That means that
+# also the "canary" images will only be pulled once. This is good for testing
+# (starting a pod multiple times will always run with the same canary image), but
+# implies that refreshing that image has to be done manually.
+#
+# As a special case, 'none' as registry removes the registry name.
+
+function image_version () {
+    yaml="$1"
+    image="$2"
+
+    # get version from `image: quay.io/k8scsi/csi-attacher:v1.0.1`, ignoring comments
+    version="$(sed -e 's/ *#.*$//' "$yaml" | grep "image:.*$image" | sed -e 's/ *#.*//' -e 's/.*://')"
+
+    # apply overrides
+    varname=$(echo $image | tr - _ | tr a-z A-Z)
+    eval version=\${${varname}_TAG:-\${IMAGE_TAG:-\$version}}
+
+    # When using canary images, we have to assume that the
+    # canary images were built from the corresponding branch.
+    case "$version" in canary) version=master;;
+                        *-canary) version="$(echo "$version" | sed -e 's/\(.*\)-canary/release-\1/')";;
+    esac
+    echo "$version"
+}
+
+# In addition, the RBAC rules can be overridden for provisioner and attacher.
+CSI_PROVISIONER_RBAC=${PROVISIONER_RBAC:-https://raw.githubusercontent.com/kubernetes-csi/external-provisioner/$(image_version "${BASE_DIR}/hostpath/csi-hostpath-provisioner.yaml" csi-provisioner)/deploy/kubernetes/rbac.yaml}
+CSI_ATTACHER_RBAC=${ATTACHER_RBAC:-https://raw.githubusercontent.com/kubernetes-csi/external-attacher/$(image_version "${BASE_DIR}/hostpath/csi-hostpath-attacher.yaml" csi-attacher)/deploy/kubernetes/rbac.yaml}
+CSI_SNAPSHOTTER_RBAC=${CSI_SNAPSHOTTER_RBAC:-https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/$(image_version "${BASE_DIR}/hostpath/csi-hostpath-snapshotter.yaml" csi-snapshotter)/deploy/kubernetes/rbac.yaml}
+
+INSTALL_CRD=${INSTALL_CRD:-"false"}
+
+run () {
+    echo "$@" >&2
+    "$@"
+}
+
+# apply CSIDriver and CSINodeInfo API objects
+if [[ "${INSTALL_CRD}" =~ ^(y|Y|yes|true)$ ]] ; then
+    echo "installing CRDs"
+    run kubectl apply -f https://raw.githubusercontent.com/kubernetes/csi-api/${K8S_RELEASE}/pkg/crd/manifests/csidriver.yaml --validate=false
+    run kubectl apply -f https://raw.githubusercontent.com/kubernetes/csi-api/${K8S_RELEASE}/pkg/crd/manifests/csinodeinfo.yaml --validate=false
+fi
+
+# rbac rules
+echo "applying RBAC rules"
+run kubectl apply -f "${CSI_PROVISIONER_RBAC}"
+run kubectl apply -f "${CSI_ATTACHER_RBAC}"
+run kubectl apply -f "${CSI_SNAPSHOTTER_RBAC}"
+
+# deploy hostpath plugin and registrar sidecar
+echo "deploying hostpath components"
+for i in ${BASE_DIR}/hostpath/*.yaml; do
+    echo "   $i"
+    modified="$(cat "$i" | while IFS= read -r line; do
+        nocomments="$(echo "$line" | sed -e 's/ *#.*$//')"
+        if echo "$nocomments" | grep -q '^\s*image:\s*'; then
+            # Split 'image: quay.io/k8scsi/csi-attacher:v1.0.1'
+            # into image (quay.io/k8scsi/csi-attacher:v1.0.1),
+            # registry (quay.io/k8scsi),
+            # name (csi-attacher),
+            # tag (v1.0.1).
+            image=$(echo "$nocomments" | sed -e 's;.*image:\s*;;')
+            registry=$(echo "$image" | sed -e 's;\(.*\)/.*;\1;')
+            name=$(echo "$image" | sed -e 's;.*/\([^:]*\).*;\1;')
+            tag=$(echo "$image" | sed -e 's;.*:;;')
+
+            # Variables are with underscores and upper case.
+            varname=$(echo $name | tr - _ | tr a-z A-Z)
+
+            # Now replace registry and/or tag, if set as env variables.
+            # If not set, the replacement is the same as the original value.
+            prefix=$(eval echo \${${varname}_REGISTRY:-${IMAGE_REGISTRY:-${registry}}}/ | sed -e 's;none/;;')
+            suffix=$(eval echo :\${${varname}_TAG:-${IMAGE_TAG:-${tag}}})
+            line="$(echo "$nocomments" | sed -e "s;$image;${prefix}${name}${suffix};")"
+            echo "        using $line" >&2
+        fi
+        echo "$line"
+    done)"
+    if ! echo "$modified" | kubectl apply -f -; then
+        echo "modified version of $i:"
+        echo "$modified"
+        exit 1
+    fi
+done
+
+# Wait until all pods are running. We have to make some assumptions
+# about the deployment here, otherwise we wouldn't know what to wait
+# for: the expectation is that we run attacher, provisioner,
+# snapshotter, socat and hostpath plugin in the default namespace.
+cnt=0
+while [ $(kubectl get pods 2>/dev/null | grep '^csi-hostpath.* Running ' | wc -l) -lt 5 ]; do
+    if [ $cnt -gt 30 ]; then
+        echo "Running pods:"
+        kubectl describe pods
+
+        echo >&2 "ERROR: hostpath deployment not ready after over 5min"
+        exit 1
+    fi
+    echo $(date +%H:%M:%S) "waiting for hostpath deployment to complete, attempt #$cnt"
+    cnt=$(($cnt + 1))
+    sleep 10
+done

deploy/kubernetes-1.14/hostpath/csi-hostpath-attacher.yaml

@@ -0,0 +1,55 @@
+kind: Service
+apiVersion: v1
+metadata:
+  name: csi-hostpath-attacher
+  labels:
+    app: csi-hostpath-attacher
+spec:
+  selector:
+    app: csi-hostpath-attacher
+  ports:
+    - name: dummy
+      port: 12345
+
+---
+kind: StatefulSet
+apiVersion: apps/v1
+metadata:
+  name: csi-hostpath-attacher
+spec:
+  serviceName: "csi-hostpath-attacher"
+  replicas: 1
+  selector:
+    matchLabels:
+      app: csi-hostpath-attacher
+  template:
+    metadata:
+      labels:
+        app: csi-hostpath-attacher
+    spec:
+      affinity:
+        podAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+          - labelSelector:
+              matchExpressions:
+              - key: app
+                operator: In
+                values:
+                - csi-hostpathplugin
+            topologyKey: kubernetes.io/hostname
+      serviceAccountName: csi-attacher
+      containers:
+        - name: csi-attacher
+          image: quay.io/k8scsi/csi-attacher:canary # TODO: replace with released version
+          args:
+            - --v=5
+            - --csi-address=/csi/csi.sock
+          volumeMounts:
+          - mountPath: /csi
+            name: socket-dir
+
+      volumes:
+        - hostPath:
+            path: /var/lib/kubelet/plugins/csi-hostpath
+            type: DirectoryOrCreate
+          name: socket-dir

deploy/kubernetes-1.14/hostpath/csi-hostpath-plugin.yaml

@@ -0,0 +1,136 @@
+# Service defined here, plus serviceName below in StatefulSet,
+# are needed only because of condition explained in
+# https://github.com/kubernetes/kubernetes/issues/69608
+
+kind: Service
+apiVersion: v1
+metadata:
+  name: csi-hostpathplugin
+  labels:
+    app: csi-hostpathplugin
+spec:
+  selector:
+    app: csi-hostpathplugin
+  ports:
+    - name: dummy
+      port: 12345
+---
+kind: StatefulSet
+apiVersion: apps/v1
+metadata:
+  name: csi-hostpathplugin
+spec:
+  serviceName: "csi-hostpathplugin"
+  # One replica only:
+  # Host path driver only works when everything runs
+  # on a single node. We achieve that by starting it once and then
+  # co-locate all other pods via inter-pod affinity
+  replicas: 1
+  selector:
+    matchLabels:
+      app: csi-hostpathplugin
+  template:
+    metadata:
+      labels:
+        app: csi-hostpathplugin
+    spec:
+      hostNetwork: true
+      containers:
+        - name: node-driver-registrar
+          image: quay.io/k8scsi/csi-node-driver-registrar:canary # TODO: replace with released version
+          lifecycle:
+            preStop:
+              exec:
+                command: ["/bin/sh", "-c", "rm -rf /registration/csi-hostpath /registration/csi-hostpath-reg.sock"]
+          args:
+            - --v=5
+            - --csi-address=/csi/csi.sock
+            - --kubelet-registration-path=/var/lib/kubelet/plugins/csi-hostpath/csi.sock
+          securityContext:
+            privileged: true
+          env:
+            - name: KUBE_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: spec.nodeName
+          volumeMounts:
+          - mountPath: /csi
+            name: socket-dir
+          - mountPath: /registration
+            name: registration-dir
+          - mountPath: /csi-data-dir
+            name: csi-data-dir
+
+        - name: hostpath
+          image: quay.io/k8scsi/hostpathplugin:canary # TODO: replace with released version
+          args:
+            - "--v=5"
+            - "--endpoint=$(CSI_ENDPOINT)"
+            - "--nodeid=$(KUBE_NODE_NAME)"
+          env:
+            - name: CSI_ENDPOINT
+              value: unix:///csi/csi.sock
+            - name: KUBE_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: spec.nodeName
+          securityContext:
+            privileged: true
+          ports:
+          - containerPort: 9898
+            name: healthz
+            protocol: TCP
+          livenessProbe:
+            failureThreshold: 5
+            httpGet:
+              path: /healthz
+              port: healthz
+            initialDelaySeconds: 10
+            timeoutSeconds: 3
+            periodSeconds: 2
+          volumeMounts:
+            - mountPath: /csi
+              name: socket-dir
+            - mountPath: /var/lib/kubelet/pods
+              mountPropagation: Bidirectional
+              name: mountpoint-dir
+            - mountPath: /var/lib/kubelet/plugins
+              mountPropagation: Bidirectional
+              name: plugins-dir
+
+        - name: liveness-probe
+          imagePullPolicy: Always
+          volumeMounts:
+          - mountPath: /csi
+            name: socket-dir
+          image: quay.io/k8scsi/livenessprobe:v1.0.2
+          args:
+          - --csi-address=/csi/csi.sock
+          - --connection-timeout=3s
+          - --health-port=9898
+
+      volumes:
+        - hostPath:
+            path: /var/lib/kubelet/plugins/csi-hostpath
+            type: DirectoryOrCreate
+          name: socket-dir
+        - hostPath:
+            path: /var/lib/kubelet/pods
+            type: DirectoryOrCreate
+          name: mountpoint-dir
+        - hostPath:
+            path: /var/lib/kubelet/plugins_registry
+            type: Directory
+          name: registration-dir
+        - hostPath:
+            path: /var/lib/kubelet/plugins
+            type: Directory
+          name: plugins-dir
+        - hostPath:
+            # 'path' is where PV data is persisted on host.
+            # using /tmp is also possible while the PVs will not available after plugin container recreation or host reboot
+            path: /var/lib/csi-hostpath-data/
+            type: DirectoryOrCreate
+          name: csi-data-dir