-
Notifications
You must be signed in to change notification settings - Fork 205
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
deploy: preliminary example for Kubernetes 1.14
The only difference is in the image versions. We still need two examples, because some CSI driver developers may need the older example for Kubernetes 1.13 if they depend on the alpha features. It's preliminary because the actual images haven't been released yet.
- Loading branch information
Showing
9 changed files
with
509 additions
and
6 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,9 +1,7 @@ | ||
The deployment for Kubernetes 1.13 uses CSI 1.0 and this is | ||
The deployment for Kubernetes 1.13 uses CSI 1.0 and thus is | ||
incompatible with older Kubernetes releases. | ||
|
||
It relies on the CRDs for CSIDriverInfo and CSINodeInfo, which are | ||
about to be replaced with builtin APIs in Kubernetes 1.14. It can be | ||
The sidecar images rely on the CRDs for CSIDriverInfo and CSINodeInfo, | ||
which were replaced with builtin APIs in Kubernetes 1.14. They can be | ||
deployed on Kubernetes 1.14 if the CRDs are installed, but features | ||
relying on these CRDs (like topology) are unlikely to work. | ||
|
||
Kubernetes 1.14 will need a different deployment. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,10 @@ | ||
The deployment for Kubernetes 1.14 uses CSI 1.0 and thus is incompatible with | ||
Kubernetes < 1.13. | ||
|
||
It uses the builtin APIs for CSIDriverInfo and CSINodeInfo that were | ||
introduced in Kubernetes 1.14, so features depending on those (like | ||
topology) will not work on Kubernetes 1.13. But because this example | ||
deployment does not enable those features, it can run on Kubernetes 1.13. | ||
|
||
WARNING: the images for Kubernetes 1.14 have not been released yet, so this | ||
example uses the "canary" images instead. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,138 @@ | ||
#!/usr/bin/env bash | ||
|
||
# This script captures the steps required to successfully | ||
# deploy the hostpath plugin driver. This should be considered | ||
# authoritative and all updates for this process should be | ||
# done here and referenced elsewhere. | ||
|
||
# The script assumes that kubectl is available on the OS path | ||
# where it is executed. | ||
|
||
set -e | ||
set -o pipefail | ||
|
||
BASE_DIR=$(dirname "$0") | ||
K8S_RELEASE=${K8S_RELEASE:-"release-1.13"} | ||
|
||
# If set, the following env variables override image registry and/or tag for each of the images. | ||
# They are named after the image name, with hyphen replaced by underscore and in upper case. | ||
# | ||
# - CSI_ATTACHER_REGISTRY | ||
# - CSI_ATTACHER_TAG | ||
# - CSI_NODE_DRIVER_REGISTRAR_REGISTRY | ||
# - CSI_NODE_DRIVER_REGISTRAR_TAG | ||
# - CSI_PROVISIONER_REGISTRY | ||
# - CSI_PROVISIONER_TAG | ||
# - CSI_SNAPSHOTTER_REGISTRY | ||
# - CSI_SNAPSHOTTER_TAG | ||
# - HOSTPATHPLUGIN_REGISTRY | ||
# - HOSTPATHPLUGIN_TAG | ||
# | ||
# Alternatively, it is possible to override all registries or tags with: | ||
# - IMAGE_REGISTRY | ||
# - IMAGE_TAG | ||
# These are used as fallback when the more specific variables are unset or empty. | ||
# | ||
# Beware that the .yaml files do not have "imagePullPolicy: Always". That means that | ||
# also the "canary" images will only be pulled once. This is good for testing | ||
# (starting a pod multiple times will always run with the same canary image), but | ||
# implies that refreshing that image has to be done manually. | ||
# | ||
# As a special case, 'none' as registry removes the registry name. | ||
|
||
function image_version () { | ||
yaml="$1" | ||
image="$2" | ||
|
||
# get version from `image: quay.io/k8scsi/csi-attacher:v1.0.1` | ||
version="$(grep "image:.*$image" "$yaml" | sed -e 's/.*:v/v/')" | ||
|
||
# apply overrides | ||
varname=$(echo $image | tr - _ | tr a-z A-Z) | ||
eval version=\${${varname}_TAG:-\${IMAGE_TAG:-\$version}} | ||
|
||
# When using canary images, we have to assume that the | ||
# canary images were built from the corresponding branch. | ||
case "$version" in canary) version=master;; | ||
*-canary) version="$(echo "$version" | sed -e 's/\(.*\)-canary/release-\1/')";; | ||
esac | ||
echo "$version" | ||
} | ||
|
||
# In addition, the RBAC rules can be overridden for provisioner and attacher. | ||
CSI_PROVISIONER_RBAC=${PROVISIONER_RBAC:-https://raw.githubusercontent.com/kubernetes-csi/external-provisioner/$(image_version "${BASE_DIR}/hostpath/csi-hostpath-provisioner.yaml" csi-provisioner)/deploy/kubernetes/rbac.yaml} | ||
CSI_ATTACHER_RBAC=${ATTACHER_RBAC:-https://raw.githubusercontent.com/kubernetes-csi/external-attacher/$(image_version "${BASE_DIR}/hostpath/csi-hostpath-attacher.yaml" csi-attacher)/deploy/kubernetes/rbac.yaml} | ||
CSI_SNAPSHOTTER_RBAC=${CSI_SNAPSHOTTER_RBAC:-https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/$(image_version "${BASE_DIR}/hostpath/csi-hostpath-snapshotter.yaml" csi-snapshotter)/deploy/kubernetes/rbac.yaml} | ||
|
||
INSTALL_CRD=${INSTALL_CRD:-"false"} | ||
|
||
run () { | ||
echo "$@" >&2 | ||
"$@" | ||
} | ||
|
||
# apply CSIDriver and CSINodeInfo API objects | ||
if [[ "${INSTALL_CRD}" =~ ^(y|Y|yes|true)$ ]] ; then | ||
echo "installing CRDs" | ||
run kubectl apply -f https://raw.githubusercontent.com/kubernetes/csi-api/${K8S_RELEASE}/pkg/crd/manifests/csidriver.yaml --validate=false | ||
run kubectl apply -f https://raw.githubusercontent.com/kubernetes/csi-api/${K8S_RELEASE}/pkg/crd/manifests/csinodeinfo.yaml --validate=false | ||
fi | ||
|
||
# rbac rules | ||
echo "applying RBAC rules" | ||
run kubectl apply -f "${CSI_PROVISIONER_RBAC}" | ||
run kubectl apply -f "${CSI_ATTACHER_RBAC}" | ||
run kubectl apply -f "${CSI_SNAPSHOTTER_RBAC}" | ||
|
||
# deploy hostpath plugin and registrar sidecar | ||
echo "deploying hostpath components" | ||
for i in ${BASE_DIR}/hostpath/*.yaml; do | ||
echo " $i" | ||
modified="$(cat "$i" | while IFS= read -r line; do | ||
if echo "$line" | grep -q '^\s*image:\s*'; then | ||
# Split 'image: quay.io/k8scsi/csi-attacher:v1.0.1' | ||
# into image (quay.io/k8scsi/csi-attacher:v1.0.1), | ||
# registry (quay.io/k8scsi), | ||
# name (csi-attacher), | ||
# tag (v1.0.1). | ||
image=$(echo "$line" | sed -e 's;.*image:\s*;;') | ||
registry=$(echo "$image" | sed -e 's;\(.*\)/.*;\1;') | ||
name=$(echo "$image" | sed -e 's;.*/\([^:]*\).*;\1;') | ||
tag=$(echo "$image" | sed -e 's;.*:;;') | ||
# Variables are with underscores and upper case. | ||
varname=$(echo $name | tr - _ | tr a-z A-Z) | ||
# Now replace registry and/or tag, if set as env variables. | ||
# If not set, the replacement is the same as the original value. | ||
prefix=$(eval echo \${${varname}_REGISTRY:-${IMAGE_REGISTRY:-${registry}}}/ | sed -e 's;none/;;') | ||
suffix=$(eval echo :\${${varname}_TAG:-${IMAGE_TAG:-${tag}}}) | ||
line="$(echo "$line" | sed -e "s;$image;${prefix}${name}${suffix};")" | ||
echo " using $line" >&2 | ||
fi | ||
echo "$line" | ||
done)" | ||
if ! echo "$modified" | kubectl apply -f -; then | ||
echo "modified version of $i:" | ||
echo "$modified" | ||
exit 1 | ||
fi | ||
done | ||
# Wait until all pods are running. We have to make some assumptions | ||
# about the deployment here, otherwise we wouldn't know what to wait | ||
# for: the expectation is that we run attacher, provisioner, | ||
# snapshotter, socat and hostpath plugin in the default namespace. | ||
cnt=0 | ||
while [ $(kubectl get pods 2>/dev/null | grep '^csi-hostpath.* Running ' | wc -l) -lt 5 ]; do | ||
if [ $cnt -gt 30 ]; then | ||
echo "Running pods:" | ||
kubectl describe pods | ||
echo >&2 "ERROR: hostpath deployment not ready after over 5min" | ||
exit 1 | ||
fi | ||
echo $(date +%H:%M:%S) "waiting for hostpath deployment to complete, attempt #$cnt" | ||
cnt=$(($cnt + 1)) | ||
sleep 10 | ||
done |
55 changes: 55 additions & 0 deletions
55
deploy/kubernetes-1.14/hostpath/csi-hostpath-attacher.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,55 @@ | ||
kind: Service | ||
apiVersion: v1 | ||
metadata: | ||
name: csi-hostpath-attacher | ||
labels: | ||
app: csi-hostpath-attacher | ||
spec: | ||
selector: | ||
app: csi-hostpath-attacher | ||
ports: | ||
- name: dummy | ||
port: 12345 | ||
|
||
--- | ||
kind: StatefulSet | ||
apiVersion: apps/v1 | ||
metadata: | ||
name: csi-hostpath-attacher | ||
spec: | ||
serviceName: "csi-hostpath-attacher" | ||
replicas: 1 | ||
selector: | ||
matchLabels: | ||
app: csi-hostpath-attacher | ||
template: | ||
metadata: | ||
labels: | ||
app: csi-hostpath-attacher | ||
spec: | ||
affinity: | ||
podAffinity: | ||
requiredDuringSchedulingIgnoredDuringExecution: | ||
- labelSelector: | ||
matchExpressions: | ||
- key: app | ||
operator: In | ||
values: | ||
- csi-hostpathplugin | ||
topologyKey: kubernetes.io/hostname | ||
serviceAccountName: csi-attacher | ||
containers: | ||
- name: csi-attacher | ||
image: quay.io/k8scsi/csi-attacher:canary # TODO: replace with released version | ||
args: | ||
- --v=5 | ||
- --csi-address=/csi/csi.sock | ||
volumeMounts: | ||
- mountPath: /csi | ||
name: socket-dir | ||
|
||
volumes: | ||
- hostPath: | ||
path: /var/lib/kubelet/plugins/csi-hostpath | ||
type: DirectoryOrCreate | ||
name: socket-dir |
136 changes: 136 additions & 0 deletions
136
deploy/kubernetes-1.14/hostpath/csi-hostpath-plugin.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,136 @@ | ||
# Service defined here, plus serviceName below in StatefulSet, | ||
# are needed only because of condition explained in | ||
# https://github.com/kubernetes/kubernetes/issues/69608 | ||
|
||
kind: Service | ||
apiVersion: v1 | ||
metadata: | ||
name: csi-hostpathplugin | ||
labels: | ||
app: csi-hostpathplugin | ||
spec: | ||
selector: | ||
app: csi-hostpathplugin | ||
ports: | ||
- name: dummy | ||
port: 12345 | ||
--- | ||
kind: StatefulSet | ||
apiVersion: apps/v1 | ||
metadata: | ||
name: csi-hostpathplugin | ||
spec: | ||
serviceName: "csi-hostpathplugin" | ||
# One replica only: | ||
# Host path driver only works when everything runs | ||
# on a single node. We achieve that by starting it once and then | ||
# co-locate all other pods via inter-pod affinity | ||
replicas: 1 | ||
selector: | ||
matchLabels: | ||
app: csi-hostpathplugin | ||
template: | ||
metadata: | ||
labels: | ||
app: csi-hostpathplugin | ||
spec: | ||
hostNetwork: true | ||
containers: | ||
- name: node-driver-registrar | ||
image: quay.io/k8scsi/csi-node-driver-registrar:canary # TODO: replace with released version | ||
lifecycle: | ||
preStop: | ||
exec: | ||
command: ["/bin/sh", "-c", "rm -rf /registration/csi-hostpath /registration/csi-hostpath-reg.sock"] | ||
args: | ||
- --v=5 | ||
- --csi-address=/csi/csi.sock | ||
- --kubelet-registration-path=/var/lib/kubelet/plugins/csi-hostpath/csi.sock | ||
securityContext: | ||
privileged: true | ||
env: | ||
- name: KUBE_NODE_NAME | ||
valueFrom: | ||
fieldRef: | ||
apiVersion: v1 | ||
fieldPath: spec.nodeName | ||
volumeMounts: | ||
- mountPath: /csi | ||
name: socket-dir | ||
- mountPath: /registration | ||
name: registration-dir | ||
- mountPath: /csi-data-dir | ||
name: csi-data-dir | ||
|
||
- name: hostpath | ||
image: quay.io/k8scsi/hostpathplugin:canary # TODO: replace with released version | ||
args: | ||
- "--v=5" | ||
- "--endpoint=$(CSI_ENDPOINT)" | ||
- "--nodeid=$(KUBE_NODE_NAME)" | ||
env: | ||
- name: CSI_ENDPOINT | ||
value: unix:///csi/csi.sock | ||
- name: KUBE_NODE_NAME | ||
valueFrom: | ||
fieldRef: | ||
apiVersion: v1 | ||
fieldPath: spec.nodeName | ||
securityContext: | ||
privileged: true | ||
ports: | ||
- containerPort: 9898 | ||
name: healthz | ||
protocol: TCP | ||
livenessProbe: | ||
failureThreshold: 5 | ||
httpGet: | ||
path: /healthz | ||
port: healthz | ||
initialDelaySeconds: 10 | ||
timeoutSeconds: 3 | ||
periodSeconds: 2 | ||
volumeMounts: | ||
- mountPath: /csi | ||
name: socket-dir | ||
- mountPath: /var/lib/kubelet/pods | ||
mountPropagation: Bidirectional | ||
name: mountpoint-dir | ||
- mountPath: /var/lib/kubelet/plugins | ||
mountPropagation: Bidirectional | ||
name: plugins-dir | ||
|
||
- name: liveness-probe | ||
imagePullPolicy: Always | ||
volumeMounts: | ||
- mountPath: /csi | ||
name: socket-dir | ||
image: quay.io/k8scsi/livenessprobe:v1.0.2 | ||
args: | ||
- --csi-address=/csi/csi.sock | ||
- --connection-timeout=3s | ||
- --health-port=9898 | ||
|
||
volumes: | ||
- hostPath: | ||
path: /var/lib/kubelet/plugins/csi-hostpath | ||
type: DirectoryOrCreate | ||
name: socket-dir | ||
- hostPath: | ||
path: /var/lib/kubelet/pods | ||
type: DirectoryOrCreate | ||
name: mountpoint-dir | ||
- hostPath: | ||
path: /var/lib/kubelet/plugins_registry | ||
type: Directory | ||
name: registration-dir | ||
- hostPath: | ||
path: /var/lib/kubelet/plugins | ||
type: Directory | ||
name: plugins-dir | ||
- hostPath: | ||
# 'path' is where PV data is persisted on host. | ||
# using /tmp is also possible while the PVs will not available after plugin container recreation or host reboot | ||
path: /var/lib/csi-hostpath-data/ | ||
type: DirectoryOrCreate | ||
name: csi-data-dir |
Oops, something went wrong.