From 0806eb940479b8ce5b53713eac851a8ec44fbb88 Mon Sep 17 00:00:00 2001 From: weipengzhu Date: Tue, 22 Sep 2020 20:05:31 +0800 Subject: [PATCH] Update CSI Deployment 1. support CentOS8 2. add Fio example 3. fixes deployment steps --- asserts/zbs-csi-driver/example/fio.yaml | 37 ++ .../v0.1.1/deploy/accounts.yaml | 89 +++++ .../v0.1.1/deploy/clusterroles.yaml | 121 ++++++ .../v0.1.1/deploy/namespace.yaml | 4 + .../v0.1.1/deploy/zbs-csi-driver.yaml | 343 ++++++++++++++++++ docs/zbs-csi-driver-deployment.md | 202 ++++++++++- 6 files changed, 777 insertions(+), 19 deletions(-) create mode 100644 asserts/zbs-csi-driver/example/fio.yaml create mode 100644 asserts/zbs-csi-driver/v0.1.1/deploy/accounts.yaml create mode 100644 asserts/zbs-csi-driver/v0.1.1/deploy/clusterroles.yaml create mode 100644 asserts/zbs-csi-driver/v0.1.1/deploy/namespace.yaml create mode 100644 asserts/zbs-csi-driver/v0.1.1/deploy/zbs-csi-driver.yaml diff --git a/asserts/zbs-csi-driver/example/fio.yaml b/asserts/zbs-csi-driver/example/fio.yaml new file mode 100644 index 00000000..1b21c28e --- /dev/null +++ b/asserts/zbs-csi-driver/example/fio.yaml @@ -0,0 +1,37 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: fio-pvc +spec: + storageClassName: zbs-csi-driver-default + volumeMode: Block + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 30Gi +--- +apiVersion: v1 +kind: Pod +metadata: + name: fio + labels: + app: fio +spec: + volumes: + - name: fio-pvc + persistentVolumeClaim: + claimName: fio-pvc + containers: + - name: fio + image: clusterhq/fio-tool + command: + - tail + args: + - '-f' + - /dev/null + imagePullPolicy: IfNotPresent + volumeDevices: + - devicePath: /mnt/fio + name: fio-pvc + restartPolicy: Always diff --git a/asserts/zbs-csi-driver/v0.1.1/deploy/accounts.yaml b/asserts/zbs-csi-driver/v0.1.1/deploy/accounts.yaml new file mode 100644 index 00000000..1381a72f --- /dev/null +++ b/asserts/zbs-csi-driver/v0.1.1/deploy/accounts.yaml @@ -0,0 +1,89 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: zbs-csi-controller-account + namespace: iomesh-system +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: zbs-csi-provisioner-binding +subjects: + - kind: ServiceAccount + name: zbs-csi-controller-account + namespace: iomesh-system +roleRef: + kind: ClusterRole + name: zbs-csi-provisioner-role + apiGroup: rbac.authorization.k8s.io +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: zbs-csi-attacher-binding +subjects: + - kind: ServiceAccount + name: zbs-csi-controller-account + namespace: iomesh-system +roleRef: + kind: ClusterRole + name: zbs-csi-attacher-role + apiGroup: rbac.authorization.k8s.io +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: zbs-csi-snapshotter-binding +subjects: + - kind: ServiceAccount + name: zbs-csi-controller-account + namespace: iomesh-system +roleRef: + kind: ClusterRole + name: zbs-csi-snapshotter-role + apiGroup: rbac.authorization.k8s.io +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: zbs-csi-resizer-binding +subjects: + - kind: ServiceAccount + name: zbs-csi-controller-account + namespace: iomesh-system +roleRef: + kind: ClusterRole + name: zbs-csi-resizer-role + apiGroup: rbac.authorization.k8s.io +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: zbs-csi-driver-controller-binding +subjects: + - kind: ServiceAccount + name: zbs-csi-controller-account + namespace: iomesh-system +roleRef: + kind: ClusterRole + name: zbs-csi-driver-role + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: zbs-csi-node-account + namespace: iomesh-system +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: zbs-csi-driver-node-binding +subjects: + - kind: ServiceAccount + name: zbs-csi-node-account + namespace: iomesh-system +roleRef: + kind: ClusterRole + name: zbs-csi-driver-role + apiGroup: rbac.authorization.k8s.io diff --git a/asserts/zbs-csi-driver/v0.1.1/deploy/clusterroles.yaml b/asserts/zbs-csi-driver/v0.1.1/deploy/clusterroles.yaml new file mode 100644 index 00000000..dbaf7086 --- /dev/null +++ b/asserts/zbs-csi-driver/v0.1.1/deploy/clusterroles.yaml @@ -0,0 +1,121 @@ +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: zbs-csi-provisioner-role + labels: + app.kubernetes.io/name: zbs-csi-driver +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["get", "list"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: zbs-csi-attacher-role + labels: + app.kubernetes.io/name: zbs-csi-driver +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["csi.storage.k8s.io"] + resources: ["csinodeinfos"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + # rename if there are conflicts + name: zbs-csi-snapshotter-role + labels: + app.kubernetes.io/name: zbs-csi-driver +rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["update"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: zbs-csi-resizer-role + labels: + app.kubernetes.io/name: zbs-csi-driver +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["patch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: zbs-csi-driver-role + labels: + app.kubernetes.io/name: zbs-csi-driver +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["list", "get"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["create", "update", "list", "get"] diff --git a/asserts/zbs-csi-driver/v0.1.1/deploy/namespace.yaml b/asserts/zbs-csi-driver/v0.1.1/deploy/namespace.yaml new file mode 100644 index 00000000..96df2224 --- /dev/null +++ b/asserts/zbs-csi-driver/v0.1.1/deploy/namespace.yaml @@ -0,0 +1,4 @@ +kind: Namespace +apiVersion: v1 +metadata: + name: iomesh-system diff --git a/asserts/zbs-csi-driver/v0.1.1/deploy/zbs-csi-driver.yaml b/asserts/zbs-csi-driver/v0.1.1/deploy/zbs-csi-driver.yaml new file mode 100644 index 00000000..9469db4e --- /dev/null +++ b/asserts/zbs-csi-driver/v0.1.1/deploy/zbs-csi-driver.yaml @@ -0,0 +1,343 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: zbs-csi-driver-default + annotations: + storageclass.kubernetes.io/is-default-class: "true" +provisioner: zbs-csi-driver.iomesh.com +reclaimPolicy: Retain +allowVolumeExpansion: true +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: zbs-csi-driver-controller-plugin + namespace: iomesh-system +spec: + replicas: 3 + selector: + matchLabels: + app: zbs-csi-driver-controller-plugin + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + template: + metadata: + labels: + app: zbs-csi-driver-controller-plugin + spec: + hostNetwork: true + serviceAccountName: zbs-csi-controller-account + containers: + - name: csi-snapshotter + image: quay.io/k8scsi/csi-snapshotter:v2.1.1 + imagePullPolicy: IfNotPresent + args: + - "--csi-address=$(ADDRESS)" + - "--leader-election" + - "--leader-election-namespace=$(NAMESPACE)" + - "--v=5" + env: + - name: ADDRESS + value: /csi/csi.sock + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: csi-provisioner + image: quay.io/k8scsi/csi-provisioner:v1.6.0 + args: + - "--csi-address=$(ADDRESS)" + - "--enable-leader-election" + - "--leader-election-type=leases" + - "--leader-election-namespace=$(NAMESPACE)" + - "--v=5" + env: + - name: ADDRESS + value: /csi/csi.sock + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + imagePullPolicy: IfNotPresent + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: csi-attacher + image: quay.io/k8scsi/csi-attacher:v2.2.0 + args: + - "--csi-address=$(ADDRESS)" + - "--leader-election" + - "--leader-election-namespace=$(NAMESPACE)" + - "--v=5" + env: + - name: ADDRESS + value: /csi/csi.sock + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + imagePullPolicy: IfNotPresent + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: csi-resizer + image: quay.io/k8scsi/csi-resizer:v0.5.0 + args: + - "--csi-address=$(ADDRESS)" + - "--leader-election" + - "--leader-election-namespace=$(NAMESPACE)" + - "--v=5" + env: + - name: ADDRESS + value: /csi/csi.sock + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: socket-dir + mountPath: /csi + imagePullPolicy: IfNotPresent + - name: liveness-probe + image: quay.io/k8scsi/livenessprobe:v1.1.0 + args: + - --csi-address=/csi/csi.sock + - --health-port=9811 + imagePullPolicy: IfNotPresent + volumeMounts: + - mountPath: /csi + name: socket-dir + - name: zbs-csi-driver + image: iomesh/zbs-csi-driver:v0.1.1 + ports: + - containerPort: 9811 + name: healthz + protocol: TCP + livenessProbe: + failureThreshold: 10 + initialDelaySeconds: 20 + timeoutSeconds: 10 + periodSeconds: 20 + httpGet: + path: /healthz + port: healthz + args: + - "--csi_addr=unix:///csi/csi.sock" + - "--driver_name=zbs-csi-driver.iomesh.com" + - "--role=controller" + # zbs-cluster-vip:10206 + - "--meta_proxy=" + # unique cluster id + - "--cluster_id=" + # HCL / EXTERNAL + - "--deployment_mode=EXTERNAL" + - "--namespace=$(NAMESPACE)" + - "--node_map=zbs-csi-node-map" + - "--v=5" + imagePullPolicy: IfNotPresent + volumeMounts: + - name: socket-dir + mountPath: /csi + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: NODE_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumes: + - name: socket-dir + emptyDir: {} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: zbs-csi-node-map + namespace: iomesh-system +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: zbs-csi-driver-node-plugin + namespace: iomesh-system +spec: + selector: + matchLabels: + app: zbs-csi-driver-node-plugin + template: + metadata: + labels: + app: zbs-csi-driver-node-plugin + spec: + serviceAccountName: zbs-csi-node-account + # for zbs-cluster-vip + hostNetwork: true + containers: + - name: driver-registrar + securityContext: + privileged: true + image: quay.io/k8scsi/csi-node-driver-registrar:v1.0.2 + args: + - --v=5 + - --csi-address=/csi/csi.sock + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + lifecycle: + preStop: + exec: + command: + [ + "/bin/sh", + "-c", + "rm -rf /registration/zbs-csi-driver.iomesh.com /registration/zbs-csi-driver.iomesh.com-reg.sock", + ] + env: + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/zbs-csi-driver.iomesh.com/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + - name: liveness-probe + image: quay.io/k8scsi/livenessprobe:v1.1.0 + args: + - --csi-address=/csi/csi.sock + - --health-port=9809 + imagePullPolicy: IfNotPresent + volumeMounts: + - mountPath: /csi + name: socket-dir + - name: zbs-csi-driver + image: iomesh/zbs-csi-driver:v0.1.1 + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + allowPrivilegeEscalation: true + ports: + - containerPort: 9809 + name: healthz + protocol: TCP + - containerPort: 9810 + name: livenessport + protocol: TCP + livenessProbe: + failureThreshold: 10 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 20 + timeoutSeconds: 10 + periodSeconds: 20 + args: + - "--csi_addr=unix:///csi/csi.sock" + - "--role=node" + - "--driver_name=zbs-csi-driver.iomesh.com" + # zbs-cluster-vip:10206 + - "--meta_proxy=" + - "--namespace=$(NAMESPACE)" + # unique cluster id + - "--cluster_id=" + # HCI / EXTERNAL + - "--deployment_mode=EXTERNAL" + - "--node_map=zbs-csi-node-map" + # for EXTERNAL mode, iscsi_portal=zbs-cluster-vip:3260 + # for HCI mode, iscsi_portal=127.0.0.1:3260 + - "--iscsi_portal=" + - "--liveness_port=9810" + - "--v=5" + imagePullPolicy: IfNotPresent + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: pods-mount-dir + mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + - name: device-dir + mountPath: /dev + - name: iscsi-dir + mountPath: /etc/iscsi + - name: iscsiadm + mountPath: /sbin/iscsiadm + - name: lsscsi + mountPath: /usr/bin/lsscsi + - name: iscsi-lib + mountPath: /var/lib/iscsi + - name: lib-dir + mountPath: /lib/modules + - name: lib64-dir + mountPath: /lib64 + # for CentOS8 + #- name: iscsi-lock + # mountPath: /run/lock/iscsi + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: NODE_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumes: + - name: socket-dir + hostPath: + path: /var/lib/kubelet/plugins/zbs-csi-driver.iomesh.com + type: DirectoryOrCreate + - name: pods-mount-dir + hostPath: + path: /var/lib/kubelet + type: Directory + - name: device-dir + hostPath: + path: /dev + type: Directory + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + - name: iscsi-dir + hostPath: + path: /etc/iscsi + type: Directory + - name: iscsi-lib + hostPath: + path: /var/lib/iscsi + type: Directory + - name: iscsiadm + hostPath: + path: /sbin/iscsiadm + type: File + - name: lsscsi + hostPath: + path: /usr/bin/lsscsi + type: File + - name: lib-dir + hostPath: + path: /lib/modules + type: Directory + - name: lib64-dir + hostPath: + path: /lib64 + type: Directory + # for CentOS8 + #- name: iscsi-lock + # hostPath: + # path: /run/lock/iscsi + # type: Directory \ No newline at end of file diff --git a/docs/zbs-csi-driver-deployment.md b/docs/zbs-csi-driver-deployment.md index 78fe7274..25fe6ef4 100644 --- a/docs/zbs-csi-driver-deployment.md +++ b/docs/zbs-csi-driver-deployment.md @@ -8,17 +8,46 @@ This topic explains how to install ZBS CSI Driver with kubernetes. Follow the st ## Env -- Centos7 +- CentOS7 / CentOS8 - Kubernetes v1.17 or higher +- ZBS v4.5.0-rc14 (image version SMTXOS-4.5.0-B5-el7-2009231556-x86_64) or higher + ## Setup Kubernetes If there is no kubernetes cluster，please refer to [Installing Kubernetes](https://kubernetes.io/docs/setup/production-environment/tools/). ### Enable Kubernetes features -1. Enable feature gates on each `kube-apiserver`: `--feature-gates=CSINodeInfo=true,CSIDriverRegistry=true,CSIBlockVolume=true,VolumeSnapshotDataSource=true,VolumePVCDataSource=true,VolumePVCDataSource=true,ExpandCSIVolumes=true,ExpandInUsePersistentVolumes=true` and `--allow-privileged=true`. +Enable the CSI related features to ensure that the driver works normally. + +After a feature is GA, the feature gate will be removed in the next few versions. Please refer to **[feature-gates](https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/)** to selectively enable features. + +| Feature Gate | Default | Stage | Since | Until | +| ---------------------------- | ------- | ----- | ----- | ----- | +| CSINodeInfo | false | Alpha | 1.12 | 1.13 | +| CSINodeInfo | true | Beta | 1.14 | 1.16 | +| CSINodeInfo | true | GA | 1.17 | - | +| CSIDriverRegistry | false | Alpha | 1.12 | 1.13 | +| CSIDriverRegistry | true | Beta | 1.14 | 1.16 | +| CSIDriverRegistry | true | GA | 1.17 | - | +| VolumeSnapshotDataSource | false | Alpha | 1.12 | 1.16 | +| VolumeSnapshotDataSource | true | Beta | 1.17 | - | +| VolumePVCDataSource | false | Alpha | 1.15 | 1.15 | +| VolumePVCDataSource | true | Beta | 1.16 | 1.17 | +| VolumePVCDataSource | true | GA | 1.18 | - | +| ExpandCSIVolumes | false | Alpha | 1.14 | 1.15 | +| ExpandCSIVolumes | true | Beta | 1.16 | - | +| CSIBlockVolume | false | Alpha | 1.11 | 1.13 | +| CSIBlockVolume | true | Beta | 1.14 | 1.17 | +| CSIBlockVolume | true | GA | 1.18 | - | +| ExpandInUsePersistentVolumes | false | Beta | 1.11 | 1.14 | +| ExpandInUsePersistentVolumes | true | Beta | 1.15 | - | + +For Kubernetes 1.17, we can open all feature gates. + +1. Enable feature gates on each `kube-apiserver`: `--feature-gates=CSINodeInfo=true,CSIDriverRegistry=true,CSIBlockVolume=true,VolumeSnapshotDataSource=true,VolumePVCDataSource=true,ExpandCSIVolumes=true,ExpandInUsePersistentVolumes=true` and `--allow-privileged=true` ```yaml # /etc/kubernetes/manifests/kube-apiserver.yaml @@ -31,17 +60,17 @@ spec: containers: - command: - kube-apiserver - - --feature-gates=CSINodeInfo=true,CSIDriverRegistry=true,CSIBlockVolume=true,VolumeSnapshotDataSource=true, VolumePVCDataSource=true,VolumePVCDataSource=true,ExpandCSIVolumes=true,ExpandInUsePersistentVolumes=true + - --feature-gates=CSINodeInfo=true,CSIDriverRegistry=true,CSIBlockVolume=true,VolumeSnapshotDataSource=true,VolumePVCDataSource=true,ExpandCSIVolumes=true,ExpandInUsePersistentVolumes=true - --allow-privileged=true ``` -2. Enable feature gates on each `kubelet`: `--feature-gates=CSINodeInfo=true,CSIDriverRegistry=true,CSIBlockVolume=true,VolumeSnapshotDataSource=true, VolumePVCDataSource=true,VolumePVCDataSource=true,ExpandCSIVolumes=true,ExpandInUsePersistentVolumes=true` and `--allow-privileged=true`. +2. Enable feature gates on each `kubelet`: `--feature-gates=CSINodeInfo=true,CSIDriverRegistry=true,CSIBlockVolume=true,VolumeSnapshotDataSource=true,VolumePVCDataSource=true,ExpandCSIVolumes=true,ExpandInUsePersistentVolumes=true`. ```yaml # /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf # Note: This dropin only works with kubeadm and kubelet v1.11+ [Service] -Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf --feature-gates=CSINodeInfo=true,CSIDriverRegistry=true,CSIBlockVolume=true,VolumeSnapshotDataSource=true,VolumePVCDataSource=true,VolumePVCDataSource=true,ExpandCSIVolumes=true,ExpandInUsePersistentVolumes=true --allow-privileged=true" +Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf --feature-gates=CSINodeInfo=true,CSIDriverRegistry=true,CSIBlockVolume=true,VolumeSnapshotDataSource=true,VolumePVCDataSource=true,ExpandCSIVolumes=true,ExpandInUsePersistentVolumes=true" Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml" # This is a file that "kubeadm init" and "kubeadm join" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env @@ -61,53 +90,119 @@ systemctl daemon-reload systemctl restart kubelet ``` +4. Wait kubelet and kube-apiserver ready + +```sh +systemctl status kubelet +``` + +``` +● kubelet.service - kubelet: The Kubernetes Node Agent + Loaded: loaded (/usr/lib/systemd/system/kubelet.service; enabled; vendor preset: disabled) + Drop-In: /usr/lib/systemd/system/kubelet.service.d + └─10-kubeadm.conf + Active: active (running) since Mon 2020-09-23 14:36:18 CST; +``` + +```sh +kubectl wait --for=condition=Ready pod/kube-apiserver- -n kube-system +``` + +``` +pod/kube-apiserver- condition met +``` + ### Deploy Common Snapshot Controller +The volume snapshot controller management is similar to pv/pvc controller, it manages the snapshot CRDs. +Regardless of the number CSI drivers deployed on the cluster, there must be only one instance of the volume snapshot controller running and one set of volume snapshot CRDs installed per cluster. + 1. Download **[external-controller repo](https://github.com/kubernetes-csi/external-snapshotter/tree/release-2.1)** ```sh wget https://github.com/kubernetes-csi/external-snapshotter/archive/release-2.1.zip -unzip external-snapshotter-release-2.1.zip +unzip release-2.1.zip && cd external-snapshotter-release-2.1 ``` 2. Create Snapshot Beta CRD ```sh -kubectl create -f external-snapshotter-release-2.1/config/crd +kubectl create -f ./config/crd ``` 3. Install Common Snapshot Controller ```sh -kubectl apply -f external-snapshotter-release-2.1/deploy/kubernetes/snapshot-controller +kubectl apply -f ./deploy/kubernetes/snapshot-controller ``` +> **_Note:_ replace with the namespace you want for your controller, e.g. kube-system** + 4. Verify ```sh -kubectl get statefulsets.apps snapshot-controller +watch kubectl get statefulset snapshot-controller -n +``` + +``` +NAME READY AGE +snapshot-controller 1/1 32s ``` ## Setup ZBS Cluster -Configure a `zbs-cluster-vip` +1. Ensure that the kubernetes cluster can access the ZBS cluster through the access network + +2. Configure `zbs-cluster-vip` in the access network segment + +```sh +zbs-task vip set iscsi +``` ## Setup open-iscsi +1. Install open-iscsi on each kubernetes node + ```sh yum install iscsi-initiator-utils ``` +2. Ensure that the node.startup option of /etc/iscsi/iscsid.conf is manual + +```sh +sed -i 's/^node.startup = automatic$/node.startup = manual/' /etc/iscsi/iscsid.conf +``` + +3. Disable selinux + +```sh +setenforce 0 +sed -i 's/^SELINUX=enforcing$/SELINUX=permissive/' /etc/selinux/config +``` + +4. Enable and start `iscsid` + +```sh +systemctl enable --now iscsid +``` + ## Deploy zbs-csi-driver -Obtain the `kubernetes-cluster-id` from the cluster administrator or the cluster management system and download **[zbs-csi-driver-deploy](https://github.com/iomesh/zbs-csi-driver/blob/master/deploy)**. +1. Obtain the `kubernetes-cluster-id` from the cluster administrator or the cluster management system > **_Note:_ `kubernetes-cluster-id` should be unique and cannot be modified**. -1. Configure controller plugin +2. Download **[zbs-csi-driver-deploy](https://github.com/iomesh/iomesh-docs/tree/master/asserts/zbs-csi-driver/v0.1.1/deploy)** + +3. Configure controller plugin ```yaml # deploy/zbs-csi-driver.yaml + spec: + # for zbs-cluster-vip + hostNetwork: true + serviceAccountName: zbs-csi-controller-account + - containers: - name: zbs-csi-driver image: iomesh/zbs-csi-driver:v0.1.1 args: @@ -118,10 +213,13 @@ Obtain the `kubernetes-cluster-id` from the cluster administrator or the clust - "--deployment_mode=EXTERNAL" ``` -2. Configure node plugin +4. Configure node plugin + +If the OS is CentOS8, you need to mount iscsi-lock. ```yaml # deploy/zbs-csi-driver.yaml + containers: - name: zbs-csi-driver image: iomesh/zbs-csi-driver:v0.1.1 args: @@ -131,11 +229,20 @@ Obtain the `kubernetes-cluster-id` from the cluster administrator or the clust - "--cluster_id=kubernetes-cluster-id" - "--iscsi_portal=zbs-cluster-vip:3260" - "--deployment_mode=EXTERNAL" + volumeMounts: + # - name: iscsi-lock + # mountPath: /run/lock/iscsi + volumes: + # - name: iscsi-lock + # hostPath: + # path: /run/lock/iscsi + # type: Directory + ``` > **_Note:_ For HCI Deployment, `deployment_mode` is `HCI` , `iscsi_portal` is `127.0.0.1:3260`** -3. Configure StorageClass +5. Configure StorageClass ```yaml # deploy/zbs-csi-driver.yaml @@ -147,20 +254,77 @@ metadata: provisioner: zbs-csi-driver.iomesh.com reclaimPolicy: Retain allowVolumeExpansion: true +parameters: + csi.storage.k8s.io/fstype: "ext4" + replicaFactor: "1" + thinProvision: "true" ``` -4. Deploy +6. Deploy ```sh kubectl apply -f ./deploy ``` -5. Verify +7. Wait for ready ```sh -kubectl get pod -n iomesh-system +watch kubectl get pod -n iomesh-system ``` -## Example +``` +Every 2.0s: kubectl get pod -n iomesh-system Wed Sep 23 14:33:52 2020 +NAME READY STATUS RESTARTS AGE +zbs-csi-driver-controller-plugin-5dbfb48d5c-2sk97 6/6 Running 0 42s +zbs-csi-driver-controller-plugin-5dbfb48d5c-cfhwt 6/6 Running 0 42s +zbs-csi-driver-controller-plugin-5dbfb48d5c-drl7s 6/6 Running 0 42s +zbs-csi-driver-node-plugin-25585 3/3 Running 0 39s +zbs-csi-driver-node-plugin-fscsp 3/3 Running 0 30s +zbs-csi-driver-node-plugin-g4c4v 3/3 Running 0 39s +``` + +## Examples + +### Fio +1. kubectl apply -f https://github.com/iomesh/iomesh-docs/tree/master/asserts/zbs-csi-driver/example/fio.yaml -TODO +2. Wait fio-pvc bound and fio pod ready + +```sh +watch kubectl get pvc fio-pvc +``` + +``` +Every 2.0s: kubectl get pvc fio-pvc localhost.localdomain: Wed Sep 23 14:40:03 2020 + +NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE +fio-pvc Bound pvc-d7916b34-50cd-49bd-86f9-5287db1265cb 30Gi RWO zbs-csi-driver-default 15s + +``` + +```sh +kubectl wait --for=condition=Ready pod/fio +``` + +``` +pod/fio condition met +``` + +3. Run test + +```sh +kubectl exec -it fio sh + +fio --name fio --filename=/mnt/fio --bs=256k --rw=write --ioengine=libaio --direct=1 --iodepth=128 --numjobs=1 --size=$(blockdev --getsize64 /mnt/fio) + +fio --name fio --filename=/mnt/fio --bs=4k --rw=randread --ioengine=libaio --direct=1 --iodepth=128 --numjobs=1 --size=$(blockdev --getsize64 /mnt/fio) +``` + +4. Cleanup + +```sh +kubectl delete pod fio +kubectl delete pvc fio-pvc +# You need to delete pv when reclaimPolicy is Retain +kubectl delete pvc-b0d74bab-2d1a-4727-a236-47c93840545f +```