feat(block): adding block volume support for ZFSPV (#102)

This commit adds the support for creating a Raw Block Volume request using volumemode as block in PVC :-

```
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
  name: block-claim
spec:
  volumeMode: Block
  storageClassName: zfspv-block
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 5Gi
```

The driver will create a zvol for this volume and bind mount the block device at the given path.

Signed-off-by: Pawan <pawan@mayadata.io>

deploy/sample/fio-block.yaml

@@ -0,0 +1,50 @@
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: zfspv-block
+allowVolumeExpansion: true
+parameters:
+  poolname: "zfspv-pool"
+provisioner: zfs.csi.openebs.io
+---
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: block-claim
+spec:
+  volumeMode: Block
+  storageClassName: zfspv-block
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 5Gi
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: fiob
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      name: fiob
+  template:
+    metadata:
+      labels:
+        name: fiob
+    spec:
+      containers:
+        - resources:
+          name: perfrunner
+          image: openebs/tests-fio
+          imagePullPolicy: IfNotPresent
+          command: ["/bin/bash"]
+          args: ["-c", "while true ;do sleep 50; done"]
+          volumeDevices:
+            - devicePath: /dev/xvda
+              name: storage
+      volumes:
+        - name: storage
+          persistentVolumeClaim:
+            claimName: block-claim

deploy/yamls/zfs-driver.yaml

@@ -791,7 +791,7 @@ spec:
             - name: libnvpair
               mountPath: /lib/libnvpair.so.1
             - name: pods-mount-dir
-              mountPath: /var/lib/kubelet/pods
+              mountPath: /var/lib/kubelet/
               # needed so that any mounts setup inside this container are
               # propagated back to the host machine.
               mountPropagation: "Bidirectional"
@@ -838,6 +838,6 @@ spec:
             type: DirectoryOrCreate
         - name: pods-mount-dir
           hostPath:
-            path: /var/lib/kubelet/pods
+            path: /var/lib/kubelet/
             type: Directory
 ---

deploy/zfs-operator.yaml

@@ -1248,7 +1248,7 @@ spec:
             - name: libnvpair
               mountPath: /lib/libnvpair.so.1
             - name: pods-mount-dir
-              mountPath: /var/lib/kubelet/pods
+              mountPath: /var/lib/kubelet/
               # needed so that any mounts setup inside this container are
               # propagated back to the host machine.
               mountPropagation: "Bidirectional"
@@ -1295,6 +1295,6 @@ spec:
             type: DirectoryOrCreate
         - name: pods-mount-dir
           hostPath:
-            path: /var/lib/kubelet/pods
+            path: /var/lib/kubelet/
             type: Directory
 ---

pkg/driver/agent.go

@@ -110,14 +110,18 @@ func (ns *node) NodePublishVolume(
 
 	vol, mountInfo, err := GetVolAndMountInfo(req)
 	if err != nil {
-		goto PublishVolumeResponse
+		return nil, status.Error(codes.Internal, err.Error())
 	}
-	// attempt mount operation on the requested path
-	if err = zfs.MountVolume(vol, mountInfo); err != nil {
-		goto PublishVolumeResponse
+	// If the access type is block, do nothing for stage
+	switch req.GetVolumeCapability().GetAccessType().(type) {
+	case *csi.VolumeCapability_Block:
+		// attempt block mount operation on the requested path
+		err = zfs.MountBlock(vol, mountInfo)
+	case *csi.VolumeCapability_Mount:
+		// attempt filesystem mount operation on the requested path
+		err = zfs.MountFilesystem(vol, mountInfo)
 	}
 
-PublishVolumeResponse:
 	if err != nil {
 		return nil, status.Error(codes.Internal, err.Error())
 	}

pkg/zfs/mount.go

@@ -208,11 +208,37 @@ func MountDataset(vol *apis.ZFSVolume, mount *apis.MountInfo) error {
 }
 
 // MountVolume mounts the disk to the specified path
-func MountVolume(vol *apis.ZFSVolume, mount *apis.MountInfo) error {
+func MountFilesystem(vol *apis.ZFSVolume, mount *apis.MountInfo) error {
 	switch vol.Spec.VolumeType {
 	case VOLTYPE_DATASET:
 		return MountDataset(vol, mount)
 	default:
 		return MountZvol(vol, mount)
 	}
 }
+
+func MountBlock(vol *apis.ZFSVolume, mountinfo *apis.MountInfo) error {
+	target := mountinfo.MountPath
+	devicePath := ZFS_DEVPATH + vol.Spec.PoolName + "/" + vol.Name
+	mountopt := []string{"bind"}
+
+	mounter := &mount.SafeFormatAndMount{Interface: mount.New(""), Exec: mount.NewOsExec()}
+
+	// Create the mount point as a file since bind mount device node requires it to be a file
+	err := mounter.MakeFile(target)
+	if err != nil {
+		return status.Errorf(codes.Internal, "Could not create target file %q: %v", target, err)
+	}
+
+	// do the bind mount of the zvol device at the target path
+	if err := mounter.Mount(devicePath, target, "", mountopt); err != nil {
+		if removeErr := os.Remove(target); removeErr != nil {
+			return status.Errorf(codes.Internal, "Could not remove mount target %q: %v", target, removeErr)
+		}
+		return status.Errorf(codes.Internal, "mount failed at %v err : %v", target, err)
+	}
+
+	logrus.Infof("NodePublishVolume mounted block device %s at %s", devicePath, target)
+
+	return nil
+}

tests/container/container.go

@@ -254,6 +254,29 @@ func (b *Builder) WithVolumeMountsNew(volumeMounts []corev1.VolumeMount) *Builde
 	return b
 }
 
+// WithVolumeDevicesNew sets the command arguments of the container
+func (b *Builder) WithVolumeDevicesNew(volumeDevices []corev1.VolumeDevice) *Builder {
+	if volumeDevices == nil {
+		b.errors = append(
+			b.errors,
+			errors.New("failed to build container object: nil volumeDevices"),
+		)
+		return b
+	}
+
+	if len(volumeDevices) == 0 {
+		b.errors = append(
+			b.errors,
+			errors.New("failed to build container object: missing volumeDevices"),
+		)
+		return b
+	}
+	newvolumeDevices := []corev1.VolumeDevice{}
+	newvolumeDevices = append(newvolumeDevices, volumeDevices...)
+	b.con.VolumeDevices = newvolumeDevices
+	return b
+}
+
 // WithImagePullPolicy sets the image pull policy of the container
 func (b *Builder) WithImagePullPolicy(policy corev1.PullPolicy) *Builder {
 	if len(policy) == 0 {

tests/provision_test.go

@@ -57,7 +57,20 @@ func zvolCreationTest() {
 	By("Deleting storage class", deleteStorageClass)
 }
 
+func blockVolCreationTest() {
+	By("Creating default storage class", createStorageClass)
+	By("creating and verifying PVC bound status", createAndVerifyBlockPVC)
+
+	By("Creating and deploying app pod", createDeployVerifyBlockApp)
+	By("verifying ZFSVolume object", VerifyZFSVolume)
+	By("verifying ZFSVolume property change", VerifyZFSVolumePropEdit)
+	By("Deleting application deployment", deleteAppDeployment)
+	By("Deleting pvc", deletePVC)
+	By("Deleting storage class", deleteStorageClass)
+}
+
 func volumeCreationTest() {
 	By("Running dataset creation test", datasetCreationTest)
 	By("Running zvol creation test", zvolCreationTest)
+	By("Running block volume creation test", blockVolCreationTest)
 }

tests/pvc/build.go

@@ -179,3 +179,13 @@ func (b *Builder) Build() (*corev1.PersistentVolumeClaim, error) {
 	}
 	return b.pvc.object, nil
 }
+
+// WithVolumeMode sets the VolumeMode field in PVC with provided arguments
+func (b *Builder) WithVolumeMode(volumemode *corev1.PersistentVolumeMode) *Builder {
+	if volumemode == nil {
+		b.errs = append(b.errs, errors.New("failed to build PVC object: missing volumemode"))
+		return b
+	}
+	b.pvc.object.Spec.VolumeMode = volumemode
+	return b
+}

tests/utils.go

@@ -127,6 +127,27 @@ func createExt4StorageClass() {
 	Expect(err).To(BeNil(), "while creating a ext4 storageclass {%s}", scName)
 }
 
+func createStorageClass() {
+	var (
+		err error
+	)
+
+	parameters := map[string]string{
+		"poolname": POOLNAME,
+	}
+
+	By("building a default storage class")
+	scObj, err = sc.NewBuilder().
+		WithGenerateName(scName).
+		WithParametersNew(parameters).
+		WithProvisioner(ZFSProvisioner).Build()
+	Expect(err).ShouldNot(HaveOccurred(),
+		"while building default storageclass obj with prefix {%s}", scName)
+
+	scObj, err = SCClient.Create(scObj)
+	Expect(err).To(BeNil(), "while creating a default storageclass {%s}", scName)
+}
+
 func createZfsStorageClass() {
 	var (
 		err error
@@ -322,6 +343,53 @@ func createAndVerifyPVC() {
 	)
 }
 
+func createAndVerifyBlockPVC() {
+	var (
+		err     error
+		pvcName = "zfspv-pvc"
+	)
+
+	volmode := corev1.PersistentVolumeBlock
+
+	By("building a pvc")
+	pvcObj, err = pvc.NewBuilder().
+		WithName(pvcName).
+		WithNamespace(OpenEBSNamespace).
+		WithStorageClass(scObj.Name).
+		WithAccessModes(accessModes).
+		WithVolumeMode(&volmode).
+		WithCapacity(capacity).Build()
+	Expect(err).ShouldNot(
+		HaveOccurred(),
+		"while building pvc {%s} in namespace {%s}",
+		pvcName,
+		OpenEBSNamespace,
+	)
+
+	By("creating above pvc")
+	pvcObj, err = PVCClient.WithNamespace(OpenEBSNamespace).Create(pvcObj)
+	Expect(err).To(
+		BeNil(),
+		"while creating pvc {%s} in namespace {%s}",
+		pvcName,
+		OpenEBSNamespace,
+	)
+
+	By("verifying pvc status as bound")
+
+	status := IsPVCBoundEventually(pvcName)
+	Expect(status).To(Equal(true),
+		"while checking status equal to bound")
+
+	pvcObj, err = PVCClient.WithNamespace(OpenEBSNamespace).Get(pvcObj.Name, metav1.GetOptions{})
+	Expect(err).To(
+		BeNil(),
+		"while retrieving pvc {%s} in namespace {%s}",
+		pvcName,
+		OpenEBSNamespace,
+	)
+}
+
 func resizeAndVerifyPVC() {
 	var (
 		err     error
@@ -427,6 +495,75 @@ func createAndDeployAppPod() {
 	)
 }
 
+func createAndDeployBlockAppPod() {
+	var err error
+	By("building a busybox app pod deployment using above zfs volume")
+	deployObj, err = deploy.NewBuilder().
+		WithName(appName).
+		WithNamespace(OpenEBSNamespace).
+		WithLabelsNew(
+			map[string]string{
+				"app": "busybox",
+			},
+		).
+		WithSelectorMatchLabelsNew(
+			map[string]string{
+				"app": "busybox",
+			},
+		).
+		WithPodTemplateSpecBuilder(
+			pts.NewBuilder().
+				WithLabelsNew(
+					map[string]string{
+						"app": "busybox",
+					},
+				).
+				WithContainerBuilders(
+					container.NewBuilder().
+						WithImage("busybox").
+						WithName("busybox").
+						WithImagePullPolicy(corev1.PullIfNotPresent).
+						WithCommandNew(
+							[]string{
+								"sh",
+								"-c",
+								"date > /mnt/datadir/date.txt; sync; sleep 5; sync; tail -f /dev/null;",
+							},
+						).
+						WithVolumeDevicesNew(
+							[]corev1.VolumeDevice{
+								corev1.VolumeDevice{
+									Name:       "datavol1",
+									DevicePath: "/dev/xvda",
+								},
+							},
+						),
+				).
+				WithVolumeBuilders(
+					k8svolume.NewBuilder().
+						WithName("datavol1").
+						WithPVCSource(pvcObj.Name),
+				),
+		).
+		Build()
+
+	Expect(err).ShouldNot(HaveOccurred(), "while building app deployement {%s}", appName)
+
+	deployObj, err = DeployClient.WithNamespace(OpenEBSNamespace).Create(deployObj)
+	Expect(err).ShouldNot(
+		HaveOccurred(),
+		"while creating pod {%s} in namespace {%s}",
+		appName,
+		OpenEBSNamespace,
+	)
+}
+
+func createDeployVerifyBlockApp() {
+	By("creating and deploying app pod", createAndDeployBlockAppPod)
+	time.Sleep(30 * time.Second)
+	By("verifying app pod is running", verifyAppPodRunning)
+}
+
 func verifyAppPodRunning() {
 	var err error
 	appPod, err = PodClient.WithNamespace(OpenEBSNamespace).

unreleased/102-pawanpraka1

@@ -0,0 +1 @@
+adding RAW Block Volume support for ZFSPV