[WIP] rbd: add additional space for encrypted volumes

e2e/rbd.go

@@ -27,10 +27,12 @@ import (
 
 	. "github.com/onsi/ginkgo/v2"
 	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/wait"
 	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/cloud-provider/volume/helpers"
 	"k8s.io/kubernetes/test/e2e/framework"
 	e2edebug "k8s.io/kubernetes/test/e2e/framework/debug"
 	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
@@ -1983,6 +1985,214 @@ var _ = Describe("RBD", func() {
 				}
 			})
 
+			By("create/resize/clone/restore a encrypted block pvc and verify the image size", func() {
+				err := deleteResource(rbdExamplePath + "storageclass.yaml")
+				if err != nil {
+					framework.Failf("failed to delete storageclass: %v", err)
+				}
+				err = createRBDStorageClass(
+					f.ClientSet,
+					f,
+					defaultSCName,
+					nil,
+					map[string]string{"encrypted": "true", "encryptionType": util.EncryptionTypeBlock.String()},
+					deletePolicy)
+				if err != nil {
+					framework.Failf("failed to create storageclass: %v", err)
+				}
+
+				err = createRBDSnapshotClass(f)
+				if err != nil {
+					framework.Failf("failed to create storageclass: %v", err)
+				}
+				defer func() {
+					err = deleteRBDSnapshotClass()
+					if err != nil {
+						framework.Failf("failed to delete VolumeSnapshotClass: %v", err)
+					}
+				}()
+
+				var (
+					imageSize       uint64
+					resizeImageSize uint64
+					sizeInBytes     int64
+				)
+
+				//nolint:goconst // The string "1Gi" is used multiple times in rbd.go, so it's not a const value.
+				pvcSize := "1Gi"
+				if sizeInBytes, err = helpers.RoundUpToB(resource.MustParse(pvcSize)); err != nil {
+					framework.Failf("failed to parse pvc size: %v", err)
+				}
+				imageSize = uint64(sizeInBytes) + util.Luks2HeaderSize
+
+				pvc, err := loadPVC(rawPvcPath)
+				if err != nil {
+					framework.Failf("failed to load PVC: %v", err)
+				}
+				pvc.Namespace = f.UniqueName
+				pvc.Spec.Resources.Requests[v1.ResourceStorage] = resource.MustParse(pvcSize)
+
+				app, err := loadApp(rawAppPath)
+				if err != nil {
+					framework.Failf("failed to load application: %v", err)
+				}
+				labelKey := "app"
+				labelValue := "rbd-pod-block-encrypted"
+				opt := metav1.ListOptions{
+					LabelSelector: fmt.Sprintf("%s=%s", labelKey, labelValue),
+				}
+
+				app.Labels = map[string]string{labelKey: labelValue}
+				app.Namespace = f.UniqueName
+				err = createPVCAndApp("", f, pvc, app, deployTimeout)
+				if err != nil {
+					framework.Failf("failed to create PVC and application: %v", err)
+				}
+
+				// validate created backend rbd images
+				err = validateImageSize(f, pvc, imageSize)
+				if err != nil {
+					framework.Failf("failed to validate image size: %v", err)
+				}
+				err = checkDeviceSize(app, f, &opt, pvcSize)
+				if err != nil {
+					framework.Failf("failed to validate device size: %v", err)
+				}
+
+				// create clone PVC and validate the image size
+				labelValueClonePod := "rbd-pod-block-encrypted-clone"
+				optClonePod := metav1.ListOptions{
+					LabelSelector: fmt.Sprintf("%s=%s", labelKey, labelValueClonePod),
+				}
+
+				pvcClone, err := loadPVC(pvcBlockSmartClonePath)
+				if err != nil {
+					framework.Failf("failed to load PVC: %v", err)
+				}
+				pvcClone.Spec.DataSource.Name = pvc.Name
+				pvcClone.Spec.Resources.Requests[v1.ResourceStorage] = resource.MustParse(pvcSize)
+				pvcClone.Namespace = f.UniqueName
+
+				appClone, err := loadApp(appBlockSmartClonePath)
+				if err != nil {
+					framework.Failf("failed to load application: %v", err)
+				}
+				appClone.Namespace = f.UniqueName
+				appClone.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvcClone.Name
+				appClone.Labels = map[string]string{labelKey: labelValueClonePod}
+
+				err = createPVCAndApp("", f, pvcClone, appClone, deployTimeout)
+				if err != nil {
+					framework.Failf("failed to create clone PVC and application : %v", err)
+				}
+
+				err = validateImageSize(f, pvcClone, imageSize)
+				if err != nil {
+					framework.Failf("failed to validate image size: %v", err)
+				}
+				err = checkDeviceSize(appClone, f, &optClonePod, pvcSize)
+				if err != nil {
+					framework.Failf("failed to validate device size: %v", err)
+				}
+
+				// create snapshot and restore PVC and validate the image size
+				labelValueRestorePod := "rbd-pod-block-encrypted-restore"
+				optRestorePod := metav1.ListOptions{
+					LabelSelector: fmt.Sprintf("%s=%s", labelKey, labelValueRestorePod),
+				}
+				snap := getSnapshot(snapshotPath)
+				snap.Namespace = f.UniqueName
+				snap.Spec.Source.PersistentVolumeClaimName = &pvc.Name
+
+				err = createSnapshot(&snap, deployTimeout)
+				if err != nil {
+					framework.Failf("failed to create snapshot: %v", err)
+				}
+
+				pvcRestore, err := loadPVC(pvcBlockRestorePath)
+				if err != nil {
+					framework.Failf("failed to load PVC: %v", err)
+				}
+				pvcRestore.Spec.DataSource.Name = snap.Name
+				pvcRestore.Spec.VolumeMode = pvc.Spec.VolumeMode
+				pvcRestore.Spec.Resources.Requests[v1.ResourceStorage] = resource.MustParse(pvcSize)
+				pvcRestore.Namespace = f.UniqueName
+
+				appRestore, err := loadApp(appBlockRestorePath)
+				if err != nil {
+					framework.Failf("failed to load application: %v", err)
+				}
+				appRestore.Namespace = f.UniqueName
+				appRestore.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvcRestore.Name
+				appRestore.Labels = map[string]string{labelKey: labelValueRestorePod}
+
+				err = createPVCAndApp("", f, pvcRestore, appRestore, deployTimeout)
+				if err != nil {
+					framework.Failf("failed to create clone PVC and application : %v", err)
+				}
+
+				err = validateImageSize(f, pvcRestore, imageSize)
+				if err != nil {
+					framework.Failf("failed to validate image size: %v", err)
+				}
+				err = checkDeviceSize(appRestore, f, &optRestorePod, pvcSize)
+				if err != nil {
+					framework.Failf("failed to validate device size: %v", err)
+				}
+
+				// resize PVC and validate the image size
+				resizePVCSize := "2Gi"
+				if sizeInBytes, err = helpers.RoundUpToB(resource.MustParse(resizePVCSize)); err != nil {
+					framework.Failf("failed to parse resize pvc size: %v", err)
+				}
+				resizeImageSize = uint64(sizeInBytes) + util.Luks2HeaderSize
+
+				err = expandPVCSize(f.ClientSet, pvc, resizePVCSize, deployTimeout)
+				if err != nil {
+					framework.Failf("failed to expand pvc size: %v", err)
+				}
+				// wait for application pod to come up after resize
+				err = waitForPodInRunningState(app.Name, app.Namespace, f.ClientSet, deployTimeout, noError)
+				if err != nil {
+					framework.Failf("timeout waiting for pod to be in running state: %v", err)
+				}
+
+				err = validateImageSize(f, pvc, resizeImageSize)
+				if err != nil {
+					framework.Failf("failed to validate image size after resize: %v", err)
+				}
+				err = checkDeviceSize(app, f, &opt, resizePVCSize)
+				if err != nil {
+					framework.Failf("failed to validate device size after resize: %v", err)
+				}
+
+				// delete resources
+				err = deletePVCAndApp("", f, pvc, app)
+				if err != nil {
+					framework.Failf("failed to delete pvc and app: %v", err)
+				}
+				err = deletePVCAndApp("", f, pvcClone, appClone)
+				if err != nil {
+					framework.Failf("failed to delete clone pvc and app: %v", err)
+				}
+				err = deletePVCAndApp("", f, pvcRestore, appRestore)
+				if err != nil {
+					framework.Failf("failed to delete clone pvc and app: %v", err)
+				}
+				err = deleteSnapshot(&snap, deployTimeout)
+				if err != nil {
+					framework.Failf("failed to delete snapshot: %v", err)
+				}
+				err = deleteResource(rbdExamplePath + "storageclass.yaml")
+				if err != nil {
+					framework.Failf("failed to delete storageclass: %v", err)
+				}
+
+				// validate created backend rbd images
+				validateRBDImageCount(f, 0, defaultRBDPool)
+				validateOmapCount(f, 0, rbdType, defaultRBDPool, snapsType)
+			})
+
 			ByFileAndBlockEncryption("create a PVC and bind it to an app using rbd-nbd mounter with encryption", func(
 				validator encryptionValidateFunc, _ validateFunc, encType util.EncryptionType,
 			) {

e2e/rbd_helper.go

@@ -1067,6 +1067,7 @@ type imageInfo struct {
 	StripeUnit  int    `json:"stripe_unit"`
 	StripeCount int    `json:"stripe_count"`
 	ObjectSize  int    `json:"object_size"`
+	Size        uint64 `json:"size"`
 }
 
 // getImageInfo queries rbd about the given image and returns its metadata, and returns
@@ -1126,3 +1127,22 @@ func validateStripe(f *framework.Framework,
 
 	return nil
 }
+
+// validateImageSize validates the size of the image.
+func validateImageSize(f *framework.Framework, pvc *v1.PersistentVolumeClaim, imageSize uint64) error {
+	imageData, err := getImageInfoFromPVC(pvc.Namespace, pvc.Name, f)
+	if err != nil {
+		return err
+	}
+
+	imgInfo, err := getImageInfo(f, imageData.imageName, defaultRBDPool)
+	if err != nil {
+		return err
+	}
+
+	if imgInfo.Size != imageSize {
+		return fmt.Errorf("image %s size %d does not match expected %d", imgInfo.Name, imgInfo.Size, imageSize)
+	}
+
+	return nil
+}

internal/rbd/controllerserver.go

@@ -1221,6 +1221,17 @@ func (cs *ControllerServer) CreateSnapshot(
 		return nil, status.Error(codes.Internal, err.Error())
 	}
 
+	err = vol.Connect(cr)
+	if err != nil {
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+	defer vol.Destroy(ctx)
+
+	err = vol.getImageInfo()
+	if err != nil {
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+
 	return &csi.CreateSnapshotResponse{
 		Snapshot: &csi.Snapshot{
 			SizeBytes:      vol.VolSize,
@@ -1281,6 +1292,17 @@ func cloneFromSnapshot(
 		}
 	}
 
+	err = rbdSnap.Connect(cr)
+	if err != nil {
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+	defer rbdSnap.Destroy(ctx)
+
+	err = rbdSnap.getImageInfo()
+	if err != nil {
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+
 	return &csi.CreateSnapshotResponse{
 		Snapshot: &csi.Snapshot{
 			SizeBytes:      rbdSnap.VolSize,

internal/rbd/encryption.go

@@ -57,6 +57,9 @@ const (
 	metadataDEK    = "rbd.csi.ceph.com/dek"
 	oldMetadataDEK = ".rbd.csi.ceph.com/dek"
 
+	// luks2 header size metadata key.
+	luks2HeaderSizeKey = "rbd.csi.ceph.com/luks2HeaderSize"
+
 	encryptionPassphraseSize = 20
 
 	// rbdDefaultEncryptionType is the default to use when the
@@ -124,6 +127,11 @@ func (ri *rbdImage) setupBlockEncryption(ctx context.Context) error {
 		return err
 	}
 
+	err = ri.SetMetadata(luks2HeaderSizeKey, strconv.FormatUint(util.Luks2HeaderSize, 10))
+	if err != nil {
+		return fmt.Errorf("failed to save %s metadata on image: %w", luks2HeaderSizeKey, err)
+	}
+
 	err = ri.ensureEncryptionMetadataSet(rbdImageEncryptionPrepared)
 	if err != nil {
 		log.ErrorLog(ctx, "failed to save encryption status, deleting "+

internal/rbd/rbd_util.go

@@ -452,8 +452,16 @@ func createImage(ctx context.Context, pOpts *rbdVolume, cr *util.Credentials) er
 		return fmt.Errorf("failed to get IOContext: %w", err)
 	}
 
-	err = librbd.CreateImage(pOpts.ioctx, pOpts.RbdImageName,
-		uint64(util.RoundOffVolSize(pOpts.VolSize)*helpers.MiB), options)
+	size := uint64(util.RoundOffVolSize(pOpts.VolSize) * helpers.MiB)
+	if pOpts.isBlockEncrypted() {
+		// When a block-mode PVC is created with encryption enabled,
+		// some space is reserved for the LUKS2 header.
+		// Add the LUKS2 header size to the image size so that the user has at least
+		// the requested size.
+		size += util.Luks2HeaderSize
+	}
+
+	err = librbd.CreateImage(pOpts.ioctx, pOpts.RbdImageName, size, options)
 	if err != nil {
 		return fmt.Errorf("failed to create rbd image: %w", err)
 	}
@@ -1607,6 +1615,26 @@ func (ri *rbdImage) GetImageCreationTime() (*timestamppb.Timestamp, error) {
 	return ri.CreatedAt, nil
 }
 
+// getLuks2HeaderSizeSet returns the value of the LUKS2 header size
+// set in the image metadata.
+func (ri *rbdImage) getLuks2HeaderSizeSet() (uint64, error) {
+	value, err := ri.GetMetadata(luks2HeaderSizeKey)
+	if err != nil {
+		if !errors.Is(err, librbd.ErrNotFound) {
+			return 0, err
+		}
+
+		return 0, nil
+	}
+
+	headerSize, parseErr := strconv.ParseUint(value, 10, 64)
+	if parseErr != nil {
+		return 0, parseErr
+	}
+
+	return headerSize, nil
+}
+
 // getImageInfo queries rbd about the given image and returns its metadata, and returns
 // ErrImageNotFound if provided image is not found.
 func (ri *rbdImage) getImageInfo() error {
@@ -1623,6 +1651,14 @@ func (ri *rbdImage) getImageInfo() error {
 	// TODO: can rv.VolSize not be a uint64? Or initialize it to -1?
 	ri.VolSize = int64(imageInfo.Size)
 
+	// If the luks2HeaderSizeKey metadata is set
+	// reduce the extra size of the LUKS header from the image size.
+	headerSize, err := ri.getLuks2HeaderSizeSet()
+	if err != nil {
+		return err
+	}
+	ri.VolSize -= int64(headerSize)
+
 	features, err := image.GetFeatures()
 	if err != nil {
 		return err
@@ -1871,7 +1907,17 @@ func (ri *rbdImage) resize(newSize int64) error {
 	}
 	defer image.Close()
 
-	err = image.Resize(uint64(util.RoundOffVolSize(newSize) * helpers.MiB))
+	size := uint64(util.RoundOffVolSize(newSize) * helpers.MiB)
+
+	// If the luks2HeaderSizeKey metadata is set
+	// add the extra size of the LUKS header to the image size.
+	headerSize, err := ri.getLuks2HeaderSizeSet()
+	if err != nil {
+		return err
+	}
+	size += headerSize
+
+	err = image.Resize(size)
 	if err != nil {
 		return err
 	}