core: cephblockpoolRadosNamespace cleanup

Clean up pool images and snapshots in the
radosnamespace

Signed-off-by: sp98 <sapillai@redhat.com>

Documentation/Storage-Configuration/ceph-teardown.md

@@ -200,6 +200,7 @@ Once the cleanup job is completed successfully, Rook will remove the finalizers
 
 This cleanup is supported only for the following custom resources:
 
-| Custom Resource                                | Ceph Resources to be cleaned up |
-| --------                                       | ------- |
-| CephFilesystemSubVolumeGroup                   | CSI stored RADOS OMAP details for pvc/volumesnapshots, subvolume snapshots, subvolume clones, subvolumes |
+| Custom Resource                      | Ceph Resources to be cleaned up |
+| --------                             | ------- |
+| CephFilesystemSubVolumeGroup         | CSI stored RADOS OMAP details for pvc/volumesnapshots, subvolume snapshots, subvolume clones, subvolumes |
+| CephBlockPoolRadosNamespace          | Images and snapshots in the rados namespace|

cmd/rook/ceph/cleanup.go

@@ -56,6 +56,12 @@ var cleanUpSubVolumeGroupCmd = &cobra.Command{
 	Short: "Starts the cleanup process of a CephFilesystemSubVolumeGroup",
 }
 
+var cleanUpRadosNamespaceCmd = &cobra.Command{
+	// the subcommand matches CRD kind of the custom resource to be cleaned up
+	Use:   "CephBlockPoolRadosNamespace",
+	Short: "Starts the cleanup process for a CephBlockPoolRadosNamespace",
+}
+
 func init() {
 	cleanUpCmd.Flags().StringVar(&dataDirHostPath, "data-dir-host-path", "", "dataDirHostPath on the node")
 	cleanUpCmd.Flags().StringVar(&namespaceDir, "namespace-dir", "", "dataDirHostPath on the node")
@@ -68,10 +74,11 @@ func init() {
 
 	flags.SetFlagsFromEnv(cleanUpSubVolumeGroupCmd.Flags(), rook.RookEnvVarPrefix)
 
-	cleanUpCmd.AddCommand(cleanUpHostCmd, cleanUpSubVolumeGroupCmd)
+	cleanUpCmd.AddCommand(cleanUpHostCmd, cleanUpSubVolumeGroupCmd, cleanUpRadosNamespaceCmd)
 
 	cleanUpHostCmd.RunE = startHostCleanUp
 	cleanUpSubVolumeGroupCmd.RunE = startSubVolumeGroupCleanUp
+	cleanUpRadosNamespaceCmd.RunE = startRadosNamespaceCleanup
 }
 
 func startHostCleanUp(cmd *cobra.Command, args []string) error {
@@ -140,3 +147,30 @@ func startSubVolumeGroupCleanUp(cmd *cobra.Command, args []string) error {
 
 	return nil
 }
+
+func startRadosNamespaceCleanup(cmd *cobra.Command, args []string) error {
+	rook.SetLogLevel()
+	rook.LogStartupInfo(cleanUpRadosNamespaceCmd.Flags())
+
+	ctx := cmd.Context()
+	context := createContext()
+	namespace := os.Getenv(k8sutil.PodNamespaceEnvVar)
+	clusterInfo := client.AdminClusterInfo(ctx, namespace, "")
+
+	poolName := os.Getenv(opcontroller.CephBlockPoolNameEnv)
+	if poolName == "" {
+		rook.TerminateFatal(fmt.Errorf("cephblockpool name is not available in the pod environment variables"))
+	}
+
+	radosNamespace := os.Getenv(opcontroller.CephBlockPoolRadosNamespaceEnv)
+	if radosNamespace == "" {
+		rook.TerminateFatal(fmt.Errorf("cephblockpool radosNamespace is not available in the pod environment variables"))
+	}
+
+	err := cleanup.RadosNamespaceCleanup(context, clusterInfo, poolName, radosNamespace)
+	if err != nil {
+		rook.TerminateFatal(fmt.Errorf("failed to cleanup cephBlockPoolRadosNamespace %q resources in the pool %q. %v", radosNamespace, poolName, err))
+	}
+
+	return nil
+}

pkg/daemon/ceph/cleanup/radosnamespace.go

@@ -0,0 +1,56 @@
+/*
+Copyright 2024 The Rook Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cleanup
+
+import (
+	"github.com/pkg/errors"
+	"github.com/rook/rook/pkg/clusterd"
+	"github.com/rook/rook/pkg/daemon/ceph/client"
+	cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
+)
+
+func RadosNamespaceCleanup(context *clusterd.Context, clusterInfo *client.ClusterInfo, poolName, radosNamespace string) error {
+	logger.Infof("starting clean up of CephBlockPoolRadosNamespace %q resources in blockpool %q", radosNamespace, poolName)
+
+	images, err := cephclient.ListImagesInRadosNamespace(context, clusterInfo, poolName, radosNamespace)
+	if err != nil {
+		return errors.Wrapf(err, "failed to list images in blockpool %q in rados namespace %q", poolName, radosNamespace)
+	}
+
+	var retErr error
+	for _, image := range images {
+		err := cephclient.PurgeImageSnapshots(context, clusterInfo, poolName, image.Name, radosNamespace)
+		if err != nil {
+			retErr = errors.Wrapf(err, "failed to purge snapshots of the image %q in pool %q.", image.Name, poolName)
+			logger.Error(retErr)
+		}
+
+		err = cephclient.DeleteImageInRadosNamespace(context, clusterInfo, image.Name, poolName, radosNamespace)
+		if err != nil {
+			retErr = errors.Wrapf(err, "failed to delete image %q in pool %q.", image.Name, poolName)
+			logger.Error(retErr)
+		}
+	}
+
+	if retErr != nil {
+		logger.Errorf("failed to clean up CephBlockPoolRadosNamespace %q resources in blockpool %q", radosNamespace, poolName)
+		return retErr
+	}
+
+	logger.Infof("successfully cleaned up CephBlockPoolRadosNamespace %q resources in blockpool %q", radosNamespace, poolName)
+	return nil
+}

pkg/daemon/ceph/cleanup/radosnamespace_test.go

@@ -0,0 +1,80 @@
+/*
+Copyright 2024 The Rook Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cleanup
+
+import (
+	"testing"
+
+	"github.com/pkg/errors"
+	"github.com/rook/rook/pkg/clusterd"
+	cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
+	exectest "github.com/rook/rook/pkg/util/exec/test"
+	"github.com/stretchr/testify/assert"
+)
+
+const (
+	mockImageLSResponse = `[{"image":"csi-vol-136268e8-5386-4453-a6bd-9dca381d187d","id":"16e35cfa56a7","size":1073741824,"format":2}]`
+)
+
+func TestRadosNamespace(t *testing.T) {
+	clusterInfo := cephclient.AdminTestClusterInfo("mycluster")
+	poolName := "test-pool"
+	radosNamespace := "test-namespace"
+
+	t.Run("no images in rados namespace", func(t *testing.T) {
+		executor := &exectest.MockExecutor{}
+		executor.MockExecuteCommandWithOutput = func(command string, args ...string) (string, error) {
+			logger.Infof("Command: %s %v", command, args)
+			if args[0] == "ls" && args[1] == "-l" {
+				assert.Equal(t, poolName, args[2])
+				return "", nil
+			}
+			return "", errors.New("unknown command")
+		}
+		context := &clusterd.Context{Executor: executor}
+		err := RadosNamespaceCleanup(context, clusterInfo, poolName, radosNamespace)
+		assert.NoError(t, err)
+	})
+
+	t.Run("images available in rados namespace", func(t *testing.T) {
+		executor := &exectest.MockExecutor{}
+		executor.MockExecuteCommandWithOutput = func(command string, args ...string) (string, error) {
+			logger.Infof("Command: %s %v", command, args)
+			// list all subvolumes in subvolumegroup
+			if args[0] == "ls" && args[1] == "-l" {
+				assert.Equal(t, poolName, args[2])
+				return mockImageLSResponse, nil
+			}
+			if args[0] == "snap" && args[1] == "purge" {
+				assert.Equal(t, "test-pool/csi-vol-136268e8-5386-4453-a6bd-9dca381d187d", args[2])
+				assert.Equal(t, "--namespace", args[3])
+				assert.Equal(t, radosNamespace, args[4])
+				return "", nil
+			}
+			if args[0] == "rm" {
+				assert.Equal(t, "test-pool/csi-vol-136268e8-5386-4453-a6bd-9dca381d187d", args[1])
+				assert.Equal(t, "--namespace", args[2])
+				assert.Equal(t, radosNamespace, args[3])
+				return "", nil
+			}
+			return "", errors.New("unknown command")
+		}
+		context := &clusterd.Context{Executor: executor}
+		err := RadosNamespaceCleanup(context, clusterInfo, poolName, radosNamespace)
+		assert.NoError(t, err)
+	})
+}

pkg/daemon/ceph/client/image.go

@@ -41,8 +41,15 @@ type CephBlockImage struct {
 	InfoName string `json:"name"`
 }
 
-func ListImages(context *clusterd.Context, clusterInfo *ClusterInfo, poolName string) ([]CephBlockImage, error) {
+func ListImagesInPool(context *clusterd.Context, clusterInfo *ClusterInfo, poolName string) ([]CephBlockImage, error) {
+	return ListImagesInRadosNamespace(context, clusterInfo, poolName, "")
+}
+
+func ListImagesInRadosNamespace(context *clusterd.Context, clusterInfo *ClusterInfo, poolName, namespace string) ([]CephBlockImage, error) {
 	args := []string{"ls", "-l", poolName}
+	if namespace != "" {
+		args = append(args, "--namespace", namespace)
+	}
 	cmd := NewRBDCommand(context, clusterInfo, args)
 	cmd.JsonOutput = true
 	buf, err := cmd.Run()
@@ -68,6 +75,20 @@ func ListImages(context *clusterd.Context, clusterInfo *ClusterInfo, poolName st
 	return images, nil
 }
 
+// PurgeImageSnapshots purges all the snapshots of a given image in a pool
+func PurgeImageSnapshots(context *clusterd.Context, clusterInfo *ClusterInfo, poolName, imageName, namespace string) error {
+	args := []string{"snap", "purge", getImageSpec(imageName, poolName)}
+	if namespace != "" {
+		args = append(args, "--namespace", namespace)
+	}
+	cmd := NewRBDCommand(context, clusterInfo, args)
+	_, err := cmd.Run()
+	if err != nil {
+		return errors.Wrapf(err, "failed to purge all snapshots of image %q in pool %q", imageName, poolName)
+	}
+	return nil
+}
+
 // CreateImage creates a block storage image.
 // If dataPoolName is not empty, the image will use poolName as the metadata pool and the dataPoolname for data.
 // If size is zero an empty image will be created. Otherwise, an image will be
@@ -116,16 +137,22 @@ func CreateImage(context *clusterd.Context, clusterInfo *ClusterInfo, name, pool
 	return &CephBlockImage{Name: name, Size: newSizeBytes}, nil
 }
 
-func DeleteImage(context *clusterd.Context, clusterInfo *ClusterInfo, name, poolName string) error {
+func DeleteImageInPool(context *clusterd.Context, clusterInfo *ClusterInfo, name, poolName string) error {
+	return DeleteImageInRadosNamespace(context, clusterInfo, name, poolName, "")
+}
+
+func DeleteImageInRadosNamespace(context *clusterd.Context, clusterInfo *ClusterInfo, name, poolName, namespace string) error {
 	logger.Infof("deleting rbd image %q from pool %q", name, poolName)
 	imageSpec := getImageSpec(name, poolName)
 	args := []string{"rm", imageSpec}
+	if namespace != "" {
+		args = append(args, "--namespace", namespace)
+	}
 	buf, err := NewRBDCommand(context, clusterInfo, args).Run()
 	if err != nil {
 		return errors.Wrapf(err, "failed to delete image %s in pool %s, output: %s",
 			name, poolName, string(buf))
 	}
-
 	return nil
 }
 

pkg/daemon/ceph/client/image_test.go

@@ -187,15 +187,15 @@ func TestListImageLogLevelInfo(t *testing.T) {
 	}
 
 	clusterInfo := AdminTestClusterInfo("mycluster")
-	images, err = ListImages(context, clusterInfo, "pool1")
+	images, err = ListImagesInPool(context, clusterInfo, "pool1")
 	assert.Nil(t, err)
 	assert.NotNil(t, images)
 	assert.True(t, len(images) == 3)
 	assert.True(t, listCalled)
 	listCalled = false
 
 	emptyListResult = true
-	images, err = ListImages(context, clusterInfo, "pool1")
+	images, err = ListImagesInPool(context, clusterInfo, "pool1")
 	assert.Nil(t, err)
 	assert.NotNil(t, images)
 	assert.True(t, len(images) == 0)
@@ -251,15 +251,15 @@ func TestListImageLogLevelDebug(t *testing.T) {
 	}
 
 	clusterInfo := AdminTestClusterInfo("mycluster")
-	images, err = ListImages(context, clusterInfo, "pool1")
+	images, err = ListImagesInPool(context, clusterInfo, "pool1")
 	assert.Nil(t, err)
 	assert.NotNil(t, images)
 	assert.True(t, len(images) == 3)
 	assert.True(t, listCalled)
 	listCalled = false
 
 	emptyListResult = true
-	images, err = ListImages(context, clusterInfo, "pool1")
+	images, err = ListImagesInPool(context, clusterInfo, "pool1")
 	assert.Nil(t, err)
 	assert.NotNil(t, images)
 	assert.True(t, len(images) == 0)

pkg/operator/ceph/controller/cleanup.go

@@ -40,6 +40,10 @@ const (
 	CephFSNameEnv               = "FILESYSTEM_NAME"
 	CSICephFSRadosNamesaceEnv   = "CSI_CEPHFS_RADOS_NAMESPACE"
 	CephFSMetaDataPoolNameEnv   = "METADATA_POOL_NAME"
+
+	// cephblockpoolradosnamespace env resources
+	CephBlockPoolNameEnv           = "BLOCKPOOL_NAME"
+	CephBlockPoolRadosNamespaceEnv = "RADOS_NAMESPACE"
 )
 
 // ResourceCleanup defines an rook ceph resource to be cleaned up

pkg/operator/ceph/pool/radosnamespace/controller.go

@@ -183,6 +183,12 @@ func (r *ReconcileCephBlockPoolRadosNamespace) reconcile(request reconcile.Reque
 		if cephCluster.Spec.External.Enable {
 			logger.Warning("external rados namespace %q deletion is not supported, delete it manually", namespacedName)
 		} else {
+			if opcontroller.ForceDeleteRequested(cephBlockPoolRadosNamespace.GetAnnotations()) {
+				cleanupErr := r.cleanup(cephBlockPoolRadosNamespace, &cephCluster)
+				if cleanupErr != nil {
+					return reconcile.Result{}, errors.Wrapf(cleanupErr, "failed to create clean up job for radosNamespace resources %q", namespacedName)
+				}
+			}
 			err := r.deleteRadosNamespace(cephBlockPoolRadosNamespace)
 			if err != nil {
 				if strings.Contains(err.Error(), opcontroller.UninitializedCephConfigError) {
@@ -356,3 +362,18 @@ func buildClusterID(cephBlockPoolRadosNamespace *cephv1.CephBlockPoolRadosNamesp
 	clusterID := fmt.Sprintf("%s-%s-block-%s", cephBlockPoolRadosNamespace.Namespace, cephBlockPoolRadosNamespace.Spec.BlockPoolName, getRadosNamespaceName(cephBlockPoolRadosNamespace))
 	return k8sutil.Hash(clusterID)
 }
+
+func (r *ReconcileCephBlockPoolRadosNamespace) cleanup(radosNamespace *cephv1.CephBlockPoolRadosNamespace, cephCluster *cephv1.CephCluster) error {
+	logger.Infof("starting cleanup of the ceph resources for radosNamesapce %q in namespace %q", radosNamespace.Name, radosNamespace.Namespace)
+	cleanupConfig := map[string]string{
+		opcontroller.CephBlockPoolNameEnv:           radosNamespace.Spec.BlockPoolName,
+		opcontroller.CephBlockPoolRadosNamespaceEnv: getRadosNamespaceName(radosNamespace),
+	}
+	cleanup := opcontroller.NewResourceCleanup(radosNamespace, cephCluster, r.opConfig.Image, cleanupConfig)
+	jobName := k8sutil.TruncateNodeNameForJob("cleanup-radosnamespace-%s", fmt.Sprintf("%s-%s", radosNamespace.Spec.BlockPoolName, radosNamespace.Name))
+	err := cleanup.StartJob(r.clusterInfo.Context, r.context.Clientset, jobName)
+	if err != nil {
+		return errors.Wrapf(err, "failed to run clean up job to clean the ceph resources in radosNamesapce %q", radosNamespace.Name)
+	}
+	return nil
+}

tests/framework/clients/block.go

@@ -161,7 +161,7 @@ func (b *BlockOperation) ListAllImages(clusterInfo *client.ClusterInfo) ([]Block
 func (b *BlockOperation) ListImagesInPool(clusterInfo *client.ClusterInfo, poolName string) ([]BlockImage, error) {
 	// for each pool, get further details about all the images in the pool
 	images := []BlockImage{}
-	cephImages, err := client.ListImages(b.k8sClient.MakeContext(), clusterInfo, poolName)
+	cephImages, err := client.ListImagesInPool(b.k8sClient.MakeContext(), clusterInfo, poolName)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get images from pool %s: %+v", poolName, err)
 	}
@@ -182,7 +182,7 @@ func (b *BlockOperation) ListImagesInPool(clusterInfo *client.ClusterInfo, poolN
 // DeleteBlockImage Function to list all the blocks created/being managed by rook
 func (b *BlockOperation) DeleteBlockImage(clusterInfo *client.ClusterInfo, image BlockImage) error {
 	context := b.k8sClient.MakeContext()
-	return client.DeleteImage(context, clusterInfo, image.Name, image.PoolName)
+	return client.DeleteImageInPool(context, clusterInfo, image.Name, image.PoolName)
 }
 
 // CreateClientPod starts a pod that should have a block PVC.