Merge pull request #6867 from openshift-cherrypick-robot/cherry-pick-…

…6845-to-release-4.12

[release-4.12] OCPBUGS-7607: IBMCloud: Handle COS reclamations

pkg/destroy/ibmcloud/cloudobjectstorage.go

@@ -4,13 +4,70 @@ import (
 	"fmt"
 	"net/http"
 
+	"github.com/IBM/platform-services-go-sdk/resourcecontrollerv2"
 	"github.com/pkg/errors"
 )
 
-const cosTypeName = "cos instance"
+const (
+	cosTypeName = "cos instance"
+	// reclamationReclaim will delete the resource, reclaim it.
+	reclamationReclaim = "reclaim"
+)
+
+// Resource ID collected via following command using IBM Cloud CLI:
+// $ ibmcloud catalog service cloud-object-storage --output json | jq -r '.[].id' .
 const cosResourceID = "dff97f5c-bc5e-4455-b470-411c3edbe49c"
 
-// listCOSInstances lists COS service instances
+// findCOSInstanceReclamation Checks current reclamations for one that matches the COS instance name.
+func (o *ClusterUninstaller) findCOSInstanceReclamation(instance cloudResource) (*resourcecontrollerv2.Reclamation, error) {
+	o.Logger.Debugf("Searching for COS reclamations for instance %s", instance.name)
+	ctx, cancel := o.contextWithTimeout()
+	defer cancel()
+
+	reclamationOptions := o.controllerSvc.NewListReclamationsOptions()
+	resources, _, err := o.controllerSvc.ListReclamationsWithContext(ctx, reclamationOptions)
+	if err != nil {
+		return nil, errors.Wrapf(err, "Failed listing reclamations for instance %s", instance.name)
+	}
+
+	o.Logger.Debugf("Checking reclamations that match instance %s", instance.name)
+	for _, reclamation := range resources.Resources {
+		getOptions := o.controllerSvc.NewGetResourceInstanceOptions(*reclamation.ResourceInstanceID)
+		cosInstance, _, err := o.controllerSvc.GetResourceInstanceWithContext(ctx, getOptions)
+		if err != nil {
+			return nil, errors.Wrapf(err, "Failed checking reclamation %s", *reclamation.ResourceInstanceID)
+		}
+		if *cosInstance.Name == instance.name {
+			o.Logger.Debugf("Found COS instance reclamation %s - %s", instance.name, *reclamation.ID)
+			return &reclamation, nil
+		}
+	}
+
+	return nil, nil
+}
+
+// reclaimCOSInstanceReclamation reclaims (deletes) a reclamation from a deleted COS instance.
+func (o *ClusterUninstaller) reclaimCOSInstanceReclamation(reclamationID string) error {
+	o.Logger.Debugf("Reclaming COS instance reclamation %s", reclamationID)
+	ctx, cancel := o.contextWithTimeout()
+	defer cancel()
+
+	options := o.controllerSvc.NewRunReclamationActionOptions(reclamationID, reclamationReclaim)
+	_, response, err := o.controllerSvc.RunReclamationActionWithContext(ctx, options)
+	if err != nil {
+		// If reclaim attempt failed because the reclamation doesn't exist (404) don't return an error
+		if response != nil && response.StatusCode == http.StatusNotFound {
+			o.Logger.Debugf("Reclamation not found, it has likely already been reclaimed %s", reclamationID)
+			return nil
+		}
+		return errors.Wrapf(err, "Failed to reclaim COS instance reclamation %s", reclamationID)
+	}
+
+	o.Logger.Infof("Reclaimed %s", reclamationID)
+	return nil
+}
+
+// listCOSInstances lists COS service instances.
 func (o *ClusterUninstaller) listCOSInstances() (cloudResources, error) {
 	o.Logger.Debugf("Listing COS instances")
 	ctx, cancel := o.contextWithTimeout()
@@ -50,23 +107,19 @@ func (o *ClusterUninstaller) listCOSInstances() (cloudResources, error) {
 }
 
 func (o *ClusterUninstaller) deleteCOSInstance(item cloudResource) error {
-	o.Logger.Debugf("Deleting COS instance %q", item.name)
+	o.Logger.Debugf("Deleting COS instance %s", item.name)
 	ctx, cancel := o.contextWithTimeout()
 	defer cancel()
 
 	options := o.controllerSvc.NewDeleteResourceInstanceOptions(item.id)
 	options.SetRecursive(true)
 	details, err := o.controllerSvc.DeleteResourceInstanceWithContext(ctx, options)
-
-	if err != nil && details != nil && details.StatusCode == http.StatusNotFound {
-		// The resource is gone
-		o.deletePendingItems(item.typeName, []cloudResource{item})
-		o.Logger.Infof("Deleted COS instance %q", item.name)
-		return nil
-	}
-
-	if err != nil && details != nil && details.StatusCode != http.StatusNotFound {
-		return errors.Wrapf(err, "Failed to delete COS instance %s", item.name)
+	if err != nil {
+		// If the deletion attempt failed because the COS instance doesn't exist (404) don't return an error
+		if details != nil && details.StatusCode == http.StatusNotFound {
+			return nil
+		}
+		return errors.Wrapf(err, "Failed to delete COS Instance %s", item.name)
 	}
 
 	return nil
@@ -84,9 +137,22 @@ func (o *ClusterUninstaller) destroyCOSInstances() error {
 
 	for _, item := range items {
 		if _, ok := found[item.key]; !ok {
+			// Check if a COS reclamation was created for the deleted instance
+			reclamation, err := o.findCOSInstanceReclamation(item)
+			if err != nil {
+				return err
+			}
+			if reclamation != nil {
+				err = o.reclaimCOSInstanceReclamation(*reclamation.ID)
+				if err != nil {
+					return err
+				}
+				continue
+			}
+			o.Logger.Debugf("No reclamations found for COS instance %s", item.name)
 			// This item has finished deletion.
 			o.deletePendingItems(item.typeName, []cloudResource{item})
-			o.Logger.Infof("Deleted COS instance %q", item.name)
+			o.Logger.Infof("Deleted COS instance %s", item.name)
 			continue
 		}
 		err = o.deleteCOSInstance(item)

pkg/destroy/ibmcloud/ibmcloud.go

@@ -131,13 +131,17 @@ func (o *ClusterUninstaller) destroyCluster() error {
 	}{{
 		{name: "Stop instances", execute: o.stopInstances},
 	}, {
+		// Instances must occur before LB cleanup
 		{name: "Instances", execute: o.destroyInstances},
 		{name: "Disks", execute: o.destroyDisks},
 	}, {
+		// LB's must occur before Subnet cleanup
 		{name: "Load Balancers", execute: o.destroyLoadBalancers},
 	}, {
 		{name: "Subnets", execute: o.destroySubnets},
 	}, {
+		// Public Gateways must occur before FIP's cleanup
+		// Security Groups must occur before VPC cleanup
 		{name: "Images", execute: o.destroyImages},
 		{name: "Public Gateways", execute: o.destroyPublicGateways},
 		{name: "Security Groups", execute: o.destroySecurityGroups},
@@ -147,10 +151,14 @@ func (o *ClusterUninstaller) destroyCluster() error {
 		{name: "Dedicated Hosts", execute: o.destroyDedicatedHosts},
 		{name: "VPCs", execute: o.destroyVPCs},
 	}, {
+		// IAM must occur before COS cleanup
+		{name: "IAM Authorizations", execute: o.destroyIAMAuthorizations},
+	}, {
+		// COS must occur before RG cleanup
 		{name: "Cloud Object Storage Instances", execute: o.destroyCOSInstances},
 		{name: "Dedicated Host Groups", execute: o.destroyDedicatedHostGroups},
+	}, {
 		{name: "DNS Records", execute: o.destroyDNSRecords},
-		{name: "IAM Authorizations", execute: o.destroyIAMAuthorizations},
 		{name: "Resource Groups", execute: o.destroyResourceGroups},
 	}}