Run CAPI Backup on workload cluster during upgrade

pkg/task/task.go

@@ -27,29 +27,29 @@ type Task interface {
 
 // Command context maintains the mutable and shared entities.
 type CommandContext struct {
-	ClientFactory             interfaces.ClientFactory
-	Bootstrapper              interfaces.Bootstrapper
-	Provider                  providers.Provider
-	ClusterManager            interfaces.ClusterManager
-	GitOpsManager             interfaces.GitOpsManager
-	Validations               interfaces.Validator
-	Writer                    filewriter.FileWriter
-	EksdInstaller             interfaces.EksdInstaller
-	PackageInstaller          interfaces.PackageInstaller
-	EksdUpgrader              interfaces.EksdUpgrader
-	ClusterUpgrader           interfaces.ClusterUpgrader
-	ClusterCreator            interfaces.ClusterCreator
-	CAPIManager               interfaces.CAPIManager
-	ClusterSpec               *cluster.Spec
-	CurrentClusterSpec        *cluster.Spec
-	UpgradeChangeDiff         *types.ChangeDiff
-	BootstrapCluster          *types.Cluster
-	ManagementCluster         *types.Cluster
-	WorkloadCluster           *types.Cluster
-	Profiler                  *Profiler
-	OriginalError             error
-	ManagementClusterStateDir string
-	ForceCleanup              bool
+	ClientFactory         interfaces.ClientFactory
+	Bootstrapper          interfaces.Bootstrapper
+	Provider              providers.Provider
+	ClusterManager        interfaces.ClusterManager
+	GitOpsManager         interfaces.GitOpsManager
+	Validations           interfaces.Validator
+	Writer                filewriter.FileWriter
+	EksdInstaller         interfaces.EksdInstaller
+	PackageInstaller      interfaces.PackageInstaller
+	EksdUpgrader          interfaces.EksdUpgrader
+	ClusterUpgrader       interfaces.ClusterUpgrader
+	ClusterCreator        interfaces.ClusterCreator
+	CAPIManager           interfaces.CAPIManager
+	ClusterSpec           *cluster.Spec
+	CurrentClusterSpec    *cluster.Spec
+	UpgradeChangeDiff     *types.ChangeDiff
+	BootstrapCluster      *types.Cluster
+	ManagementCluster     *types.Cluster
+	WorkloadCluster       *types.Cluster
+	Profiler              *Profiler
+	OriginalError         error
+	BackupClusterStateDir string
+	ForceCleanup          bool
 }
 
 func (c *CommandContext) SetError(err error) {
@@ -132,7 +132,7 @@ func (tr *taskRunner) RunTask(ctx context.Context, commandContext *CommandContex
 	var checkpointInfo CheckpointInfo
 	var err error
 
-	commandContext.ManagementClusterStateDir = fmt.Sprintf("cluster-state-backup-%s", time.Now().Format("2006-01-02T15_04_05"))
+	commandContext.BackupClusterStateDir = fmt.Sprintf("%s-backup-%s", commandContext.ClusterSpec.Cluster.Name, time.Now().Format("2006-01-02T15_04_05"))
 	commandContext.Profiler = &Profiler{
 		metrics: make(map[string]map[string]time.Duration),
 		starts:  make(map[string]map[string]time.Time),

pkg/workflows/management/post_cluster_upgrade.go

@@ -23,7 +23,7 @@ func (s *postClusterUpgrade) Run(ctx context.Context, commandContext *task.Comma
 	}
 
 	logger.Info("Cleaning up backup resources")
-	capiObjectFile := filepath.Join(commandContext.ManagementCluster.Name, commandContext.ManagementClusterStateDir)
+	capiObjectFile := filepath.Join(commandContext.ManagementCluster.Name, commandContext.BackupClusterStateDir)
 	if err := os.RemoveAll(capiObjectFile); err != nil {
 		logger.Info(fmt.Sprintf("management cluster CAPI backup file not found: %v", err))
 	}

pkg/workflows/management/pre_cluster_upgrade.go

@@ -12,12 +12,12 @@ type preClusterUpgrade struct{}
 
 // Run preClusterUpgrade implements steps to be performed before management cluster's upgrade.
 func (s *preClusterUpgrade) Run(ctx context.Context, commandContext *task.CommandContext) task.Task {
-	// Take best effort CAPI backup of workload cluster without filter.
-	// If that errors, then take CAPI backup filtering on only workload cluster.
+	// Take best effort CAPI backup of management cluster without filter.
+	// If that errors, then take CAPI backup filtering on only management cluster.
 	logger.Info("Backing up management cluster's resources before upgrading")
-	err := commandContext.ClusterManager.BackupCAPI(ctx, commandContext.ManagementCluster, commandContext.ManagementClusterStateDir, "")
+	err := commandContext.ClusterManager.BackupCAPI(ctx, commandContext.ManagementCluster, commandContext.BackupClusterStateDir, "")
 	if err != nil {
-		err = commandContext.ClusterManager.BackupCAPIWaitForInfrastructure(ctx, commandContext.ManagementCluster, commandContext.ManagementClusterStateDir, commandContext.ManagementCluster.Name)
+		err = commandContext.ClusterManager.BackupCAPIWaitForInfrastructure(ctx, commandContext.ManagementCluster, commandContext.BackupClusterStateDir, commandContext.ManagementCluster.Name)
 		if err != nil {
 			commandContext.SetError(err)
 			return &workflows.CollectMgmtClusterDiagnosticsTask{}

pkg/workflows/management/upgrade_test.go

@@ -106,7 +106,7 @@ func newUpgradeManagementTest(t *testing.T) *upgradeManagementTestSetup {
 			s.Cluster.Name = "management"
 			s.Cluster.Spec.DatacenterRef.Kind = v1alpha1.VSphereDatacenterKind
 		}),
-		managementStatePath: fmt.Sprintf("cluster-state-backup-%s", time.Now().Format("2006-01-02T15_04_05")),
+		managementStatePath: fmt.Sprintf("%s-backup-%s", "management", time.Now().Format("2006-01-02T15_04_05")),
 	}
 }
 

pkg/workflows/upgrade.go

@@ -459,9 +459,9 @@ func (s *moveManagementToBootstrapTask) Run(ctx context.Context, commandContext
 	// Take best effort CAPI backup of workload cluster without filter.
 	// If that errors, then take CAPI backup filtering on only workload cluster.
 	logger.Info("Backing up workload cluster's management resources before moving to bootstrap cluster")
-	err := commandContext.ClusterManager.BackupCAPI(ctx, commandContext.WorkloadCluster, commandContext.ManagementClusterStateDir, "")
+	err := commandContext.ClusterManager.BackupCAPI(ctx, commandContext.WorkloadCluster, commandContext.BackupClusterStateDir, "")
 	if err != nil {
-		err = commandContext.ClusterManager.BackupCAPIWaitForInfrastructure(ctx, commandContext.WorkloadCluster, commandContext.ManagementClusterStateDir, commandContext.WorkloadCluster.Name)
+		err = commandContext.ClusterManager.BackupCAPIWaitForInfrastructure(ctx, commandContext.WorkloadCluster, commandContext.BackupClusterStateDir, commandContext.WorkloadCluster.Name)
 		if err != nil {
 			commandContext.SetError(err)
 			return &CollectDiagnosticsTask{}
@@ -521,7 +521,7 @@ func (s *upgradeWorkloadClusterTask) Run(ctx context.Context, commandContext *ta
 		// Take backup of bootstrap cluster capi components
 		if commandContext.BootstrapCluster != nil {
 			logger.Info("Backing up management components from bootstrap cluster")
-			err := commandContext.ClusterManager.BackupCAPIWaitForInfrastructure(ctx, commandContext.BootstrapCluster, fmt.Sprintf("bootstrap-%s", commandContext.ManagementClusterStateDir), commandContext.WorkloadCluster.Name)
+			err := commandContext.ClusterManager.BackupCAPIWaitForInfrastructure(ctx, commandContext.BootstrapCluster, fmt.Sprintf("bootstrap-%s", commandContext.BackupClusterStateDir), commandContext.WorkloadCluster.Name)
 			if err != nil {
 				logger.Info("Bootstrap management component backup failed, use existing workload cluster backup", "error", err)
 			}
@@ -701,7 +701,7 @@ func (s *deleteBootstrapClusterTask) Run(ctx context.Context, commandContext *ta
 			logger.Info(fmt.Sprintf("%v", err))
 		}
 
-		capiObjectFile := filepath.Join(commandContext.BootstrapCluster.Name, commandContext.ManagementClusterStateDir)
+		capiObjectFile := filepath.Join(commandContext.BootstrapCluster.Name, commandContext.BackupClusterStateDir)
 		if err := os.RemoveAll(capiObjectFile); err != nil {
 			logger.Info(fmt.Sprintf("management cluster CAPI backup file not found: %v", err))
 		}

pkg/workflows/upgrade_test.go

@@ -93,7 +93,7 @@ func newUpgradeTest(t *testing.T) *upgradeTestSetup {
 		ctx:                 context.Background(),
 		newClusterSpec:      test.NewClusterSpec(func(s *cluster.Spec) { s.Cluster.Name = "cluster-name" }),
 		workloadCluster:     &types.Cluster{Name: "workload"},
-		managementStatePath: fmt.Sprintf("cluster-state-backup-%s", time.Now().Format("2006-01-02T15_04_05")),
+		managementStatePath: fmt.Sprintf("%s-backup-%s", "cluster-name", time.Now().Format("2006-01-02T15_04_05")),
 	}
 }
 

pkg/workflows/workload/post_cluster_upgrade.go

@@ -0,0 +1,38 @@
+package workload
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/aws/eks-anywhere/pkg/logger"
+	"github.com/aws/eks-anywhere/pkg/task"
+)
+
+type postClusterUpgrade struct{}
+
+// Run postClusterUpgrade implements steps to be performed after the upgrade process.
+func (s *postClusterUpgrade) Run(ctx context.Context, commandContext *task.CommandContext) task.Task {
+	logger.Info("Cleaning up backup resources")
+	capiObjectFile := filepath.Join(commandContext.ManagementCluster.Name, commandContext.BackupClusterStateDir)
+	if err := os.RemoveAll(capiObjectFile); err != nil {
+		logger.Info(fmt.Sprintf("workload cluster CAPI backup file not found: %v", err))
+	}
+
+	return nil
+}
+
+func (s *postClusterUpgrade) Name() string {
+	return "post-cluster-upgrade"
+}
+
+func (s *postClusterUpgrade) Checkpoint() *task.CompletedTask {
+	return &task.CompletedTask{
+		Checkpoint: nil,
+	}
+}
+
+func (s *postClusterUpgrade) Restore(ctx context.Context, commandContext *task.CommandContext, completedTask *task.CompletedTask) (task.Task, error) {
+	return nil, nil
+}

pkg/workflows/workload/pre_cluster_upgrade.go

@@ -0,0 +1,38 @@
+package workload
+
+import (
+	"context"
+
+	"github.com/aws/eks-anywhere/pkg/logger"
+	"github.com/aws/eks-anywhere/pkg/task"
+	"github.com/aws/eks-anywhere/pkg/workflows"
+)
+
+type preClusterUpgrade struct{}
+
+// Run preClusterUpgrade implements steps to be performed before workload cluster's upgrade.
+func (s *preClusterUpgrade) Run(ctx context.Context, commandContext *task.CommandContext) task.Task {
+	// Take CAPI backup filtering on only current workload cluster.
+	logger.Info("Backing up workload cluster's resources before upgrading")
+	err := commandContext.ClusterManager.BackupCAPI(ctx, commandContext.ManagementCluster, commandContext.BackupClusterStateDir, commandContext.WorkloadCluster.Name)
+	if err != nil {
+		commandContext.SetError(err)
+		return &workflows.CollectMgmtClusterDiagnosticsTask{}
+	}
+
+	return &upgradeCluster{}
+}
+
+func (s *preClusterUpgrade) Name() string {
+	return "pre-cluster-upgrade"
+}
+
+func (s *preClusterUpgrade) Checkpoint() *task.CompletedTask {
+	return &task.CompletedTask{
+		Checkpoint: nil,
+	}
+}
+
+func (s *preClusterUpgrade) Restore(ctx context.Context, commandContext *task.CommandContext, completedTask *task.CompletedTask) (task.Task, error) {
+	return &upgradeCluster{}, nil
+}

pkg/workflows/workload/upgrade_test.go

@@ -2,9 +2,11 @@ package workload_test
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"os"
 	"testing"
+	"time"
 
 	"github.com/golang/mock/gomock"
 
@@ -21,22 +23,23 @@ import (
 )
 
 type upgradeTestSetup struct {
-	t                  *testing.T
-	clusterManager     *mocks.MockClusterManager
-	gitOpsManager      *mocks.MockGitOpsManager
-	provider           *providermocks.MockProvider
-	writer             *writermocks.MockFileWriter
-	validator          *mocks.MockValidator
-	eksd               *mocks.MockEksdInstaller
-	packageInstaller   *mocks.MockPackageInstaller
-	clusterUpgrader    *mocks.MockClusterUpgrader
-	datacenterConfig   providers.DatacenterConfig
-	machineConfigs     []providers.MachineConfig
-	ctx                context.Context
-	currentClusterSpec *cluster.Spec
-	clusterSpec        *cluster.Spec
-	workloadCluster    *types.Cluster
-	workload           *workload.Upgrade
+	t                     *testing.T
+	clusterManager        *mocks.MockClusterManager
+	gitOpsManager         *mocks.MockGitOpsManager
+	provider              *providermocks.MockProvider
+	writer                *writermocks.MockFileWriter
+	validator             *mocks.MockValidator
+	eksd                  *mocks.MockEksdInstaller
+	packageInstaller      *mocks.MockPackageInstaller
+	clusterUpgrader       *mocks.MockClusterUpgrader
+	datacenterConfig      providers.DatacenterConfig
+	machineConfigs        []providers.MachineConfig
+	ctx                   context.Context
+	currentClusterSpec    *cluster.Spec
+	clusterSpec           *cluster.Spec
+	workloadCluster       *types.Cluster
+	workload              *workload.Upgrade
+	backupClusterStateDir string
 }
 
 func newUpgradeTest(t *testing.T) *upgradeTestSetup {
@@ -96,7 +99,8 @@ func newUpgradeTest(t *testing.T) *upgradeTestSetup {
 			s.ManagementCluster = &types.Cluster{Name: "management"}
 			s.Cluster.Spec.KubernetesVersion = v1alpha1.Kube128
 		}),
-		workloadCluster: &types.Cluster{Name: "workload"},
+		workloadCluster:       &types.Cluster{Name: "workload"},
+		backupClusterStateDir: fmt.Sprintf("%s-backup-%s", "workload", time.Now().Format("2006-01-02T15_04_05")),
 	}
 }
 
@@ -137,6 +141,12 @@ func (c *upgradeTestSetup) expectPreflightValidationsToPass() {
 	c.validator.EXPECT().PreflightValidations(c.ctx).Return(nil)
 }
 
+func (c *upgradeTestSetup) expectBackupWorkloadFromCluster(err error) {
+	gomock.InOrder(
+		c.clusterManager.EXPECT().BackupCAPI(c.ctx, c.clusterSpec.ManagementCluster, c.backupClusterStateDir, c.workloadCluster.Name).Return(err),
+	)
+}
+
 func (c *upgradeTestSetup) expectSaveLogsManagement() {
 	c.clusterManager.EXPECT().SaveLogsManagementCluster(c.ctx, c.clusterSpec, c.clusterSpec.ManagementCluster)
 	c.expectWrite()
@@ -154,6 +164,7 @@ func TestUpgradeRunSuccess(t *testing.T) {
 	test.expectPreflightValidationsToPass()
 	test.expectDatacenterConfig()
 	test.expectMachineConfigs()
+	test.expectBackupWorkloadFromCluster(nil)
 	test.expectUpgradeWorkloadCluster(nil)
 	test.expectWriteWorkloadClusterConfig(nil)
 
@@ -171,6 +182,7 @@ func TestUpgradeRunUpgradeFail(t *testing.T) {
 	test.expectPreflightValidationsToPass()
 	test.expectDatacenterConfig()
 	test.expectMachineConfigs()
+	test.expectBackupWorkloadFromCluster(nil)
 	test.expectUpgradeWorkloadCluster(fmt.Errorf("boom"))
 	test.expectSaveLogsManagement()
 
@@ -210,6 +222,23 @@ func TestUpgradeRunValidateFail(t *testing.T) {
 	}
 }
 
+func TestUpgradeWorkloadRunBackupFailed(t *testing.T) {
+	features.ClearCache()
+	os.Setenv(features.UseControllerForCli, "true")
+	test := newUpgradeTest(t)
+	test.expectSetup()
+	test.expectPreflightValidationsToPass()
+	test.expectDatacenterConfig()
+	test.expectMachineConfigs()
+	test.expectBackupWorkloadFromCluster(errors.New(""))
+	test.expectSaveLogsManagement()
+
+	err := test.run()
+	if err == nil {
+		t.Fatalf("Upgrade.Run() err = %v, want err = nil", err)
+	}
+}
+
 func TestUpgradeRunWriteClusterConfigFail(t *testing.T) {
 	features.ClearCache()
 	os.Setenv(features.UseControllerForCli, "true")
@@ -218,6 +247,7 @@ func TestUpgradeRunWriteClusterConfigFail(t *testing.T) {
 	test.expectPreflightValidationsToPass()
 	test.expectDatacenterConfig()
 	test.expectMachineConfigs()
+	test.expectBackupWorkloadFromCluster(nil)
 	test.expectUpgradeWorkloadCluster(nil)
 	test.expectWriteWorkloadClusterConfig(fmt.Errorf("boom"))
 	test.expectWrite()

pkg/workflows/workload/validate.go

@@ -69,7 +69,7 @@ func (s *setAndValidateUpgradeWorkloadTask) Run(ctx context.Context, commandCont
 		commandContext.SetError(err)
 		return nil
 	}
-	return &upgradeCluster{}
+	return &preClusterUpgrade{}
 }
 
 func (s *setAndValidateUpgradeWorkloadTask) providerValidation(ctx context.Context, commandContext *task.CommandContext) []validations.Validation {

pkg/workflows/workload/writeclusterconfig.go

@@ -28,6 +28,9 @@
 	if commandContext.OriginalError == nil {
 		logger.MarkSuccess(successMsg)
 	}
+	if commandContext.CurrentClusterSpec != nil {
+		return &postClusterUpgrade{}
+	}
 	return nil
 }
 
@@ -42,5 +45,8 @@
 }
 
 func (s *writeClusterConfig) Restore(ctx context.Context, commandContext *task.CommandContext, completedTask *task.CompletedTask) (task.Task, error) {
+	if commandContext.CurrentClusterSpec == nil {
+		return &postClusterUpgrade{}, nil
+	}
 	return nil, nil
 }