Suspend CronJob but keep instant backup working while backup is paused (#1024)

Author: hossainemruz

apis/stash/v1beta1/backup_session_types.go

@@ -57,7 +57,6 @@ const (
 	BackupSessionRunning   BackupSessionPhase = "Running"
 	BackupSessionSucceeded BackupSessionPhase = "Succeeded"
 	BackupSessionFailed    BackupSessionPhase = "Failed"
-	BackupSessionSkipped   BackupSessionPhase = "Skipped"
 	BackupSessionUnknown   BackupSessionPhase = "Unknown"
 )
 

pkg/backup/backupsession.go

@@ -207,11 +207,6 @@ func (c *BackupSessionController) processBackupSession(key string) error {
 }
 
 func (c *BackupSessionController) startBackupProcess(backupSession *api_v1beta1.BackupSession, invoker apis.Invoker, targetInfo apis.TargetInfo) (*restic.BackupOutput, error) {
-	// skip if backup invoker is paused
-	if invoker.Paused {
-		log.Infof("Skipping processing BackupSession %s/%s. Reason: Backup invoker is paused.", backupSession.Namespace, backupSession.Name)
-		return nil, nil
-	}
 
 	// if BackupSession already has been processed for this host then skip further processing
 	if c.isBackupTakenForThisHost(backupSession, targetInfo.Target) {
@@ -509,8 +504,7 @@ func (c *BackupSessionController) isBackupTakenForThisHost(backupSession *api_v1
 
 	// if overall backupSession phase is "Succeeded" or "Failed" or "Skipped" then it has been processed already
 	if backupSession.Status.Phase == api_v1beta1.BackupSessionSucceeded ||
-		backupSession.Status.Phase == api_v1beta1.BackupSessionFailed ||
-		backupSession.Status.Phase == api_v1beta1.BackupSessionSkipped {
+		backupSession.Status.Phase == api_v1beta1.BackupSessionFailed {
 		return true
 	}
 

pkg/cmds/create_backupsession.go

@@ -106,15 +106,6 @@ func (opt *options) createBackupSession() error {
 		OwnerReferences: []metav1.OwnerReference{},
 	}
 
-	// skip if backup invoker paused
-	if invoker.Paused {
-		msg := fmt.Sprintf("Skipping creating BackupSession. Reason: Backup invoker %s/%s is paused.", opt.namespace, invoker.ObjectMeta.Name)
-		log.Infoln(msg)
-
-		// write event to backup invoker denoting that backup session has been skipped
-		return writeBackupSessionSkippedEvent(opt.k8sClient, invoker.ObjectRef, msg)
-	}
-
 	wc := util.WorkloadClients{
 		KubeClient:       opt.k8sClient,
 		StashClient:      opt.stashClient,

pkg/controller/backup_configuration.go

@@ -29,6 +29,7 @@ import (
 	"stash.appscode.dev/stash/pkg/util"
 
 	"github.com/appscode/go/log"
+	"github.com/appscode/go/types"
 	"github.com/golang/glog"
 	batch_v1beta1 "k8s.io/api/batch/v1beta1"
 	core "k8s.io/api/core/v1"
@@ -111,12 +112,6 @@ func (c *StashController) applyBackupInvokerReconciliationLogic(invoker apis.Inv
 		if err != nil {
 			return err
 		}
-		// skip if BackupBatch paused
-		if invoker.Paused {
-			log.Infof("Skipping processing for invoker %s %s/%s. Reason: The invoker is paused.", invoker.ObjectRef.Kind, invoker.ObjectMeta.Namespace, invoker.ObjectMeta.Name)
-			return nil
-		}
-
 		for _, targetInfo := range invoker.TargetsInfo {
 			if targetInfo.Target != nil && invoker.Driver != api_v1beta1.VolumeSnapshotter &&
 				util.BackupModel(targetInfo.Target.Ref.Kind) == apis.ModelSidecar {
@@ -255,6 +250,7 @@ func (c *StashController) EnsureBackupTriggeringCronJob(invoker apis.Invoker) er
 		core_util.EnsureOwnerReference(&in.ObjectMeta, invoker.OwnerRef)
 
 		in.Spec.Schedule = invoker.Schedule
+		in.Spec.Suspend = types.BoolP(invoker.Paused) // this ensure that the CronJob is suspended when the backup invoker is paused.
 		in.Spec.JobTemplate.Labels = invoker.Labels
 		// ensure that job gets deleted on completion
 		in.Spec.JobTemplate.Labels[apis.KeyDeleteJobOnCompletion] = apis.AllowDeletingJobOnCompletion

pkg/controller/backup_session.go

@@ -152,15 +152,6 @@ func (c *StashController) applyBackupSessionReconciliationLogic(backupSession *a
 	} else if phase == api_v1beta1.BackupSessionRunning {
 		log.Infof("Skipping processing BackupSession %s/%s. Reason: phase is %q.", backupSession.Namespace, backupSession.Name, backupSession.Status.Phase)
 		return nil
-	} else if phase == api_v1beta1.BackupSessionSkipped {
-		log.Infof("Skipping processing BackupSession %s/%s. Reason: phase is %q.", backupSession.Namespace, backupSession.Name, backupSession.Status.Phase)
-		return nil
-	}
-
-	// skip if backup invoker is paused
-	if invoker.Paused {
-		log.Infof("Skipping processing BackupSession %s/%s. Reason: %s is paused.", backupSession.Namespace, backupSession.Name, invoker.ObjectRef.Kind)
-		return c.setBackupSessionSkipped(backupSession, fmt.Sprintf("backup invoker %s %s/%s is paused", invoker.ObjectRef.Kind, invoker.ObjectRef.Namespace, invoker.ObjectRef.Name))
 	}
 
 	// if preBackup hook exist, then execute preBackupHook
@@ -427,29 +418,6 @@ func (c *StashController) setBackupSessionFailed(invoker apis.Invoker, backupSes
 	return errors.NewAggregate([]error{backupErr, err})
 }
 
-func (c *StashController) setBackupSessionSkipped(backupSession *api_v1beta1.BackupSession, reason string) error {
-	// set BackupSession phase to "Skipped"
-	_, err := stash_util.UpdateBackupSessionStatus(c.stashClient.StashV1beta1(), backupSession, func(in *api_v1beta1.BackupSessionStatus) *api_v1beta1.BackupSessionStatus {
-		in.Phase = api_v1beta1.BackupSessionSkipped
-		in.Targets = backupSession.Status.Targets
-		return in
-	})
-	if err != nil {
-		return err
-	}
-
-	// write skip event
-	_, err = eventer.CreateEvent(
-		c.kubeClient,
-		eventer.EventSourceBackupSessionController,
-		backupSession,
-		core.EventTypeWarning,
-		eventer.EventReasonBackupSessionSkipped,
-		reason,
-	)
-	return err
-}
-
 func (c *StashController) setTargetPhaseRunning(target *api_v1beta1.BackupTarget, driver api_v1beta1.Snapshotter, backupSession *api_v1beta1.BackupSession) (*api_v1beta1.BackupSession, error) {
 	// find out the total number of hosts in target that will be backed up in this backup session
 	totalHosts, err := c.getTotalHosts(target, backupSession.Namespace, driver)

test/e2e/framework/backup_configuration.go

@@ -87,6 +87,34 @@ func (f *Framework) EventuallyCronJobCreated(meta metav1.ObjectMeta) GomegaAsync
 	)
 }
 
+func (f *Framework) EventuallyCronJobSuspended(meta metav1.ObjectMeta) GomegaAsyncAssertion {
+	return Eventually(
+		func() bool {
+			cronJob, err := f.KubeClient.BatchV1beta1().CronJobs(meta.Namespace).Get(meta.Name, metav1.GetOptions{})
+			if err != nil {
+				return false
+			}
+			return *cronJob.Spec.Suspend
+		},
+		time.Minute*2,
+		time.Second*5,
+	)
+}
+
+func (f *Framework) EventuallyCronJobResumed(meta metav1.ObjectMeta) GomegaAsyncAssertion {
+	return Eventually(
+		func() bool {
+			cronJob, err := f.KubeClient.BatchV1beta1().CronJobs(meta.Namespace).Get(meta.Name, metav1.GetOptions{})
+			if err != nil {
+				return false
+			}
+			return !*cronJob.Spec.Suspend
+		},
+		time.Minute*2,
+		time.Second*5,
+	)
+}
+
 func (f *Framework) EventuallyBackupConfigurationCreated(meta metav1.ObjectMeta) GomegaAsyncAssertion {
 	return Eventually(
 		func() bool {

test/e2e/framework/backup_session.go

@@ -19,6 +19,7 @@ package framework
 import (
 	"fmt"
 	"strings"
+	"time"
 
 	"stash.appscode.dev/stash/apis"
 	"stash.appscode.dev/stash/apis/stash/v1beta1"
@@ -47,7 +48,6 @@ func (f *Framework) EventuallyBackupProcessCompleted(meta metav1.ObjectMeta) Gom
 			}
 			if bs.Status.Phase == v1beta1.BackupSessionSucceeded ||
 				bs.Status.Phase == v1beta1.BackupSessionFailed ||
-				bs.Status.Phase == v1beta1.BackupSessionSkipped ||
 				bs.Status.Phase == v1beta1.BackupSessionUnknown {
 				return true
 			}
@@ -103,3 +103,47 @@ func (fi *Invocation) TriggerInstantBackup(objMeta metav1.ObjectMeta, invokerRef
 
 	return fi.StashClient.StashV1beta1().BackupSessions(backupSession.Namespace).Create(backupSession)
 }
+
+func (fi *Invocation) EventuallyBackupCount(invokerMeta metav1.ObjectMeta, invokerKind string) GomegaAsyncAssertion {
+	return Eventually(func() int64 {
+		count, err := fi.GetSuccessfulBackupSessionCount(invokerMeta, invokerKind)
+		if err != nil {
+			return 0
+		}
+		return count
+	}, 3*time.Minute, 5*time.Second)
+}
+
+func (fi *Invocation) GetSuccessfulBackupSessionCount(invokerMeta metav1.ObjectMeta, invokerKind string) (int64, error) {
+	backupSessions, err := fi.StashClient.StashV1beta1().BackupSessions(fi.namespace).List(metav1.ListOptions{})
+	if err != nil {
+		return 0, err
+	}
+
+	count := int64(0)
+	for _, bs := range backupSessions.Items {
+		if bs.Spec.Invoker.Kind == invokerKind &&
+			bs.Spec.Invoker.Name == invokerMeta.Name &&
+			bs.Status.Phase == v1beta1.BackupSessionSucceeded {
+			count++
+		}
+	}
+	return count, nil
+}
+
+func (fi *Invocation) EventuallyRunningBackupCompleted(invokerMeta metav1.ObjectMeta, invokerKind string) GomegaAsyncAssertion {
+	return Eventually(func() bool {
+		backupSessions, err := fi.StashClient.StashV1beta1().BackupSessions(fi.namespace).List(metav1.ListOptions{})
+		if err != nil {
+			return false
+		}
+		for _, bs := range backupSessions.Items {
+			if bs.Spec.Invoker.Kind == invokerKind &&
+				bs.Spec.Invoker.Name == invokerMeta.Name &&
+				bs.Status.Phase == v1beta1.BackupSessionRunning {
+				return false
+			}
+		}
+		return true
+	}, 3*time.Minute, 5*time.Second)
+}