add backoff limit to jobs, also correct job controller to update stat…

…us correctly for pgbasebackup jobs
CrunchyData · Sep 28, 2018 · c8265b7 · c8265b7
1 parent 9232ec7
commit c8265b7
Show file tree

Hide file tree

Showing 8 changed files with 44 additions and 11 deletions.
diff --git a/apis/cr/v1/common.go b/apis/cr/v1/common.go
@@ -57,3 +57,12 @@ type PgContainerResources struct {
 	LimitsMemory   string `json:"limitsmemory"`
 	LimitsCPU      string `json:"limitscpu"`
 }
+
+// JobCompletedStatus ....
+const JobCompletedStatus = "completed"
+
+// JobSubmittedStatus ....
+const JobSubmittedStatus = "submitted"
+
+// JobErrorStatus ....
+const JobErrorStatus = "error"
diff --git a/conf/postgres-operator/backrest-job.json b/conf/postgres-operator/backrest-job.json
@@ -11,6 +11,7 @@
                 }
     },
     "spec": {
+	"backoffLimit": 0,
         "template": {
             "metadata": {
                 "name": "backrest-{{.Command}}-{{.ClusterName}}",

diff --git a/conf/postgres-operator/backrest-restore-job.json b/conf/postgres-operator/backrest-restore-job.json
@@ -9,6 +9,7 @@
         }
     },
     "spec": {
+	"backoffLimit": 0,
         "template": {
             "metadata": {
                 "name": "{{.RestoreName}}",

diff --git a/conf/postgres-operator/backup-job.json b/conf/postgres-operator/backup-job.json
@@ -10,6 +10,7 @@
 	}
     },
     "spec": {
+        "backoffLimit": 0,
         "template": {
             "metadata": {
                 "name": "{{.Name}}",

diff --git a/conf/postgres-operator/cluster/1/cluster-upgrade-job-1.json b/conf/postgres-operator/cluster/1/cluster-upgrade-job-1.json
@@ -5,6 +5,7 @@
         "name": "upgrade-{{.Name}}"
     },
     "spec": {
+	"backoffLimit": 0,
         "template": {
             "metadata": {
                 "name": "{{.Name}}",

diff --git a/conf/postgres-operator/rmdata-job.json b/conf/postgres-operator/rmdata-job.json
@@ -12,6 +12,7 @@
 	}
     },
     "spec": {
+	"backoffLimit": 0,
         "template": {
             "metadata": {
                 "name": "rmdata-{{.Name}}",

diff --git a/controller/jobcontroller.go b/controller/jobcontroller.go
@@ -87,14 +87,15 @@ func (c *JobController) onAdd(obj interface{}) {
 func (c *JobController) onUpdate(oldObj, newObj interface{}) {
 	job := newObj.(*apiv1.Job)
 	log.Debugf("[JobCONTROLLER] OnUpdate %s active=%d succeeded=%d conditions=[%v]", job.ObjectMeta.SelfLink, job.Status.Active, job.Status.Succeeded, job.Status.Conditions)
+	var err error
 	//label is "pgrmdata" and Status of Succeeded
 	labels := job.GetObjectMeta().GetLabels()
 	if job.Status.Succeeded > 0 && labels[util.LABEL_RMDATA] != "" {
 		log.Debugf("rmdata job labels=[%v]", labels)
 		log.Debugf("got a pgrmdata job status=%d", job.Status.Succeeded)
 		//remove the pvc referenced by that job
 		log.Debugf("deleting pvc " + labels["claimName"])
-		err := pvc.Delete(c.JobClientset, labels["claimName"], c.Namespace)
+		err = pvc.Delete(c.JobClientset, labels["claimName"], c.Namespace)
 		if err != nil {
 			log.Error(err)
 		}
@@ -104,21 +105,25 @@ func (c *JobController) onUpdate(oldObj, newObj interface{}) {
 		kubeapi.Deletepgtasks(c.JobClient, util.LABEL_RMDATA+"=true", c.Namespace)
 		kubeapi.DeleteJobs(c.JobClientset, util.LABEL_PG_CLUSTER+"="+job.ObjectMeta.Labels[util.LABEL_PG_CLUSTER], c.Namespace)
 
-	} else if job.Status.Succeeded > 0 && labels[util.LABEL_PGBACKUP] != "" {
-		log.Debugf("got a pgbackup job status=%d", job.Status.Succeeded)
-		log.Debugf("update the status to completed here for pgbackup %s\n ", labels[util.LABEL_PG_DATABASE])
-		dbname := job.ObjectMeta.Labels[util.LABEL_PG_DATABASE]
-
-		err := util.Patch(c.JobClient, "/spec/backupstatus", crv1.UpgradeCompletedStatus, "pgbackups", dbname, c.Namespace)
-
+	} else if labels[util.LABEL_PGBACKUP] != "" {
+		dbname := job.ObjectMeta.Labels[util.LABEL_PG_CLUSTER]
+		status := crv1.JobCompletedStatus
+		log.Debugf("got a pgbackup job status=%d for %s", job.Status.Succeeded, dbname)
+		if job.Status.Succeeded == 0 {
+			status = crv1.JobErrorStatus
+		}
+		err = util.Patch(c.JobClient, "/spec/backupstatus", status, "pgbackups", dbname, c.Namespace)
 		if err != nil {
 			log.Error("error in patching pgbackup " + labels["pg-database"] + err.Error())
 		}
-
-	} else if job.Status.Succeeded > 0 && labels[util.LABEL_BACKREST] != "" {
+	} else if labels[util.LABEL_BACKREST] != "" {
 		log.Debugf("got a backrest job status=%d", job.Status.Succeeded)
 		log.Debugf("update the status to completed here for backrest %s\n ", labels[util.LABEL_PG_DATABASE])
-		err := util.Patch(c.JobClient, "/spec/backreststatus", crv1.UpgradeCompletedStatus, "pgtasks", job.ObjectMeta.SelfLink, c.Namespace)
+		status := crv1.JobCompletedStatus
+		if job.Status.Succeeded == 0 {
+			status = crv1.JobErrorStatus
+		}
+		err = util.Patch(c.JobClient, "/spec/backreststatus", status, "pgtasks", job.ObjectMeta.SelfLink, c.Namespace)
 		if err != nil {
 			log.Error("error in patching pgtask " + job.ObjectMeta.SelfLink + err.Error())
 		}

diff --git a/operator/backup/backup.go b/operator/backup/backup.go
@@ -28,6 +28,7 @@ import (
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"os"
+	"time"
 )
 
 type jobTemplateFields struct {
@@ -123,4 +124,17 @@ func DeleteBackupBase(clientset *kubernetes.Clientset, client *rest.RESTClient,
 		log.Error("error deleting Job " + jobName + err.Error())
 		return
 	}
+
+	//make sure job is actually reporting as deleted
+	for i := 0; i < 5; i++ {
+		_, found := kubeapi.GetJob(clientset, jobName, namespace)
+		if !found {
+			break
+		}
+		if err != nil {
+			log.Error(err)
+		}
+		log.Debugf("waiting for backup job to report being deleted")
+		time.Sleep(time.Second * time.Duration(3))
+	}
 }