fix: recreate cronjob after change on schedule (#2688)

tsuru · Apr 4, 2024 · 6083928 · 6083928
1 parent 410c713
commit 6083928
Show file tree

Hide file tree

Showing 2 changed files with 127 additions and 0 deletions.
diff --git a/provision/kubernetes/job.go b/provision/kubernetes/job.go
@@ -24,6 +24,7 @@ import (
 	k8sErrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/utils/ptr"
 
 	jobTypes "github.com/tsuru/tsuru/types/job"
 )
@@ -130,6 +131,21 @@ func ensureCronjob(ctx context.Context, client *ClusterClient, job *jobTypes.Job
 		return errors.WithStack(err)
 	}
 
+	// when the schedule suffer changes some cronjobs may suffer a unexpected execution
+	// for these reason we decided to recreate the entire cronjob to avoid this
+	if existingCronjob != nil && existingCronjob.Spec.Schedule != job.Spec.Schedule {
+		propagationPolicy := metav1.DeletePropagationForeground
+		err = client.BatchV1().CronJobs(namespace).Delete(ctx, existingCronjob.Name, metav1.DeleteOptions{
+			GracePeriodSeconds: ptr.To[int64](0),
+			PropagationPolicy:  &propagationPolicy,
+		})
+
+		if err != nil {
+			return errors.WithStack(err)
+		}
+		existingCronjob = nil
+	}
+
 	concurrencyPolicy := ""
 	if job.Spec.ConcurrencyPolicy != nil {
 		concurrencyPolicy = *job.Spec.ConcurrencyPolicy

diff --git a/provision/kubernetes/job_test.go b/provision/kubernetes/job_test.go
@@ -24,6 +24,7 @@ import (
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	k8sTypes "k8s.io/apimachinery/pkg/types"
+	"k8s.io/utils/ptr"
 )
 
 func (s *S) TestProvisionerCreateCronJob(c *check.C) {
@@ -439,6 +440,116 @@ func (s *S) TestProvisionerUpdateCronJob(c *check.C) {
 				},
 			},
 		},
+		{
+			name: "changing the schedule cronjob",
+			setup: func() {
+				cj := jobTypes.Job{
+					Name:      "myjob",
+					TeamOwner: s.team.Name,
+					Pool:      "test-default",
+					Spec: jobTypes.JobSpec{
+						Schedule: "* * * * *",
+						Container: jobTypes.ContainerInfo{
+							OriginalImageSrc: "ubuntu:latest",
+							Command:          []string{"echo", "hello world"},
+						},
+					},
+				}
+				err := s.p.EnsureJob(context.TODO(), &cj)
+				waitCron()
+				c.Assert(err, check.IsNil)
+			},
+			scenario: func() {
+				newCJ := jobTypes.Job{
+					Name:      "myjob",
+					TeamOwner: s.team.Name,
+					Pool:      "test-default",
+					Plan:      app.Plan{Name: "c4m2"},
+					Spec: jobTypes.JobSpec{
+						Schedule: "*/2 * * * *",
+						Container: jobTypes.ContainerInfo{
+							OriginalImageSrc: "ubuntu:latest",
+							Command:          []string{"echo", "hello world"},
+						},
+					},
+				}
+				err := s.p.EnsureJob(context.TODO(), &newCJ)
+				waitCron()
+				c.Assert(err, check.IsNil)
+			},
+			expectedTarget: &batchv1.CronJob{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "myjob",
+					Namespace: "default",
+					Labels: map[string]string{
+						"app.kubernetes.io/component":  "job",
+						"app.kubernetes.io/managed-by": "tsuru",
+						"app.kubernetes.io/name":       "tsuru-job",
+						"app.kubernetes.io/instance":   "myjob",
+						"tsuru.io/is-tsuru":            "true",
+						"tsuru.io/is-service":          "true",
+						"tsuru.io/job-name":            "myjob",
+						"tsuru.io/job-pool":            "test-default",
+						"tsuru.io/job-team":            "admin",
+						"tsuru.io/is-job":              "true",
+						"tsuru.io/job-manual":          "false",
+						"tsuru.io/is-build":            "false",
+						"tsuru.io/is-deploy":           "false",
+					},
+					Annotations: map[string]string{},
+				},
+				Spec: batchv1.CronJobSpec{
+					Schedule: "*/2 * * * *",
+					Suspend:  ptr.To[bool](false),
+					JobTemplate: batchv1.JobTemplateSpec{
+						Spec: batchv1.JobSpec{
+							TTLSecondsAfterFinished: func() *int32 { defaultTTL := int32(86400); return &defaultTTL }(),
+							ActiveDeadlineSeconds:   func() *int64 { r := int64(60 * 60); return &r }(),
+							Template: corev1.PodTemplateSpec{
+								ObjectMeta: metav1.ObjectMeta{
+									Labels: map[string]string{
+										"app.kubernetes.io/component":  "job",
+										"app.kubernetes.io/managed-by": "tsuru",
+										"app.kubernetes.io/name":       "tsuru-job",
+										"app.kubernetes.io/instance":   "myjob",
+										"tsuru.io/is-tsuru":            "true",
+										"tsuru.io/is-service":          "true",
+										"tsuru.io/job-name":            "myjob",
+										"tsuru.io/job-pool":            "test-default",
+										"tsuru.io/job-team":            "admin",
+										"tsuru.io/is-job":              "true",
+										"tsuru.io/job-manual":          "false",
+										"tsuru.io/is-build":            "false",
+										"tsuru.io/is-deploy":           "false",
+									},
+									Annotations: map[string]string{},
+								},
+								Spec: corev1.PodSpec{
+									ServiceAccountName: "job-myjob",
+									Containers: []corev1.Container{
+										{
+											Name:    "job",
+											Image:   "ubuntu:latest",
+											Command: []string{"echo", "hello world"},
+											Env:     []corev1.EnvVar{},
+											Resources: corev1.ResourceRequirements{
+												Limits: corev1.ResourceList{
+													corev1.ResourceEphemeralStorage: resource.MustParse("100Mi"),
+												},
+												Requests: corev1.ResourceList{
+													corev1.ResourceEphemeralStorage: *resource.NewQuantity(0, resource.DecimalSI),
+												},
+											},
+										},
+									},
+									RestartPolicy: "OnFailure",
+								},
+							},
+						},
+					},
+				},
+			},
+		},
 	}
 	for _, tt := range tests {
 		tt.setup()