fix(health): remove CronJob progressing/suspended status (#24430)

Signed-off-by: Alexandre Gaudreault <alexandre_gaudreault@intuit.com>

Author: agaudreault

docs/operator-manual/upgrading/3.1-3.2.md

@@ -40,20 +40,25 @@ The `ManifestRequest` and `RepoServerAppDetailsQuery` messages are used by the f
 
 ## CronJob Health
 
-This realease introduce the addition of CronJob's health, a longtime omitted heath status for a native Kubernetes resource.
-The health of a CronJob is based on whether or not Jobs are currently running, and if the last completed Job was successful.
+After the upgrade, Application's status may transition to `Degraded` depending on the CronJob health.
 
-After the upgrade, Application's status may transition to `Degraded`, `Progressing` or `Suspended` depending on the CronJob health.
+!!! note "CronJob with running jobs"
 
-If the CronJob is permanently `Suspended`, then the aggregated health of the Application will now be `Suspended` instead of `Healthy`.
-If the CronJob is permanently running jobs, then the aggregated health of the Application will now be `Progressing` instead of `Healthy`.
+    If the CronJob is `Degraded` and a new job is scheduled, the health will change to `Healthy` until the active job completes.
+    This may cause your application to go from `Degraded` to `Healthy` to `Degraded` again. The CronJob status does not contain enough
+    information to infer the health of the last completed job if there are active jobs.
 
-If you do not want your CronJob to affect the Application's aggregated Health, you can configure the annotation
-`argocd.argoproj.io/ignore-healthcheck: "true"` on the CronJob resource.
+    If the CronJob constantly has active jobs, the health will be constantly `Healthy` even if the last job failed.
+
+!!! note "CronJob with suspended state"
 
-The health can also be configured globally using the `resource.customizations.health.batch_CronJob` configuration to change the default behaviour.
+    If the CronJob is in a suspended state, the CronJob status will remain Healthy. You can override this behaviour by configuring the
+    health check using the `resource.customizations.health.batch_CronJob` key in the argocd-cm ConfigMap.
 
-## Breaking Changes
+    If you decide to do so and the CronJob is `Suspended`, then the aggregated health of the Application will now be `Suspended` instead of `Healthy`.
+
+If you do not want your CronJob to affect the Application's aggregated Health, you can configure the annotation
+`argocd.argoproj.io/ignore-healthcheck: "true"` on the CronJob resource.
 
 ## Sanitized project API response
 

resource_customizations/batch/CronJob/health.lua

@@ -1,36 +1,52 @@
 hs = {}
 
 if obj.spec.suspend == true then
-    hs.status = "Suspended"
+    -- Set to Healthy insted of Suspended until bug is resolved
+    -- See https://github.com/argoproj/argo-cd/issues/24428
+    hs.status = "Healthy"
     hs.message = "CronJob is Suspended"
     return hs
 end
 
 if obj.status ~= nil then
-    if obj.status.active ~= nil and table.getn(obj.status.active) > 0 then
-        -- We could be Progressing very often, depending on the Cron schedule, which would bubble up
-        -- to the Application health. If this is undesired, the annotation `argocd.argoproj.io/ignore-healthcheck: "true"`
-        -- can be added on the CronJob.
-        hs.status = "Progressing"
-        hs.message = string.format("Waiting for %d Jobs to complete", table.getn(obj.status.active))
-        return hs
-    end
+    if obj.status.lastScheduleTime ~= nil then
+
+        -- Job is running its first execution and has not yet reported any success
+        if obj.status.lastSuccessfulTime == nil then
+            -- Set to healthy even if it may be degraded, because we dont know
+            -- if it was not yet executed or if it never succeeded
+            hs.status = "Healthy"
+            hs.message = "The Cronjob never completed succesfully. It may not be healthy"
+            return hs
+        end
+
+
+        -- Job is progressing, so lastScheduleTime will always be grater than lastSuccessfulTime
+        -- Set to healthy since we do not know if it is Degraded
+        -- See https://github.com/argoproj/argo-cd/issues/24429
+        if obj.status.active ~= nil and table.getn(obj.status.active) > 0 then
+            hs.status = "Healthy"
+            hs.message = "The job is running. Its last execution may not have been successful"
+            return hs
+        end
 
     -- If the CronJob has no active jobs and the lastSuccessfulTime < lastScheduleTime
     -- then we know it failed the last execution
-    if obj.status.lastScheduleTime ~= nil then
-        -- No issue comparing time as text
-        if obj.status.lastSuccessfulTime == nil or obj.status.lastSuccessfulTime < obj.status.lastScheduleTime then
+        if obj.status.lastSuccessfulTime ~= nil and obj.status.lastSuccessfulTime < obj.status.lastScheduleTime then
             hs.status = "Degraded"
             hs.message = "CronJob has not completed its last execution successfully"
             return hs
         end
+
         hs.message = "CronJob has completed its last execution successfully"
+        hs.status = "Healthy"
+        return hs
     end
 
     -- There is no way to know if as CronJob missed its execution based on status
     -- so we assume Healthy even if a cronJob is not getting scheduled.
     -- https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/#job-creation
+    hs.message = "CronJob has not been scheduled yet"
     hs.status = "Healthy"
     return hs
 end

resource_customizations/batch/CronJob/health_test.yaml

@@ -5,21 +5,21 @@ tests:
     inputPath: testdata/healthy.yaml
   - healthStatus:
       status: Healthy
-      message: ''
+      message: CronJob has not been scheduled yet
     inputPath: testdata/never-scheduled.yaml
   - healthStatus:
       status: Degraded
       message: CronJob has not completed its last execution successfully
     inputPath: testdata/degraded.yaml
   - healthStatus:
-      status: Degraded
-      message: CronJob has not completed its last execution successfully
+      status: Healthy
+      message: The Cronjob never completed succesfully. It may not be healthy
     inputPath: testdata/never-succeeded.yaml
   - healthStatus:
-      status: Progressing
-      message: Waiting for 1 Jobs to complete
+      status: Healthy
+      message: The job is running. Its last execution may not have been successful
     inputPath: testdata/active.yaml
   - healthStatus:
-      status: Suspended
+      status: Healthy
       message: CronJob is Suspended
     inputPath: testdata/suspended.yaml