Add logging around certain ways a task can fail

This adds some logging for cases, where the task executing process dies unexpectedly (e.g. OOM) and is unable to properly set the task state and error reason. fixes #9666
pulp · Jan 14, 2022 · ebb5aef · ebb5aef
1 parent d767beb
commit ebb5aef
Show file tree

Hide file tree

Showing 2 changed files with 16 additions and 0 deletions.
diff --git a/CHANGES/9666.bugfix b/CHANGES/9666.bugfix
@@ -0,0 +1 @@
+Added proper logging around certain ways a task could fail.
diff --git a/pulpcore/tasking/pulpcore_worker.py b/pulpcore/tasking/pulpcore_worker.py
@@ -301,6 +301,21 @@ def supervise_task(self, task):
                         cancel_reason = "Aborted during worker shutdown."
                         break
             task_process.join()
+            if not cancel_state and task_process.exitcode != 0:
+                _logger.warn(
+                    _("Task process for %s exited with non zero exitcode %i."),
+                    task.pk,
+                    task_process.exitcode,
+                )
+                cancel_state = TASK_STATES.FAILED
+                if task_process.exitcode < 0:
+                    cancel_reason = "Killed by signal {sig_num}.".format(
+                        sig_num=-task_process.exitcode
+                    )
+                else:
+                    cancel_reason = "Task process died unexpectedly with exitcode {code}.".format(
+                        code=task_process.exitcode
+                    )
             if cancel_state:
                 self.cancel_abandoned_task(task, cancel_state, cancel_reason)
         if task.reserved_resources_record: