Fix another scenario where canceled tasks could be marked failed

closes: #7980
https://pulp.plan.io/issues/7980

CHANGES/7980.bugfix

@@ -0,0 +1 @@
+Fixed a scenario where canceled tasks could be marked failed.

pulpcore/tasking/util.py

@@ -51,6 +51,9 @@ def cancel(task_id):
     task_status.state = TASK_STATES.CANCELED
     task_status.save()
 
+    resource_job.cancel()
+    job.cancel()
+
     try:
         send_stop_job_command(redis_conn, job.get_id())
         send_stop_job_command(redis_conn, resource_job.get_id())
@@ -61,9 +64,6 @@ def cancel(task_id):
     # A hack to ensure that we aren't deleting resources still being used by the workhorse
     time.sleep(0.5)
 
-    resource_job.delete()
-    job.delete()
-
     with transaction.atomic():
         for report in task_status.progress_reports.all():
             if report.state not in TASK_FINAL_STATES:

pulpcore/tasking/worker.py

@@ -122,13 +122,16 @@ def handle_job_failure(self, job, **kwargs):
         try:
             task = Task.objects.get(pk=job.get_id())
             task.release_resources()
-        except Task.DoesNotExist:
-            pass
-        else:
             exc_type, exc, tb = sys.exc_info()
-            task.set_failed(exc, tb)
-
-        return super().handle_job_failure(job, **kwargs)
+            res = super().handle_job_failure(job, **kwargs)
+            # job "is_stopped" state determined during super().handle_job_failure()
+            if not job.is_stopped:
+                # stopped jobs go onto the failed queue in RQ, so we need to ignore those
+                # to avoid overwriting the task status
+                task.set_failed(exc, tb)
+            return res
+        except Task.DoesNotExist:
+            return super().handle_job_failure(job, **kwargs)
 
     def handle_job_success(self, job, queue, started_job_registry):
         """

requirements.txt

@@ -23,6 +23,6 @@ psycopg2>=2.7,<2.9
 PyYAML>=5.1.1,<5.5.0
 python-gnupg~=0.4.7
 redis>=3.4.0
-rq>=1.7,<1.8
+rq~=1.8.0
 setuptools>=39.2.0
 whitenoise>=5.0.0,<5.3.0