From fb7c3ebd266046ca739446b7868e8e8f82646fc6 Mon Sep 17 00:00:00 2001 From: Brian Tiemann Date: Thu, 21 Nov 2024 09:51:09 -0500 Subject: [PATCH 1/5] Wait until job1 exists in Redis before enqueueing job2 --- netbox/netbox/tests/test_jobs.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/netbox/netbox/tests/test_jobs.py b/netbox/netbox/tests/test_jobs.py index 52a7bd97af..eb8afe3db4 100644 --- a/netbox/netbox/tests/test_jobs.py +++ b/netbox/netbox/tests/test_jobs.py @@ -1,8 +1,11 @@ +import time from datetime import timedelta +from redis import Redis from django.test import TestCase from django.utils import timezone from django_rq import get_queue +from rq.job import Job as RQJob from ..jobs import * from core.models import DataSource, Job @@ -121,7 +124,15 @@ def test_enqueue_once_with_enqueue(self): def test_enqueue_once_after_enqueue(self): instance = DataSource() + redis = Redis() job1 = TestJobRunner.enqueue(instance, schedule_at=self.get_schedule_at()) + job1_rq = None + max_sleep = 5 + sleep_count = 0 + while job1_rq is None and sleep_count < max_sleep: + job1_rq = RQJob.fetch(str(job1.job_id), connection=redis) + time.sleep(1) + sleep_count += 1 job2 = TestJobRunner.enqueue_once(instance, schedule_at=self.get_schedule_at(2)) self.assertNotEqual(job1, job2) From aa9d4a1f1bcd5271bdb36ff949ad8da27c866713 Mon Sep 17 00:00:00 2001 From: Brian Tiemann Date: Thu, 21 Nov 2024 09:56:34 -0500 Subject: [PATCH 2/5] Job can exist but not have status --- netbox/netbox/tests/test_jobs.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/netbox/netbox/tests/test_jobs.py b/netbox/netbox/tests/test_jobs.py index eb8afe3db4..25af6d142c 100644 --- a/netbox/netbox/tests/test_jobs.py +++ b/netbox/netbox/tests/test_jobs.py @@ -126,11 +126,10 @@ def test_enqueue_once_after_enqueue(self): instance = DataSource() redis = Redis() job1 = TestJobRunner.enqueue(instance, schedule_at=self.get_schedule_at()) - job1_rq = None + job1_rq = RQJob.fetch(str(job1.job_id), connection=redis) max_sleep = 5 sleep_count = 0 - while job1_rq is None and sleep_count < max_sleep: - job1_rq = RQJob.fetch(str(job1.job_id), connection=redis) + while not job1_rq.get_status() and sleep_count < max_sleep: time.sleep(1) sleep_count += 1 job2 = TestJobRunner.enqueue_once(instance, schedule_at=self.get_schedule_at(2)) From c14542c8cc2cce22e370b954367ade6bd58f7539 Mon Sep 17 00:00:00 2001 From: Brian Tiemann Date: Thu, 21 Nov 2024 10:02:34 -0500 Subject: [PATCH 3/5] Catch InvalidJobOperation and use as trigger for retry --- netbox/netbox/tests/test_jobs.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/netbox/netbox/tests/test_jobs.py b/netbox/netbox/tests/test_jobs.py index 25af6d142c..eea14bfae7 100644 --- a/netbox/netbox/tests/test_jobs.py +++ b/netbox/netbox/tests/test_jobs.py @@ -5,7 +5,7 @@ from django.test import TestCase from django.utils import timezone from django_rq import get_queue -from rq.job import Job as RQJob +from rq.job import Job as RQJob, InvalidJobOperation from ..jobs import * from core.models import DataSource, Job @@ -127,11 +127,15 @@ def test_enqueue_once_after_enqueue(self): redis = Redis() job1 = TestJobRunner.enqueue(instance, schedule_at=self.get_schedule_at()) job1_rq = RQJob.fetch(str(job1.job_id), connection=redis) + job1_status = None max_sleep = 5 sleep_count = 0 - while not job1_rq.get_status() and sleep_count < max_sleep: - time.sleep(1) - sleep_count += 1 + while job1_status is None and sleep_count < max_sleep: + try: + job1_status = job1_rq.get_status() + except InvalidJobOperation: + time.sleep(1) + sleep_count += 1 job2 = TestJobRunner.enqueue_once(instance, schedule_at=self.get_schedule_at(2)) self.assertNotEqual(job1, job2) From 4595eafd3805e561aa905bde9cdc2fe0f9e573f5 Mon Sep 17 00:00:00 2001 From: Brian Tiemann Date: Thu, 21 Nov 2024 10:28:36 -0500 Subject: [PATCH 4/5] Catch InvalidJobOperation when deleting/canceling job --- netbox/core/models/jobs.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/netbox/core/models/jobs.py b/netbox/core/models/jobs.py index 3cfea3e2a3..82bfd72c89 100644 --- a/netbox/core/models/jobs.py +++ b/netbox/core/models/jobs.py @@ -9,6 +9,7 @@ from django.urls import reverse from django.utils import timezone from django.utils.translation import gettext as _ +from rq.exceptions import InvalidJobOperation from core.choices import JobStatusChoices from core.models import ObjectType @@ -158,7 +159,11 @@ def delete(self, *args, **kwargs): job = queue.fetch_job(str(self.job_id)) if job: - job.cancel() + try: + job.cancel() + except InvalidJobOperation: + # Job may raise this exception from get_status() if missing from Redis + pass def start(self): """ From d4fbce571824a9d8aebac8192551f2c5e352f849 Mon Sep 17 00:00:00 2001 From: Brian Tiemann Date: Thu, 21 Nov 2024 11:29:15 -0500 Subject: [PATCH 5/5] Remove testing code --- netbox/netbox/tests/test_jobs.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/netbox/netbox/tests/test_jobs.py b/netbox/netbox/tests/test_jobs.py index eea14bfae7..52a7bd97af 100644 --- a/netbox/netbox/tests/test_jobs.py +++ b/netbox/netbox/tests/test_jobs.py @@ -1,11 +1,8 @@ -import time from datetime import timedelta -from redis import Redis from django.test import TestCase from django.utils import timezone from django_rq import get_queue -from rq.job import Job as RQJob, InvalidJobOperation from ..jobs import * from core.models import DataSource, Job @@ -124,18 +121,7 @@ def test_enqueue_once_with_enqueue(self): def test_enqueue_once_after_enqueue(self): instance = DataSource() - redis = Redis() job1 = TestJobRunner.enqueue(instance, schedule_at=self.get_schedule_at()) - job1_rq = RQJob.fetch(str(job1.job_id), connection=redis) - job1_status = None - max_sleep = 5 - sleep_count = 0 - while job1_status is None and sleep_count < max_sleep: - try: - job1_status = job1_rq.get_status() - except InvalidJobOperation: - time.sleep(1) - sleep_count += 1 job2 = TestJobRunner.enqueue_once(instance, schedule_at=self.get_schedule_at(2)) self.assertNotEqual(job1, job2)