Merge pull request #3113 from mytardis/celery-lowercase-settings

Update Celery settings to a new format

docs/admin/task_priorities.rst

@@ -17,10 +17,10 @@ the Celery app in ``tardis/celery.py``::
   DEFAULT_TASK_PRIORITY = 5
   DEFAULT_EMAIL_TASK_PRIORITY = 10
   
-  CELERY_DEFAULT_QUEUE = 'celery'
+  CELERY_TASK_DEFAULT_QUEUE = 'celery'
   # The 'x-max-priority' argument will only be respected by the RabbitMQ broker,
   # which is the recommended broker for MyTardis:
-  CELERY_QUEUES = (
+  CELERY_TASK_QUEUES = (
       Queue('celery', Exchange('celery'),
             routing_key='celery',
             queue_arguments={'x-max-priority': MAX_TASK_PRIORITY}),
@@ -30,7 +30,7 @@ the Celery app in ``tardis/celery.py``::
   ----------------
   ...
   tardis_app = Celery('tardis')
-  tardis_app.config_from_object('django.conf:settings')
+  tardis_app.config_from_object('django.conf:settings', namespace='CELERY')
   ...
 
 Celery's ``apply_async`` method's ``shadow`` argument is used to annotate storage

tardis/apps/hsm/api.py

@@ -100,7 +100,7 @@ def recall_dfo(self, request, **kwargs):
                 args=[dfo.id, request.user.id],
                 priority=dfo.priority)
         except HsmException as err:
-            # We would only see an exception here if CELERY_ALWAYS_EAGER is
+            # We would only see an exception here if CELERY_TASK_ALWAYS_EAGER is
             # True, making the task run synchronously
             logger.error("Recall failed for DFO %s: %s" % (dfo.id, str(err)))
             return JsonResponse({

tardis/celery.py

@@ -3,5 +3,5 @@
 from django.apps import apps  # pylint: disable=wrong-import-order
 
 tardis_app = Celery('tardis')
-tardis_app.config_from_object('django.conf:settings')
+tardis_app.config_from_object('django.conf:settings', namespace='CELERY')
 tardis_app.autodiscover_tasks(lambda: [n.name for n in apps.get_app_configs()])

tardis/default_settings/celery_settings.py

@@ -2,37 +2,51 @@
 from datetime import timedelta
 from kombu import Exchange, Queue
 
-CELERY_IMPORTS = ('tardis.tardis_portal.tasks',)
+# https://docs.celeryproject.org/en/stable/userguide/configuration.html
+# Celery will still be able to read old configuration files until Celery 6.0.
+# Afterwards, support for the old configuration files will be removed.
 
 # Using Celery for asynchronous task processing requires a broker e.g. RabbitMQ
 # Use a strong password for production
-BROKER_URL = 'amqp://guest:guest@localhost:5672//'
+CELERY_BROKER_URL = 'amqp://%(user)s:%(password)s@%(host)s:%(port)s/%(vhost)s' % {
+    'host': 'localhost',
+    'port': 5672,
+    'user': 'guest',
+    'password': 'guest',
+    'vhost': '/'
+}
 
-# For local development, you can force Celery tasks to run synchronously:
-# CELERY_ALWAYS_EAGER = True
-# CELERY_EAGER_PROPAGATES_EXCEPTIONS = True
+# Where to send task state and results
+CELERY_RESULT_BACKEND = 'rpc://'
+
+# List of modules to import when the Celery worker starts
+CELERY_IMPORTS = ('tardis.tardis_portal.tasks',)
 
 # These settings help with task prioritization:
-CELERY_ACKS_LATE = True
-CELERYD_PREFETCH_MULTIPLIER = 1
+CELERY_TASK_ACKS_LATE = True
+CELERY_WORKER_PREFETCH_MULTIPLIER = 1
 
 MAX_TASK_PRIORITY = 10
 DEFAULT_TASK_PRIORITY = 5
 DEFAULT_EMAIL_TASK_PRIORITY = 10
 
-CELERY_DEFAULT_QUEUE = 'celery'
+CELERY_TASK_DEFAULT_QUEUE = 'celery'
 # The 'x-max-priority' argument will only be respected by the RabbitMQ broker,
 # which is the recommended broker for MyTardis:
-CELERY_QUEUES = (
+CELERY_TASK_QUEUES = (
     Queue('celery', Exchange('celery'),
           routing_key='celery',
           queue_arguments={'x-max-priority': MAX_TASK_PRIORITY}),
 )
 
-CELERYBEAT_SCHEDULE = {
+CELERY_BEAT_SCHEDULE = {
     "verify-files": {
         "task": "tardis_portal.verify_dfos",
         "schedule": timedelta(seconds=300),
         "kwargs": {'priority': DEFAULT_TASK_PRIORITY}
     }
 }
+
+# For local development, you can force Celery tasks to run synchronously:
+# CELERY_TASK_ALWAYS_EAGER = True
+# CELERY_TASK_EAGER_PROPAGATES = True

tardis/tardis_portal/tests/api/test_datafile_resource.py

@@ -188,8 +188,8 @@ def guess_mime(filename):
     def test_download_file(self):
         '''
         Re-run the upload test in order to create a verified file to
-        download - it will be verified immediately becase
-        CELERY_ALWAYS_EAGER is True in test_settings.py
+        download - it will be verified immediately because
+        CELERY_TASK_ALWAYS_EAGER is True in test_settings.py
 
         Then download the file, check the HTTP status code and check
         the file content.

tardis/tardis_portal/tests/models/test_datafile.py

@@ -41,7 +41,7 @@ def _build(dataset, filename, url=None):
                 storage_box=datafile.get_default_storage_box(),
                 uri=url)
             dfo.save()
-            # Tests are run with CELERY_ALWAYS_EAGER = True,
+            # Tests are run with CELERY_TASK_ALWAYS_EAGER = True,
             # so saving a DFO will trigger an immediate attempt
             # to verify the DFO which will trigger an attempt
             # to apply filters because we are overriding the

tardis/tardis_portal/tests/test_copy_move.py

@@ -74,7 +74,7 @@ def test_copy(self):
         copy.verified = False
         copy.save()
         copy = self.dfo.copy_file(self.second_box)
-        # Thanks to CELERY_ALWAYS_EAGER = True in test_settings.py:
+        # Thanks to CELERY_TASK_ALWAYS_EAGER = True in test_settings.py:
         self.assertTrue(copy.verified)
 
         # Now let's delete the copy in self.second_box:

tardis/test_settings.py

@@ -28,11 +28,12 @@
 }
 
 # During testing it's always eager
-CELERY_ALWAYS_EAGER = True
-BROKER_URL = 'memory://'
-CELERY_EAGER_PROPAGATES_EXCEPTIONS = True
+CELERY_TASK_ALWAYS_EAGER = True
+CELERY_TASK_EAGER_PROPAGATES = True
+CELERY_BROKER_URL = 'memory://'
+
 tardis_app = Celery('tardis')
-tardis_app.config_from_object('django.conf:settings')
+tardis_app.config_from_object('django.conf:settings', namespace='CELERY')
 tardis_app.autodiscover_tasks(lambda: [n.name for n in apps.get_app_configs()])
 
 TEMPLATES[0]['DIRS'].append(