Set up 2 Celery queues, for long and short running tasks

files/run_worker.sh

@@ -26,6 +26,10 @@ install -m 0400 /packit-ssh/config .
 grep -q pkgs.fedoraproject.org known_hosts || ssh-keyscan pkgs.fedoraproject.org >>known_hosts
 popd
 
+# define queues to serve
+DEFAULT_QUEUES="short-running,long-running"
+QUEUES="${QUEUES:-$DEFAULT_QUEUES}"
+
 if [[ "${CELERY_COMMAND}" == "beat" ]]; then
     # when using the database backend, celery beat must be running for the results to be expired.
     # https://docs.celeryproject.org/en/stable/userguide/periodic-tasks.html#starting-the-scheduler
@@ -35,5 +39,5 @@ elif [[ "${CELERY_COMMAND}" == "worker" ]]; then
     # concurrency: Number of concurrent worker processes/threads/green threads executing tasks.
     # prefetch-multiplier: How many messages to prefetch at a time multiplied by the number of concurrent processes.
     # http://docs.celeryproject.org/en/latest/userguide/optimizing.html#prefetch-limits
-    exec celery --app="${APP}" worker --loglevel=${LOGLEVEL} --concurrency=1 --prefetch-multiplier=1
+    exec celery --app="${APP}" worker --loglevel=${LOGLEVEL} --concurrency=1 --prefetch-multiplier=1 --queues="${QUEUES}"
 fi

packit_service/celerizer.py

@@ -7,6 +7,7 @@
 from celery import Celery
 from lazy_object_proxy import Proxy
 
+from packit_service.constants import CELERY_DEFAULT_QUEUE_NAME
 from packit_service.models import get_pg_url
 from packit_service.sentry_integration import configure_sentry
 
@@ -41,6 +42,8 @@ def celery_app(self):
             days = int(getenv("CELERY_RESULT_EXPIRES", "30"))
             # https://docs.celeryproject.org/en/latest/userguide/configuration.html#result-expires
             self._celery_app.conf.result_expires = timedelta(days=days)
+            # https://docs.celeryproject.org/en/latest/userguide/configuration.html#std-setting-task_default_queue
+            self._celery_app.conf.task_default_queue = CELERY_DEFAULT_QUEUE_NAME
 
         return self._celery_app
 

packit_service/constants.py

@@ -54,6 +54,8 @@
     "denied": "denied",
 }
 
+CELERY_DEFAULT_QUEUE_NAME = "short-running"
+
 
 class KojiBuildState(Enum):
     """