Celery: Removed separate queue for translation memory

It was necessary with Whoosh based backend to avoid concurrency issues,
but now with database backend engine, this can be processed by normal
workers.

docs/admin/upgrade.rst

@@ -142,6 +142,7 @@ Notable configuration or dependencies changes:
 * There are some new and updated requirements.
 * There are several changes in :file:`settings_example.py`, most notable new middleware and changed application ordering.
 * The keys for JSON based formats no longer include leading dot. The strings are adjusted during the database migration, but external components might need adjustment in case you rely on keys in exports or API.
+* The Celery configuration was changed to no longer use ``memory`` queue. Please adjust your startup scripts and :setting:`CELERY_TASK_ROUTES`.
 
 .. seealso:: :ref:`generic-upgrade-instructions`
 

docs/changes.rst

@@ -23,6 +23,7 @@ Not yet released.
 * Added check for ECMAScript template literals.
 * Added option to watch a component.
 * Removed leading dot from JSON unit keys.
+* Removed separate Celery queue for translation memory.
 
 Weblate 4.1.1
 -------------

weblate/examples/celery

@@ -3,11 +3,10 @@
 # Simple wrapper to start/stop celery workers
 
 python -m celery -A weblate.utils multi $1 \
-    search notify celery memory backup translate \
+    search notify celery backup translate \
     --pidfile=$PWD/weblate-%n.pid \
     --logfile=$PWD/weblate-%n%I.log --loglevel=DEBUG \
     --beat:celery --concurrency:celery=4 --queues:celery=celery --prefetch-multiplier:celery=4 \
     --concurrency:notify=4 --queues:notify=notify --prefetch-multiplier:notify=10 \
-    --concurrency:memory=1 --queues:memory=memory --prefetch-multiplier:memory=4 \
-    --concurrency:translate=2 --queues:translate=translate --prefetch-multiplier:translate=4 \
+    --concurrency:translate=4 --queues:translate=translate --prefetch-multiplier:translate=4 \
     --concurrency:backup=1 --queues:backup=backup  --prefetch-multiplier:backup=2

weblate/examples/celery-weblate.conf

@@ -1,5 +1,5 @@
 # Name of nodes to start
-CELERYD_NODES="celery notify memory backup translate"
+CELERYD_NODES="celery notify backup translate"
 
 # Absolute or relative path to the 'celery' command:
 CELERY_BIN="/home/weblate/weblate-env/bin/celery"
@@ -12,7 +12,6 @@ CELERY_APP="weblate.utils"
 # increase concurency if you get weblate.E019
 CELERYD_OPTS="--beat:celery --concurrency:celery=4 --queues:celery=celery --prefetch-multiplier:celery=4 \
     --concurrency:notify=4 --queues:notify=notify --prefetch-multiplier:notify=10 \
-    --concurrency:memory=1 --queues:memory=memory --prefetch-multiplier:memory=4 \
     --concurrency:translate=4 --queues:translate=translate --prefetch-multiplier:translate=4 \
     --concurrency:backup=1 --queues:backup=backup  --prefetch-multiplier:backup=2"
 

weblate/settings_docker.py

@@ -1096,7 +1096,6 @@
 CELERY_BEAT_SCHEDULE_FILENAME = os.path.join(DATA_DIR, "celery", "beat-schedule")
 CELERY_TASK_ROUTES = {
     "weblate.trans.tasks.auto_translate": {"queue": "translate"},
-    "weblate.memory.tasks.*": {"queue": "memory"},
     "weblate.accounts.tasks.notify_*": {"queue": "notify"},
     "weblate.accounts.tasks.send_mails": {"queue": "notify"},
     "weblate.utils.tasks.settings_backup": {"queue": "backup"},

weblate/settings_example.py

@@ -863,7 +863,6 @@
 CELERY_BEAT_SCHEDULE_FILENAME = os.path.join(DATA_DIR, "celery", "beat-schedule")
 CELERY_TASK_ROUTES = {
     "weblate.trans.tasks.auto_translate": {"queue": "translate"},
-    "weblate.memory.tasks.*": {"queue": "memory"},
     "weblate.accounts.tasks.notify_*": {"queue": "notify"},
     "weblate.accounts.tasks.send_mails": {"queue": "notify"},
     "weblate.utils.tasks.settings_backup": {"queue": "backup"},