From c75c2bb12bf106e8e6f6206f7c948773b4709d30 Mon Sep 17 00:00:00 2001 From: Nokome Bentley Date: Tue, 13 Oct 2020 17:52:12 +1300 Subject: [PATCH] fix(*): Downgrade to Celery 4.4.6 Celery v4.4.7 introduced a regression in setting of configuration options which still seems to exist in v5.0.0. See https://github.com/celery/celery/issues/6285 and others. --- manager/requirements.txt | 4 ++-- overseer/requirements.txt | 4 ++-- scheduler/requirements.txt | 4 ++-- worker/Dockerfile | 6 +----- worker/Makefile | 6 +----- worker/requirements.txt | 4 ++-- worker/worker.py | 2 -- 7 files changed, 10 insertions(+), 20 deletions(-) diff --git a/manager/requirements.txt b/manager/requirements.txt index fc130d78a..a5c79aa08 100644 --- a/manager/requirements.txt +++ b/manager/requirements.txt @@ -1,6 +1,6 @@ -amqp==5.0.1 +amqp==2.6.1 brotli==1.0.9 -celery==5.0.0 +celery==4.4.6 customidenticon==0.1.5 dj_database_url==0.5.0 dj-stripe==2.3.0 diff --git a/overseer/requirements.txt b/overseer/requirements.txt index 63088ee65..8cfb74924 100644 --- a/overseer/requirements.txt +++ b/overseer/requirements.txt @@ -1,4 +1,4 @@ -amqp==5.0.1 -celery==5.0.0 +amqp==2.6.1 +celery==4.4.6 httpx==0.16.1 prometheus_client==0.8.0 diff --git a/scheduler/requirements.txt b/scheduler/requirements.txt index bbd80f9a5..4d5c9deea 100644 --- a/scheduler/requirements.txt +++ b/scheduler/requirements.txt @@ -1,5 +1,5 @@ -amqp==5.0.1 -celery==5.0.0 +amqp==2.6.1 +celery==4.4.6 django-celery-beat==2.0.0 dj-database-url==0.5.0 psycopg2-binary==2.8.6 diff --git a/worker/Dockerfile b/worker/Dockerfile index 73bde21e4..f1313544d 100644 --- a/worker/Dockerfile +++ b/worker/Dockerfile @@ -64,8 +64,4 @@ USER worker # Use the array form of CMD to ensure that the celery process has PID 1 # so that it will receive SIGTERM for graceful shutdown -CMD ["celery", "--app=worker", "worker", \ - "--events", \ - "--heartbeat-interval=60", \ - "--prefetch-multiplier=1", \ - "--concurrency=10" ] +CMD ["celery", "--app=worker", "worker", "--heartbeat-interval=60"] diff --git a/worker/Makefile b/worker/Makefile index d89572bc4..60233c2ee 100644 --- a/worker/Makefile +++ b/worker/Makefile @@ -42,15 +42,11 @@ run: venv BROKER_URL=amqp://stencila:password@localhost:5672/stencila \ CACHE_URL=redis://stencila:password@localhost:6379/0 \ WORKER_CONCURRENCY=3 \ - $(VE)/celery \ - --app worker \ - worker \ - --events \ + $(VE)/celery --app=worker worker \ --loglevel DEBUG \ --hostname worker-$$(date +'%s')@%h \ --heartbeat-interval 60 - # Build Docker image build: docker build --tag stencila/hub-worker . diff --git a/worker/requirements.txt b/worker/requirements.txt index 1ae835892..64c3ac0c3 100644 --- a/worker/requirements.txt +++ b/worker/requirements.txt @@ -1,5 +1,5 @@ -amqp==5.0.1 -celery==5.0.0 +amqp==2.6.1 +celery==4.4.6 google-api-python-client==1.12.3 kubernetes==11.0.0 lxml==4.5.2 diff --git a/worker/worker.py b/worker/worker.py index 275f00921..779a8399c 100644 --- a/worker/worker.py +++ b/worker/worker.py @@ -29,8 +29,6 @@ "worker", broker=os.environ["BROKER_URL"], backend=os.environ["CACHE_URL"], ) app.conf.update( - # In Celery 5.0 it appear necessary to also set (some of?) these options - # in the command line invocation # List of queues to subscribe to task_queues=[ Queue(queue, exchange=queue, routing_key=queue)