support for limiting number of workers and total custom celery worker…

… invocation from an env var
jay-johnson · Jul 26, 2018 · c7f4c8a · c7f4c8a
1 parent 658289b
commit c7f4c8a
Show file tree

Hide file tree

Showing 3 changed files with 16 additions and 7 deletions.
diff --git a/envs/k8-splunk.env b/envs/k8-splunk.env
@@ -20,19 +20,20 @@ export HTTP_X_FORWARDED_PROTOCOL_KEY=HTTP_X_FORWARDED_PROTO
 export COLLECT_STATICS=1
 export CELERY_ENABLED=1
 export CACHEOPS_ENABLED=1
-export NUM_WORKERS=4
+export ANTINEX_API_NUM_WORKERS=4
 export ANTINEX_WORKER_ENABLED=1
 export ANTINEX_WORKER_ONLY=0
+export ANTINEX_WORKER_SSL_ENABLED=0
 export ANTINEX_DELIVERY_MODE=persistent
 export ANTINEX_EXCHANGE_NAME=webapp.predict.requests
 export ANTINEX_EXCHANGE_TYPE=topic
 export ANTINEX_QUEUE_NAME=webapp.predict.requests
-export ANTINEX_WORKER_SSL_ENABLED=0
 export ANTINEX_CA_CERTS=
 export ANTINEX_KEYFILE=
 export ANTINEX_CERTFILE=
 export ANTINEX_TLS_PROTOCOL=
 export ANTINEX_KEYFILE=
+export ANTINEX_CORE_NUM_WORKERS=1
 export DJANGO_ALLOWED_HOSTS=*
 export USE_ENV=k8-splunk
 export USE_VENV=/opt/venv

diff --git a/envs/k8.env b/envs/k8.env
@@ -20,19 +20,20 @@ export HTTP_X_FORWARDED_PROTOCOL_KEY=HTTP_X_FORWARDED_PROTO
 export COLLECT_STATICS=1
 export CELERY_ENABLED=1
 export CACHEOPS_ENABLED=1
-export NUM_WORKERS=4
+export ANTINEX_API_NUM_WORKERS=4
 export ANTINEX_WORKER_ENABLED=1
 export ANTINEX_WORKER_ONLY=0
+export ANTINEX_WORKER_SSL_ENABLED=0
 export ANTINEX_DELIVERY_MODE=persistent
 export ANTINEX_EXCHANGE_NAME=webapp.predict.requests
 export ANTINEX_EXCHANGE_TYPE=topic
 export ANTINEX_QUEUE_NAME=webapp.predict.requests
-export ANTINEX_WORKER_SSL_ENABLED=0
 export ANTINEX_CA_CERTS=
 export ANTINEX_KEYFILE=
 export ANTINEX_CERTFILE=
 export ANTINEX_TLS_PROTOCOL=
 export ANTINEX_KEYFILE=
+export ANTINEX_CORE_NUM_WORKERS=1
 export DJANGO_ALLOWED_HOSTS=*
 export USE_ENV=k8
 export USE_VENV=/opt/venv

diff --git a/run-worker.sh b/run-worker.sh
@@ -77,6 +77,10 @@ if [[ "${SHARED_LOG_CFG}" != "" ]]; then
     echo ""
 fi
 
+if  [[ "${ANTINEX_API_NUM_WORKERS}" != "" ]]; then
+    num_workers=${ANTINEX_API_NUM_WORKERS}
+fi
+
 # Use the WORKER_EXTRA_ARGS to pass in specific args:
 # http://docs.celeryproject.org/en/latest/reference/celery.bin.worker.html
 #
@@ -86,12 +90,15 @@ fi
 # --without-gossip
 # --without-mingle
 
-if [[ "${num_workers}" == "1" ]]; then
-    echo "Starting Worker=${worker_module}"
+if [[ "${ANTINEX_API_WORKER_ARGS}" != "" ]]; then
+    echo "Launching custom api worker=${ANTINEX_API_WORKER_ARGS}"
+    celery worker ${ANTINEX_API_WORKER_ARGS}
+elif [[ "${num_workers}" == "1" ]]; then
+    echo "Starting worker=${worker_module}"
     echo "celery worker -A ${worker_module} -c ${num_workers} -l ${log_level} -n ${worker_name} -Q ${custom_queues} ${WORKER_EXTRA_ARGS}"
     celery worker -A $worker_module -c ${num_workers} -l ${log_level} -n ${worker_name} -Q $custom_queues ${WORKER_EXTRA_ARGS}
 else
-    echo "Starting Workers=${worker_module}"
+    echo "Starting workers=${worker_module}"
     echo "celery worker -A ${worker_module} -c ${num_workers} -l ${log_level} -n ${worker_name} --logfile=${log_file} -Q ${custom_queues} ${WORKER_EXTRA_ARGS}"
     celery worker -A $worker_module -c ${num_workers} -l ${log_level} -n ${worker_name} --logfile=${log_file} -Q ${custom_queues} ${WORKER_EXTRA_ARGS}
 fi