From ba7e8f28f3f4f8131caf76fcdf9408984ac69e1f Mon Sep 17 00:00:00 2001 From: Reupen Shah Date: Wed, 11 Mar 2020 16:39:40 +0000 Subject: [PATCH] Add workaround for Django 3 memory leak under gevent We've been suffering from a problematic memory leak since upgrading to Django 3 which is believed to be caused by https://github.com/django/asgiref/issues/144. The memory leak is causing Gunicorn worker processes to exhaust available memory and crash and restart. This adds a workaround which is inactive by default. This is so it can be we can verify that it fixes the problem in e.g. the dev environment. --- README.md | 1 + .../django-3-gevent-workaround.internal.md | 1 + config/gunicorn.py | 20 ++++++++++++++++--- 3 files changed, 19 insertions(+), 3 deletions(-) create mode 100644 changelog/django-3-gevent-workaround.internal.md diff --git a/README.md b/README.md index 97546b2fbd..046a2f7a42 100644 --- a/README.md +++ b/README.md @@ -302,6 +302,7 @@ Data Hub API can run on any Heroku-style platform. Configuration is performed vi | `GUNICORN_ACCESS_LOG_FORMAT` | No | | | `GUNICORN_ENABLE_ASYNC_PSYCOPG2` | No | Whether to enable asynchronous psycopg2 when the worker class is 'gevent' (default=True). | | `GUNICORN_ENABLE_STATSD` | No | Whether to enable Gunicorn StatD instrumentation (default=False). | +| `GUNICORN_PATCH_ASGIREF` | No | Whether to enable a workaround for https://github.com/django/asgiref/issues/144 when the worker class is 'gevent' (default=False). | | `GUNICORN_WORKER_CLASS` | No | [Type of Gunicorn worker.](http://docs.gunicorn.org/en/stable/settings.html#worker-class) Uses async workers via gevent by default. | | `GUNICORN_WORKER_CONNECTIONS` | No | Maximum no. of connections for async workers (default=10). | | `INTERACTION_ADMIN_CSV_IMPORT_MAX_SIZE` | No | Maximum file size in bytes for interaction admin CSV uploads (default=2MB). | diff --git a/changelog/django-3-gevent-workaround.internal.md b/changelog/django-3-gevent-workaround.internal.md new file mode 100644 index 0000000000..8903b179d8 --- /dev/null +++ b/changelog/django-3-gevent-workaround.internal.md @@ -0,0 +1 @@ +A workaround was added for a memory leak when using Django 3 with gevent. This is inactive by default. diff --git a/config/gunicorn.py b/config/gunicorn.py index f8b7b9dcde..d0044ad34e 100644 --- a/config/gunicorn.py +++ b/config/gunicorn.py @@ -41,6 +41,9 @@ _enable_async_psycopg2 = ( os.environ.get('GUNICORN_ENABLE_ASYNC_PSYCOPG2', 'true').lower() in ('true', '1') ) +_patch_asgiref = ( + os.environ.get('GUNICORN_PATCH_ASGIREF', 'false').lower() in ('true', '1') +) def post_fork(server, worker): @@ -49,6 +52,17 @@ def post_fork(server, worker): Enables async processing in Psycopg2 if GUNICORN_ENABLE_ASYNC_PSYCOPG2 is set. """ - if worker_class == 'gevent' and _enable_async_psycopg2: - patch_psycopg() - worker.log.info('Enabled async Psycopg2') + if worker_class == 'gevent': + if _enable_async_psycopg2: + patch_psycopg() + worker.log.info('Enabled async Psycopg2') + + # Temporary workaround for https://github.com/django/asgiref/issues/144. + # Essentially reverts part of + # https://github.com/django/django/commit/a415ce70bef6d91036b00dd2c8544aed7aeeaaed. + if _patch_asgiref: + import asgiref.local + import threading + + asgiref.local.Local = lambda **kwargs: threading.local() + worker.log.info('Patched asgiref.local.Local')