From a7b52fae3e426b9150b3ce085efc6661689f9eb5 Mon Sep 17 00:00:00 2001 From: Irving Popovetsky Date: Mon, 2 Mar 2020 10:41:15 -0800 Subject: [PATCH 1/5] add the django-health-check module and a healthz endpoint Signed-off-by: Irving Popovetsky --- poetry.lock | 17 ++++++++++++++++- pyproject.toml | 1 + src/operationcode_backend/urls.py | 1 + src/settings/components/base.py | 4 ++++ 4 files changed, 22 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index 8821ab7e..19e27c18 100644 --- a/poetry.lock +++ b/poetry.lock @@ -299,6 +299,17 @@ version = "2.2" Django = ">=1.11" sqlparse = ">=0.2.0" +[[package]] +category = "main" +description = "Run checks on services like databases, queue servers, celery processes, etc." +name = "django-health-check" +optional = false +python-versions = "*" +version = "3.12.1" + +[package.dependencies] +django = ">=1.11" + [[package]] category = "main" description = "Django reCaptcha v2 field/widget" @@ -1196,7 +1207,7 @@ docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"] testing = ["jaraco.itertools"] [metadata] -content-hash = "ad880e105ed81a7b5a1f2dca231852823d95d2a7e8bc61c3584ada41e7cb762e" +content-hash = "61babe314304f60419b60f31a5983a3094a54ba90a6672fe1cb226eb90c95ff3" python-versions = "^3.7" [metadata.files] @@ -1369,6 +1380,10 @@ django-debug-toolbar = [ {file = "django-debug-toolbar-2.2.tar.gz", hash = "sha256:eabbefe89881bbe4ca7c980ff102e3c35c8e8ad6eb725041f538988f2f39a943"}, {file = "django_debug_toolbar-2.2-py3-none-any.whl", hash = "sha256:ff94725e7aae74b133d0599b9bf89bd4eb8f5d2c964106e61d11750228c8774c"}, ] +django-health-check = [ + {file = "django-health-check-3.12.1.tar.gz", hash = "sha256:0563827e003d25fd4d9ebbd7467dea5f390435628d645aaa63f8889deaded73a"}, + {file = "django_health_check-3.12.1-py2.py3-none-any.whl", hash = "sha256:9e6b7d93d4902901474efd4e25d31b5aaea7563b570c0260adce52cd3c3a9e36"}, +] django-recaptcha2 = [ {file = "django-recaptcha2-1.4.1.tar.gz", hash = "sha256:c0b43851b05c6bf6ebb5ecc890c13ccedacd9bb33d64b4291c74dd6fcbc89366"}, {file = "django_recaptcha2-1.4.1-py3-none-any.whl", hash = "sha256:9ea90db0cec502741be1066c09ec1b8e02a73162a319a042e78e67c4605087af"}, diff --git a/pyproject.toml b/pyproject.toml index b0e8f5e6..50027a66 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,7 @@ sentry-sdk = "^0.10.1" six = "^1.12" honeycomb-beeline = "^2.11.4" django-allow-cidr = "^0.3.1" +django-health-check = "^3.12.1" [tool.poetry.dev-dependencies] bandit = "^1.6" diff --git a/src/operationcode_backend/urls.py b/src/operationcode_backend/urls.py index c1e5e894..1ceae76c 100644 --- a/src/operationcode_backend/urls.py +++ b/src/operationcode_backend/urls.py @@ -35,6 +35,7 @@ name="schema-swagger-ui", ), path("redoc/", schema_view.with_ui("redoc", cache_timeout=0), name="schema-redoc"), + path("healthz", include('health_check.urls')), ] ############################################## diff --git a/src/settings/components/base.py b/src/settings/components/base.py index 1f847d41..7dd27476 100644 --- a/src/settings/components/base.py +++ b/src/settings/components/base.py @@ -57,6 +57,10 @@ # temp frontend apps "widget_tweaks", "snowpenguin.django.recaptcha2", + # django-health-check + # https://django-health-check.readthedocs.io/en/latest/ + 'health_check', # required + 'health_check.db', # stock Django health checkers ] ROOT_URLCONF = "operationcode_backend.urls" From a1ae614d44b1440e35388ed94f2a5e914fe60cbe Mon Sep 17 00:00:00 2001 From: Irving Popovetsky Date: Mon, 2 Mar 2020 15:39:53 -0800 Subject: [PATCH 2/5] Add a custom_sampler to ignore all traces to the health check Signed-off-by: Irving Popovetsky --- src/gunicorn_config.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/gunicorn_config.py b/src/gunicorn_config.py index f42ec756..2e2d8ae3 100644 --- a/src/gunicorn_config.py +++ b/src/gunicorn_config.py @@ -217,6 +217,19 @@ def worker_int(worker): def worker_abort(worker): worker.log.info("worker received SIGABRT signal") +def sampler(fields): + request_path = fields.get('request.path') + response_code = fields.get('response.status_code') + + # never sample errors + if response_code and response_code >= 500: + return True, 1 + else: + # never capture healthy health checks + if request_path == '/healthz': + return False, 0 + # catchall + return True, 1 # Added for Honeycomb instrumentation def post_worker_init(worker): @@ -229,5 +242,7 @@ def post_worker_init(worker): beeline.init( writekey=os.getenv("HONEYCOMB_WRITEKEY"), dataset=os.getenv("HONEYCOMB_DATASET"), + service_name='backend', + sampler_hook=sampler, debug=False, ) From d3e1b3ab892430b9a3468abf7a27aaef94b0275b Mon Sep 17 00:00:00 2001 From: Irving Popovetsky Date: Mon, 2 Mar 2020 15:42:36 -0800 Subject: [PATCH 3/5] make sure to add honeycomb auto-instrumentation for dev environments or else youll have a bad day trying to troubleshoot Signed-off-by: Irving Popovetsky --- src/settings/environments/development.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/settings/environments/development.py b/src/settings/environments/development.py index 837675e3..72f9243b 100644 --- a/src/settings/environments/development.py +++ b/src/settings/environments/development.py @@ -13,3 +13,7 @@ INSTALLED_APPS += ("debug_toolbar",) if "debug_toolbar.middleware.DebugToolbarMiddleware" not in MIDDLEWARE: MIDDLEWARE += ("debug_toolbar.middleware.DebugToolbarMiddleware",) + +# Honeycomb beeline auto-instrumentation +if "beeline.middleware.django.HoneyMiddleware" not in MIDDLEWARE: # noqa: F821 + MIDDLEWARE += ("beeline.middleware.django.HoneyMiddleware",) # noqa: F821 From 6cdc536840cd244be69c853f10d0ec1989777091 Mon Sep 17 00:00:00 2001 From: Irving Popovetsky Date: Mon, 2 Mar 2020 15:43:06 -0800 Subject: [PATCH 4/5] Correct the ALLOWED_HOST values for prod and staging Signed-off-by: Irving Popovetsky --- src/settings/environments/production.py | 2 +- src/settings/environments/staging.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/settings/environments/production.py b/src/settings/environments/production.py index cae32d19..2feacc01 100644 --- a/src/settings/environments/production.py +++ b/src/settings/environments/production.py @@ -3,7 +3,7 @@ from settings.components import config from settings.components.base import DATABASES -ALLOWED_HOSTS = ["operationcode.org", "pybot.operationcode.org"] +ALLOWED_HOSTS = ["api.operationcode.org"] DEBUG = False if config("EXTRA_HOSTS", default=""): diff --git a/src/settings/environments/staging.py b/src/settings/environments/staging.py index d89b2978..baebff0e 100644 --- a/src/settings/environments/staging.py +++ b/src/settings/environments/staging.py @@ -3,7 +3,7 @@ from settings.components import config from settings.components.base import DATABASES -ALLOWED_HOSTS = ["operationcode.org", "api.staging.operationcode.org"] +ALLOWED_HOSTS = ["api.staging.operationcode.org"] DEBUG = False if config("EXTRA_HOSTS", default=""): From d1078d463f426a5c52bd8842fc34a4a9e572d2bf Mon Sep 17 00:00:00 2001 From: Irving Popovetsky Date: Mon, 2 Mar 2020 15:51:38 -0800 Subject: [PATCH 5/5] linter did its thing Signed-off-by: Irving Popovetsky --- src/gunicorn_config.py | 28 +++++++++++++++------------- src/operationcode_backend/urls.py | 2 +- src/settings/components/base.py | 4 ++-- 3 files changed, 18 insertions(+), 16 deletions(-) diff --git a/src/gunicorn_config.py b/src/gunicorn_config.py index 2e2d8ae3..e8a43cdc 100644 --- a/src/gunicorn_config.py +++ b/src/gunicorn_config.py @@ -217,19 +217,21 @@ def worker_int(worker): def worker_abort(worker): worker.log.info("worker received SIGABRT signal") + def sampler(fields): - request_path = fields.get('request.path') - response_code = fields.get('response.status_code') - - # never sample errors - if response_code and response_code >= 500: - return True, 1 - else: - # never capture healthy health checks - if request_path == '/healthz': - return False, 0 - # catchall - return True, 1 + request_path = fields.get("request.path") + response_code = fields.get("response.status_code") + + # never sample errors + if response_code and response_code >= 500: + return True, 1 + else: + # never capture healthy health checks + if request_path == "/healthz": + return False, 0 + # catchall + return True, 1 + # Added for Honeycomb instrumentation def post_worker_init(worker): @@ -242,7 +244,7 @@ def post_worker_init(worker): beeline.init( writekey=os.getenv("HONEYCOMB_WRITEKEY"), dataset=os.getenv("HONEYCOMB_DATASET"), - service_name='backend', + service_name="backend", sampler_hook=sampler, debug=False, ) diff --git a/src/operationcode_backend/urls.py b/src/operationcode_backend/urls.py index 1ceae76c..f54a032d 100644 --- a/src/operationcode_backend/urls.py +++ b/src/operationcode_backend/urls.py @@ -35,7 +35,7 @@ name="schema-swagger-ui", ), path("redoc/", schema_view.with_ui("redoc", cache_timeout=0), name="schema-redoc"), - path("healthz", include('health_check.urls')), + path("healthz", include("health_check.urls")), ] ############################################## diff --git a/src/settings/components/base.py b/src/settings/components/base.py index 7dd27476..acec825a 100644 --- a/src/settings/components/base.py +++ b/src/settings/components/base.py @@ -59,8 +59,8 @@ "snowpenguin.django.recaptcha2", # django-health-check # https://django-health-check.readthedocs.io/en/latest/ - 'health_check', # required - 'health_check.db', # stock Django health checkers + "health_check", # required + "health_check.db", # stock Django health checkers ] ROOT_URLCONF = "operationcode_backend.urls"