From 9bc1547b9236e14cccf30328f9561f8d9d65552e Mon Sep 17 00:00:00 2001 From: Graham Beckley Date: Wed, 2 Aug 2023 13:04:16 -0400 Subject: [PATCH 01/14] Add tox config for FastAPI 100 --- setup.py | 2 ++ tests/constraints/fastapi-0.100.txt | 1 + tests/requirements/fastapi.txt | 3 +++ tox.ini | 6 +++++- 4 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 tests/constraints/fastapi-0.100.txt create mode 100644 tests/requirements/fastapi.txt diff --git a/setup.py b/setup.py index d1d6ead..2232e56 100644 --- a/setup.py +++ b/setup.py @@ -31,6 +31,7 @@ def read(*parts): "Framework :: Django :: 4.0", "Framework :: Django :: 4.1", "Framework :: Flask", + "Framework :: FastAPI", "Intended Audience :: Developers", "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)", "Programming Language :: Python", @@ -47,6 +48,7 @@ def read(*parts): "django": ["django"], "flask": ["flask", "blinker"], "sanic": ["sanic"], + "fastapi": ["fastapi", "asgiref"], }, zip_safe=False, python_requires=">=3.7,<4", diff --git a/tests/constraints/fastapi-0.100.txt b/tests/constraints/fastapi-0.100.txt new file mode 100644 index 0000000..c34c72e --- /dev/null +++ b/tests/constraints/fastapi-0.100.txt @@ -0,0 +1 @@ +fastapi>=0.100,<0.101 \ No newline at end of file diff --git a/tests/requirements/fastapi.txt b/tests/requirements/fastapi.txt new file mode 100644 index 0000000..18476e7 --- /dev/null +++ b/tests/requirements/fastapi.txt @@ -0,0 +1,3 @@ +fastapi +asgiref +httpx \ No newline at end of file diff --git a/tox.ini b/tox.ini index a1d99fe..6b9718d 100644 --- a/tox.ini +++ b/tox.ini @@ -6,6 +6,7 @@ envlist = py38-docs py{37,38,39,310,311}-dj32 py{38,39,310,311}-dj{40,41,42} + py{37,38,39,310,311}-fa100 py{37,38,39,310,311}-fl{20,21,22} py{37,38,39,310,311}-s{21,22} @@ -26,12 +27,14 @@ setenv = deps = -rtests/requirements/default.txt dj{32,40,41,42}: -rtests/requirements/django.txt + fa100: -rtests/requirements/fastapi.txt fl{20,21,22}: -rtests/requirements/flask.txt s{21,22}: -rtests/requirements/sanic.txt dj32: -ctests/constraints/django-3.2.txt dj40: -ctests/constraints/django-4.0.txt dj41: -ctests/constraints/django-4.1.txt dj42: -ctests/constraints/django-4.2.txt + fa100: -ctests/constraints/fastapi-0.100.txt fl20: -ctests/constraints/flask-2.0.txt fl21: -ctests/constraints/flask-2.1.txt fl22: -ctests/constraints/flask-2.2.txt @@ -40,7 +43,8 @@ deps = commands = python --version dj{32,40,41,42}: pytest tests/core/ tests/django --nomigrations {posargs:} - fl{012,10,11,20,21,22}: pytest tests/core/ tests/flask/ {posargs:} + fa{100}: pytest tests/core/ tests/fastapi/ {posargs:} + fl{20,21,22}: pytest tests/core/ tests/flask/ {posargs:} s{20,21,22}: pytest tests/core/ tests/sanic/ {posargs:} [testenv:py38-docs] From 978f0670681bcac1acb7a4883e188e20eb00c713 Mon Sep 17 00:00:00 2001 From: Graham Beckley Date: Wed, 2 Aug 2023 13:05:25 -0400 Subject: [PATCH 02/14] Add dockerflow router to package root --- src/dockerflow/fastapi/__init__.py | 1 + src/dockerflow/fastapi/router.py | 11 +++++++++ tests/fastapi/test_fastapi.py | 37 ++++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+) create mode 100644 src/dockerflow/fastapi/__init__.py create mode 100644 src/dockerflow/fastapi/router.py create mode 100644 tests/fastapi/test_fastapi.py diff --git a/src/dockerflow/fastapi/__init__.py b/src/dockerflow/fastapi/__init__.py new file mode 100644 index 0000000..6521c06 --- /dev/null +++ b/src/dockerflow/fastapi/__init__.py @@ -0,0 +1 @@ +from .router import dockerflow_router diff --git a/src/dockerflow/fastapi/router.py b/src/dockerflow/fastapi/router.py new file mode 100644 index 0000000..820ad84 --- /dev/null +++ b/src/dockerflow/fastapi/router.py @@ -0,0 +1,11 @@ +from fastapi import Request, Response +from fastapi.routing import APIRouter + + +dockerflow_router = APIRouter(tags=["Dockerflow"]) + + +@dockerflow_router.get("/__lbheartbeat__") +@dockerflow_router.head("/__lbheartbeat__") +def lbheartbeat(): + return {"status": "ok"} diff --git a/tests/fastapi/test_fastapi.py b/tests/fastapi/test_fastapi.py new file mode 100644 index 0000000..ee8655a --- /dev/null +++ b/tests/fastapi/test_fastapi.py @@ -0,0 +1,37 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, you can obtain one at http://mozilla.org/MPL/2.0/. +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from dockerflow.fastapi import dockerflow_router + + +def create_app(): + app = FastAPI() + app.include_router(dockerflow_router) + app.add_middleware(MozlogRequestSummaryLogger) + return app + + +@pytest.fixture +def app(): + return create_app() + + +@pytest.fixture +def client(app): + return TestClient(app) + + +def test_lbheartbeat_get(client): + response = client.get("/__lbheartbeat__") + assert response.status_code == 200 + assert response.json() == {"status": "ok"} + + +def test_lbheartbeat_head(client): + response = client.head("/__lbheartbeat__") + assert response.status_code == 200 + assert response.content == b"" From 566052ca5c1f41f4b383251b29fc1de4cf805c39 Mon Sep 17 00:00:00 2001 From: Graham Beckley Date: Wed, 2 Aug 2023 13:09:23 -0400 Subject: [PATCH 03/14] Add request summary middleware --- src/dockerflow/fastapi/__init__.py | 1 + src/dockerflow/fastapi/middleware.py | 77 ++++++++++++++++++++++++++++ tests/fastapi/test_fastapi.py | 22 +++++++- 3 files changed, 99 insertions(+), 1 deletion(-) create mode 100644 src/dockerflow/fastapi/middleware.py diff --git a/src/dockerflow/fastapi/__init__.py b/src/dockerflow/fastapi/__init__.py index 6521c06..3319269 100644 --- a/src/dockerflow/fastapi/__init__.py +++ b/src/dockerflow/fastapi/__init__.py @@ -1 +1,2 @@ from .router import dockerflow_router +from .middleware import MozlogRequestSummaryLogger diff --git a/src/dockerflow/fastapi/middleware.py b/src/dockerflow/fastapi/middleware.py new file mode 100644 index 0000000..3444095 --- /dev/null +++ b/src/dockerflow/fastapi/middleware.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +import logging +import sys +import time +from typing import Any, Dict + +from asgiref.typing import ( + ASGI3Application, + ASGIReceiveCallable, + ASGISendCallable, + ASGISendEvent, + HTTPScope, +) + +from ..logging import JsonLogFormatter + + +class MozlogRequestSummaryLogger: + def __init__( + self, + app: ASGI3Application, + logger: logging.Logger | None = None, + ) -> None: + self.app = app + if logger is None: + self.logger = logging.getLogger("request.summary") + self.logger.setLevel(logging.INFO) + handler = logging.StreamHandler(sys.stdout) + handler.setLevel(logging.INFO) + handler.setFormatter(JsonLogFormatter) + self.logger.addHandler(handler) + else: + self.logger = logger + + async def __call__( + self, scope: HTTPScope, receive: ASGIReceiveCallable, send: ASGISendCallable + ) -> None: + if scope["type"] != "http": + return await self.app(scope, receive, send) + + info = dict(request_headers={}, response={}) + + async def inner_send(message: ASGISendEvent) -> None: + if message["type"] == "http.response.start": + info["response"] = message + + await send(message) + + try: + info["start_time"] = time.time() + await self.app(scope, receive, inner_send) + except Exception as exc: + info["response"]["status"] = 500 + raise exc + finally: + info["end_time"] = time.time() + self._log(scope, info) + + def _log(self, scope: HTTPScope, info) -> None: + self.logger.info("", extra=self._format(scope, info)) + + def _format(self, scope: HTTPScope, info) -> Dict[str, Any]: + for name, value in scope["headers"]: + header_key = name.decode("latin1").lower() + header_val = value.decode("latin1") + info["request_headers"][header_key] = header_val + + request_duration_ms = (info["end_time"] - info["start_time"]) * 1000.0 + return { + "agent": info["request_headers"].get("user-agent", ""), + "path": scope["path"], + "method": scope["method"], + "code": info["response"]["status"], + "lang": info["request_headers"].get("accept-language"), + "t": int(request_duration_ms), + } diff --git a/tests/fastapi/test_fastapi.py b/tests/fastapi/test_fastapi.py index ee8655a..29f5553 100644 --- a/tests/fastapi/test_fastapi.py +++ b/tests/fastapi/test_fastapi.py @@ -1,11 +1,13 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, you can obtain one at http://mozilla.org/MPL/2.0/. +import logging + import pytest from fastapi import FastAPI from fastapi.testclient import TestClient -from dockerflow.fastapi import dockerflow_router +from dockerflow.fastapi import MozlogRequestSummaryLogger, dockerflow_router def create_app(): @@ -35,3 +37,21 @@ def test_lbheartbeat_head(client): response = client.head("/__lbheartbeat__") assert response.status_code == 200 assert response.content == b"" + + +def test_mozlog(client, caplog): + client.get( + "/__lbheartbeat__", + headers={ + "User-Agent": "dockerflow/tests", + "Accept-Language": "en-US", + }, + ) + record = caplog.records[0] + + assert record.levelno == logging.INFO + assert record.agent == "dockerflow/tests" + assert record.lang == "en-US" + assert record.method == "GET" + assert record.path == "/__lbheartbeat__" + assert isinstance(record.t, int) From 81417055635d60de7dd9e4d42ae674d5ecca08f6 Mon Sep 17 00:00:00 2001 From: Graham Beckley Date: Wed, 2 Aug 2023 13:44:18 -0400 Subject: [PATCH 04/14] Add version endpoint and tests --- src/dockerflow/fastapi/router.py | 14 ++++++++++++ tests/fastapi/test_fastapi.py | 37 ++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/src/dockerflow/fastapi/router.py b/src/dockerflow/fastapi/router.py index 820ad84..ba3dd1f 100644 --- a/src/dockerflow/fastapi/router.py +++ b/src/dockerflow/fastapi/router.py @@ -1,7 +1,10 @@ +import os + from fastapi import Request, Response from fastapi.routing import APIRouter +from ..version import get_version dockerflow_router = APIRouter(tags=["Dockerflow"]) @@ -9,3 +12,14 @@ @dockerflow_router.head("/__lbheartbeat__") def lbheartbeat(): return {"status": "ok"} + + +@dockerflow_router.get("/__version__") +def version(request: Request): + if getattr(request.app.state, "APP_DIR", None): + root = request.app.state.APP_DIR + elif os.getenv("APP_DIR"): + root = os.getenv("APP_DIR") + else: + root = "/app" + return get_version(root) diff --git a/tests/fastapi/test_fastapi.py b/tests/fastapi/test_fastapi.py index 29f5553..c9d3780 100644 --- a/tests/fastapi/test_fastapi.py +++ b/tests/fastapi/test_fastapi.py @@ -1,6 +1,7 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, you can obtain one at http://mozilla.org/MPL/2.0/. +import json import logging import pytest @@ -55,3 +56,39 @@ def test_mozlog(client, caplog): assert record.method == "GET" assert record.path == "/__lbheartbeat__" assert isinstance(record.t, int) + + +VERSION_CONTENT = {"foo": "bar"} + + +def test_version_app_state(client, tmp_path, app): + version_path = tmp_path / "version.json" + version_path.write_text(json.dumps(VERSION_CONTENT)) + + app.state.APP_DIR = tmp_path.resolve() + response = client.get("/__version__") + assert response.status_code == 200 + assert response.json() == VERSION_CONTENT + + +def test_version_env_var(client, tmp_path, monkeypatch): + version_path = tmp_path / "version.json" + version_path.write_text(json.dumps(VERSION_CONTENT)) + + monkeypatch.setenv("APP_DIR", tmp_path.resolve()) + + response = client.get("/__version__") + assert response.status_code == 200 + assert response.json() == VERSION_CONTENT + + +def test_version_default(client, mocker): + mock_get_version = mocker.MagicMock(return_value=VERSION_CONTENT) + mocker.patch("dockerflow.fastapi.router.get_version", mock_get_version) + + response = client.get("/__version__") + assert response.status_code == 200 + assert response.json() == VERSION_CONTENT + mock_get_version.assert_called_with("/app") + + From 7ef60d5e1420626cf0401e95475b402afdb9f7b0 Mon Sep 17 00:00:00 2001 From: Graham Beckley Date: Wed, 2 Aug 2023 13:45:10 -0400 Subject: [PATCH 05/14] Add initial heartbeat route, check mechanism, and tests --- src/dockerflow/fastapi/__init__.py | 1 + src/dockerflow/fastapi/checks.py | 49 ++++++++++++++++++++++++++++++ src/dockerflow/fastapi/router.py | 29 ++++++++++++++++++ tests/fastapi/test_fastapi.py | 30 +++++++++++++++++- 4 files changed, 108 insertions(+), 1 deletion(-) create mode 100644 src/dockerflow/fastapi/checks.py diff --git a/src/dockerflow/fastapi/__init__.py b/src/dockerflow/fastapi/__init__.py index 3319269..2ad2cfc 100644 --- a/src/dockerflow/fastapi/__init__.py +++ b/src/dockerflow/fastapi/__init__.py @@ -1,2 +1,3 @@ from .router import dockerflow_router from .middleware import MozlogRequestSummaryLogger +from .checks import check diff --git a/src/dockerflow/fastapi/checks.py b/src/dockerflow/fastapi/checks.py new file mode 100644 index 0000000..50fb306 --- /dev/null +++ b/src/dockerflow/fastapi/checks.py @@ -0,0 +1,49 @@ +from dataclasses import dataclass +import functools +import logging +from typing import Dict, List, Tuple + +from ..checks import level_to_text + +logger = logging.getLogger(__name__) + + +@dataclass +class CheckDetail: + status: str + level: int + messages: Dict[int, str] + + +registered_checks = dict() + + +def check(func, name=None): + if name is None: + name = func.__name__ + + logger.debug("Registered Dockerflow check %s", name) + + @functools.wraps(func) + def decorated_function(*args, **kwargs): + logger.debug("Called Dockerflow check %s", name) + return func(*args, **kwargs) + + registered_checks[name] = decorated_function + return decorated_function + + +def _heartbeat_check_detail(check): + errors = check() + level = max([0] + [e.level for e in errors]) + return CheckDetail( + status=level_to_text(level), level=level, messages={e.id: e.msg for e in errors} + ) + + +def run_checks(): + check_details: List[Tuple[str, CheckDetail]] = [] + for name, check in registered_checks.items(): + detail = _heartbeat_check_detail(check) + check_details.append((name, detail)) + return check_details diff --git a/src/dockerflow/fastapi/router.py b/src/dockerflow/fastapi/router.py index ba3dd1f..883921d 100644 --- a/src/dockerflow/fastapi/router.py +++ b/src/dockerflow/fastapi/router.py @@ -5,6 +5,9 @@ from ..version import get_version +from .checks import run_checks +from dockerflow import checks + dockerflow_router = APIRouter(tags=["Dockerflow"]) @@ -14,6 +17,32 @@ def lbheartbeat(): return {"status": "ok"} +@dockerflow_router.get("/__heartbeat__") +@dockerflow_router.head("/__heartbeat__") +def heartbeat(response: Response): + check_results = run_checks() + details = {} + statuses = {} + level = 0 + + for name, detail in check_results: + statuses[name] = detail.status + level = max(level, detail.level) + if detail.level > 0: + details[name] = detail + + if level < checks.ERROR: + response.status_code = 200 + else: + response.status_code = 500 + + return { + "status": checks.level_to_text(level), + "checks": statuses, + "details": details, + } + + @dockerflow_router.get("/__version__") def version(request: Request): if getattr(request.app.state, "APP_DIR", None): diff --git a/tests/fastapi/test_fastapi.py b/tests/fastapi/test_fastapi.py index c9d3780..70323ee 100644 --- a/tests/fastapi/test_fastapi.py +++ b/tests/fastapi/test_fastapi.py @@ -8,7 +8,8 @@ from fastapi import FastAPI from fastapi.testclient import TestClient -from dockerflow.fastapi import MozlogRequestSummaryLogger, dockerflow_router +from dockerflow.fastapi import MozlogRequestSummaryLogger, check, dockerflow_router +from dockerflow.checks import Error def create_app(): @@ -92,3 +93,30 @@ def test_version_default(client, mocker): mock_get_version.assert_called_with("/app") +def test_heartbeat_get(client): + @check + def return_error(): + return [Error("BOOM", id="foo")] + + response = client.get("/__heartbeat__") + assert response.status_code == 500 + assert response.json() == { + "status": "error", + "checks": {"return_error": "error"}, + "details": { + "return_error": { + "level": 40, + "messages": {"foo": "BOOM"}, + "status": "error", + } + }, + } + + +def test_heartbeat_head(client): + @check + def return_error(): + return [Error("BOOM", id="foo")] + + response = client.head("/__heartbeat__") + assert response.content == b"" From 40e67c387037057b3ec1d131a30bfa84038fe813 Mon Sep 17 00:00:00 2001 From: Graham Beckley Date: Wed, 2 Aug 2023 15:07:24 -0400 Subject: [PATCH 06/14] Refactor package structure, names of things - Define "router" in package init, register view functions defined in views module - Add test for check with custom name - use better name for check operations --- src/dockerflow/fastapi/__init__.py | 17 ++++++++++++++--- src/dockerflow/fastapi/checks.py | 9 ++++++--- .../fastapi/{router.py => views.py} | 14 +++----------- tests/fastapi/test_fastapi.py | 19 +++++++++++++++---- 4 files changed, 38 insertions(+), 21 deletions(-) rename src/dockerflow/fastapi/{router.py => views.py} (71%) diff --git a/src/dockerflow/fastapi/__init__.py b/src/dockerflow/fastapi/__init__.py index 2ad2cfc..8d83259 100644 --- a/src/dockerflow/fastapi/__init__.py +++ b/src/dockerflow/fastapi/__init__.py @@ -1,3 +1,14 @@ -from .router import dockerflow_router -from .middleware import MozlogRequestSummaryLogger -from .checks import check +from fastapi import APIRouter +from fastapi.routing import APIRoute + +from .checks import register_heartbeat_check # noqa +from .views import heartbeat, lbheartbeat, version + +router = APIRouter( + tags=["Dockerflow"], + routes=[ + APIRoute("/__lbheartbeat__", endpoint=lbheartbeat, methods=["GET", "HEAD"]), + APIRoute("/__heartbeat__", endpoint=heartbeat, methods=["GET", "HEAD"]), + APIRoute("/__version__", endpoint=version, methods=["GET"]), + ], +) diff --git a/src/dockerflow/fastapi/checks.py b/src/dockerflow/fastapi/checks.py index 50fb306..cf969d7 100644 --- a/src/dockerflow/fastapi/checks.py +++ b/src/dockerflow/fastapi/checks.py @@ -1,6 +1,6 @@ -from dataclasses import dataclass import functools import logging +from dataclasses import dataclass from typing import Dict, List, Tuple from ..checks import level_to_text @@ -18,7 +18,10 @@ class CheckDetail: registered_checks = dict() -def check(func, name=None): +def register_heartbeat_check(func=None, *, name=None): + if func is None: + return functools.partial(register_heartbeat_check, name=name) + if name is None: name = func.__name__ @@ -41,7 +44,7 @@ def _heartbeat_check_detail(check): ) -def run_checks(): +def run_heartbeat_checks(): check_details: List[Tuple[str, CheckDetail]] = [] for name, check in registered_checks.items(): detail = _heartbeat_check_detail(check) diff --git a/src/dockerflow/fastapi/router.py b/src/dockerflow/fastapi/views.py similarity index 71% rename from src/dockerflow/fastapi/router.py rename to src/dockerflow/fastapi/views.py index 883921d..bafcd61 100644 --- a/src/dockerflow/fastapi/router.py +++ b/src/dockerflow/fastapi/views.py @@ -1,26 +1,19 @@ import os from fastapi import Request, Response -from fastapi.routing import APIRouter - -from ..version import get_version -from .checks import run_checks from dockerflow import checks -dockerflow_router = APIRouter(tags=["Dockerflow"]) +from ..version import get_version +from .checks import run_heartbeat_checks -@dockerflow_router.get("/__lbheartbeat__") -@dockerflow_router.head("/__lbheartbeat__") def lbheartbeat(): return {"status": "ok"} -@dockerflow_router.get("/__heartbeat__") -@dockerflow_router.head("/__heartbeat__") def heartbeat(response: Response): - check_results = run_checks() + check_results = run_heartbeat_checks() details = {} statuses = {} level = 0 @@ -43,7 +36,6 @@ def heartbeat(response: Response): } -@dockerflow_router.get("/__version__") def version(request: Request): if getattr(request.app.state, "APP_DIR", None): root = request.app.state.APP_DIR diff --git a/tests/fastapi/test_fastapi.py b/tests/fastapi/test_fastapi.py index 70323ee..6f9782c 100644 --- a/tests/fastapi/test_fastapi.py +++ b/tests/fastapi/test_fastapi.py @@ -8,8 +8,10 @@ from fastapi import FastAPI from fastapi.testclient import TestClient -from dockerflow.fastapi import MozlogRequestSummaryLogger, check, dockerflow_router from dockerflow.checks import Error +from dockerflow.fastapi import register_heartbeat_check +from dockerflow.fastapi import router as dockerflow_router +from dockerflow.fastapi.middleware import MozlogRequestSummaryLogger def create_app(): @@ -85,7 +87,7 @@ def test_version_env_var(client, tmp_path, monkeypatch): def test_version_default(client, mocker): mock_get_version = mocker.MagicMock(return_value=VERSION_CONTENT) - mocker.patch("dockerflow.fastapi.router.get_version", mock_get_version) + mocker.patch("dockerflow.fastapi.views.get_version", mock_get_version) response = client.get("/__version__") assert response.status_code == 200 @@ -94,7 +96,7 @@ def test_version_default(client, mocker): def test_heartbeat_get(client): - @check + @register_heartbeat_check def return_error(): return [Error("BOOM", id="foo")] @@ -114,9 +116,18 @@ def return_error(): def test_heartbeat_head(client): - @check + @register_heartbeat_check def return_error(): return [Error("BOOM", id="foo")] response = client.head("/__heartbeat__") assert response.content == b"" + + +def test_heartbeat_custom_name(client): + @register_heartbeat_check(name="my_check_name") + def return_error(): + return [Error("BOOM", id="foo")] + + response = client.get("/__heartbeat__") + assert response.json()["checks"]["my_check_name"] From 717ec5baf8532c1bf5995e6422773c44937aaba6 Mon Sep 17 00:00:00 2001 From: Mathieu Leplatre Date: Wed, 14 Feb 2024 15:27:34 +0100 Subject: [PATCH 07/14] Leverage centralized checks (#85) --- src/dockerflow/fastapi/__init__.py | 1 - src/dockerflow/fastapi/checks.py | 52 ------------------------------ src/dockerflow/fastapi/views.py | 30 +++++++---------- tests/fastapi/test_fastapi.py | 15 ++++----- 4 files changed, 19 insertions(+), 79 deletions(-) delete mode 100644 src/dockerflow/fastapi/checks.py diff --git a/src/dockerflow/fastapi/__init__.py b/src/dockerflow/fastapi/__init__.py index 8d83259..8181e18 100644 --- a/src/dockerflow/fastapi/__init__.py +++ b/src/dockerflow/fastapi/__init__.py @@ -1,7 +1,6 @@ from fastapi import APIRouter from fastapi.routing import APIRoute -from .checks import register_heartbeat_check # noqa from .views import heartbeat, lbheartbeat, version router = APIRouter( diff --git a/src/dockerflow/fastapi/checks.py b/src/dockerflow/fastapi/checks.py deleted file mode 100644 index cf969d7..0000000 --- a/src/dockerflow/fastapi/checks.py +++ /dev/null @@ -1,52 +0,0 @@ -import functools -import logging -from dataclasses import dataclass -from typing import Dict, List, Tuple - -from ..checks import level_to_text - -logger = logging.getLogger(__name__) - - -@dataclass -class CheckDetail: - status: str - level: int - messages: Dict[int, str] - - -registered_checks = dict() - - -def register_heartbeat_check(func=None, *, name=None): - if func is None: - return functools.partial(register_heartbeat_check, name=name) - - if name is None: - name = func.__name__ - - logger.debug("Registered Dockerflow check %s", name) - - @functools.wraps(func) - def decorated_function(*args, **kwargs): - logger.debug("Called Dockerflow check %s", name) - return func(*args, **kwargs) - - registered_checks[name] = decorated_function - return decorated_function - - -def _heartbeat_check_detail(check): - errors = check() - level = max([0] + [e.level for e in errors]) - return CheckDetail( - status=level_to_text(level), level=level, messages={e.id: e.msg for e in errors} - ) - - -def run_heartbeat_checks(): - check_details: List[Tuple[str, CheckDetail]] = [] - for name, check in registered_checks.items(): - detail = _heartbeat_check_detail(check) - check_details.append((name, detail)) - return check_details diff --git a/src/dockerflow/fastapi/views.py b/src/dockerflow/fastapi/views.py index bafcd61..8aa1d59 100644 --- a/src/dockerflow/fastapi/views.py +++ b/src/dockerflow/fastapi/views.py @@ -5,7 +5,6 @@ from dockerflow import checks from ..version import get_version -from .checks import run_heartbeat_checks def lbheartbeat(): @@ -13,27 +12,22 @@ def lbheartbeat(): def heartbeat(response: Response): - check_results = run_heartbeat_checks() - details = {} - statuses = {} - level = 0 - - for name, detail in check_results: - statuses[name] = detail.status - level = max(level, detail.level) - if detail.level > 0: - details[name] = detail - - if level < checks.ERROR: + check_results = checks.run_checks( + checks.get_checks().items(), + ) + + payload = { + "status": checks.level_to_text(check_results.level), + "checks": check_results.statuses, + "details": check_results.details, + } + + if check_results.level < checks.ERROR: response.status_code = 200 else: response.status_code = 500 - return { - "status": checks.level_to_text(level), - "checks": statuses, - "details": details, - } + return payload def version(request: Request): diff --git a/tests/fastapi/test_fastapi.py b/tests/fastapi/test_fastapi.py index 6f9782c..e86bf08 100644 --- a/tests/fastapi/test_fastapi.py +++ b/tests/fastapi/test_fastapi.py @@ -8,8 +8,7 @@ from fastapi import FastAPI from fastapi.testclient import TestClient -from dockerflow.checks import Error -from dockerflow.fastapi import register_heartbeat_check +from dockerflow import checks from dockerflow.fastapi import router as dockerflow_router from dockerflow.fastapi.middleware import MozlogRequestSummaryLogger @@ -96,9 +95,9 @@ def test_version_default(client, mocker): def test_heartbeat_get(client): - @register_heartbeat_check + @checks.register def return_error(): - return [Error("BOOM", id="foo")] + return [checks.Error("BOOM", id="foo")] response = client.get("/__heartbeat__") assert response.status_code == 500 @@ -116,18 +115,18 @@ def return_error(): def test_heartbeat_head(client): - @register_heartbeat_check + @checks.register def return_error(): - return [Error("BOOM", id="foo")] + return [checks.Error("BOOM", id="foo")] response = client.head("/__heartbeat__") assert response.content == b"" def test_heartbeat_custom_name(client): - @register_heartbeat_check(name="my_check_name") + @checks.register(name="my_check_name") def return_error(): - return [Error("BOOM", id="foo")] + return [checks.Error("BOOM", id="foo")] response = client.get("/__heartbeat__") assert response.json()["checks"]["my_check_name"] From a5757bea7858c671bc914116358d4fc5365cc251 Mon Sep 17 00:00:00 2001 From: Mathieu Leplatre Date: Wed, 14 Feb 2024 15:33:32 +0100 Subject: [PATCH 08/14] Do not test on Python 3.7 --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 860d32a..6fa1ce5 100644 --- a/tox.ini +++ b/tox.ini @@ -7,7 +7,7 @@ envlist = py{37,38,39,310}-dj32 py{38,39,310,311}-dj{40,41,42} py{310,311}-dj{50} - py{37,38,39,310,311}-fa100 + py{38,39,310,311}-fa100 py{37,38,39,310,311}-fl{20,21,22} py{37,38,39,310,311}-s{21,22} From c0b9033d22058b5e17470bafc9a653ed88cbef01 Mon Sep 17 00:00:00 2001 From: Mathieu Leplatre Date: Wed, 14 Feb 2024 18:22:16 +0100 Subject: [PATCH 09/14] Add 'taskName' to excluded fields --- src/dockerflow/logging.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/dockerflow/logging.py b/src/dockerflow/logging.py index d4e260b..081d808 100644 --- a/src/dockerflow/logging.py +++ b/src/dockerflow/logging.py @@ -65,6 +65,7 @@ class JsonLogFormatter(logging.Formatter): "processName", "relativeCreated", "stack_info", + "taskName", "thread", "threadName", ) From f5ce41a6684e0d3442903781b95d9ea4b36860b3 Mon Sep 17 00:00:00 2001 From: Mathieu Leplatre Date: Wed, 14 Feb 2024 21:27:52 +0100 Subject: [PATCH 10/14] Test with python 3.12 --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 9e20287..7a4fd28 100644 --- a/tox.ini +++ b/tox.ini @@ -7,7 +7,7 @@ envlist = py{38,39,310}-dj32 py{38,39,310,311}-dj{40,41,42} py{310,311}-dj{50} - py{38,39,310,311}-fa100 + py{38,39,310,311,312}-fa100 py{38,39,310,311}-fl{20,21,22} py{38,39,310,311}-s{21,22} From 2dec3f7c3bfd80333eac5db3bec173b53ce047ec Mon Sep 17 00:00:00 2001 From: Mathieu Leplatre Date: Wed, 14 Feb 2024 21:29:22 +0100 Subject: [PATCH 11/14] Improve test coverage of FastAPI --- src/dockerflow/fastapi/middleware.py | 9 ++++----- tests/fastapi/test_fastapi.py | 16 ++++++++++++++-- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/src/dockerflow/fastapi/middleware.py b/src/dockerflow/fastapi/middleware.py index 3444095..9be8b45 100644 --- a/src/dockerflow/fastapi/middleware.py +++ b/src/dockerflow/fastapi/middleware.py @@ -24,14 +24,13 @@ def __init__( ) -> None: self.app = app if logger is None: - self.logger = logging.getLogger("request.summary") - self.logger.setLevel(logging.INFO) + logger = logging.getLogger("request.summary") + logger.setLevel(logging.INFO) handler = logging.StreamHandler(sys.stdout) handler.setLevel(logging.INFO) handler.setFormatter(JsonLogFormatter) - self.logger.addHandler(handler) - else: - self.logger = logger + logger.addHandler(handler) + self.logger = logger async def __call__( self, scope: HTTPScope, receive: ASGIReceiveCallable, send: ASGISendCallable diff --git a/tests/fastapi/test_fastapi.py b/tests/fastapi/test_fastapi.py index e86bf08..b2cd8b1 100644 --- a/tests/fastapi/test_fastapi.py +++ b/tests/fastapi/test_fastapi.py @@ -56,10 +56,21 @@ def test_mozlog(client, caplog): assert record.agent == "dockerflow/tests" assert record.lang == "en-US" assert record.method == "GET" + assert record.code == 200 assert record.path == "/__lbheartbeat__" assert isinstance(record.t, int) +def test_mozlog_failure(client, mocker, caplog): + mocker.patch("dockerflow.fastapi.views.get_version", side_effect=ValueError("crash")) + + with pytest.raises(ValueError): + client.get("/__version__") + + record = caplog.records[0] + assert record.code == 500 + + VERSION_CONTENT = {"foo": "bar"} @@ -116,10 +127,11 @@ def return_error(): def test_heartbeat_head(client): @checks.register - def return_error(): - return [checks.Error("BOOM", id="foo")] + def return_sucess(): + return [checks.Info("Nice", id="foo")] response = client.head("/__heartbeat__") + assert response.status_code == 200 assert response.content == b"" From 5160629e486b78659bdda6708ad33ca890c259ec Mon Sep 17 00:00:00 2001 From: Mathieu Leplatre Date: Wed, 14 Feb 2024 22:04:52 +0100 Subject: [PATCH 12/14] Blackify --- tests/fastapi/test_fastapi.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/fastapi/test_fastapi.py b/tests/fastapi/test_fastapi.py index b2cd8b1..4b14775 100644 --- a/tests/fastapi/test_fastapi.py +++ b/tests/fastapi/test_fastapi.py @@ -62,7 +62,9 @@ def test_mozlog(client, caplog): def test_mozlog_failure(client, mocker, caplog): - mocker.patch("dockerflow.fastapi.views.get_version", side_effect=ValueError("crash")) + mocker.patch( + "dockerflow.fastapi.views.get_version", side_effect=ValueError("crash") + ) with pytest.raises(ValueError): client.get("/__version__") From 0ec8876925c4f32e2efabe41e943535e8f2e856d Mon Sep 17 00:00:00 2001 From: Mathieu Leplatre Date: Tue, 20 Feb 2024 17:39:41 +0100 Subject: [PATCH 13/14] FastAPI integration docs (#95) * Add FastAPI documentation * Update docs/fastapi.rst Co-authored-by: grahamalama * Mention APP_DIR for the version file --------- Co-authored-by: grahamalama --- docs/api/fastapi.rst | 11 + docs/fastapi.rst | 325 +++++++++++++++++++++++++++++ docs/index.rst | 12 +- docs/requirements.txt | 1 + src/dockerflow/fastapi/__init__.py | 1 + 5 files changed, 345 insertions(+), 5 deletions(-) create mode 100644 docs/api/fastapi.rst create mode 100644 docs/fastapi.rst diff --git a/docs/api/fastapi.rst b/docs/api/fastapi.rst new file mode 100644 index 0000000..9290b6d --- /dev/null +++ b/docs/api/fastapi.rst @@ -0,0 +1,11 @@ +FastAPI +======= + +This documents the various FastAPI specific functionality but doesn't cover +internals of the extension. + +Extension +--------- + +.. automodule:: dockerflow.fastapi + :members: router diff --git a/docs/fastapi.rst b/docs/fastapi.rst new file mode 100644 index 0000000..9f15112 --- /dev/null +++ b/docs/fastapi.rst @@ -0,0 +1,325 @@ +FastAPI +======= + +The package ``dockerflow.fastapi`` package implements various tools to support +`FastAPI`_ based projects that want to follow the Dockerflow specs: + +- A Python logging formatter following the `mozlog`_ format. + +- A FastAPI extension implements: + + - Emitting of `request.summary`_ log records based on request specific data. + + - Views for health monitoring: + + - ``/__version__`` - Serves a ``version.json`` file + + - ``/__heartbeat__`` - Runs the configured Dockerflow checks + + - ``/__lbheartbeat__`` - Retuns a HTTP 200 response + + - Hooks to add custom Dockerflow checks. + +.. _`FastAPI`: https://fastapi.tiangolo.com +.. _`mozlog`: https://github.com/mozilla-services/Dockerflow/blob/main/docs/mozlog.md +.. _`request.summary`: https://github.com/mozilla-services/Dockerflow/blob/main/docs/mozlog.md#application-request-summary-type-requestsummary + +.. seealso:: + + For more information see the :doc:`API documentation ` for + the ``dockerflow.fastapi`` module. + +Setup +----- + +To install ``python-dockerflow``'s FastAPI support please follow these steps: + +#. In your code where your FastAPI application lives set up the dockerflow FastAPI + extension:: + + from fastapi import FastAPI + from dockerflow.fastapi import router + from dockerflow.fastapi.middleware import MozlogRequestSummaryLogger + + app = FastAPI() + app.include_router(router) + app.add_middleware(MozlogRequestSummaryLogger) + +#. Make sure the app root path is set correctly as this will be used + to locate the ``version.json`` file that is generated by + your CI or another process during deployment. + + .. seealso:: :ref:`fastapi-versions` for more information + +#. Configure logging to use the ``JsonLogFormatter`` logging formatter for the + ``request.summary`` logger (you may have to extend your existing logging + configuration), see :ref:`fastapi-logging` for more information. + +.. _fastapi-config: + +Configuration +------------- + +.. epigraph:: + + Accept its configuration through environment variables. + +There are several options to handle configuration values through +environment variables when configuring FastAPI. + +``pydantic-settings`` +~~~~~~~~~~~~~~~~~~~~~ + +The simplest is to use `pydantic-settings`_ that will load settings from +environment variables or secrets files, and turn them into model instance. + +.. code-block:: python + + from pydantic_settings import BaseSettings + + class Settings(BaseSettings): + port: int = 8000 + + settings = Settings() + +.. _pydantic-settings: https://docs.pydantic.dev/latest/concepts/pydantic_settings/ + +.. _fastapi-serving: + +``PORT`` +-------- + +.. epigraph:: + + Listen on environment variable ``$PORT`` for HTTP requests. + +Depending on which ASGI server you are using to run your Python application +there are different ways to accept the :envvar:`PORT` as the port to launch +your application with. + +It's recommended to use port ``8000`` by default. + +Uvicorn +~~~~~~~ + +.. code-block:: python + + import uvicorn + + from myapp import Settings + + settings = Settings() + + if __name__ == "__main__": + server = uvicorn.Server( + uvicorn.Config( + "myapp:app", + host=settings.host, + port=settings.port, + reload=settings.app_reload, + log_config=None, + ) + ) + server.run() + +.. _fastapi-versions: + +Versions +-------- + +.. epigraph:: + + Must have a JSON version object at /app/version.json. + +Dockerflow requires writing a `version object`_ to the file +``/app/version.json`` as seen from the docker container to be served under +the URL path ``/__version__``. + +.. note:: + + The default ``/app`` location can be customized using the ``APP_DIR`` + environment variable. + +To facilitate this python-dockerflow comes with a FastAPI view to read the +file under path the parent directory of the app root. See the +:class:`FastAPI API docs <~fastapi.FastAPI>` for more information about the +app root path. + +.. _version object: https://github.com/mozilla-services/Dockerflow/blob/main/docs/version_object.md + +.. _fastapi-health: + +Health monitoring +----------------- + +Health monitoring happens via three different views following the Dockerflow_ +spec: + +.. http:get:: /__version__ + + The view that serves the :ref:`version information `. + + **Example request**: + + .. sourcecode:: http + + GET /__version__ HTTP/1.1 + Host: example.com + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Vary: Accept-Encoding + Content-Type: application/json + + { + "commit": "52ce614fbf99540a1bf6228e36be6cef63b4d73b", + "version": "2017.11.0", + "source": "https://github.com/mozilla/telemetry-analysis-service", + "build": "https://circleci.com/gh/mozilla/telemetry-analysis-service/2223" + } + + :statuscode 200: no error + :statuscode 404: a version.json wasn't found + +.. http:get:: /__heartbeat__ + + The heartbeat view will go through the list of registered Dockerflow + checks, run each check and add their results to a JSON response. + + The view will return HTTP responses with either an status code of 200 if + all checks ran successfully or 500 if there was one or more warnings or + errors returned by the checks. + + Here's an example of a check that handles various levels of exceptions + from an external storage system with different check message:: + + from dockerflow import checks + + @checks.register + def storage_reachable(): + result = [] + try: + acme.storage.ping() + except SlowConnectionException as exc: + result.append(checks.Warning(exc.msg, id='acme.health.0002')) + except StorageException as exc: + result.append(checks.Error(exc.msg, id='acme.health.0001')) + return result + + **Example request**: + + .. sourcecode:: http + + GET /__heartbeat__ HTTP/1.1 + Host: example.com + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 500 Internal Server Error + Vary: Accept-Encoding + Content-Type: application/json + + { + "status": "warning", + "checks": { + "check_debug": "ok", + "check_sts_preload": "warning" + }, + "details": { + "check_sts_preload": { + "status": "warning", + "level": 30, + "messages": { + "security.W021": "You have not set the SECURE_HSTS_PRELOAD setting to True. Without this, your site cannot be submitted to the browser preload list." + } + } + } + } + + :statuscode 200: no error + :statuscode 500: there was an error + +.. http:get:: /__lbheartbeat__ + + The view that simply returns a successful HTTP response so that a load + balancer in front of the application can check that the web application + has started up. + + **Example request**: + + .. sourcecode:: http + + GET /__lbheartbeat__ HTTP/1.1 + Host: example.com + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Vary: Accept-Encoding + Content-Type: application/json + + :statuscode 200: no error + +.. _Dockerflow: https://github.com/mozilla-services/Dockerflow + +.. _fastapi-logging: + +Logging +------- + +Dockerflow provides a :class:`~dockerflow.logging.JsonLogFormatter` Python +logging formatter class. + +To use it, put something like this **BEFORE** your FastAPI app is initialized +for at least the ``request.summary`` logger: + +.. code-block:: python + + from logging.conf import dictConfig + + dictConfig({ + 'version': 1, + 'formatters': { + 'json': { + '()': 'dockerflow.logging.JsonLogFormatter', + 'logger_name': 'myproject' + } + }, + 'handlers': { + 'console': { + 'level': 'DEBUG', + 'class': 'logging.StreamHandler', + 'formatter': 'json' + }, + }, + 'loggers': { + 'request.summary': { + 'handlers': ['console'], + 'level': 'DEBUG', + }, + } + }) + +.. _fastapi-static: + +Static content +-------------- + +We recommend using default `FastAPI features `_ for static files: + +.. code-block:: python + + from fastapi.staticfiles import StaticFiles + + SRC_DIR = Path(__file__).parent + + app = FastAPI() + + app.mount("/static", StaticFiles(directory=SRC_DIR / "static"), name="static") diff --git a/docs/index.rst b/docs/index.rst index 6e2d320..933073b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -24,17 +24,17 @@ Features environment Accept its configuration through environment variables. - See: :ref:`Django `, :ref:`Flask `, :ref:`Sanic ` + See: :ref:`Django `, :ref:`FastAPI `, :ref:`Flask `, :ref:`Sanic ` port Listen on environment variable ``$PORT`` for HTTP requests. - See: :ref:`Django `, :ref:`Flask `, :ref:`Sanic ` + See: :ref:`Django `, :ref:`FastAPI `, :ref:`Flask `, :ref:`Sanic ` version Must have a JSON version object at ``/app/version.json``. - See: :ref:`Django `, :ref:`Flask `, :ref:`Sanic ` + See: :ref:`Django `, :ref:`FastAPI `, :ref:`Flask `, :ref:`Sanic ` health @@ -44,18 +44,19 @@ Features * Respond to ``/__lbheartbeat__`` with an HTTP 200. This is for load balancer checks and should not check backing services. - See: :ref:`Django `, :ref:`Flask `, :ref:`Sanic ` + See: :ref:`Django `, :ref:`FastAPI `, :ref:`Flask `, :ref:`Sanic ` logging Send text logs to ``stdout`` or ``stderr``. See: :ref:`Generic `, :ref:`Django `, + :ref:`FastAPI `, :ref:`Flask `, :ref:`Sanic ` static content Serve its own static content. See: - :ref:`Django `, :ref:`Flask `, :ref:`Flask ` + :ref:`Django `, logging:ref:`FastAPI `, :ref:`Flask ` Contents -------- @@ -69,6 +70,7 @@ Contents changelog logging django + fastapi flask sanic api/index diff --git a/docs/requirements.txt b/docs/requirements.txt index 6b83b8c..cf151c7 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,5 +1,6 @@ -r ../tests/requirements/default.txt -r ../tests/requirements/docs.txt -r ../tests/requirements/django.txt +-r ../tests/requirements/fastapi.txt -r ../tests/requirements/flask.txt -r ../tests/requirements/sanic.txt diff --git a/src/dockerflow/fastapi/__init__.py b/src/dockerflow/fastapi/__init__.py index 8181e18..19596b0 100644 --- a/src/dockerflow/fastapi/__init__.py +++ b/src/dockerflow/fastapi/__init__.py @@ -11,3 +11,4 @@ APIRoute("/__version__", endpoint=version, methods=["GET"]), ], ) +"""This router adds the Dockerflow views.""" From 8ba84ef9f93416abdd1c20152861ba2848247220 Mon Sep 17 00:00:00 2001 From: Mathieu Leplatre Date: Wed, 21 Feb 2024 11:16:10 +0100 Subject: [PATCH 14/14] Update tox.ini Co-authored-by: grahamalama --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index f7d370a..e76aa0d 100644 --- a/tox.ini +++ b/tox.ini @@ -45,7 +45,7 @@ deps = commands = python --version dj{32,40,41,42,50}: pytest --no-migrations -o DJANGO_SETTINGS_MODULE=tests.django.settings -o django_find_project=false {posargs:tests/core/ tests/django} - fa{100}: pytest tests/core/ tests/fastapi/ {posargs:} + fa{100}: pytest {posargs: tests/core/ tests/fastapi/} fl{20,21,22}: pytest {posargs:tests/core/ tests/flask/} s{21,22}: pytest {posargs:tests/core/ tests/sanic/}