diff --git a/docs/api/fastapi.rst b/docs/api/fastapi.rst new file mode 100644 index 0000000..9290b6d --- /dev/null +++ b/docs/api/fastapi.rst @@ -0,0 +1,11 @@ +FastAPI +======= + +This documents the various FastAPI specific functionality but doesn't cover +internals of the extension. + +Extension +--------- + +.. automodule:: dockerflow.fastapi + :members: router diff --git a/docs/fastapi.rst b/docs/fastapi.rst new file mode 100644 index 0000000..9f15112 --- /dev/null +++ b/docs/fastapi.rst @@ -0,0 +1,325 @@ +FastAPI +======= + +The package ``dockerflow.fastapi`` package implements various tools to support +`FastAPI`_ based projects that want to follow the Dockerflow specs: + +- A Python logging formatter following the `mozlog`_ format. + +- A FastAPI extension implements: + + - Emitting of `request.summary`_ log records based on request specific data. + + - Views for health monitoring: + + - ``/__version__`` - Serves a ``version.json`` file + + - ``/__heartbeat__`` - Runs the configured Dockerflow checks + + - ``/__lbheartbeat__`` - Retuns a HTTP 200 response + + - Hooks to add custom Dockerflow checks. + +.. _`FastAPI`: https://fastapi.tiangolo.com +.. _`mozlog`: https://github.com/mozilla-services/Dockerflow/blob/main/docs/mozlog.md +.. _`request.summary`: https://github.com/mozilla-services/Dockerflow/blob/main/docs/mozlog.md#application-request-summary-type-requestsummary + +.. seealso:: + + For more information see the :doc:`API documentation ` for + the ``dockerflow.fastapi`` module. + +Setup +----- + +To install ``python-dockerflow``'s FastAPI support please follow these steps: + +#. In your code where your FastAPI application lives set up the dockerflow FastAPI + extension:: + + from fastapi import FastAPI + from dockerflow.fastapi import router + from dockerflow.fastapi.middleware import MozlogRequestSummaryLogger + + app = FastAPI() + app.include_router(router) + app.add_middleware(MozlogRequestSummaryLogger) + +#. Make sure the app root path is set correctly as this will be used + to locate the ``version.json`` file that is generated by + your CI or another process during deployment. + + .. seealso:: :ref:`fastapi-versions` for more information + +#. Configure logging to use the ``JsonLogFormatter`` logging formatter for the + ``request.summary`` logger (you may have to extend your existing logging + configuration), see :ref:`fastapi-logging` for more information. + +.. _fastapi-config: + +Configuration +------------- + +.. epigraph:: + + Accept its configuration through environment variables. + +There are several options to handle configuration values through +environment variables when configuring FastAPI. + +``pydantic-settings`` +~~~~~~~~~~~~~~~~~~~~~ + +The simplest is to use `pydantic-settings`_ that will load settings from +environment variables or secrets files, and turn them into model instance. + +.. code-block:: python + + from pydantic_settings import BaseSettings + + class Settings(BaseSettings): + port: int = 8000 + + settings = Settings() + +.. _pydantic-settings: https://docs.pydantic.dev/latest/concepts/pydantic_settings/ + +.. _fastapi-serving: + +``PORT`` +-------- + +.. epigraph:: + + Listen on environment variable ``$PORT`` for HTTP requests. + +Depending on which ASGI server you are using to run your Python application +there are different ways to accept the :envvar:`PORT` as the port to launch +your application with. + +It's recommended to use port ``8000`` by default. + +Uvicorn +~~~~~~~ + +.. code-block:: python + + import uvicorn + + from myapp import Settings + + settings = Settings() + + if __name__ == "__main__": + server = uvicorn.Server( + uvicorn.Config( + "myapp:app", + host=settings.host, + port=settings.port, + reload=settings.app_reload, + log_config=None, + ) + ) + server.run() + +.. _fastapi-versions: + +Versions +-------- + +.. epigraph:: + + Must have a JSON version object at /app/version.json. + +Dockerflow requires writing a `version object`_ to the file +``/app/version.json`` as seen from the docker container to be served under +the URL path ``/__version__``. + +.. note:: + + The default ``/app`` location can be customized using the ``APP_DIR`` + environment variable. + +To facilitate this python-dockerflow comes with a FastAPI view to read the +file under path the parent directory of the app root. See the +:class:`FastAPI API docs <~fastapi.FastAPI>` for more information about the +app root path. + +.. _version object: https://github.com/mozilla-services/Dockerflow/blob/main/docs/version_object.md + +.. _fastapi-health: + +Health monitoring +----------------- + +Health monitoring happens via three different views following the Dockerflow_ +spec: + +.. http:get:: /__version__ + + The view that serves the :ref:`version information `. + + **Example request**: + + .. sourcecode:: http + + GET /__version__ HTTP/1.1 + Host: example.com + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Vary: Accept-Encoding + Content-Type: application/json + + { + "commit": "52ce614fbf99540a1bf6228e36be6cef63b4d73b", + "version": "2017.11.0", + "source": "https://github.com/mozilla/telemetry-analysis-service", + "build": "https://circleci.com/gh/mozilla/telemetry-analysis-service/2223" + } + + :statuscode 200: no error + :statuscode 404: a version.json wasn't found + +.. http:get:: /__heartbeat__ + + The heartbeat view will go through the list of registered Dockerflow + checks, run each check and add their results to a JSON response. + + The view will return HTTP responses with either an status code of 200 if + all checks ran successfully or 500 if there was one or more warnings or + errors returned by the checks. + + Here's an example of a check that handles various levels of exceptions + from an external storage system with different check message:: + + from dockerflow import checks + + @checks.register + def storage_reachable(): + result = [] + try: + acme.storage.ping() + except SlowConnectionException as exc: + result.append(checks.Warning(exc.msg, id='acme.health.0002')) + except StorageException as exc: + result.append(checks.Error(exc.msg, id='acme.health.0001')) + return result + + **Example request**: + + .. sourcecode:: http + + GET /__heartbeat__ HTTP/1.1 + Host: example.com + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 500 Internal Server Error + Vary: Accept-Encoding + Content-Type: application/json + + { + "status": "warning", + "checks": { + "check_debug": "ok", + "check_sts_preload": "warning" + }, + "details": { + "check_sts_preload": { + "status": "warning", + "level": 30, + "messages": { + "security.W021": "You have not set the SECURE_HSTS_PRELOAD setting to True. Without this, your site cannot be submitted to the browser preload list." + } + } + } + } + + :statuscode 200: no error + :statuscode 500: there was an error + +.. http:get:: /__lbheartbeat__ + + The view that simply returns a successful HTTP response so that a load + balancer in front of the application can check that the web application + has started up. + + **Example request**: + + .. sourcecode:: http + + GET /__lbheartbeat__ HTTP/1.1 + Host: example.com + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Vary: Accept-Encoding + Content-Type: application/json + + :statuscode 200: no error + +.. _Dockerflow: https://github.com/mozilla-services/Dockerflow + +.. _fastapi-logging: + +Logging +------- + +Dockerflow provides a :class:`~dockerflow.logging.JsonLogFormatter` Python +logging formatter class. + +To use it, put something like this **BEFORE** your FastAPI app is initialized +for at least the ``request.summary`` logger: + +.. code-block:: python + + from logging.conf import dictConfig + + dictConfig({ + 'version': 1, + 'formatters': { + 'json': { + '()': 'dockerflow.logging.JsonLogFormatter', + 'logger_name': 'myproject' + } + }, + 'handlers': { + 'console': { + 'level': 'DEBUG', + 'class': 'logging.StreamHandler', + 'formatter': 'json' + }, + }, + 'loggers': { + 'request.summary': { + 'handlers': ['console'], + 'level': 'DEBUG', + }, + } + }) + +.. _fastapi-static: + +Static content +-------------- + +We recommend using default `FastAPI features `_ for static files: + +.. code-block:: python + + from fastapi.staticfiles import StaticFiles + + SRC_DIR = Path(__file__).parent + + app = FastAPI() + + app.mount("/static", StaticFiles(directory=SRC_DIR / "static"), name="static") diff --git a/docs/index.rst b/docs/index.rst index 6e2d320..933073b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -24,17 +24,17 @@ Features environment Accept its configuration through environment variables. - See: :ref:`Django `, :ref:`Flask `, :ref:`Sanic ` + See: :ref:`Django `, :ref:`FastAPI `, :ref:`Flask `, :ref:`Sanic ` port Listen on environment variable ``$PORT`` for HTTP requests. - See: :ref:`Django `, :ref:`Flask `, :ref:`Sanic ` + See: :ref:`Django `, :ref:`FastAPI `, :ref:`Flask `, :ref:`Sanic ` version Must have a JSON version object at ``/app/version.json``. - See: :ref:`Django `, :ref:`Flask `, :ref:`Sanic ` + See: :ref:`Django `, :ref:`FastAPI `, :ref:`Flask `, :ref:`Sanic ` health @@ -44,18 +44,19 @@ Features * Respond to ``/__lbheartbeat__`` with an HTTP 200. This is for load balancer checks and should not check backing services. - See: :ref:`Django `, :ref:`Flask `, :ref:`Sanic ` + See: :ref:`Django `, :ref:`FastAPI `, :ref:`Flask `, :ref:`Sanic ` logging Send text logs to ``stdout`` or ``stderr``. See: :ref:`Generic `, :ref:`Django `, + :ref:`FastAPI `, :ref:`Flask `, :ref:`Sanic ` static content Serve its own static content. See: - :ref:`Django `, :ref:`Flask `, :ref:`Flask ` + :ref:`Django `, logging:ref:`FastAPI `, :ref:`Flask ` Contents -------- @@ -69,6 +70,7 @@ Contents changelog logging django + fastapi flask sanic api/index diff --git a/docs/requirements.txt b/docs/requirements.txt index 6b83b8c..cf151c7 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,5 +1,6 @@ -r ../tests/requirements/default.txt -r ../tests/requirements/docs.txt -r ../tests/requirements/django.txt +-r ../tests/requirements/fastapi.txt -r ../tests/requirements/flask.txt -r ../tests/requirements/sanic.txt diff --git a/setup.py b/setup.py index 004c002..1c1c3cb 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,9 @@ def read(*parts): "Framework :: Django :: 4.0", "Framework :: Django :: 4.1", "Framework :: Django :: 4.2", + "Framework :: Django :: 5.0", "Framework :: Flask", + "Framework :: FastAPI", "Intended Audience :: Developers", "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)", "Programming Language :: Python", @@ -48,6 +50,7 @@ def read(*parts): "django": ["django"], "flask": ["flask", "blinker"], "sanic": ["sanic"], + "fastapi": ["fastapi", "asgiref"], }, zip_safe=False, python_requires=">=3.7,<4", diff --git a/src/dockerflow/fastapi/__init__.py b/src/dockerflow/fastapi/__init__.py new file mode 100644 index 0000000..19596b0 --- /dev/null +++ b/src/dockerflow/fastapi/__init__.py @@ -0,0 +1,14 @@ +from fastapi import APIRouter +from fastapi.routing import APIRoute + +from .views import heartbeat, lbheartbeat, version + +router = APIRouter( + tags=["Dockerflow"], + routes=[ + APIRoute("/__lbheartbeat__", endpoint=lbheartbeat, methods=["GET", "HEAD"]), + APIRoute("/__heartbeat__", endpoint=heartbeat, methods=["GET", "HEAD"]), + APIRoute("/__version__", endpoint=version, methods=["GET"]), + ], +) +"""This router adds the Dockerflow views.""" diff --git a/src/dockerflow/fastapi/middleware.py b/src/dockerflow/fastapi/middleware.py new file mode 100644 index 0000000..9be8b45 --- /dev/null +++ b/src/dockerflow/fastapi/middleware.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +import logging +import sys +import time +from typing import Any, Dict + +from asgiref.typing import ( + ASGI3Application, + ASGIReceiveCallable, + ASGISendCallable, + ASGISendEvent, + HTTPScope, +) + +from ..logging import JsonLogFormatter + + +class MozlogRequestSummaryLogger: + def __init__( + self, + app: ASGI3Application, + logger: logging.Logger | None = None, + ) -> None: + self.app = app + if logger is None: + logger = logging.getLogger("request.summary") + logger.setLevel(logging.INFO) + handler = logging.StreamHandler(sys.stdout) + handler.setLevel(logging.INFO) + handler.setFormatter(JsonLogFormatter) + logger.addHandler(handler) + self.logger = logger + + async def __call__( + self, scope: HTTPScope, receive: ASGIReceiveCallable, send: ASGISendCallable + ) -> None: + if scope["type"] != "http": + return await self.app(scope, receive, send) + + info = dict(request_headers={}, response={}) + + async def inner_send(message: ASGISendEvent) -> None: + if message["type"] == "http.response.start": + info["response"] = message + + await send(message) + + try: + info["start_time"] = time.time() + await self.app(scope, receive, inner_send) + except Exception as exc: + info["response"]["status"] = 500 + raise exc + finally: + info["end_time"] = time.time() + self._log(scope, info) + + def _log(self, scope: HTTPScope, info) -> None: + self.logger.info("", extra=self._format(scope, info)) + + def _format(self, scope: HTTPScope, info) -> Dict[str, Any]: + for name, value in scope["headers"]: + header_key = name.decode("latin1").lower() + header_val = value.decode("latin1") + info["request_headers"][header_key] = header_val + + request_duration_ms = (info["end_time"] - info["start_time"]) * 1000.0 + return { + "agent": info["request_headers"].get("user-agent", ""), + "path": scope["path"], + "method": scope["method"], + "code": info["response"]["status"], + "lang": info["request_headers"].get("accept-language"), + "t": int(request_duration_ms), + } diff --git a/src/dockerflow/fastapi/views.py b/src/dockerflow/fastapi/views.py new file mode 100644 index 0000000..8aa1d59 --- /dev/null +++ b/src/dockerflow/fastapi/views.py @@ -0,0 +1,40 @@ +import os + +from fastapi import Request, Response + +from dockerflow import checks + +from ..version import get_version + + +def lbheartbeat(): + return {"status": "ok"} + + +def heartbeat(response: Response): + check_results = checks.run_checks( + checks.get_checks().items(), + ) + + payload = { + "status": checks.level_to_text(check_results.level), + "checks": check_results.statuses, + "details": check_results.details, + } + + if check_results.level < checks.ERROR: + response.status_code = 200 + else: + response.status_code = 500 + + return payload + + +def version(request: Request): + if getattr(request.app.state, "APP_DIR", None): + root = request.app.state.APP_DIR + elif os.getenv("APP_DIR"): + root = os.getenv("APP_DIR") + else: + root = "/app" + return get_version(root) diff --git a/tests/constraints/fastapi-0.100.txt b/tests/constraints/fastapi-0.100.txt new file mode 100644 index 0000000..c34c72e --- /dev/null +++ b/tests/constraints/fastapi-0.100.txt @@ -0,0 +1 @@ +fastapi>=0.100,<0.101 \ No newline at end of file diff --git a/tests/fastapi/test_fastapi.py b/tests/fastapi/test_fastapi.py new file mode 100644 index 0000000..4b14775 --- /dev/null +++ b/tests/fastapi/test_fastapi.py @@ -0,0 +1,146 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, you can obtain one at http://mozilla.org/MPL/2.0/. +import json +import logging + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from dockerflow import checks +from dockerflow.fastapi import router as dockerflow_router +from dockerflow.fastapi.middleware import MozlogRequestSummaryLogger + + +def create_app(): + app = FastAPI() + app.include_router(dockerflow_router) + app.add_middleware(MozlogRequestSummaryLogger) + return app + + +@pytest.fixture +def app(): + return create_app() + + +@pytest.fixture +def client(app): + return TestClient(app) + + +def test_lbheartbeat_get(client): + response = client.get("/__lbheartbeat__") + assert response.status_code == 200 + assert response.json() == {"status": "ok"} + + +def test_lbheartbeat_head(client): + response = client.head("/__lbheartbeat__") + assert response.status_code == 200 + assert response.content == b"" + + +def test_mozlog(client, caplog): + client.get( + "/__lbheartbeat__", + headers={ + "User-Agent": "dockerflow/tests", + "Accept-Language": "en-US", + }, + ) + record = caplog.records[0] + + assert record.levelno == logging.INFO + assert record.agent == "dockerflow/tests" + assert record.lang == "en-US" + assert record.method == "GET" + assert record.code == 200 + assert record.path == "/__lbheartbeat__" + assert isinstance(record.t, int) + + +def test_mozlog_failure(client, mocker, caplog): + mocker.patch( + "dockerflow.fastapi.views.get_version", side_effect=ValueError("crash") + ) + + with pytest.raises(ValueError): + client.get("/__version__") + + record = caplog.records[0] + assert record.code == 500 + + +VERSION_CONTENT = {"foo": "bar"} + + +def test_version_app_state(client, tmp_path, app): + version_path = tmp_path / "version.json" + version_path.write_text(json.dumps(VERSION_CONTENT)) + + app.state.APP_DIR = tmp_path.resolve() + response = client.get("/__version__") + assert response.status_code == 200 + assert response.json() == VERSION_CONTENT + + +def test_version_env_var(client, tmp_path, monkeypatch): + version_path = tmp_path / "version.json" + version_path.write_text(json.dumps(VERSION_CONTENT)) + + monkeypatch.setenv("APP_DIR", tmp_path.resolve()) + + response = client.get("/__version__") + assert response.status_code == 200 + assert response.json() == VERSION_CONTENT + + +def test_version_default(client, mocker): + mock_get_version = mocker.MagicMock(return_value=VERSION_CONTENT) + mocker.patch("dockerflow.fastapi.views.get_version", mock_get_version) + + response = client.get("/__version__") + assert response.status_code == 200 + assert response.json() == VERSION_CONTENT + mock_get_version.assert_called_with("/app") + + +def test_heartbeat_get(client): + @checks.register + def return_error(): + return [checks.Error("BOOM", id="foo")] + + response = client.get("/__heartbeat__") + assert response.status_code == 500 + assert response.json() == { + "status": "error", + "checks": {"return_error": "error"}, + "details": { + "return_error": { + "level": 40, + "messages": {"foo": "BOOM"}, + "status": "error", + } + }, + } + + +def test_heartbeat_head(client): + @checks.register + def return_sucess(): + return [checks.Info("Nice", id="foo")] + + response = client.head("/__heartbeat__") + assert response.status_code == 200 + assert response.content == b"" + + +def test_heartbeat_custom_name(client): + @checks.register(name="my_check_name") + def return_error(): + return [checks.Error("BOOM", id="foo")] + + response = client.get("/__heartbeat__") + assert response.json()["checks"]["my_check_name"] diff --git a/tests/requirements/fastapi.txt b/tests/requirements/fastapi.txt new file mode 100644 index 0000000..18476e7 --- /dev/null +++ b/tests/requirements/fastapi.txt @@ -0,0 +1,3 @@ +fastapi +asgiref +httpx \ No newline at end of file diff --git a/tox.ini b/tox.ini index 7d1bf23..e76aa0d 100644 --- a/tox.ini +++ b/tox.ini @@ -7,6 +7,7 @@ envlist = py{38,39,310}-dj32 py{38,39,310,311}-dj{40,41,42} py{310,311,312}-dj{50} + py{38,39,310,311,312}-fa100 py{38,39,310,311}-fl{20,21,22} py{38,39,310,311}-s{21,22} @@ -27,6 +28,7 @@ setenv = deps = -rtests/requirements/default.txt dj{32,40,41,42,50}: -rtests/requirements/django.txt + fa100: -rtests/requirements/fastapi.txt fl{20,21,22}: -rtests/requirements/flask.txt s{21,22}: -rtests/requirements/sanic.txt dj32: -ctests/constraints/django-3.2.txt @@ -34,6 +36,7 @@ deps = dj41: -ctests/constraints/django-4.1.txt dj42: -ctests/constraints/django-4.2.txt dj50: -ctests/constraints/django-5.0.txt + fa100: -ctests/constraints/fastapi-0.100.txt fl20: -ctests/constraints/flask-2.0.txt fl21: -ctests/constraints/flask-2.1.txt fl22: -ctests/constraints/flask-2.2.txt @@ -42,6 +45,7 @@ deps = commands = python --version dj{32,40,41,42,50}: pytest --no-migrations -o DJANGO_SETTINGS_MODULE=tests.django.settings -o django_find_project=false {posargs:tests/core/ tests/django} + fa{100}: pytest {posargs: tests/core/ tests/fastapi/} fl{20,21,22}: pytest {posargs:tests/core/ tests/flask/} s{21,22}: pytest {posargs:tests/core/ tests/sanic/}