Port Datasette from Sanic to ASGI + Uvicorn (#518)

Datasette now uses ASGI internally, and no longer depends on Sanic.

It now uses Uvicorn as the underlying HTTP server.

This was thirteen months in the making... for full details see the issue:

#272

And for a full sequence of commits plus commentary, see the pull request:

#518

datasette/app.py

@@ -1,11 +1,9 @@
 import asyncio
 import collections
 import hashlib
-import json
 import os
 import sys
 import threading
-import time
 import traceback
 import urllib.parse
 from concurrent import futures
@@ -14,10 +12,8 @@
 import click
 from markupsafe import Markup
 from jinja2 import ChoiceLoader, Environment, FileSystemLoader, PrefixLoader
-from sanic import Sanic, response
-from sanic.exceptions import InvalidUsage, NotFound
 
-from .views.base import DatasetteError, ureg
+from .views.base import DatasetteError, ureg, AsgiRouter
 from .views.database import DatabaseDownload, DatabaseView
 from .views.index import IndexView
 from .views.special import JsonDataView
@@ -36,7 +32,16 @@
     sqlite_timelimit,
     to_css_class,
 )
-from .tracer import capture_traces, trace
+from .utils.asgi import (
+    AsgiLifespan,
+    NotFound,
+    asgi_static,
+    asgi_send,
+    asgi_send_html,
+    asgi_send_json,
+    asgi_send_redirect,
+)
+from .tracer import trace, AsgiTracer
 from .plugins import pm, DEFAULT_PLUGINS
 from .version import __version__
 
@@ -126,8 +131,8 @@
 DEFAULT_CONFIG = {option.name: option.default for option in CONFIG_OPTIONS}
 
 
-async def favicon(request):
-    return response.text("")
+async def favicon(scope, receive, send):
+    await asgi_send(send, "", 200)
 
 
 class Datasette:
@@ -413,6 +418,7 @@ def versions(self):
                 "full": sys.version,
             },
             "datasette": datasette_version,
+            "asgi": "3.0",
             "sqlite": {
                 "version": sqlite_version,
                 "fts_versions": fts_versions,
@@ -543,21 +549,7 @@ def register_renderers(self):
             self.renderers[renderer["extension"]] = renderer["callback"]
 
     def app(self):
-        class TracingSanic(Sanic):
-            async def handle_request(self, request, write_callback, stream_callback):
-                if request.args.get("_trace"):
-                    request["traces"] = []
-                    request["trace_start"] = time.time()
-                    with capture_traces(request["traces"]):
-                        await super().handle_request(
-                            request, write_callback, stream_callback
-                        )
-                else:
-                    await super().handle_request(
-                        request, write_callback, stream_callback
-                    )
-
-        app = TracingSanic(__name__)
+        "Returns an ASGI app function that serves the whole of Datasette"
         default_templates = str(app_root / "datasette" / "templates")
         template_paths = []
         if self.template_dir:
@@ -588,134 +580,127 @@ async def handle_request(self, request, write_callback, stream_callback):
         pm.hook.prepare_jinja2_environment(env=self.jinja_env)
 
         self.register_renderers()
+
+        routes = []
+
+        def add_route(view, regex):
+            routes.append((regex, view))
+
         # Generate a regex snippet to match all registered renderer file extensions
         renderer_regex = "|".join(r"\." + key for key in self.renderers.keys())
 
-        app.add_route(IndexView.as_view(self), r"/<as_format:(\.jsono?)?$>")
+        add_route(IndexView.as_asgi(self), r"/(?P<as_format>(\.jsono?)?$)")
         # TODO: /favicon.ico and /-/static/ deserve far-future cache expires
-        app.add_route(favicon, "/favicon.ico")
-        app.static("/-/static/", str(app_root / "datasette" / "static"))
+        add_route(favicon, "/favicon.ico")
+
+        add_route(
+            asgi_static(app_root / "datasette" / "static"), r"/-/static/(?P<path>.*)$"
+        )
         for path, dirname in self.static_mounts:
-            app.static(path, dirname)
+            add_route(asgi_static(dirname), r"/" + path + "/(?P<path>.*)$")
+
         # Mount any plugin static/ directories
         for plugin in get_plugins(pm):
             if plugin["static_path"]:
-                modpath = "/-/static-plugins/{}/".format(plugin["name"])
-                app.static(modpath, plugin["static_path"])
-        app.add_route(
-            JsonDataView.as_view(self, "metadata.json", lambda: self._metadata),
-            r"/-/metadata<as_format:(\.json)?$>",
+                modpath = "/-/static-plugins/{}/(?P<path>.*)$".format(plugin["name"])
+                add_route(asgi_static(plugin["static_path"]), modpath)
+        add_route(
+            JsonDataView.as_asgi(self, "metadata.json", lambda: self._metadata),
+            r"/-/metadata(?P<as_format>(\.json)?)$",
         )
-        app.add_route(
-            JsonDataView.as_view(self, "versions.json", self.versions),
-            r"/-/versions<as_format:(\.json)?$>",
+        add_route(
+            JsonDataView.as_asgi(self, "versions.json", self.versions),
+            r"/-/versions(?P<as_format>(\.json)?)$",
         )
-        app.add_route(
-            JsonDataView.as_view(self, "plugins.json", self.plugins),
-            r"/-/plugins<as_format:(\.json)?$>",
+        add_route(
+            JsonDataView.as_asgi(self, "plugins.json", self.plugins),
+            r"/-/plugins(?P<as_format>(\.json)?)$",
         )
-        app.add_route(
-            JsonDataView.as_view(self, "config.json", lambda: self._config),
-            r"/-/config<as_format:(\.json)?$>",
+        add_route(
+            JsonDataView.as_asgi(self, "config.json", lambda: self._config),
+            r"/-/config(?P<as_format>(\.json)?)$",
         )
-        app.add_route(
-            JsonDataView.as_view(self, "databases.json", self.connected_databases),
-            r"/-/databases<as_format:(\.json)?$>",
+        add_route(
+            JsonDataView.as_asgi(self, "databases.json", self.connected_databases),
+            r"/-/databases(?P<as_format>(\.json)?)$",
         )
-        app.add_route(
-            DatabaseDownload.as_view(self), r"/<db_name:[^/]+?><as_db:(\.db)$>"
+        add_route(
+            DatabaseDownload.as_asgi(self), r"/(?P<db_name>[^/]+?)(?P<as_db>\.db)$"
         )
-        app.add_route(
-            DatabaseView.as_view(self),
-            r"/<db_name:[^/]+?><as_format:(" + renderer_regex + r"|.jsono|\.csv)?$>",
+        add_route(
+            DatabaseView.as_asgi(self),
+            r"/(?P<db_name>[^/]+?)(?P<as_format>"
+            + renderer_regex
+            + r"|.jsono|\.csv)?$",
         )
-        app.add_route(
-            TableView.as_view(self), r"/<db_name:[^/]+>/<table_and_format:[^/]+?$>"
+        add_route(
+            TableView.as_asgi(self),
+            r"/(?P<db_name>[^/]+)/(?P<table_and_format>[^/]+?$)",
         )
-        app.add_route(
-            RowView.as_view(self),
-            r"/<db_name:[^/]+>/<table:[^/]+?>/<pk_path:[^/]+?><as_format:("
+        add_route(
+            RowView.as_asgi(self),
+            r"/(?P<db_name>[^/]+)/(?P<table>[^/]+?)/(?P<pk_path>[^/]+?)(?P<as_format>"
             + renderer_regex
-            + r")?$>",
+            + r")?$",
         )
         self.register_custom_units()
 
-        # On 404 with a trailing slash redirect to path without that slash:
-        # pylint: disable=unused-variable
-        @app.middleware("response")
-        def redirect_on_404_with_trailing_slash(request, original_response):
-            if original_response.status == 404 and request.path.endswith("/"):
-                path = request.path.rstrip("/")
-                if request.query_string:
-                    path = "{}?{}".format(path, request.query_string)
-                return response.redirect(path)
-
-        @app.middleware("response")
-        async def add_traces_to_response(request, response):
-            if request.get("traces") is None:
-                return
-            traces = request["traces"]
-            trace_info = {
-                "request_duration_ms": 1000 * (time.time() - request["trace_start"]),
-                "sum_trace_duration_ms": sum(t["duration_ms"] for t in traces),
-                "num_traces": len(traces),
-                "traces": traces,
-            }
-            if "text/html" in response.content_type and b"</body>" in response.body:
-                extra = json.dumps(trace_info, indent=2)
-                extra_html = "<pre>{}</pre></body>".format(extra).encode("utf8")
-                response.body = response.body.replace(b"</body>", extra_html)
-            elif "json" in response.content_type and response.body.startswith(b"{"):
-                data = json.loads(response.body.decode("utf8"))
-                if "_trace" not in data:
-                    data["_trace"] = trace_info
-                    response.body = json.dumps(data).encode("utf8")
-
-        @app.exception(Exception)
-        def on_exception(request, exception):
-            title = None
-            help = None
-            if isinstance(exception, NotFound):
-                status = 404
-                info = {}
-                message = exception.args[0]
-            elif isinstance(exception, InvalidUsage):
-                status = 405
-                info = {}
-                message = exception.args[0]
-            elif isinstance(exception, DatasetteError):
-                status = exception.status
-                info = exception.error_dict
-                message = exception.message
-                if exception.messagge_is_html:
-                    message = Markup(message)
-                title = exception.title
-            else:
-                status = 500
-                info = {}
-                message = str(exception)
-                traceback.print_exc()
-            templates = ["500.html"]
-            if status != 500:
-                templates = ["{}.html".format(status)] + templates
-            info.update(
-                {"ok": False, "error": message, "status": status, "title": title}
-            )
-            if request is not None and request.path.split("?")[0].endswith(".json"):
-                r = response.json(info, status=status)
-
-            else:
-                template = self.jinja_env.select_template(templates)
-                r = response.html(template.render(info), status=status)
-            if self.cors:
-                r.headers["Access-Control-Allow-Origin"] = "*"
-            return r
-
-        # First time server starts up, calculate table counts for immutable databases
-        @app.listener("before_server_start")
-        async def setup_db(app, loop):
+        async def setup_db():
+            # First time server starts up, calculate table counts for immutable databases
             for dbname, database in self.databases.items():
                 if not database.is_mutable:
                     await database.table_counts(limit=60 * 60 * 1000)
 
-        return app
+        return AsgiLifespan(
+            AsgiTracer(DatasetteRouter(self, routes)), on_startup=setup_db
+        )
+
+
+class DatasetteRouter(AsgiRouter):
+    def __init__(self, datasette, routes):
+        self.ds = datasette
+        super().__init__(routes)
+
+    async def handle_404(self, scope, receive, send):
+        # If URL has a trailing slash, redirect to URL without it
+        path = scope.get("raw_path", scope["path"].encode("utf8"))
+        if path.endswith(b"/"):
+            path = path.rstrip(b"/")
+            if scope["query_string"]:
+                path += b"?" + scope["query_string"]
+            await asgi_send_redirect(send, path.decode("latin1"))
+        else:
+            await super().handle_404(scope, receive, send)
+
+    async def handle_500(self, scope, receive, send, exception):
+        title = None
+        if isinstance(exception, NotFound):
+            status = 404
+            info = {}
+            message = exception.args[0]
+        elif isinstance(exception, DatasetteError):
+            status = exception.status
+            info = exception.error_dict
+            message = exception.message
+            if exception.messagge_is_html:
+                message = Markup(message)
+            title = exception.title
+        else:
+            status = 500
+            info = {}
+            message = str(exception)
+            traceback.print_exc()
+        templates = ["500.html"]
+        if status != 500:
+            templates = ["{}.html".format(status)] + templates
+        info.update({"ok": False, "error": message, "status": status, "title": title})
+        headers = {}
+        if self.ds.cors:
+            headers["Access-Control-Allow-Origin"] = "*"
+        if scope["path"].split("?")[0].endswith(".json"):
+            await asgi_send_json(send, info, status=status, headers=headers)
+        else:
+            template = self.ds.jinja_env.select_template(templates)
+            await asgi_send_html(
+                send, template.render(info), status=status, headers=headers
+            )

datasette/cli.py

@@ -1,4 +1,5 @@
 import asyncio
+import uvicorn
 import click
 from click import formatting
 from click_default_group import DefaultGroup
@@ -354,4 +355,4 @@ def serve(
     asyncio.get_event_loop().run_until_complete(ds.run_sanity_checks())
 
     # Start the server
-    ds.app().run(host=host, port=port, debug=debug)
+    uvicorn.run(ds.app(), host=host, port=port, log_level="info")

datasette/renderer.py

@@ -88,5 +88,5 @@ def json_renderer(args, data, view_name):
         content_type = "text/plain"
     else:
         body = json.dumps(data, cls=CustomJSONEncoder)
-        content_type = "application/json"
+        content_type = "application/json; charset=utf-8"
     return {"body": body, "status_code": status_code, "content_type": content_type}