Skip to content
Permalink
Browse files

Port Datasette from Sanic to ASGI + Uvicorn (#518)

Datasette now uses ASGI internally, and no longer depends on Sanic.

It now uses Uvicorn as the underlying HTTP server.

This was thirteen months in the making... for full details see the issue:

#272

And for a full sequence of commits plus commentary, see the pull request:

#518
  • Loading branch information...
simonw committed Jun 24, 2019
1 parent 35429f9 commit ba8db9679f3bd2454c9e76e7e6c352126848b57a
@@ -1,11 +1,9 @@
import asyncio
import collections
import hashlib
import json
import os
import sys
import threading
import time
import traceback
import urllib.parse
from concurrent import futures
@@ -14,10 +12,8 @@
import click
from markupsafe import Markup
from jinja2 import ChoiceLoader, Environment, FileSystemLoader, PrefixLoader
from sanic import Sanic, response
from sanic.exceptions import InvalidUsage, NotFound

from .views.base import DatasetteError, ureg
from .views.base import DatasetteError, ureg, AsgiRouter
from .views.database import DatabaseDownload, DatabaseView
from .views.index import IndexView
from .views.special import JsonDataView
@@ -36,7 +32,16 @@
sqlite_timelimit,
to_css_class,
)
from .tracer import capture_traces, trace
from .utils.asgi import (
AsgiLifespan,
NotFound,
asgi_static,
asgi_send,
asgi_send_html,
asgi_send_json,
asgi_send_redirect,
)
from .tracer import trace, AsgiTracer
from .plugins import pm, DEFAULT_PLUGINS
from .version import __version__

@@ -126,8 +131,8 @@
DEFAULT_CONFIG = {option.name: option.default for option in CONFIG_OPTIONS}


async def favicon(request):
return response.text("")
async def favicon(scope, receive, send):
await asgi_send(send, "", 200)


class Datasette:
@@ -413,6 +418,7 @@ def versions(self):
"full": sys.version,
},
"datasette": datasette_version,
"asgi": "3.0",
"sqlite": {
"version": sqlite_version,
"fts_versions": fts_versions,
@@ -543,21 +549,7 @@ def register_renderers(self):
self.renderers[renderer["extension"]] = renderer["callback"]

def app(self):
class TracingSanic(Sanic):
async def handle_request(self, request, write_callback, stream_callback):
if request.args.get("_trace"):
request["traces"] = []
request["trace_start"] = time.time()
with capture_traces(request["traces"]):
await super().handle_request(
request, write_callback, stream_callback
)
else:
await super().handle_request(
request, write_callback, stream_callback
)

app = TracingSanic(__name__)
"Returns an ASGI app function that serves the whole of Datasette"
default_templates = str(app_root / "datasette" / "templates")
template_paths = []
if self.template_dir:
@@ -588,134 +580,127 @@ class TracingSanic(Sanic):
pm.hook.prepare_jinja2_environment(env=self.jinja_env)

self.register_renderers()

routes = []

def add_route(view, regex):
routes.append((regex, view))

# Generate a regex snippet to match all registered renderer file extensions
renderer_regex = "|".join(r"\." + key for key in self.renderers.keys())

app.add_route(IndexView.as_view(self), r"/<as_format:(\.jsono?)?$>")
add_route(IndexView.as_asgi(self), r"/(?P<as_format>(\.jsono?)?$)")
# TODO: /favicon.ico and /-/static/ deserve far-future cache expires
app.add_route(favicon, "/favicon.ico")
app.static("/-/static/", str(app_root / "datasette" / "static"))
add_route(favicon, "/favicon.ico")

add_route(
asgi_static(app_root / "datasette" / "static"), r"/-/static/(?P<path>.*)$"
)
for path, dirname in self.static_mounts:
app.static(path, dirname)
add_route(asgi_static(dirname), r"/" + path + "/(?P<path>.*)$")

# Mount any plugin static/ directories
for plugin in get_plugins(pm):
if plugin["static_path"]:
modpath = "/-/static-plugins/{}/".format(plugin["name"])
app.static(modpath, plugin["static_path"])
app.add_route(
JsonDataView.as_view(self, "metadata.json", lambda: self._metadata),
r"/-/metadata<as_format:(\.json)?$>",
modpath = "/-/static-plugins/{}/(?P<path>.*)$".format(plugin["name"])
add_route(asgi_static(plugin["static_path"]), modpath)
add_route(
JsonDataView.as_asgi(self, "metadata.json", lambda: self._metadata),
r"/-/metadata(?P<as_format>(\.json)?)$",
)
app.add_route(
JsonDataView.as_view(self, "versions.json", self.versions),
r"/-/versions<as_format:(\.json)?$>",
add_route(
JsonDataView.as_asgi(self, "versions.json", self.versions),
r"/-/versions(?P<as_format>(\.json)?)$",
)
app.add_route(
JsonDataView.as_view(self, "plugins.json", self.plugins),
r"/-/plugins<as_format:(\.json)?$>",
add_route(
JsonDataView.as_asgi(self, "plugins.json", self.plugins),
r"/-/plugins(?P<as_format>(\.json)?)$",
)
app.add_route(
JsonDataView.as_view(self, "config.json", lambda: self._config),
r"/-/config<as_format:(\.json)?$>",
add_route(
JsonDataView.as_asgi(self, "config.json", lambda: self._config),
r"/-/config(?P<as_format>(\.json)?)$",
)
app.add_route(
JsonDataView.as_view(self, "databases.json", self.connected_databases),
r"/-/databases<as_format:(\.json)?$>",
add_route(
JsonDataView.as_asgi(self, "databases.json", self.connected_databases),
r"/-/databases(?P<as_format>(\.json)?)$",
)
app.add_route(
DatabaseDownload.as_view(self), r"/<db_name:[^/]+?><as_db:(\.db)$>"
add_route(
DatabaseDownload.as_asgi(self), r"/(?P<db_name>[^/]+?)(?P<as_db>\.db)$"
)
app.add_route(
DatabaseView.as_view(self),
r"/<db_name:[^/]+?><as_format:(" + renderer_regex + r"|.jsono|\.csv)?$>",
add_route(
DatabaseView.as_asgi(self),
r"/(?P<db_name>[^/]+?)(?P<as_format>"
+ renderer_regex
+ r"|.jsono|\.csv)?$",
)
app.add_route(
TableView.as_view(self), r"/<db_name:[^/]+>/<table_and_format:[^/]+?$>"
add_route(
TableView.as_asgi(self),
r"/(?P<db_name>[^/]+)/(?P<table_and_format>[^/]+?$)",
)
app.add_route(
RowView.as_view(self),
r"/<db_name:[^/]+>/<table:[^/]+?>/<pk_path:[^/]+?><as_format:("
add_route(
RowView.as_asgi(self),
r"/(?P<db_name>[^/]+)/(?P<table>[^/]+?)/(?P<pk_path>[^/]+?)(?P<as_format>"
+ renderer_regex
+ r")?$>",
+ r")?$",
)
self.register_custom_units()

# On 404 with a trailing slash redirect to path without that slash:
# pylint: disable=unused-variable
@app.middleware("response")
def redirect_on_404_with_trailing_slash(request, original_response):
if original_response.status == 404 and request.path.endswith("/"):
path = request.path.rstrip("/")
if request.query_string:
path = "{}?{}".format(path, request.query_string)
return response.redirect(path)

@app.middleware("response")
async def add_traces_to_response(request, response):
if request.get("traces") is None:
return
traces = request["traces"]
trace_info = {
"request_duration_ms": 1000 * (time.time() - request["trace_start"]),
"sum_trace_duration_ms": sum(t["duration_ms"] for t in traces),
"num_traces": len(traces),
"traces": traces,
}
if "text/html" in response.content_type and b"</body>" in response.body:
extra = json.dumps(trace_info, indent=2)
extra_html = "<pre>{}</pre></body>".format(extra).encode("utf8")
response.body = response.body.replace(b"</body>", extra_html)
elif "json" in response.content_type and response.body.startswith(b"{"):
data = json.loads(response.body.decode("utf8"))
if "_trace" not in data:
data["_trace"] = trace_info
response.body = json.dumps(data).encode("utf8")

@app.exception(Exception)
def on_exception(request, exception):
title = None
help = None
if isinstance(exception, NotFound):
status = 404
info = {}
message = exception.args[0]
elif isinstance(exception, InvalidUsage):
status = 405
info = {}
message = exception.args[0]
elif isinstance(exception, DatasetteError):
status = exception.status
info = exception.error_dict
message = exception.message
if exception.messagge_is_html:
message = Markup(message)
title = exception.title
else:
status = 500
info = {}
message = str(exception)
traceback.print_exc()
templates = ["500.html"]
if status != 500:
templates = ["{}.html".format(status)] + templates
info.update(
{"ok": False, "error": message, "status": status, "title": title}
)
if request is not None and request.path.split("?")[0].endswith(".json"):
r = response.json(info, status=status)

else:
template = self.jinja_env.select_template(templates)
r = response.html(template.render(info), status=status)
if self.cors:
r.headers["Access-Control-Allow-Origin"] = "*"
return r

# First time server starts up, calculate table counts for immutable databases
@app.listener("before_server_start")
async def setup_db(app, loop):
async def setup_db():
# First time server starts up, calculate table counts for immutable databases
for dbname, database in self.databases.items():
if not database.is_mutable:
await database.table_counts(limit=60 * 60 * 1000)

return app
return AsgiLifespan(
AsgiTracer(DatasetteRouter(self, routes)), on_startup=setup_db
)


class DatasetteRouter(AsgiRouter):
def __init__(self, datasette, routes):
self.ds = datasette
super().__init__(routes)

async def handle_404(self, scope, receive, send):
# If URL has a trailing slash, redirect to URL without it
path = scope.get("raw_path", scope["path"].encode("utf8"))
if path.endswith(b"/"):
path = path.rstrip(b"/")
if scope["query_string"]:
path += b"?" + scope["query_string"]
await asgi_send_redirect(send, path.decode("latin1"))
else:
await super().handle_404(scope, receive, send)

async def handle_500(self, scope, receive, send, exception):
title = None
if isinstance(exception, NotFound):
status = 404
info = {}
message = exception.args[0]
elif isinstance(exception, DatasetteError):
status = exception.status
info = exception.error_dict
message = exception.message
if exception.messagge_is_html:
message = Markup(message)
title = exception.title
else:
status = 500
info = {}
message = str(exception)
traceback.print_exc()
templates = ["500.html"]
if status != 500:
templates = ["{}.html".format(status)] + templates
info.update({"ok": False, "error": message, "status": status, "title": title})
headers = {}
if self.ds.cors:
headers["Access-Control-Allow-Origin"] = "*"
if scope["path"].split("?")[0].endswith(".json"):
await asgi_send_json(send, info, status=status, headers=headers)
else:
template = self.ds.jinja_env.select_template(templates)
await asgi_send_html(
send, template.render(info), status=status, headers=headers
)
@@ -1,4 +1,5 @@
import asyncio
import uvicorn
import click
from click import formatting
from click_default_group import DefaultGroup
@@ -354,4 +355,4 @@ def serve(
asyncio.get_event_loop().run_until_complete(ds.run_sanity_checks())

# Start the server
ds.app().run(host=host, port=port, debug=debug)
uvicorn.run(ds.app(), host=host, port=port, log_level="info")
@@ -88,5 +88,5 @@ def json_renderer(args, data, view_name):
content_type = "text/plain"
else:
body = json.dumps(data, cls=CustomJSONEncoder)
content_type = "application/json"
content_type = "application/json; charset=utf-8"
return {"body": body, "status_code": status_code, "content_type": content_type}

0 comments on commit ba8db96

Please sign in to comment.
You can’t perform that action at this time.