Fix plus test for unicode characters in custom query name, closes #558

datasette/utils/asgi.py

@@ -45,9 +45,10 @@ def host(self):
 
     @property
     def path(self):
-        return (
-            self.scope.get("raw_path", self.scope["path"].encode("latin-1"))
-        ).decode("latin-1")
+        if "raw_path" in self.scope:
+            return self.scope["raw_path"].decode("latin-1")
+        else:
+            return self.scope["path"].decode("utf-8")
 
     @property
     def query_string(self):

tests/fixtures.py

@@ -12,7 +12,7 @@
 import string
 import tempfile
 import time
-from urllib.parse import unquote
+from urllib.parse import unquote, quote
 
 
 # This temp file is used by one of the plugin config tests
@@ -49,18 +49,20 @@ async def _get(self, path, allow_redirects=True, redirect_count=0, method="GET")
         if "?" in path:
             path, _, query_string = path.partition("?")
             query_string = query_string.encode("utf8")
-        instance = ApplicationCommunicator(
-            self.asgi_app,
-            {
-                "type": "http",
-                "http_version": "1.0",
-                "method": method,
-                "path": unquote(path),
-                "raw_path": path.encode("ascii"),
-                "query_string": query_string,
-                "headers": [[b"host", b"localhost"]],
-            },
-        )
+        if "%" in path:
+            raw_path = path.encode("latin-1")
+        else:
+            raw_path = quote(path, safe="/:,").encode("latin-1")
+        scope = {
+            "type": "http",
+            "http_version": "1.0",
+            "method": method,
+            "path": unquote(path),
+            "raw_path": raw_path,
+            "query_string": query_string,
+            "headers": [[b"host", b"localhost"]],
+        }
+        instance = ApplicationCommunicator(self.asgi_app, scope)
         await instance.send_input({"type": "http.request"})
         # First message back should be response.start with headers and status
         messages = []
@@ -291,6 +293,7 @@ def generate_sortable_rows(num):
                 },
             },
             "queries": {
+                "𝐜𝐢𝐭𝐢𝐞𝐬": "select id, name from facet_cities order by id limit 1;",
                 "pragma_cache_size": "PRAGMA cache_size;",
                 "neighborhood_search": {
                     "sql": """

tests/test_api.py

@@ -1613,6 +1613,11 @@ def test_infinity_returned_as_invalid_json_if_requested(app_client):
     ] == response.json
 
 
+def test_custom_query_with_unicode_characters(app_client):
+    response = app_client.get("/fixtures/𝐜𝐢𝐭𝐢𝐞𝐬.json?_shape=array")
+    assert [{"id": 1, "name": "San Francisco"}] == response.json
+
+
 def test_trace(app_client):
     response = app_client.get("/fixtures/simple_primary_key.json?_trace=1")
     data = response.json