From 825b8a2557dc2b42d008c102cc1477662094b172 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 15 Jun 2018 08:52:43 -0700 Subject: [PATCH] .json?_labels=1 to expand foreign keys, refs #233 Output looks something like this: { "rowid": 233, "TreeID": 121240, "qLegalStatus": { "value" 2, "label": "Private" } "qSpecies": { "value": 16, "label": "Sycamore" } "qAddress": "91 Commonwealth Ave", ... } --- datasette/utils.py | 18 ++++++++++++ datasette/views/table.py | 59 ++++++++++++++++++++++++++++++++++++++-- docs/json_api.rst | 4 +++ tests/test_api.py | 29 ++++++++++++++++++++ 4 files changed, 107 insertions(+), 3 deletions(-) diff --git a/datasette/utils.py b/datasette/utils.py index eb31475da7..11588abca2 100644 --- a/datasette/utils.py +++ b/datasette/utils.py @@ -1,4 +1,5 @@ from contextlib import contextmanager +from collections import OrderedDict import base64 import hashlib import imp @@ -800,3 +801,20 @@ def path_with_format(request, format, extra_qs=None): elif request.query_string: path = "{}?{}".format(path, request.query_string) return path + + +class CustomRow(OrderedDict): + # Loose imitation of sqlite3.Row which offers + # both index-based AND key-based lookups + def __init__(self, columns): + self.columns = columns + + def __getitem__(self, key): + if isinstance(key, int): + return super().__getitem__(self.columns[key]) + else: + return super().__getitem__(key) + + def __iter__(self): + for column in self.columns: + yield self[column] diff --git a/datasette/views/table.py b/datasette/views/table.py index 272902fb48..aec34cfb21 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -1,3 +1,4 @@ +from collections import namedtuple import sqlite3 import urllib @@ -6,6 +7,7 @@ from sanic.request import RequestParameters from datasette.utils import ( + CustomRow, Filters, InterruptedError, compound_keys_after_sql, @@ -25,17 +27,35 @@ class RowTableShared(BaseView): - def sortable_columns_for_table(self, name, table, use_rowid): - table_metadata = self.table_metadata(name, table) + def sortable_columns_for_table(self, database, table, use_rowid): + table_metadata = self.table_metadata(database, table) if "sortable_columns" in table_metadata: sortable_columns = set(table_metadata["sortable_columns"]) else: - table_info = self.ds.inspect()[name]["tables"].get(table) or {} + table_info = self.ds.inspect()[database]["tables"].get(table) or {} sortable_columns = set(table_info.get("columns", [])) if use_rowid: sortable_columns.add("rowid") return sortable_columns + def expandable_columns(self, database, table): + # Returns list of (fk_dict, label_column) pairs for that table + tables = self.ds.inspect()[database].get("tables", {}) + table_info = tables.get(table) + if not table_info: + return [] + expandables = [] + for fk in table_info["foreign_keys"]["outgoing"]: + label_column = ( + self.table_metadata( + database, fk["other_table"] + ).get("label_column") + or tables.get(fk["other_table"], {}).get("label_column") + ) + if label_column: + expandables.append((fk, label_column)) + return expandables + async def expand_foreign_keys(self, database, table, column, values): "Returns dict mapping (column, value) -> label" labeled_fks = {} @@ -610,6 +630,38 @@ async def data(self, request, name, hash, table): if use_rowid and filter_columns[0] == "rowid": filter_columns = filter_columns[1:] + # Expand labeled columns if requested + labeled_columns = [] + if request.raw_args.get("_labels", None): + expandable_columns = self.expandable_columns(name, table) + expanded_labels = {} + for fk, label_column in expandable_columns: + column = fk["column"] + labeled_columns.append(column) + # Gather the values + column_index = columns.index(column) + values = [row[column_index] for row in rows] + # Expand them + expanded_labels.update(await self.expand_foreign_keys( + name, table, column, values + )) + if expanded_labels: + # Rewrite the rows + new_rows = [] + for row in rows: + new_row = CustomRow(columns) + for column in row.keys(): + value = row[column] + if (column, value) in expanded_labels: + new_row[column] = { + 'value': value, + 'label': expanded_labels[(column, value)] + } + else: + new_row[column] = value + new_rows.append(new_row) + rows = new_rows + # Pagination next link next_value = None next_url = None @@ -762,6 +814,7 @@ async def extra_template(): "truncated": results.truncated, "table_rows_count": table_rows_count, "filtered_table_rows_count": filtered_table_rows_count, + "labeled_columns": labeled_columns, "columns": columns, "primary_keys": pks, "units": units, diff --git a/docs/json_api.rst b/docs/json_api.rst index 379186a10e..da89c48b26 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -193,3 +193,7 @@ The Datasette table view takes a number of special querystring arguments: ``?_next=TOKEN`` Pagination by continuation token - pass the token that was returned in the ``"next"`` property by the previous page. + +``?_labels=1`` + Indicates that you would like to expand any foreign key references. These + will be exposed in the JSON as ``{"value": 3, "label": "Monterey"}``. diff --git a/tests/test_api.py b/tests/test_api.py index af1fe4c5a2..0ba6e3c6dc 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1096,6 +1096,35 @@ def test_suggest_facets_off(): ).json["suggested_facets"] +def test_expand_labels(app_client): + response = app_client.get( + "/test_tables/facetable.json?_shape=object&_labels=1&_size=2" + "&neighborhood__contains=c" + ) + assert { + "2": { + "pk": 2, + "planet_int": 1, + "state": "CA", + "city_id": { + "value": 1, + "label": "San Francisco" + }, + "neighborhood": "Dogpatch" + }, + "13": { + "pk": 13, + "planet_int": 1, + "state": "MI", + "city_id": { + "value": 3, + "label": "Detroit" + }, + "neighborhood": "Corktown" + } + } == response.json + + @pytest.mark.parametrize('path,expected_cache_control', [ ("/test_tables/facetable.json", "max-age=31536000"), ("/test_tables/facetable.json?_ttl=invalid", "max-age=31536000"),