Merge pull request #163 from CasperWA/close_94_handle_include_query_p…

…aram

Implement `include` query parameter.

NOTE: It currently validates against `optimade.routers.ENTRY_COLLECTIONS`,
so paths are not implemented, and the models only implement
part of `ENTRY_COLLECTIONS`, i.e., `references` and `structures`.

When updating how relationships are handled, this now all depends
on `ENTRY_COLLECTIONS`.

openapi/index_openapi.json

@@ -183,6 +183,18 @@
             },
             "name": "page_below",
             "in": "query"
+          },
+          {
+            "description": "The JSON API concept of returning [compound documents](https://jsonapi.org/format/1.0/#document-compound-documents) by utilizing the `include` query parameter as specified by [JSON API 1.0](https://jsonapi.org/format/1.0/#fetching-includes).\n\nAll related resource objects MUST be returned as part of an array value for the top-level `included` field, see section [JSON Response Schema: Common Fields](https://github.com/Materials-Consortia/OPTiMaDe/blob/develop/optimade.rst#json-response-schema-common-fields).\n\nThe value of `include` MUST be a comma-separated list of \"relationship paths\", as defined in the [JSON API](https://jsonapi.org/format/1.0/#fetching-includes). If relationship paths are not supported, or a server is unable to identify a relationship path a `400 Bad Request` response MUST be made.\n\nThe **default value** for `include` is `references`. This means `references` entries MUST always be included under the top-level field `included` as default, since a server assumes if `include` is not specified by a client in the request, it is still specified as `include=references`. Note, if a client explicitly specifies `include` and leaves out `references`, `references` resource objects MUST NOT be included under the top-level field `included`, as per the definition of `included`, see section [JSON Response Schema: Common Fields](https://github.com/Materials-Consortia/OPTiMaDe/blob/develop/optimade.rst#json-response-schema-common-fields).\n\n- **Note**: A query with the parameter `include` set to the empty string means no related resource objects are to be returned under the top-level field `included`.",
+            "required": false,
+            "schema": {
+              "title": "Include",
+              "type": "string",
+              "description": "The JSON API concept of returning [compound documents](https://jsonapi.org/format/1.0/#document-compound-documents) by utilizing the `include` query parameter as specified by [JSON API 1.0](https://jsonapi.org/format/1.0/#fetching-includes).\n\nAll related resource objects MUST be returned as part of an array value for the top-level `included` field, see section [JSON Response Schema: Common Fields](https://github.com/Materials-Consortia/OPTiMaDe/blob/develop/optimade.rst#json-response-schema-common-fields).\n\nThe value of `include` MUST be a comma-separated list of \"relationship paths\", as defined in the [JSON API](https://jsonapi.org/format/1.0/#fetching-includes). If relationship paths are not supported, or a server is unable to identify a relationship path a `400 Bad Request` response MUST be made.\n\nThe **default value** for `include` is `references`. This means `references` entries MUST always be included under the top-level field `included` as default, since a server assumes if `include` is not specified by a client in the request, it is still specified as `include=references`. Note, if a client explicitly specifies `include` and leaves out `references`, `references` resource objects MUST NOT be included under the top-level field `included`, as per the definition of `included`, see section [JSON Response Schema: Common Fields](https://github.com/Materials-Consortia/OPTiMaDe/blob/develop/optimade.rst#json-response-schema-common-fields).\n\n- **Note**: A query with the parameter `include` set to the empty string means no related resource objects are to be returned under the top-level field `included`.",
+              "default": "references"
+            },
+            "name": "include",
+            "in": "query"
           }
         ],
         "responses": {

optimade/server/entry_collections.py

@@ -10,8 +10,8 @@
 from optimade.models import EntryResource
 
 from .config import CONFIG
-from .deps import EntryListingQueryParams, SingleEntryQueryParams
 from .mappers import ResourceMapper
+from .query_params import EntryListingQueryParams, SingleEntryQueryParams
 
 
 if CONFIG.use_real_mongo:

optimade/server/exception_handlers.py

@@ -29,13 +29,16 @@ def general_exception(
     except AttributeError:
         status_code = kwargs.get("status_code", 500)
 
+    try:
+        title = exc.title
+    except AttributeError:
+        title = str(exc.__class__.__name__)
+
     detail = getattr(exc, "detail", str(exc))
 
     errors = kwargs.get("errors", None)
     if not errors:
-        errors = [
-            Error(detail=detail, status=status_code, title=str(exc.__class__.__name__))
-        ]
+        errors = [Error(detail=detail, status=status_code, title=title)]
 
     try:
         response = ErrorResponse(

optimade/server/deps.py → optimade/server/query_params.py

@@ -1,4 +1,3 @@
-# pylint: disable=line-too-long
 from fastapi import Query
 from pydantic import EmailStr  # pylint: disable=no-name-in-module
 
@@ -91,6 +90,25 @@ def __init__(
             description="RECOMMENDED for use with _value-based_ pagination: using `page_above`/`page_below` and `page_limit` is RECOMMENDED.",
             ge=0,
         ),
+        include: str = Query(
+            "references",
+            description="The JSON API concept of returning [compound documents](https://jsonapi.org/format/1.0/#document-compound-documents) "
+            "by utilizing the `include` query parameter as specified by [JSON API 1.0](https://jsonapi.org/format/1.0/#fetching-includes).\n\n"
+            "All related resource objects MUST be returned as part of an array value for the top-level `included` field, see section "
+            "[JSON Response Schema: Common Fields](https://github.com/Materials-Consortia/OPTiMaDe/blob/develop/optimade.rst#json-response-"
+            "schema-common-fields).\n\n"
+            'The value of `include` MUST be a comma-separated list of "relationship paths", as defined in the '
+            "[JSON API](https://jsonapi.org/format/1.0/#fetching-includes). If relationship paths are not supported, or a server is unable "
+            "to identify a relationship path a `400 Bad Request` response MUST be made.\n\n"
+            "The **default value** for `include` is `references`. This means `references` entries MUST always be included under the top-level "
+            "field `included` as default, since a server assumes if `include` is not specified by a client in the request, it is still specified "
+            "as `include=references`. Note, if a client explicitly specifies `include` and leaves out `references`, `references` resource objects "
+            "MUST NOT be included under the top-level field `included`, as per the definition of `included`, see section "
+            "[JSON Response Schema: Common Fields](https://github.com/Materials-Consortia/OPTiMaDe/blob/develop/optimade.rst#json-response-"
+            "schema-common-fields).\n\n"
+            "- **Note**: A query with the parameter `include` set to the empty string means no related resource objects are to be returned under the "
+            "top-level field `included`.",
+        ),
     ):
         self.filter = filter
         self.response_format = response_format
@@ -103,6 +121,7 @@ def __init__(
         self.page_cursor = page_cursor
         self.page_above = page_above
         self.page_below = page_below
+        self.include = include
 
 
 class SingleEntryQueryParams:
@@ -129,7 +148,27 @@ def __init__(
             "**Example**: http://example.com/optimade/v0.9/structures?response_fields=last_modified,nsites",
             regex=r"([a-z_][a-z_0-9]*(,[a-z_][a-z_0-9]*)*)?",
         ),
+        include: str = Query(
+            "references",
+            description="The JSON API concept of returning [compound documents](https://jsonapi.org/format/1.0/#document-compound-documents) "
+            "by utilizing the `include` query parameter as specified by [JSON API 1.0](https://jsonapi.org/format/1.0/#fetching-includes).\n\n"
+            "All related resource objects MUST be returned as part of an array value for the top-level `included` field, see section "
+            "[JSON Response Schema: Common Fields](https://github.com/Materials-Consortia/OPTiMaDe/blob/develop/optimade.rst#json-response-"
+            "schema-common-fields).\n\n"
+            'The value of `include` MUST be a comma-separated list of "relationship paths", as defined in the '
+            "[JSON API](https://jsonapi.org/format/1.0/#fetching-includes). If relationship paths are not supported, or a server is unable "
+            "to identify a relationship path a `400 Bad Request` response MUST be made.\n\n"
+            "The **default value** for `include` is `references`. This means `references` entries MUST always be included under the top-level "
+            "field `included` as default, since a server assumes if `include` is not specified by a client in the request, it is still specified "
+            "as `include=references`. Note, if a client explicitly specifies `include` and leaves out `references`, `references` resource objects "
+            "MUST NOT be included under the top-level field `included`, as per the definition of `included`, see section "
+            "[JSON Response Schema: Common Fields](https://github.com/Materials-Consortia/OPTiMaDe/blob/develop/optimade.rst#json-response-"
+            "schema-common-fields).\n\n"
+            "- **Note**: A query with the parameter `include` set to the empty string means no related resource objects are to be returned under the "
+            "top-level field `included`.",
+        ),
     ):
         self.response_format = response_format
         self.email_address = email_address
         self.response_fields = response_fields
+        self.include = include

optimade/server/routers/links.py

@@ -5,9 +5,9 @@
 
 from optimade.models import ErrorResponse, LinksResponse, LinksResource
 from optimade.server.config import CONFIG
-from optimade.server.deps import EntryListingQueryParams
 from optimade.server.entry_collections import MongoCollection, client
 from optimade.server.mappers import LinksMapper
+from optimade.server.query_params import EntryListingQueryParams
 
 from .utils import get_entries
 

optimade/server/routers/references.py

@@ -10,9 +10,9 @@
     ReferenceResponseOne,
 )
 from optimade.server.config import CONFIG
-from optimade.server.deps import EntryListingQueryParams, SingleEntryQueryParams
 from optimade.server.entry_collections import MongoCollection, client
 from optimade.server.mappers import ReferenceMapper
+from optimade.server.query_params import EntryListingQueryParams, SingleEntryQueryParams
 
 from .utils import get_entries, get_single_entry
 

optimade/server/routers/structures.py

@@ -10,9 +10,9 @@
     StructureResponseOne,
 )
 from optimade.server.config import CONFIG
-from optimade.server.deps import EntryListingQueryParams, SingleEntryQueryParams
 from optimade.server.entry_collections import MongoCollection, client
 from optimade.server.mappers import StructureMapper
+from optimade.server.query_params import EntryListingQueryParams, SingleEntryQueryParams
 
 from .utils import get_entries, get_single_entry
 

optimade/server/routers/utils.py

@@ -1,7 +1,7 @@
-# pylint: disable=import-outside-toplevel
+# pylint: disable=import-outside-toplevel,too-many-locals
 import urllib
 from datetime import datetime
-from typing import Union, List, Dict
+from typing import Union, List, Dict, Any
 
 from fastapi import HTTPException
 from starlette.requests import Request
@@ -18,8 +18,8 @@
 )
 
 from optimade.server.config import CONFIG
-from optimade.server.deps import EntryListingQueryParams, SingleEntryQueryParams
 from optimade.server.entry_collections import EntryCollection
+from optimade.server.query_params import EntryListingQueryParams, SingleEntryQueryParams
 
 
 ENTRY_INFO_SCHEMAS = {
@@ -34,6 +34,20 @@
 }
 
 
+class BadRequest(HTTPException):
+    """400 Bad Request"""
+
+    def __init__(
+        self,
+        status_code: int = 400,
+        detail: Any = None,
+        headers: dict = None,
+        title: str = "Bad Request",
+    ) -> None:
+        super().__init__(status_code=status_code, detail=detail, headers=headers)
+        self.title = title
+
+
 def meta_values(
     url: str,
     data_returned: int,
@@ -96,6 +110,7 @@ def handle_response_fields(
 def get_included_relationships(
     results: Union[EntryResource, List[EntryResource]],
     ENTRY_COLLECTIONS: Dict[str, EntryCollection],
+    include_param: List[str],
 ) -> Dict[str, List[EntryResource]]:
     """Filters the included relationships and makes the appropriate compound request
     to include them in the response.
@@ -104,6 +119,8 @@ def get_included_relationships(
         results: list of returned documents.
         ENTRY_COLLECTIONS: dictionary containing collections to query, with key
             based on endpoint type.
+        include_param: list of queried related resources that should be included in
+            `included`.
 
     Returns:
         Dictionary with the same keys as ENTRY_COLLECTIONS, each containing the list
@@ -115,6 +132,14 @@ def get_included_relationships(
     if not isinstance(results, list):
         results = [results]
 
+    empty_values = {'""', "''"}
+    for entry_type in include_param:
+        if entry_type not in ENTRY_COLLECTIONS and entry_type not in empty_values:
+            raise BadRequest(
+                detail=f"'{entry_type}' cannot be identified as a valid relationship type. "
+                f"Known relationship types: {sorted(ENTRY_COLLECTIONS.keys())}"
+            )
+
     endpoint_includes = defaultdict(dict)
     for doc in results:
         # convert list of references into dict by ID to only included unique IDs
@@ -127,6 +152,10 @@ def get_included_relationships(
 
         relationships = relationships.dict()
         for entry_type in ENTRY_COLLECTIONS:
+            # Skip entry type if it is not in `include_param`
+            if entry_type not in include_param:
+                continue
+
             entry_relationship = relationships.get(entry_type, {})
             if entry_relationship is not None:
                 refs = entry_relationship.get("data", [])
@@ -179,7 +208,11 @@ def get_entries(
 
     results, data_returned, more_data_available, fields = collection.find(params)
 
-    included = get_included_relationships(results, ENTRY_COLLECTIONS)
+    include = []
+    if getattr(params, "include", False):
+        include.extend(params.include.split(","))
+
+    included = get_included_relationships(results, ENTRY_COLLECTIONS, include)
 
     if more_data_available:
         # Deduce the `next` link from the current request
@@ -221,7 +254,11 @@ def get_single_entry(
     params.filter = f'id="{entry_id}"'
     results, data_returned, more_data_available, fields = collection.find(params)
 
-    included = get_included_relationships(results, ENTRY_COLLECTIONS)
+    include = []
+    if getattr(params, "include", False):
+        include.extend(params.include.split(","))
+
+    included = get_included_relationships(results, ENTRY_COLLECTIONS, include)
 
     if more_data_available:
         raise HTTPException(