Merge pull request #95 from Materials-Consortia/close_89_add_links

Add /links endpoint

optimade/models/links.py

@@ -1,8 +1,8 @@
-from pydantic import Schema, UrlStr, validator
+from pydantic import Schema, UrlStr, validator  # pylint: disable=no-name-in-module
 from typing import Union
 
-from .jsonapi import Link
-from .entries import EntryResourceAttributes, EntryResource
+from .jsonapi import Link, Attributes
+from .entries import EntryResource
 
 
 __all__ = (
@@ -14,7 +14,7 @@
 )
 
 
-class LinksResourceAttributes(EntryResourceAttributes):
+class LinksResourceAttributes(Attributes):
     """Links endpoint resource object attributes"""
 
     name: str = Schema(
@@ -27,7 +27,7 @@ class LinksResourceAttributes(EntryResourceAttributes):
         description="Human-readable description for the OPTiMaDe API implementation "
         "a client may provide in a list to an end-user.",
     )
-    base_url: Union[UrlStr, Link] = Schema(
+    base_url: Union[UrlStr, Link, None] = Schema(
         ...,
         description="JSON API links object, pointing to the base URL for this implementation",
     )
@@ -57,6 +57,10 @@ def type_must_be_in_specific_set(cls, value):
             )
         return value
 
+    @validator("relationships")
+    def relationships_must_not_be_present(cls, value):
+        raise ValueError('"relationships" is not allowed for links resources')
+
 
 class ChildResource(LinksResource):
     """A child object representing a link to an implementation exactly one layer below the current implementation"""

optimade/models/toplevel.py

@@ -1,12 +1,19 @@
 from datetime import datetime
 from typing import Union, List, Optional, Dict, Any
 
-from pydantic import BaseModel, validator, UrlStr, Schema, EmailStr
+from pydantic import (  # pylint: disable=no-name-in-module
+    BaseModel,
+    validator,
+    UrlStr,
+    Schema,
+    EmailStr,
+)
 
 from .jsonapi import Link, Meta
 from .util import NonnegativeInt
 from .baseinfo import BaseInfoResource
 from .entries import EntryInfoResource, EntryResource
+from .links import LinksResource
 from .optimade_json import Error, Success, Failure, Warnings
 from .references import ReferenceResource
 from .structures import StructureResource
@@ -21,6 +28,7 @@
     "ErrorResponse",
     "EntryInfoResponse",
     "InfoResponse",
+    "LinksResponse",
     "EntryResponseOne",
     "EntryResponseMany",
     "StructureResponseOne",
@@ -194,6 +202,10 @@ class EntryResponseMany(Success):
     included: Optional[Union[List[EntryResource], List[Dict[str, Any]]]] = Schema(...)
 
 
+class LinksResponse(EntryResponseMany):
+    data: Union[List[LinksResource], List[Dict[str, Any]]] = Schema(...)
+
+
 class StructureResponseOne(EntryResponseOne):
     data: Union[StructureResource, Dict[str, Any], None] = Schema(...)
 

optimade/server/config.ini

@@ -13,5 +13,3 @@ index_base_url = http://example.com/optimade/index
 [structures]
 band_gap :
 _mp_chemsys :
-
-[references]

optimade/server/config.py

@@ -66,9 +66,11 @@ def load_from_ini(self):
 
         self.provider_fields = {}
         for endpoint in {"structures", "references"}:
-            self.provider_fields[endpoint] = {
-                field for field, _ in config[endpoint].items() if _ == ""
-            }
+            self.provider_fields[endpoint] = (
+                {field for field, _ in config[endpoint].items() if _ == ""}
+                if endpoint in config
+                else {}
+            )
 
     def load_from_json(self):
         """ Load from the file "config.json", if it exists. """

optimade/server/deps.py

@@ -13,16 +13,16 @@ def __init__(
         self,
         *,
         filter: str = Query(  # pylint: disable=redefined-builtin
-            None,
+            "",
             description="""See [the full and latest OPTiMaDe spec](https://github.com/Materials-Consortia/OPTiMaDe/blob/develop/optimade.rst) for filter query syntax.
 
 Example: `chemical_formula = "Al" OR (prototype_formula = "AB" AND elements HAS Si, Al, O)`.
 """,
         ),
         response_format: str = Query("json"),
-        email_address: EmailStr = Query(None),
-        response_fields: str = Query(None),
-        sort: str = Query(None),
+        email_address: EmailStr = Query(""),
+        response_fields: str = Query(""),
+        sort: str = Query(""),
         page_limit: NonnegativeInt = Query(CONFIG.page_limit),
         page_offset: NonnegativeInt = Query(0),
         page_page: NonnegativeInt = Query(0),
@@ -50,8 +50,8 @@ def __init__(
         self,
         *,
         response_format: str = Query("json"),
-        email_address: EmailStr = Query(None),
-        response_fields: str = Query(None),
+        email_address: EmailStr = Query(""),
+        response_fields: str = Query(""),
     ):
         self.response_format = response_format
         self.email_address = email_address

optimade/server/entry_collections.py

@@ -35,6 +35,10 @@ def __contains__(self, entry):
     def get_attribute_fields(self) -> set:
         schema = self.resource_cls.schema()
         attributes = schema["properties"]["attributes"]
+        if "allOf" in attributes:
+            allOf = attributes.pop("allOf")
+            for dict_ in allOf:
+                attributes.update(dict_)
         if "$ref" in attributes:
             path = attributes["$ref"].split("/")[1:]
             attributes = schema.copy()
@@ -77,7 +81,7 @@ def __init__(
         self.transformer = NewMongoTransformer()
 
         self.provider = CONFIG.provider["prefix"]
-        self.provider_fields = CONFIG.provider_fields[resource_mapper.ENDPOINT]
+        self.provider_fields = CONFIG.provider_fields.get(resource_mapper.ENDPOINT, [])
         self.page_limit = CONFIG.page_limit
         self.parser = LarkParser(
             version=(0, 10, 0), variant="default"

optimade/server/main.py

@@ -9,9 +9,11 @@
 from starlette.requests import Request
 
 from optimade.models import (
+    LinksResource,
     StructureResource,
     ReferenceResource,
     InfoResponse,
+    LinksResponse,
     ErrorResponse,
     EntryInfoResponse,
     ReferenceResponseMany,
@@ -23,7 +25,7 @@
 from .deps import EntryListingQueryParams, SingleEntryQueryParams
 from .entry_collections import MongoCollection
 from .config import CONFIG
-from .mappers import StructureMapper, ReferenceMapper
+from .mappers import LinksMapper, StructureMapper, ReferenceMapper
 import optimade.server.utils as u
 
 
@@ -50,6 +52,9 @@
 references = MongoCollection(
     client[CONFIG.mongo_database]["references"], ReferenceResource, ReferenceMapper
 )
+links = MongoCollection(
+    client[CONFIG.mongo_database]["links"], LinksResource, LinksMapper
+)
 ENTRY_COLLECTIONS = {"references": references, "structures": structures}
 
 
@@ -60,6 +65,7 @@
     "references": Path(__file__)
     .resolve()
     .parent.joinpath("tests/test_references.json"),
+    "links": Path(__file__).resolve().parent.joinpath("tests/test_links.json"),
 }
 if not CONFIG.use_real_mongo and (path.exists() for path in test_paths.values()):
     import bson.json_util
@@ -72,10 +78,18 @@ def load_entries(endpoint_name: str, endpoint_collection: MongoCollection):
             endpoint_collection.collection.insert_many(
                 bson.json_util.loads(bson.json_util.dumps(data))
             )
+        if endpoint_name == "links":
+            print(
+                "adding providers.json to links from github.com/Materials-Consortia/OPTiMaDe"
+            )
+            endpoint_collection.collection.insert_many(
+                bson.json_util.loads(bson.json_util.dumps(u.get_providers()))
+            )
         print(f"done inserting test {endpoint_name}...")
 
     load_entries("structures", structures)
     load_entries("references", references)
+    load_entries("links", links)
 
 
 @app.exception_handler(StarletteHTTPException)
@@ -159,6 +173,29 @@ def get_single_reference(
     )
 
 
+@app.get(
+    "/links",
+    response_model=Union[LinksResponse, ErrorResponse],
+    response_model_skip_defaults=True,
+    tags=["Links"],
+)
+def get_links(request: Request, params: EntryListingQueryParams = Depends()):
+    for str_param in ["filter", "sort"]:
+        if getattr(params, str_param):
+            setattr(params, str_param, "")
+    for int_param in [
+        "page_offset",
+        "page_page",
+        "page_cursor",
+        "page_above",
+        "page_below",
+    ]:
+        if getattr(params, int_param):
+            setattr(params, int_param, 0)
+    params.page_limit = CONFIG.page_limit
+    return u.get_entries(links, LinksResponse, request, params)
+
+
 @app.get(
     "/info",
     response_model=Union[InfoResponse, ErrorResponse],

optimade/server/mappers/__init__.py

@@ -1,6 +1,9 @@
 # pylint: disable=undefined-variable
 from .entries import *
+from .links import *
 from .references import *
 from .structures import *
 
-__all__ = entries.__all__ + references.__all__ + structures.__all__  # noqa
+__all__ = (
+    entries.__all__ + links.__all__ + references.__all__ + structures.__all__  # noqa
+)

optimade/server/mappers/entries.py

@@ -1,23 +1,22 @@
-import abc
 from typing import Tuple
 from optimade.server.config import CONFIG
 
-
 __all__ = ("ResourceMapper",)
 
 
-class ResourceMapper(metaclass=abc.ABCMeta):
+class ResourceMapper:
     """Generic Resource Mapper"""
 
     ENDPOINT: str = ""
     ALIASES: Tuple[Tuple[str, str]] = ()
+    TOP_LEVEL_NON_ATTRIBUTES_FIELDS: set = {"id", "type", "relationships", "links"}
 
     @classmethod
     def all_aliases(cls) -> Tuple[Tuple[str, str]]:
         return (
             tuple(
                 (CONFIG.provider["prefix"] + field, field)
-                for field in CONFIG.provider_fields[cls.ENDPOINT]
+                for field in CONFIG.provider_fields.get(cls.ENDPOINT, {})
             )
             + cls.ALIASES
         )
@@ -34,13 +33,46 @@ def alias_for(cls, field: str) -> str:
         """
         return dict(cls.all_aliases()).get(field, field)
 
-    @abc.abstractclassmethod
+    @classmethod
     def map_back(cls, doc: dict) -> dict:
         """Map properties from MongoDB to OPTiMaDe
 
+        Starting from a MongoDB document ``doc``, map the DB fields to the corresponding OPTiMaDe fields.
+        Then, the fields are all added to the top-level field "attributes",
+        with the exception of other top-level fields, defined in ``cls.TOPLEVEL_NON_ATTRIBUTES_FIELDS``.
+        All fields not in ``cls.TOPLEVEL_NON_ATTRIBUTES_FIELDS`` + "attributes" will be removed.
+        Finally, the ``type`` is given the value of the specified ``cls.ENDPOINT``.
+
         :param doc: A resource object in MongoDB format
         :type doc: dict
 
         :return: A resource object in OPTiMaDe format
         :rtype: dict
         """
+        if "_id" in doc:
+            del doc["_id"]
+
+        mapping = ((real, alias) for alias, real in cls.all_aliases())
+        newdoc = {}
+        reals = {real for alias, real in cls.all_aliases()}
+        for k in doc:
+            if k not in reals:
+                newdoc[k] = doc[k]
+        for real, alias in mapping:
+            if real in doc:
+                newdoc[alias] = doc[real]
+
+        if "attributes" in newdoc:
+            raise Exception("Will overwrite doc field!")
+        attributes = newdoc.copy()
+
+        for k in cls.TOP_LEVEL_NON_ATTRIBUTES_FIELDS:
+            attributes.pop(k, None)
+        for k in list(newdoc.keys()):
+            if k not in cls.TOP_LEVEL_NON_ATTRIBUTES_FIELDS:
+                del newdoc[k]
+
+        newdoc["type"] = cls.ENDPOINT
+        newdoc["attributes"] = attributes
+
+        return newdoc

optimade/server/mappers/links.py

@@ -0,0 +1,23 @@
+from .entries import ResourceMapper
+
+__all__ = ("LinksMapper",)
+
+
+class LinksMapper(ResourceMapper):
+
+    ENDPOINT = "links"
+
+    @classmethod
+    def map_back(cls, doc: dict) -> dict:
+        """Map properties from MongoDB to OPTiMaDe
+
+        :param doc: A resource object in MongoDB format
+        :type doc: dict
+
+        :return: A resource object in OPTiMaDe format
+        :rtype: dict
+        """
+        type_ = doc["type"]
+        newdoc = super().map_back(doc)
+        newdoc["type"] = type_
+        return newdoc

optimade/server/mappers/references.py

@@ -1,45 +1,8 @@
 from .entries import ResourceMapper
 
-
 __all__ = ("ReferenceMapper",)
 
 
 class ReferenceMapper(ResourceMapper):
 
     ENDPOINT = "references"
-    ALIASES = (("id", "task_id"),)
-
-    @classmethod
-    def map_back(cls, doc: dict) -> dict:
-        """Map properties from MongoDB to OPTiMaDe
-
-        :param doc: A resource object in MongoDB format
-        :type doc: dict
-
-        :return: A resource object in OPTiMaDe format
-        :rtype: dict
-        """
-        if "_id" in doc:
-            del doc["_id"]
-        # print(doc)
-        mapping = ((real, alias) for alias, real in cls.all_aliases())
-        newdoc = {}
-        reals = {real for alias, real in cls.all_aliases()}
-        for k in doc:
-            if k not in reals:
-                newdoc[k] = doc[k]
-        for real, alias in mapping:
-            if real in doc:
-                newdoc[alias] = doc[real]
-
-        # print(newdoc)
-        if "attributes" in newdoc:
-            raise Exception("Will overwrite doc field!")
-        newdoc["attributes"] = newdoc.copy()
-        for k in {"id", "type"}:
-            newdoc["attributes"].pop(k, None)
-        for k in list(newdoc.keys()):
-            if k not in ("id", "attributes"):
-                del newdoc[k]
-        newdoc["type"] = cls.ENDPOINT
-        return newdoc