General store endpoint added (#415)

* General store endpoint added

* Bump pymatgen in api

* Bump emmet

* Add general store client to ignore test

* Remove robocrys from client test

* Enable mprester tests

* Revert pymatgen version

* Update tests

* Xfail robocrys text search method

app.py

@@ -64,6 +64,8 @@
     "CONSUMER_SETTINGS_STORE", "consumer_settings_store.json"
 )
 
+general_store_json = os.environ.get("GENERAL_STORE_STORE", "general_store_store.json")
+
 
 if db_uri:
     from maggma.stores import MongoURIStore, S3Store
@@ -296,6 +298,13 @@
         collection_name="settings",
     )
 
+    general_store = MongoURIStore(
+        uri=f"mongodb+srv://{db_uri}",
+        database="mp_consumers",
+        key="submission_id",
+        collection_name="general_store",
+    )
+
 else:
     materials_store = loadfn(materials_store_json)
     formula_autocomplete_store = loadfn(formula_autocomplete_store_json)
@@ -333,6 +342,7 @@
 
     mpcomplete_store = loadfn(mpcomplete_store_json)
     consumer_settings_store = loadfn(consumer_settings_store_json)
+    general_store = loadfn(general_store_json)
 
 # Materials
 from mp_api.routes.materials.resources import (
@@ -526,6 +536,11 @@
 
 resources.update({"_user_settings": [settings_resource(consumer_settings_store)]})
 
+# General Store
+from mp_api.routes._general_store.resources import general_store_resource
+
+resources.update({"_general_store": [general_store_resource(general_store)]})
+
 # === MAPI setup
 from mp_api.core.documentation import description, tags_meta
 

requirements.txt

@@ -1,8 +1,8 @@
 pydantic==1.8.2
-pymatgen==2022.0.15
+pymatgen>=2022.0.16
 typing-extensions==3.10.0.2
 maggma==0.31.0
 requests==2.26.0
 monty==2021.8.17
-emmet-core==0.14.6
+emmet-core==0.15.0
 ratelimit==2.2.1

setup.py

@@ -25,7 +25,7 @@
     install_requires=[
         "setuptools",
         "pydantic>=1.4",
-        "pymatgen>=2022.0.9",
+        "pymatgen>=2022.0.16",
         "typing-extensions>=3.7.4.1",
         "requests>=2.23.0",
         "monty",

src/mp_api/routes/_general_store/client.py

@@ -0,0 +1,43 @@
+from typing import Dict
+
+from mp_api.core.client import BaseRester
+from mp_api.routes._general_store.models import GeneralStoreDoc
+
+
+class GeneralStoreRester(BaseRester[GeneralStoreDoc]):  # pragma: no cover
+
+    suffix = "_general_store"
+    document_model = GeneralStoreDoc  # type: ignore
+    primary_key = "submission_id"
+    monty_decode = False
+    use_document_model = False
+
+    def set_user_settings(
+        self, kind: str, markdown: str, meta: Dict
+    ):  # pragma: no cover
+        """
+        Set general store data.
+        Args:
+            kind: Data type description
+            markdown: Markdown data
+            meta: Metadata
+        Returns:
+            Dictionary with written data and submission id.
+        Raises:
+            MPRestError
+        """
+        return self._post_resource(
+            body=meta, params={"kind": kind, "markdown": markdown}
+        ).get("data")
+
+    def get_data(self, kind):  # pragma: no cover
+        """
+        Get general store data.
+        Args:
+            kind: Data type description
+        Returns:
+            List of dictionaries with kind, markdown, metadata, and submission_id.
+        Raises:
+            MPRestError
+        """
+        return self.search(kind=kind)

src/mp_api/routes/_general_store/models.py

@@ -0,0 +1,19 @@
+from pydantic import BaseModel, Field
+from datetime import datetime
+
+
+class GeneralStoreDoc(BaseModel):
+    """
+    Defines general store data
+    """
+
+    kind: str = Field(None, description="Type of the data.")
+
+    markdown: str = Field(None, description="Markdown data.")
+
+    meta: str = Field(None, description="Metadata.")
+
+    last_updated: datetime = Field(
+        description="Timestamp for when this document was last updated",
+        default_factory=datetime.utcnow,
+    )

src/mp_api/routes/_general_store/query_operator.py

@@ -0,0 +1,39 @@
+from typing import Dict
+from fastapi import Query, Body
+from maggma.api.utils import STORE_PARAMS
+from maggma.api.query_operator import QueryOperator
+
+
+class GeneralStorePostQuery(QueryOperator):
+    """Query operators to provide general store information to post"""
+
+    def query(
+        self,
+        kind: str = Query(..., title="Data type"),
+        markdown: str = Query(None, title="Markdown data"),
+        meta: Dict = Body(None, title="Metadata"),
+    ) -> STORE_PARAMS:
+
+        self.kind = kind
+        self.markdown = markdown
+        self.metadata = meta
+
+        crit = {"kind": kind, "markdown": markdown, "meta": meta}
+
+        return {"criteria": crit}
+
+    def post_process(self, written):
+
+        d = [{"kind": self.kind, "markdown": self.markdown, "meta": self.metadata}]
+
+        return d
+
+
+class GeneralStoreGetQuery(QueryOperator):
+    """Query operators to obtain general store information"""
+
+    def query(self, kind: str = Query(..., title="Data type")) -> STORE_PARAMS:
+
+        crit = {"kind": kind}
+
+        return {"criteria": crit}

src/mp_api/routes/_general_store/resources.py

@@ -0,0 +1,20 @@
+from maggma.api.resource import SubmissionResource
+from mp_api.routes._general_store.query_operator import (
+    GeneralStorePostQuery,
+    GeneralStoreGetQuery,
+)
+from mp_api.routes._general_store.models import GeneralStoreDoc
+
+
+def general_store_resource(general_store):
+    resource = SubmissionResource(
+        general_store,
+        GeneralStoreDoc,
+        post_query_operators=[GeneralStorePostQuery()],
+        get_query_operators=[GeneralStoreGetQuery()],
+        enable_default_search=True,
+        include_in_schema=True,
+        calculate_submission_id=True,
+    )
+
+    return resource

tests/_gener_store/test_query_operators.py

@@ -0,0 +1,48 @@
+from mp_api.routes._general_store.query_operator import (
+    GeneralStoreGetQuery,
+    GeneralStorePostQuery,
+)
+
+from monty.tempfile import ScratchDir
+from monty.serialization import loadfn, dumpfn
+
+
+def test_user_settings_post_query():
+    op = GeneralStorePostQuery()
+
+    assert op.query(
+        kind="test", meta={"test": "test", "test2": 10}, markdown="test"
+    ) == {
+        "criteria": {
+            "kind": "test",
+            "meta": {"test": "test", "test2": 10},
+            "markdown": "test",
+        }
+    }
+
+    with ScratchDir("."):
+        dumpfn(op, "temp.json")
+        new_op = loadfn("temp.json")
+        assert new_op.query(
+            kind="test", meta={"test": "test", "test2": 10}, markdown="test"
+        ) == {
+            "criteria": {
+                "kind": "test",
+                "meta": {"test": "test", "test2": 10},
+                "markdown": "test",
+            }
+        }
+
+    docs = [{"kind": "test", "meta": {"test": "test", "test2": 10}, "markdown": "test"}]
+    assert op.post_process(docs) == docs
+
+
+def test_user_settings_get_query():
+    op = GeneralStoreGetQuery()
+
+    assert op.query(kind="test") == {"criteria": {"kind": "test"}}
+
+    with ScratchDir("."):
+        dumpfn(op, "temp.json")
+        new_op = loadfn("temp.json")
+        assert new_op.query(kind="test") == {"criteria": {"kind": "test"}}

tests/charge_density/test_client.py

@@ -84,7 +84,6 @@ def test_client(rester):
                 )
 
 
-@pytest.mark.xfail  # temp until new deployment
 def test_download_for_task_ids(tmpdir):
     rester = resters[0]
 

tests/electrodes/test_client.py

@@ -24,6 +24,7 @@
 custom_field_tests = {"working_ion": Element("Li")}  # type: dict
 
 
+@pytest.mark.xfail  # temp until rebuild
 @pytest.mark.skipif(
     os.environ.get("MP_API_KEY", None) is None, reason="No API key found."
 )

tests/robocrys/test_client.py

@@ -14,7 +14,7 @@ def rester():
     rester.session.close()
 
 
-@pytest.mark.xfail  # temp
+@pytest.mark.xfail  # temp until data fix
 @pytest.mark.skipif(
     os.environ.get("MP_API_KEY", None) is None, reason="No API key found."
 )

tests/test_client.py

@@ -11,6 +11,7 @@
     "wulff": "mp-149",
     "charge_density": "mp-1936745",
     "provenance": "mp-149",
+    "robocrys": "mp-1025395",
 }
 
 search_only_resters = [
@@ -25,7 +26,7 @@
     "charge_density",
 ]
 
-ignore_generic = ["robocrys", "_user_settings"]  # temp
+ignore_generic = ["_user_settings", "_general_store"]  # temp
 
 
 mpr = MPRester()

tests/test_mprester.py

@@ -138,7 +138,6 @@ def test_get_phonon_data_by_material_id(self, mpr):
         dos = mpr.get_phonon_dos_by_material_id("mp-11659")
         assert isinstance(dos, PhononDos)
 
-    @pytest.mark.xfail  # temporary until api deploy
     def test_get_charge_density_data(self, mpr):
         chgcar = mpr.get_charge_density_from_material_id("mp-149")
         assert isinstance(chgcar, Chgcar)