Decouple changes in providers repo

.github/dependabot.yml

@@ -12,3 +12,15 @@ updates:
   - shyamd
   labels:
   - dependency_updates
+- package-ecosystem: submodules
+  directory: "/"
+  schedule:
+    interval: weekly
+  open-pull-requests-limit: 10
+  target-branch: master
+  reviewers:
+  - CasperWA
+  - ml-evs
+  - shyamd
+  labels:
+  - providers_updates

.github/workflows/deps_lint.yml

@@ -37,6 +37,8 @@ jobs:
 
     steps:
     - uses: actions/checkout@v2
+      with:
+        submodules: true
 
     - name: Set up Python 3.7
       uses: actions/setup-python@v1
@@ -58,6 +60,8 @@ jobs:
 
     steps:
     - uses: actions/checkout@v2
+      with:
+        submodules: true
 
     - name: Set up Python 3.7
       uses: actions/setup-python@v1
@@ -86,6 +90,8 @@ jobs:
 
     steps:
     - uses: actions/checkout@v2
+      with:
+        submodules: true
 
     - name: Build the Docker images
       run: docker-compose build
@@ -148,6 +154,8 @@ jobs:
 
     steps:
     - uses: actions/checkout@v2
+      with:
+        submodules: true
 
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v1
@@ -199,6 +207,8 @@ jobs:
 
     steps:
     - uses: actions/checkout@v2
+      with:
+        submodules: true
 
     - uses: actions/setup-python@v1
       with:

.github/workflows/publish-on-pypi.yml

@@ -14,6 +14,8 @@ jobs:
     steps:
     - name: Checkout repository
       uses: actions/checkout@v2
+      with:
+        submodules: true
 
     - name: Set up Python 3.7
       uses: actions/setup-python@v1
@@ -94,6 +96,8 @@ jobs:
 
     steps:
     - uses: actions/checkout@v2
+      with:
+        submodules: true
 
     - uses: actions/setup-python@v1
       with:

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "providers"]
+	path = providers
+	url = git@github.com:Materials-Consortia/providers.git

Dockerfile

@@ -5,6 +5,7 @@ WORKDIR /app
 # copy repo contents
 COPY setup.py README.md ./
 COPY optimade ./optimade
+COPY providers/providers.json ./optimade/server/data/
 RUN pip install -e .[server]
 
 ARG PORT=5000

optimade/server/data/__init__.py

@@ -7,6 +7,7 @@
     "structures": "test_structures.json",
     "references": "test_references.json",
     "links": "test_links.json",
+    "providers": "providers.json",
 }
 
 

optimade/server/data/providers.json

@@ -0,0 +1 @@
+../../../providers/providers.json

optimade/server/routers/utils.py

@@ -21,7 +21,7 @@
 from optimade.server.entry_collections import EntryCollection
 from optimade.server.exceptions import BadRequest
 from optimade.server.query_params import EntryListingQueryParams, SingleEntryQueryParams
-
+from optimade.server.data import providers
 
 ENTRY_INFO_SCHEMAS = {
     "structures": StructureResource.schema,
@@ -303,14 +303,8 @@ def mongo_id_for_database(database_id: str, database_type: str) -> str:
 
 def get_providers():
     """Retrieve Materials-Consortia providers (from https://providers.optimade.org/providers.json)"""
-    import requests
-
-    mat_consortia_providers = requests.get(
-        "https://providers.optimade.org/providers.json"
-    ).json()
-
     providers_list = []
-    for provider in mat_consortia_providers.get("data", []):
+    for provider in providers.get("data", []):
         # Remove/skip "exmpl"
         if provider["id"] == "exmpl":
             continue

tests/test_setup.py

@@ -14,6 +14,7 @@
     "landing_page.html",
     "filters.txt",
     "optional_filters.txt",
+    "providers.json",
 ]