Add adapters - Base design + 'structures' (+ 'references'... sort of)

.github/aiida/profile.yaml

@@ -0,0 +1,14 @@
+---
+profile: PLACEHOLDER_PROFILE
+email: aiida@localhost
+first_name: AiiDA
+last_name: OPTIMADE
+institution: Materials-Consortia
+db_backend: PLACEHOLDER_BACKEND
+db_engine: postgresql_psycopg2
+db_host: localhost
+db_port: 5432
+db_name: PLACEHOLDER_DATABASE_NAME
+db_username: postgres
+db_password: test
+repository: PLACEHOLDER_REPOSITORY

.github/aiida/setup_aiida.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+set -ev
+
+# Replace the placeholders in configuration files with actual values
+CONFIG="${GITHUB_WORKSPACE}/.github/aiida"
+sed -i "s|PLACEHOLDER_BACKEND|${AIIDA_TEST_BACKEND}|" "${CONFIG}/profile.yaml"
+sed -i "s|PLACEHOLDER_PROFILE|test_${AIIDA_TEST_BACKEND}|" "${CONFIG}/profile.yaml"
+sed -i "s|PLACEHOLDER_DATABASE_NAME|test_${AIIDA_TEST_BACKEND}|" "${CONFIG}/profile.yaml"
+sed -i "s|PLACEHOLDER_REPOSITORY|/tmp/test_repository_test_${AIIDA_TEST_BACKEND}/|" "${CONFIG}/profile.yaml"
+
+verdi setup --config "${CONFIG}/profile.yaml"
+
+verdi profile setdefault test_${AIIDA_TEST_BACKEND}

.github/workflows/deps_eager.yml

@@ -18,13 +18,25 @@ jobs:
         python-version: [3.6, 3.7, 3.8]
 
     services:
-        mongo:
-            image: mongo:4.2
-            ports:
-            - 27017:27017
+      mongo:
+        image: mongo:4.2
+        ports:
+        - 27017:27017
+      postgres:
+        image: postgres:10
+        env:
+          POSTGRES_DB: test_django
+          POSTGRES_PASSWORD: test
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+        - 5432:5432
 
     steps:
-    - uses: actions/checkout@master
+    - uses: actions/checkout@v2
 
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v1
@@ -35,8 +47,8 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install -U setuptools
-        # Install package clean with all dependencies followed by eager update install
-        pip install -e .[testing,django,elastic]
+        # Install package clean with test dependencies followed by eager update install
+        pip install -e .[testing]
         pip install -U --upgrade-strategy eager -r .github/workflows/requirements_eager.txt
 
     - name: Run tests on updated packages
@@ -49,6 +61,20 @@ jobs:
       env:
         OPTIMADE_CI_FORCE_MONGO: 0
 
+    - name: Install adapter conversion dependencies (eagerly)
+      run: |
+        pip install -U --upgrade-strategy eager -e .[all]
+        # AiiDA-specific
+        reentry scan
+
+    - name: Setup up environment for AiiDA
+      env:
+        AIIDA_TEST_BACKEND: django
+      run: .github/aiida/setup_aiida.sh
+
+    - name: Run previously skipped tests for adapter conversion
+      run: pytest -rs --cov=./optimade/ --cov-report=xml --cov-append tests/adapters/
+
   # deps_clean-install:
   #   runs-on: ubuntu-latest
   #   strategy:
@@ -58,7 +84,7 @@ jobs:
   #       python-version: [3.6, 3.7, 3.8]
 
   #   steps:
-  #   - uses: actions/checkout@master
+  #   - uses: actions/checkout@v2
 
   #   - name: Set up Python ${{ matrix.python-version }}
   #     uses: actions/setup-python@v1

.github/workflows/deps_lint.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@master
+    - uses: actions/checkout@v2
 
     - name: Set up Python 3.7
       uses: actions/setup-python@v1
@@ -37,7 +37,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@master
+    - uses: actions/checkout@v2
 
     - name: Set up Python 3.7
       uses: actions/setup-python@v1
@@ -59,7 +59,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@master
+    - uses: actions/checkout@v2
 
     - name: Set up Python 3.7
       uses: actions/setup-python@v1
@@ -94,7 +94,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@master
+    - uses: actions/checkout@v2
 
     - name: Build the Docker images
       run: docker-compose build
@@ -138,13 +138,25 @@ jobs:
         python-version: [3.6, 3.7, 3.8]
 
     services:
-        mongo:
-            image: mongo:4.2
-            ports:
-            - 27017:27017
+      mongo:
+        image: mongo:4.2
+        ports:
+        - 27017:27017
+      postgres:
+        image: postgres:10
+        env:
+          POSTGRES_DB: test_django
+          POSTGRES_PASSWORD: test
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+        - 5432:5432
 
     steps:
-    - uses: actions/checkout@master
+    - uses: actions/checkout@v2
 
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v1
@@ -157,7 +169,7 @@ jobs:
         pip install -U setuptools
         # Install static dependencies followed by "normal" install
         python -m pip install -r .github/workflows/requirements.txt
-        pip install -e .[all]
+        pip install -e .[testing]
 
     - name: Run all tests (using a real MongoDB)
       run: pytest -rs --cov=./optimade/ --cov-report=xml
@@ -169,6 +181,20 @@ jobs:
       env:
         OPTIMADE_CI_FORCE_MONGO: 0
 
+    - name: Install adapter conversion dependencies
+      run: |
+        pip install -e .[all]
+        # AiiDA-specific
+        reentry scan
+
+    - name: Setup up environment for AiiDA
+      env:
+        AIIDA_TEST_BACKEND: django
+      run: .github/aiida/setup_aiida.sh
+
+    - name: Run previously skipped tests for adapter conversion
+      run: pytest -rs --cov=./optimade/ --cov-report=xml --cov-append tests/adapters/
+
     - name: Upload coverage to Codecov
       if: matrix.python-version == 3.7 && github.repository == 'Materials-Consortia/optimade-python-tools'
       uses: codecov/codecov-action@v1

.github/workflows/publish-on-pypi.yml

@@ -12,7 +12,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@master
+      uses: actions/checkout@v2
 
     - name: Set up Python 3.7
       uses: actions/setup-python@v1

.github/workflows/requirements.txt

@@ -1,5 +1,5 @@
-lark-parser==0.8.1
-fastapi==0.52.0
+lark-parser==0.8.5
+fastapi==0.53.1
 pydantic==1.4
 email_validator==1.0.5
 requests==2.23.0

optimade/adapters/__init__.py

@@ -0,0 +1,7 @@
+# pylint: disable=undefined-variable
+from .exceptions import *
+from .references import *
+from .structures import *
+
+
+__all__ = exceptions.__all__ + references.__all__ + structures.__all__

optimade/adapters/base.py

@@ -0,0 +1,120 @@
+import re
+from typing import Union, Dict, Callable, Any, Tuple, List
+
+from pydantic import BaseModel  # pylint: disable=no-name-in-module
+
+from optimade.models import EntryResource
+
+
+class EntryAdapter:
+    """Base class for lazy resource entry adapters
+    :param entry: JSON OPTIMADE single resource entry.
+    """
+
+    ENTRY_RESOURCE: EntryResource = EntryResource
+    _type_converters: Dict[str, Callable] = {}
+
+    def __init__(self, entry: dict):
+        self._entry = None
+        self._converted = {}
+
+        self.entry = entry
+
+        # Note that these return also the default values for otherwise non-provided properties.
+        self._common_converters = {
+            "json": self.entry.json,  # Return JSON serialized string, see https://pydantic-docs.helpmanual.io/usage/exporting_models/#modeljson
+            "dict": self.entry.dict,  # Return Python dict, see https://pydantic-docs.helpmanual.io/usage/exporting_models/#modeldict
+        }
+
+    @property
+    def entry(self):
+        """Get OPTIMADE entry"""
+        return self._entry
+
+    @entry.setter
+    def entry(self, value: dict):
+        """Set OPTIMADE entry
+        If already set, print that this can _only_ be set once.
+        """
+        if self._entry is None:
+            self._entry = self.ENTRY_RESOURCE(**value)
+        else:
+            print("entry can only be set once and is already set.")
+
+    def convert(self, format: str) -> Any:
+        """Convert OPTIMADE entry to desired format"""
+        if (
+            format not in self._type_converters
+            and format not in self._common_converters
+        ):
+            raise AttributeError(
+                f"Non-valid entry type to convert to: {format}\n"
+                f"Valid entry types: {tuple(self._type_converters.keys()) + tuple(self._common_converters.keys())}"
+            )
+
+        if self._converted.get(format, None) is None:
+            if format in self._type_converters:
+                self._converted[format] = self._type_converters[format](self.entry)
+            else:
+                self._converted[format] = self._common_converters[format]()
+
+        return self._converted[format]
+
+    @staticmethod
+    def _get_model_attributes(
+        starting_instances: Union[Tuple[BaseModel], List[BaseModel]], name: str
+    ) -> Any:
+        """Helper method for retrieving the OPTIMADE model's attribute, supporting "."-nested attributes"""
+        for res in starting_instances:
+            nested_attributes = name.split(".")
+            for nested_attribute in nested_attributes:
+                if nested_attribute in getattr(res, "__fields__", {}):
+                    res = getattr(res, nested_attribute)
+                else:
+                    res = None
+                    break
+            if res is not None:
+                return res
+        raise AttributeError
+
+    def __getattr__(self, name: str) -> Any:
+        """Get converted entry or attribute from OPTIMADE entry
+        Support any level of "."-nested OPTIMADE ENTRY_RESOURCE attributes, e.g., `attributes.species` for StuctureResource.
+        NOTE: All nested attributes must individually be subclasses of `pydantic.BaseModel`,
+        i.e., one can not access nested attributes in lists by passing a "."-nested `name` to this method,
+        e.g., `attributes.species.name` or `attributes.species[0].name` will not work for variable `name`.
+
+        Order:
+        - Try to return converted entry if using `as_<_type_converters key>`.
+        - Try to return OPTIMADE ENTRY_RESOURCE (nested) attribute.
+        - Try to return OPTIMADE ENTRY_RESOURCE.attributes (nested) attribute.
+        - Raise AttributeError
+        """
+        # as_<entry_type>
+        if name.startswith("as_"):
+            entry_type = "_".join(name.split("_")[1:])
+            return self.convert(entry_type)
+
+        # Try returning ENTRY_RESOURCE attribute
+        try:
+            res = self._get_model_attributes((self.entry, self.entry.attributes), name)
+        except AttributeError:
+            pass
+        else:
+            return res
+
+        # Non-valid attribute
+        entry_resource_name = re.match(
+            r"(<class ')([a-zA-Z_]+\.)*([a-zA-Z_]+)('>)", str(self.ENTRY_RESOURCE)
+        )
+        entry_resource_name = (
+            entry_resource_name.group(3)
+            if entry_resource_name is not None
+            else "UNKNOWN RESOURCE"
+        )
+        raise AttributeError(
+            f"Unknown attribute: {name}\n"
+            "If you want to get a converted entry as <entry_type> use `as_<entry_type>`, "
+            f"where `<entry_type>` is one of {tuple(self._type_converters.keys()) + tuple(self._common_converters.keys())}\n"
+            f"Otherwise, you can try to retrieve an OPTIMADE {entry_resource_name} attribute or property."
+        )

optimade/adapters/exceptions.py

@@ -0,0 +1,5 @@
+__all__ = ("ConversionError",)
+
+
+class ConversionError(Exception):
+    """Could not convert entry to format"""

optimade/adapters/references/__init__.py

@@ -0,0 +1,16 @@
+from typing import Union
+
+from optimade.models import ReferenceResource
+
+from optimade.adapters.base import EntryAdapter
+
+
+__all__ = ("Reference",)
+
+
+class Reference(EntryAdapter):
+    """Lazy reference resource converter
+    :param reference: a single JSON OPTIMADE reference resource entry.
+    """
+
+    ENTRY_RESOURCE = ReferenceResource

optimade/adapters/structures/__init__.py

@@ -0,0 +1,30 @@
+from typing import Union
+
+from optimade.models import StructureResource
+
+from optimade.adapters.base import EntryAdapter
+
+from .aiida import get_aiida_structure_data
+from .ase import get_ase_atoms
+from .cif import get_cif
+from .proteindatabank import get_pdb, get_pdbx_mmcif
+from .pymatgen import get_pymatgen
+
+
+__all__ = ("Structure",)
+
+
+class Structure(EntryAdapter):
+    """Lazy structure resource converter
+    :param structure: a single JSON OPTIMADE structure resource entry.
+    """
+
+    ENTRY_RESOURCE = StructureResource
+    _type_converters = {
+        "aiida_structuredata": get_aiida_structure_data,
+        "ase": get_ase_atoms,
+        "cif": get_cif,
+        "pdb": get_pdb,
+        "pdbx_mmcif": get_pdbx_mmcif,
+        "pymatgen": get_pymatgen,
+    }