Move exceptions and warnings out of server code and separate deps (#1405

)

* Move exceptions and warnings out of server code

* Separate server and non-server tests and deps

* Move exceptions to top-level

* Patch and include top-level modules in API docs

* Skip filter tests when requirements not present

* Run filtertransformer tests alongside each server backend

* Do not use server data in models tests

* Clarify test skip message

.github/workflows/ci.yml

@@ -60,6 +60,7 @@ jobs:
         pip install -U setuptools wheel
         pip install -e .
         pip install -r requirements.txt
+        pip install -r requirements-server.txt
         pip install -r requirements-dev.txt
 
     - name: Run pre-commit
@@ -85,6 +86,7 @@ jobs:
         pip install -U setuptools wheel
         pip install -e .
         pip install -r requirements.txt
+        pip install -r requirements-server.txt
         pip install -r requirements-dev.txt
 
     - name: Pass generated OpenAPI schemas through validator.swagger.io
@@ -194,25 +196,29 @@ jobs:
         pip install -r requirements-dev.txt
         pip install -r requirements-http-client.txt
 
-    - name: Run all tests (using `mongomock`)
-      run: pytest -rs -vvv --cov=./optimade/ --cov-report=xml tests/
+    - name: Run non-server tests
+      run: pytest -rs -vvv --cov=./optimade/ --cov-report=xml tests/ --ignore tests/server
+
+    - name: Install latest server dependencies
+      run: pip install -r requirements-server.txt
+
+    - name: Run server tests (using `mongomock`)
+      run: pytest -rs -vvv --cov=./optimade/ --cov-report=xml --cov-append tests/server tests/filtertransformers
       env:
         OPTIMADE_DATABASE_BACKEND: 'mongomock'
 
-
     - name: Run server tests (using a real MongoDB)
-      run: pytest -rs -vvv --cov=./optimade/ --cov-report=xml --cov-append tests/server
+      run: pytest -rs -vvv --cov=./optimade/ --cov-report=xml --cov-append tests/server tests/filtertransformers
       env:
         OPTIMADE_DATABASE_BACKEND: 'mongodb'
 
     - name: Run server tests (using Elasticsearch)
-      run: pytest -rs -vvv --cov=./optimade/ --cov-report=xml --cov-append tests/server
+      run: pytest -rs -vvv --cov=./optimade/ --cov-report=xml --cov-append tests/server tests/filtertransformers
       env:
         OPTIMADE_DATABASE_BACKEND: 'elastic'
 
     - name: Install adapter conversion dependencies
-      run: |
-        pip install -r requirements-client.txt
+      run: pip install -r requirements-client.txt
 
     - name: Setup environment for AiiDA
       env:
@@ -305,6 +311,7 @@ jobs:
         pip install -U setuptools wheel
         pip install -e .
         pip install -r requirements.txt
+        pip install -r requirements-server.txt
         pip install -r requirements-dev.txt
         pip install -r requirements-http-client.txt
         pip install -r requirements-docs.txt

docs/api_reference/exceptions.md

@@ -0,0 +1,3 @@
+# exceptions
+
+::: optimade.exceptions

docs/api_reference/utils.md

@@ -0,0 +1,3 @@
+# utils
+
+::: optimade.utils

docs/api_reference/warnings.md

@@ -0,0 +1,3 @@
+# warnings
+
+::: optimade.warnings

optimade/adapters/warnings.py

@@ -1,4 +1,4 @@
-from optimade.server.warnings import OptimadeWarning
+from optimade.warnings import OptimadeWarning
 
 __all__ = ("AdapterPackageNotFound", "ConversionWarning")
 

optimade/client/client.py

@@ -34,8 +34,8 @@
     TooManyRequestsException,
     silent_raise,
 )
+from optimade.exceptions import BadRequest
 from optimade.filterparser import LarkParser
-from optimade.server.exceptions import BadRequest
 from optimade.utils import get_all_databases
 
 ENDPOINTS = ("structures", "references", "calculations", "info", "extensions")

optimade/exceptions.py

@@ -0,0 +1,115 @@
+from abc import ABC
+from typing import Any, Dict, Optional, Tuple, Type
+
+__all__ = (
+    "OptimadeHTTPException",
+    "BadRequest",
+    "VersionNotSupported",
+    "Forbidden",
+    "NotFound",
+    "UnprocessableEntity",
+    "InternalServerError",
+    "NotImplementedResponse",
+    "POSSIBLE_ERRORS",
+)
+
+
+class OptimadeHTTPException(Exception, ABC):
+    """This abstract class can be subclassed to define
+    HTTP responses with the desired status codes, and
+    detailed error strings to represent in the JSON:API
+    error response.
+
+    This class closely follows the `starlette.HTTPException` without
+    requiring it as a dependency, so that such errors can also be
+    raised from within client code.
+
+    Attributes:
+        status_code: The HTTP status code accompanying this exception.
+        title: A descriptive title for this exception.
+        detail: An optional string containing the details of the error.
+
+    """
+
+    status_code: int
+    title: str
+    detail: Optional[str] = None
+    headers: Optional[Dict[str, Any]] = None
+
+    def __init__(
+        self, detail: Optional[str] = None, headers: Optional[dict] = None
+    ) -> None:
+        if self.status_code is None:
+            raise AttributeError(
+                "HTTPException class {self.__class__.__name__} is missing required `status_code` attribute."
+            )
+        self.detail = detail
+        self.headers = headers
+
+    def __str__(self) -> str:
+        return self.detail if self.detail is not None else self.__repr__()
+
+    def __repr__(self) -> str:
+        class_name = self.__class__.__name__
+        return f"{class_name}(status_code={self.status_code!r}, detail={self.detail!r})"
+
+
+class BadRequest(OptimadeHTTPException):
+    """400 Bad Request"""
+
+    status_code: int = 400
+    title: str = "Bad Request"
+
+
+class VersionNotSupported(OptimadeHTTPException):
+    """553 Version Not Supported"""
+
+    status_code: int = 553
+    title: str = "Version Not Supported"
+
+
+class Forbidden(OptimadeHTTPException):
+    """403 Forbidden"""
+
+    status_code: int = 403
+    title: str = "Forbidden"
+
+
+class NotFound(OptimadeHTTPException):
+    """404 Not Found"""
+
+    status_code: int = 404
+    title: str = "Not Found"
+
+
+class UnprocessableEntity(OptimadeHTTPException):
+    """422 Unprocessable Entity"""
+
+    status_code: int = 422
+    title: str = "Unprocessable Entity"
+
+
+class InternalServerError(OptimadeHTTPException):
+    """500 Internal Server Error"""
+
+    status_code: int = 500
+    title: str = "Internal Server Error"
+
+
+class NotImplementedResponse(OptimadeHTTPException):
+    """501 Not Implemented"""
+
+    status_code: int = 501
+    title: str = "Not Implemented"
+
+
+"""A tuple of the possible errors that can be returned by an OPTIMADE API."""
+POSSIBLE_ERRORS: Tuple[Type[OptimadeHTTPException], ...] = (
+    BadRequest,
+    Forbidden,
+    NotFound,
+    UnprocessableEntity,
+    InternalServerError,
+    NotImplementedResponse,
+    VersionNotSupported,
+)

optimade/filterparser/lark_parser.py

@@ -9,7 +9,7 @@
 
 from lark import Lark, Tree
 
-from optimade.server.exceptions import BadRequest
+from optimade.exceptions import BadRequest
 
 __all__ = ("ParserError", "LarkParser")
 

optimade/filtertransformers/base_transformer.py

@@ -11,9 +11,9 @@
 
 from lark import Transformer, Tree, v_args
 
-from optimade.server.exceptions import BadRequest
+from optimade.exceptions import BadRequest
 from optimade.server.mappers import BaseResourceMapper
-from optimade.server.warnings import UnknownProviderProperty
+from optimade.warnings import UnknownProviderProperty
 
 __all__ = (
     "BaseTransformer",

optimade/filtertransformers/mongo.py

@@ -11,9 +11,9 @@
 
 from lark import Token, v_args
 
+from optimade.exceptions import BadRequest
 from optimade.filtertransformers.base_transformer import BaseTransformer, Quantity
-from optimade.server.exceptions import BadRequest
-from optimade.server.warnings import TimestampNotRFCCompliant
+from optimade.warnings import TimestampNotRFCCompliant
 
 __all__ = ("MongoTransformer",)
 

optimade/models/structures.py

@@ -19,7 +19,7 @@
     StrictField,
     SupportLevel,
 )
-from optimade.server.warnings import MissingExpectedField
+from optimade.warnings import MissingExpectedField
 
 EXTENDED_CHEMICAL_SYMBOLS = set(CHEMICAL_SYMBOLS + EXTRA_SYMBOLS)
 

optimade/server/entry_collections/entry_collections.py

@@ -5,13 +5,13 @@
 
 from lark import Transformer
 
+from optimade.exceptions import BadRequest, Forbidden, NotFound
 from optimade.filterparser import LarkParser
 from optimade.models.entries import EntryResource
 from optimade.server.config import CONFIG, SupportedBackend
-from optimade.server.exceptions import BadRequest, Forbidden, NotFound
 from optimade.server.mappers import BaseResourceMapper
 from optimade.server.query_params import EntryListingQueryParams, SingleEntryQueryParams
-from optimade.server.warnings import (
+from optimade.warnings import (
     FieldValueNotRecognized,
     QueryParamNotUsed,
     UnknownProviderProperty,

optimade/server/exception_handlers.py

@@ -1,15 +1,15 @@
 import traceback
-from typing import Callable, Iterable, List, Optional, Tuple, Type
+from typing import Callable, Iterable, List, Optional, Tuple, Type, Union
 
 from fastapi import Request
 from fastapi.encoders import jsonable_encoder
 from fastapi.exceptions import RequestValidationError, StarletteHTTPException
 from lark.exceptions import VisitError
 from pydantic import ValidationError
 
+from optimade.exceptions import BadRequest, OptimadeHTTPException
 from optimade.models import ErrorResponse, ErrorSource, OptimadeError
 from optimade.server.config import CONFIG
-from optimade.server.exceptions import BadRequest
 from optimade.server.logger import LOGGER
 from optimade.server.routers.utils import JSONAPIResponse, meta_values
 
@@ -78,7 +78,8 @@ def general_exception(
 
 
 def http_exception_handler(
-    request: Request, exc: StarletteHTTPException
+    request: Request,
+    exc: Union[StarletteHTTPException, OptimadeHTTPException],
 ) -> JSONAPIResponse:
     """Handle a general HTTP Exception from Starlette
 
@@ -152,7 +153,7 @@ def grammar_not_implemented_handler(
     All errors raised during filter transformation are wrapped in the Lark `VisitError`.
     According to the OPTIMADE specification, these errors are repurposed to be 501 NotImplementedErrors.
 
-    For special exceptions, like [`BadRequest`][optimade.server.exceptions.BadRequest], we pass-through to
+    For special exceptions, like [`BadRequest`][optimade.exceptions.BadRequest], we pass-through to
     [`general_exception()`][optimade.server.exception_handlers.general_exception], since they should not
     return a 501 NotImplementedError.
 
@@ -226,6 +227,7 @@ def general_exception_handler(request: Request, exc: Exception) -> JSONAPIRespon
     ]
 ] = [
     (StarletteHTTPException, http_exception_handler),
+    (OptimadeHTTPException, http_exception_handler),
     (RequestValidationError, request_validation_exception_handler),
     (ValidationError, validation_exception_handler),
     (VisitError, grammar_not_implemented_handler),