Add ability to check openapi responses.

This can be used to ensure all API endpoints support common responses.
Path to responses.yaml needs to be provied via config `pyramid_openapi3_responses_config`

.isort.cfg

@@ -3,3 +3,10 @@ force_single_line=true
 atomic=true
 force_alphabetical_sort=true
 not_skip=__init__.py
+# https://black.readthedocs.io/en/stable/the_black_code_style.html?highlight=isort#how-black-wraps-lines
+# Search for : A compatible `.isort.cfg`
+multi_line_output=3
+include_trailing_comma=True
+force_grid_wrap=0
+use_parentheses=True
+line_length=88

Pipfile

@@ -5,6 +5,10 @@ name = "pypi"
 
 [packages]
 pyramid-openapi3 = {editable = true,path = "."}
+openapi-core = "==0.11.0"
+jsonschema = "==3.0.1"
+openapi-spec-validator = "*"
+structlog = "*"
 
 [dev-packages]
 black = "==19.3b0"

examples/todoapp/tests.py

@@ -40,9 +40,9 @@ def test_empty_POST(self):
             res.json,
             [
                 {
+                    "message": "Missing schema property: title",
                     "exception": "MissingSchemaProperty",
                     "field": "title",
-                    "message": "Missing schema property: title",
                 }
             ],
         )
@@ -54,10 +54,9 @@ def test_title_too_long(self):
             res.json,
             [
                 {
+                    "message": "Invalid schema property title: Value is longer (41) than the maximum length of 40",
                     "exception": "InvalidSchemaProperty",
                     "field": "title",
-                    "message": "Invalid schema property title: "
-                    "Value is longer (41) than the maximum length of 40",
                 }
             ],
         )

pyramid_openapi3/__init__.py

@@ -1,5 +1,6 @@
 """Configure pyramid_openapi3 addon."""
 
+from .check_openapi_responses import validate_required_responses
 from .exceptions import RequestValidationError
 from .wrappers import PyramidOpenAPIRequest
 from openapi_core import create_spec
@@ -129,6 +130,7 @@ def register() -> None:
         spec_dict = read_yaml_file(filepath)
 
         validate_spec(spec_dict)
+        validate_required_responses(spec_dict, config)
         spec = create_spec(spec_dict)
 
         def spec_view(request: Request) -> FileResponse:

pyramid_openapi3/check_openapi_responses.py

@@ -0,0 +1,84 @@
+"""Verify that endpoints have defined required responses."""
+
+from .exceptions import ResponseValidationError
+from openapi_core.schema.specs.models import Spec
+from pathlib import Path
+from pyramid.config import Configurator
+
+import openapi_core
+import structlog  # type: ignore
+import typing as t
+import yaml
+
+logger = structlog.get_logger(__name__)
+
+
+def get_config(config_path: str) -> t.Dict[str, str]:
+    """Read config from file."""
+    config = Path(config_path)
+
+    if not config.is_file():
+        raise RuntimeError(f"ERROR Config file not found on: {config}\n")
+
+    with config.open("r") as f:
+        return yaml.safe_load(f)
+
+
+def required_responses(
+    config: t.Dict, endpoint: str, method: str, has_params: bool
+) -> t.Set:
+    """Get required responses for given method on endpoint."""
+    required_resp: t.Set = set(config.get("required_responses", {}).get(method, []))
+    if has_params:
+        required_params: set = set(
+            config.get("required_responses_params", {}).get(method, [])
+        )
+        required_resp = required_resp.union(required_params)
+    allowed_missing: set = set(
+        config.get("allowed_missing_responses", {}).get(endpoint, {}).get(method, [])
+    )
+    required_resp = required_resp - allowed_missing
+    return required_resp
+
+
+def validate_required_responses(spec_dict: dict, config: Configurator) -> None:
+    """Verify that all endpoints have defined required responses."""
+    check_failed: bool = False
+    missing_responses_count: int = 0
+    errors = []
+
+    filepath: str = config.registry.settings.get("pyramid_openapi3_responses_config")
+    if not filepath:
+        logger.warning(
+            "pyramid_openapi3_responses_config not configured. Required Responses will not be validated."
+        )
+        return
+
+    responses_config: t.Dict[str, str] = get_config(filepath)
+
+    spec: Spec = openapi_core.create_spec(spec_dict)
+    for path in spec.paths.values():
+        for operation in path.operations.values():
+            operation_responses = operation.responses.keys()
+            method: str = operation.http_method
+            endpoint: str = operation.path_name
+            has_params: bool = len(operation.parameters) > 0
+            required: t.Set = required_responses(
+                responses_config, endpoint, method, has_params
+            )
+
+            missing_responses: t.Set = required - operation_responses
+            for missing_response in missing_responses:
+                check_failed = True
+                missing_responses_count += 1
+                errors.append(
+                    "ERROR missing response "
+                    f"'{missing_response}' for '{method}' request on path "
+                    f"'{endpoint}'\n"
+                )
+    if check_failed:
+        errors.append(
+            "\nFAILED: Openapi responses check: "
+            f"{missing_responses_count} missing response definitions. \n"
+        )
+        raise ResponseValidationError(errors=errors)

pyramid_openapi3/tests/test_openapi_responses.py

@@ -0,0 +1,130 @@
+"""Tests validation of openapi responses."""
+
+from pyramid.testing import testConfig
+from pyramid_openapi3.check_openapi_responses import validate_required_responses
+from pyramid_openapi3.exceptions import ResponseValidationError
+
+import pytest
+import tempfile
+import yaml
+
+VALIDATION_DOCUMENT = b"""
+required_responses:
+  get:
+    - '200'
+    - '400'
+# Additional required response definitions for endpoints with parameters
+required_responses_params:
+  get:
+    - '404'
+"""
+
+
+def test_get_config_no_file() -> None:
+    """Test get_config when config is not a file object."""
+    from pyramid_openapi3.check_openapi_responses import get_config
+
+    with pytest.raises(RuntimeError):
+        get_config("file_does_not_exist")
+
+
+def test_response_validation_error() -> None:
+    """Test that ResponseValidationError is raised when 404 is missing."""
+
+    SPEC_DOCUMENT = b"""
+        openapi: "3.0.0"
+        info:
+          version: "1.0.0"
+          title: Foo API
+        paths:
+          /foo:
+            get:
+              parameters:
+              - name: bar
+                in: query
+                schema:
+                  type: integer
+              responses:
+                "200":
+                  description: A foo
+                "400":
+                  description: Bad Request
+    """
+
+    with testConfig() as config:
+        config.include("pyramid_openapi3")
+
+        with tempfile.NamedTemporaryFile() as document:
+            document.write(VALIDATION_DOCUMENT)
+            document.seek(0)
+
+            config.add_settings(pyramid_openapi3_responses_config=document.name)
+            with pytest.raises(ResponseValidationError):
+                validate_required_responses(yaml.safe_load(SPEC_DOCUMENT), config)
+
+
+def test_happy_path() -> None:
+    """Test validation of openapi responses."""
+
+    SPEC_DOCUMENT = b"""
+        openapi: "3.0.0"
+        info:
+          version: "1.0.0"
+          title: Foo API
+        paths:
+          /foo:
+            get:
+              parameters:
+              - name: bar
+                in: query
+                schema:
+                  type: integer
+              responses:
+                "200":
+                  description: A foo
+                "400":
+                  description: Bad Request
+                "404":
+                  description: Not Found
+    """
+
+    with testConfig() as config:
+        config.include("pyramid_openapi3")
+
+        with tempfile.NamedTemporaryFile() as document:
+            document.write(VALIDATION_DOCUMENT)
+            document.seek(0)
+
+            config.add_settings(pyramid_openapi3_responses_config=document.name)
+            validate_required_responses(yaml.safe_load(SPEC_DOCUMENT), config)
+
+
+def test_no_params() -> None:
+    """Test spec without parameters."""
+
+    SPEC_DOCUMENT = b"""
+        openapi: "3.0.0"
+        info:
+          version: "1.0.0"
+          title: Foo API
+        paths:
+          /foo:
+            get:
+              responses:
+                "200":
+                  description: A foo
+                "400":
+                  description: Bad Request
+                "404":
+                  description: Not Found
+    """
+
+    with testConfig() as config:
+        config.include("pyramid_openapi3")
+
+        with tempfile.NamedTemporaryFile() as document:
+            document.write(VALIDATION_DOCUMENT)
+            document.seek(0)
+
+            config.add_settings(pyramid_openapi3_responses_config=document.name)
+            validate_required_responses(yaml.safe_load(SPEC_DOCUMENT), config)