✨Airbyte CDK: add POST method to HttpMocker (airbytehq#34001)

Co-authored-by: maxi297 <maxime@airbyte.io>

airbyte-cdk/python/airbyte_cdk/test/mock_http/mocker.py

@@ -2,16 +2,34 @@
 
 import contextlib
 import functools
+from enum import Enum
 from types import TracebackType
 from typing import Callable, List, Optional, Union
 
 import requests_mock
 from airbyte_cdk.test.mock_http import HttpRequest, HttpRequestMatcher, HttpResponse
 
 
+class SupportedHttpMethods(str, Enum):
+    GET = "get"
+    POST = "post"
+
+
 class HttpMocker(contextlib.ContextDecorator):
     """
-    WARNING: This implementation only works if the lib used to perform HTTP requests is `requests`
+    WARNING 1: This implementation only works if the lib used to perform HTTP requests is `requests`.
+
+    WARNING 2: Given multiple requests that are not mutually exclusive, the request will match the first one. This can happen in scenarios
+    where the same request is added twice (in which case there will always be an exception because we will never match the second
+    request) or in a case like this:
+    ```
+    http_mocker.get(HttpRequest(_A_URL, headers={"less_granular": "1", "more_granular": "2"}), <...>)
+    http_mocker.get(HttpRequest(_A_URL, headers={"less_granular": "1"}), <...>)
+    requests.get(_A_URL, headers={"less_granular": "1", "more_granular": "2"})
+    ```
+    In the example above, the matcher would match the second mock as requests_mock iterate over the matcher in reverse order (see
+    https://github.com/jamielennox/requests-mock/blob/c06f124a33f56e9f03840518e19669ba41b93202/requests_mock/adapter.py#L246) even
+    though the request sent is a better match for the first `http_mocker.get`.
     """
 
     def __init__(self) -> None:
@@ -30,35 +48,34 @@ def _validate_all_matchers_called(self) -> None:
             if not matcher.has_expected_match_count():
                 raise ValueError(f"Invalid number of matches for `{matcher}`")
 
-    def get(self, request: HttpRequest, responses: Union[HttpResponse, List[HttpResponse]]) -> None:
-        """
-        WARNING: Given multiple requests that are not mutually exclusive, the request will match the first one. This can happen in scenarios
-        where the same request is added twice (in which case there will always be an exception because we will never match the second
-        request) or in a case like this:
-        ```
-        http_mocker.get(HttpRequest(_A_URL, headers={"less_granular": "1", "more_granular": "2"}), <...>)
-        http_mocker.get(HttpRequest(_A_URL, headers={"less_granular": "1"}), <...>)
-        requests.get(_A_URL, headers={"less_granular": "1", "more_granular": "2"})
-        ```
-        In the example above, the matcher would match the second mock as requests_mock iterate over the matcher in reverse order (see
-        https://github.com/jamielennox/requests-mock/blob/c06f124a33f56e9f03840518e19669ba41b93202/requests_mock/adapter.py#L246) even
-        though the request sent is a better match for the first `http_mocker.get`.
-        """
+    def _mock_request_method(
+        self, method: SupportedHttpMethods, request: HttpRequest, responses: Union[HttpResponse, List[HttpResponse]]
+    ) -> None:
         if isinstance(responses, HttpResponse):
             responses = [responses]
 
         matcher = HttpRequestMatcher(request, len(responses))
         self._matchers.append(matcher)
-        self._mocker.get(
+
+        getattr(self._mocker, method)(
             requests_mock.ANY,
             additional_matcher=self._matches_wrapper(matcher),
             response_list=[{"text": response.body, "status_code": response.status_code} for response in responses],
         )
 
-    def _matches_wrapper(self, matcher: HttpRequestMatcher) -> Callable[[requests_mock.request._RequestObjectProxy], bool]:
+    def get(self, request: HttpRequest, responses: Union[HttpResponse, List[HttpResponse]]) -> None:
+        self._mock_request_method(SupportedHttpMethods.GET, request, responses)
+
+    def post(self, request: HttpRequest, responses: Union[HttpResponse, List[HttpResponse]]) -> None:
+        self._mock_request_method(SupportedHttpMethods.POST, request, responses)
+
+    @staticmethod
+    def _matches_wrapper(matcher: HttpRequestMatcher) -> Callable[[requests_mock.request._RequestObjectProxy], bool]:
         def matches(requests_mock_request: requests_mock.request._RequestObjectProxy) -> bool:
             # query_params are provided as part of `requests_mock_request.url`
-            http_request = HttpRequest(requests_mock_request.url, query_params={}, headers=requests_mock_request.headers)
+            http_request = HttpRequest(
+                requests_mock_request.url, query_params={}, headers=requests_mock_request.headers, body=requests_mock_request.body
+            )
             return matcher.matches(http_request)
 
         return matches
@@ -70,7 +87,8 @@ def assert_number_of_calls(self, request: HttpRequest, number_of_calls: int) ->
 
         assert corresponding_matchers[0].actual_number_of_matches == number_of_calls
 
-    def __call__(self, f):  # type: ignore  # trying to type that using callables provides the error `incompatible with return type "_F" in supertype "ContextDecorator"`
+    # trying to type that using callables provides the error `incompatible with return type "_F" in supertype "ContextDecorator"`
+    def __call__(self, f):  # type: ignore
         @functools.wraps(f)
         def wrapper(*args, **kwargs):  # type: ignore  # this is a very generic wrapper that does not need to be typed
             with self:
@@ -82,18 +100,21 @@ def wrapper(*args, **kwargs):  # type: ignore  # this is a very generic wrapper
                 except requests_mock.NoMockAddress as no_mock_exception:
                     matchers_as_string = "\n\t".join(map(lambda matcher: str(matcher.request), self._matchers))
                     raise ValueError(
-                        f"No matcher matches {no_mock_exception.args[0]} with headers `{no_mock_exception.request.headers}`. Matchers currently configured are:\n\t{matchers_as_string}"
+                        f"No matcher matches {no_mock_exception.args[0]} with headers `{no_mock_exception.request.headers}` "
+                        f"and body `{no_mock_exception.request.body}`. "
+                        f"Matchers currently configured are:\n\t{matchers_as_string}."
                     ) from no_mock_exception
                 except AssertionError as test_assertion:
                     assertion_error = test_assertion
 
-                # We validate the matchers before raising the assertion error because we want to show the tester if a HTTP request wasn't
+                # We validate the matchers before raising the assertion error because we want to show the tester if an HTTP request wasn't
                 # mocked correctly
                 try:
                     self._validate_all_matchers_called()
                 except ValueError as http_mocker_exception:
-                    # This seems useless as it catches ValueError and raises ValueError but without this, the prevaling error message in
-                    # the output is the function call that failed the assertion, whereas raising `ValueError(http_mocker_exception)` like we do here provides additional context for the exception.
+                    # This seems useless as it catches ValueError and raises ValueError but without this, the prevailing error message in
+                    # the output is the function call that failed the assertion, whereas raising `ValueError(http_mocker_exception)`
+                    # like we do here provides additional context for the exception.
                     raise ValueError(http_mocker_exception) from None
                 if assertion_error:
                     raise assertion_error

airbyte-cdk/python/airbyte_cdk/test/mock_http/request.py

@@ -1,5 +1,6 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 
+import json
 from typing import Any, List, Mapping, Optional, Union
 from urllib.parse import parse_qs, urlencode, urlparse
 
@@ -16,6 +17,7 @@ def __init__(
         url: str,
         query_params: Optional[Union[str, Mapping[str, Union[str, List[str]]]]] = None,
         headers: Optional[Mapping[str, str]] = None,
+        body: Optional[Union[str, bytes, Mapping[str, Any]]] = None,
     ) -> None:
         self._parsed_url = urlparse(url)
         self._query_params = query_params
@@ -25,31 +27,61 @@ def __init__(
             raise ValueError("If query params are provided as part of the url, `query_params` should be empty")
 
         self._headers = headers or {}
+        self._body = body
 
-    def _encode_qs(self, query_params: Union[str, Mapping[str, Union[str, List[str]]]]) -> str:
+    @staticmethod
+    def _encode_qs(query_params: Union[str, Mapping[str, Union[str, List[str]]]]) -> str:
         if isinstance(query_params, str):
             return query_params
         return urlencode(query_params, doseq=True)
 
     def matches(self, other: Any) -> bool:
         """
-        Note that headers only need to be a subset of `other` in order to match
+        If the body of any request is a Mapping, we compare as Mappings which means that the order is not important.
+        If the body is a string, encoding ISO-8859-1 will be assumed
+        Headers only need to be a subset of `other` in order to match
         """
         if isinstance(other, HttpRequest):
+            # if `other` is a mapping, we match as an object and formatting is not considers
+            if isinstance(self._body, Mapping) or isinstance(other._body, Mapping):
+                body_match = self._to_mapping(self._body) == self._to_mapping(other._body)
+            else:
+                body_match = self._to_bytes(self._body) == self._to_bytes(other._body)
+
             return (
                 self._parsed_url.scheme == other._parsed_url.scheme
                 and self._parsed_url.hostname == other._parsed_url.hostname
                 and self._parsed_url.path == other._parsed_url.path
                 and (
-                    ANY_QUERY_PARAMS in [self._query_params, other._query_params]
+                    ANY_QUERY_PARAMS in (self._query_params, other._query_params)
                     or parse_qs(self._parsed_url.query) == parse_qs(other._parsed_url.query)
                 )
                 and _is_subdict(other._headers, self._headers)
+                and body_match
             )
         return False
 
+    @staticmethod
+    def _to_mapping(body: Optional[Union[str, bytes, Mapping[str, Any]]]) -> Optional[Mapping[str, Any]]:
+        if isinstance(body, Mapping):
+            return body
+        elif isinstance(body, bytes):
+            return json.loads(body.decode())  # type: ignore  # assumes return type of Mapping[str, Any]
+        elif isinstance(body, str):
+            return json.loads(body)  # type: ignore  # assumes return type of Mapping[str, Any]
+        return None
+
+    @staticmethod
+    def _to_bytes(body: Optional[Union[str, bytes]]) -> bytes:
+        if isinstance(body, bytes):
+            return body
+        elif isinstance(body, str):
+            # `ISO-8859-1` is the default encoding used by requests
+            return body.encode("ISO-8859-1")
+        return b""
+
     def __str__(self) -> str:
-        return f"{self._parsed_url} with headers {self._headers})"
+        return f"{self._parsed_url} with headers {self._headers} and body {self._body!r})"
 
     def __repr__(self) -> str:
-        return f"HttpRequest(request={self._parsed_url}, headers={self._headers})"
+        return f"HttpRequest(request={self._parsed_url}, headers={self._headers}, body={self._body!r})"

airbyte-cdk/python/unit_tests/test/mock_http/test_mocker.py

@@ -10,51 +10,89 @@
 # see https://github.com/psf/requests/blob/0b4d494192de489701d3a2e32acef8fb5d3f042e/src/requests/models.py#L424-L429
 _A_URL = "http://test.com/"
 _ANOTHER_URL = "http://another-test.com/"
-_A_BODY = "a body"
-_ANOTHER_BODY = "another body"
+_A_RESPONSE_BODY = "a body"
+_ANOTHER_RESPONSE_BODY = "another body"
 _A_RESPONSE = HttpResponse("any response")
 _SOME_QUERY_PARAMS = {"q1": "query value"}
 _SOME_HEADERS = {"h1": "header value"}
+_SOME_REQUEST_BODY_MAPPING = {"first_field": "first_value", "second_field": 2}
+_SOME_REQUEST_BODY_STR = "some_request_body"
 
 
 class HttpMockerTest(TestCase):
     @HttpMocker()
-    def test_given_request_match_when_decorate_then_return_response(self, http_mocker):
+    def test_given_get_request_match_when_decorate_then_return_response(self, http_mocker):
         http_mocker.get(
             HttpRequest(_A_URL, _SOME_QUERY_PARAMS, _SOME_HEADERS),
-            HttpResponse(_A_BODY, 474),
+            HttpResponse(_A_RESPONSE_BODY, 474),
         )
 
         response = requests.get(_A_URL, params=_SOME_QUERY_PARAMS, headers=_SOME_HEADERS)
 
-        assert response.text == _A_BODY
+        assert response.text == _A_RESPONSE_BODY
         assert response.status_code == 474
 
     @HttpMocker()
-    def test_given_multiple_responses_when_decorate_then_return_response(self, http_mocker):
+    def test_given_loose_headers_matching_when_decorate_then_match(self, http_mocker):
         http_mocker.get(
             HttpRequest(_A_URL, _SOME_QUERY_PARAMS, _SOME_HEADERS),
-            [HttpResponse(_A_BODY, 1), HttpResponse(_ANOTHER_BODY, 2)],
+            HttpResponse(_A_RESPONSE_BODY, 474),
+        )
+
+        requests.get(_A_URL, params=_SOME_QUERY_PARAMS, headers=_SOME_HEADERS | {"more strict query param key": "any value"})
+
+    @HttpMocker()
+    def test_given_post_request_match_when_decorate_then_return_response(self, http_mocker):
+        http_mocker.post(
+            HttpRequest(_A_URL, _SOME_QUERY_PARAMS, _SOME_HEADERS, _SOME_REQUEST_BODY_STR),
+            HttpResponse(_A_RESPONSE_BODY, 474),
+        )
+
+        response = requests.post(_A_URL, params=_SOME_QUERY_PARAMS, headers=_SOME_HEADERS, data=_SOME_REQUEST_BODY_STR)
+
+        assert response.text == _A_RESPONSE_BODY
+        assert response.status_code == 474
+
+    @HttpMocker()
+    def test_given_multiple_responses_when_decorate_get_request_then_return_response(self, http_mocker):
+        http_mocker.get(
+            HttpRequest(_A_URL, _SOME_QUERY_PARAMS, _SOME_HEADERS),
+            [HttpResponse(_A_RESPONSE_BODY, 1), HttpResponse(_ANOTHER_RESPONSE_BODY, 2)],
         )
 
         first_response = requests.get(_A_URL, params=_SOME_QUERY_PARAMS, headers=_SOME_HEADERS)
         second_response = requests.get(_A_URL, params=_SOME_QUERY_PARAMS, headers=_SOME_HEADERS)
 
-        assert first_response.text == _A_BODY
+        assert first_response.text == _A_RESPONSE_BODY
+        assert first_response.status_code == 1
+        assert second_response.text == _ANOTHER_RESPONSE_BODY
+        assert second_response.status_code == 2
+
+    @HttpMocker()
+    def test_given_multiple_responses_when_decorate_post_request_then_return_response(self, http_mocker):
+        http_mocker.post(
+            HttpRequest(_A_URL, _SOME_QUERY_PARAMS, _SOME_HEADERS, _SOME_REQUEST_BODY_STR),
+            [HttpResponse(_A_RESPONSE_BODY, 1), HttpResponse(_ANOTHER_RESPONSE_BODY, 2)],
+        )
+
+        first_response = requests.post(_A_URL, params=_SOME_QUERY_PARAMS, headers=_SOME_HEADERS, data=_SOME_REQUEST_BODY_STR)
+        second_response = requests.post(_A_URL, params=_SOME_QUERY_PARAMS, headers=_SOME_HEADERS, data=_SOME_REQUEST_BODY_STR)
+
+        assert first_response.text == _A_RESPONSE_BODY
         assert first_response.status_code == 1
-        assert second_response.text == _ANOTHER_BODY
+        assert second_response.text == _ANOTHER_RESPONSE_BODY
         assert second_response.status_code == 2
 
     @HttpMocker()
     def test_given_more_requests_than_responses_when_decorate_then_raise_error(self, http_mocker):
         http_mocker.get(
             HttpRequest(_A_URL, _SOME_QUERY_PARAMS, _SOME_HEADERS),
-            [HttpResponse(_A_BODY, 1), HttpResponse(_ANOTHER_BODY, 2)],
+            [HttpResponse(_A_RESPONSE_BODY, 1), HttpResponse(_ANOTHER_RESPONSE_BODY, 2)],
         )
 
         last_response = [requests.get(_A_URL, params=_SOME_QUERY_PARAMS, headers=_SOME_HEADERS) for _ in range(10)][-1]
 
-        assert last_response.text == _ANOTHER_BODY
+        assert last_response.text == _ANOTHER_RESPONSE_BODY
         assert last_response.status_code == 2
 
     @HttpMocker()