diff --git a/src/apify_client/_http_client.py b/src/apify_client/_http_client.py index d847144d..46898f6f 100644 --- a/src/apify_client/_http_client.py +++ b/src/apify_client/_http_client.py @@ -11,12 +11,12 @@ from urllib.parse import urlencode import impit -from apify_shared.utils import ignore_docs, is_content_type_json, is_content_type_text, is_content_type_xml +from apify_shared.utils import ignore_docs from apify_client._logging import log_context, logger_name from apify_client._statistics import Statistics from apify_client._utils import is_retryable_error, retry_with_exp_backoff, retry_with_exp_backoff_async -from apify_client.errors import ApifyApiError, InvalidResponseBodyError +from apify_client.errors import ApifyApiError if TYPE_CHECKING: from collections.abc import Callable @@ -65,25 +65,6 @@ def __init__( self.stats = stats or Statistics() - @staticmethod - def _maybe_parse_response(response: impit.Response) -> Any: - if response.status_code == HTTPStatus.NO_CONTENT: - return None - - content_type = '' - if 'content-type' in response.headers: - content_type = response.headers['content-type'].split(';')[0].strip() - - try: - if is_content_type_json(content_type): - return jsonlib.loads(response.text) - elif is_content_type_xml(content_type) or is_content_type_text(content_type): # noqa: RET505 - return response.text - else: - return response.content - except ValueError as err: - raise InvalidResponseBodyError(response) from err - @staticmethod def _parse_params(params: dict | None) -> dict | None: if params is None: @@ -159,7 +140,6 @@ def call( data: Any = None, json: JSONSerializable | None = None, stream: bool | None = None, - parse_response: bool | None = True, timeout_secs: int | None = None, ) -> impit.Response: log_context.method.set(method) @@ -167,9 +147,6 @@ def call( self.stats.calls += 1 - if stream and parse_response: - raise ValueError('Cannot stream response and parse it at the same time!') - headers, params, content = self._prepare_request_call(headers, params, data, json) impit_client = self.impit_client @@ -198,11 +175,6 @@ def _make_request(stop_retrying: Callable, attempt: int) -> impit.Response: # If response status is < 300, the request was successful, and we can return the result if response.status_code < 300: # noqa: PLR2004 logger.debug('Request successful', extra={'status_code': response.status_code}) - if not stream: - _maybe_parsed_body = ( - self._maybe_parse_response(response) if parse_response else response.content - ) - setattr(response, '_maybe_parsed_body', _maybe_parsed_body) # noqa: B010 return response @@ -247,7 +219,6 @@ async def call( data: Any = None, json: JSONSerializable | None = None, stream: bool | None = None, - parse_response: bool | None = True, timeout_secs: int | None = None, ) -> impit.Response: log_context.method.set(method) @@ -255,9 +226,6 @@ async def call( self.stats.calls += 1 - if stream and parse_response: - raise ValueError('Cannot stream response and parse it at the same time!') - headers, params, content = self._prepare_request_call(headers, params, data, json) impit_async_client = self.impit_async_client @@ -283,11 +251,6 @@ async def _make_request(stop_retrying: Callable, attempt: int) -> impit.Response # If response status is < 300, the request was successful, and we can return the result if response.status_code < 300: # noqa: PLR2004 logger.debug('Request successful', extra={'status_code': response.status_code}) - if not stream: - _maybe_parsed_body = ( - self._maybe_parse_response(response) if parse_response else response.content - ) - setattr(response, '_maybe_parsed_body', _maybe_parsed_body) # noqa: B010 return response diff --git a/src/apify_client/_utils.py b/src/apify_client/_utils.py index 9c422815..4b3057cf 100644 --- a/src/apify_client/_utils.py +++ b/src/apify_client/_utils.py @@ -2,7 +2,7 @@ import asyncio import base64 -import json +import json as jsonlib import random import time from collections.abc import Callable @@ -10,13 +10,21 @@ from typing import TYPE_CHECKING, Any, TypeVar, cast import impit -from apify_shared.utils import is_file_or_bytes, maybe_extract_enum_member_value +from apify_shared.utils import ( + is_content_type_json, + is_content_type_text, + is_content_type_xml, + is_file_or_bytes, + maybe_extract_enum_member_value, +) from apify_client.errors import InvalidResponseBodyError if TYPE_CHECKING: from collections.abc import Awaitable + from impit import Response + from apify_client.errors import ApifyApiError PARSE_DATE_FIELDS_MAX_DEPTH = 3 @@ -136,7 +144,7 @@ def encode_webhook_list_to_base64(webhooks: list[dict]) -> str: webhook_representation['headersTemplate'] = webhook['headers_template'] data.append(webhook_representation) - return base64.b64encode(json.dumps(data).encode('utf-8')).decode('ascii') + return base64.b64encode(jsonlib.dumps(data).encode('utf-8')).decode('ascii') def encode_key_value_store_record_value(value: Any, content_type: str | None = None) -> tuple[Any, str]: @@ -149,11 +157,32 @@ def encode_key_value_store_record_value(value: Any, content_type: str | None = N content_type = 'application/json; charset=utf-8' if 'application/json' in content_type and not is_file_or_bytes(value) and not isinstance(value, str): - value = json.dumps(value, ensure_ascii=False, indent=2, allow_nan=False, default=str).encode('utf-8') + value = jsonlib.dumps(value, ensure_ascii=False, indent=2, allow_nan=False, default=str).encode('utf-8') return (value, content_type) +def maybe_parse_response(response: Response) -> Any: + if response.status_code == HTTPStatus.NO_CONTENT: + return None + + content_type = '' + if 'content-type' in response.headers: + content_type = response.headers['content-type'].split(';')[0].strip() + + try: + if is_content_type_json(content_type): + response_data = jsonlib.loads(response.text) + elif is_content_type_xml(content_type) or is_content_type_text(content_type): + response_data = response.text + else: + response_data = response.content + except ValueError as err: + raise InvalidResponseBodyError(response) from err + else: + return response_data + + def is_retryable_error(exc: Exception) -> bool: """Check if the given error is retryable.""" return isinstance( diff --git a/src/apify_client/clients/resource_clients/dataset.py b/src/apify_client/clients/resource_clients/dataset.py index c109aa0d..80dfc65a 100644 --- a/src/apify_client/clients/resource_clients/dataset.py +++ b/src/apify_client/clients/resource_clients/dataset.py @@ -413,7 +413,6 @@ def get_items_as_bytes( url=self._url('items'), method='GET', params=request_params, - parse_response=False, ) return response.content @@ -507,7 +506,6 @@ def stream_items( method='GET', params=request_params, stream=True, - parse_response=False, ) yield response finally: @@ -862,7 +860,6 @@ async def get_items_as_bytes( url=self._url('items'), method='GET', params=request_params, - parse_response=False, ) return response.content @@ -956,7 +953,6 @@ async def stream_items( method='GET', params=request_params, stream=True, - parse_response=False, ) yield response finally: diff --git a/src/apify_client/clients/resource_clients/key_value_store.py b/src/apify_client/clients/resource_clients/key_value_store.py index d65f0ef6..c199c39b 100644 --- a/src/apify_client/clients/resource_clients/key_value_store.py +++ b/src/apify_client/clients/resource_clients/key_value_store.py @@ -7,7 +7,12 @@ from apify_shared.utils import filter_out_none_values_recursively, ignore_docs, parse_date_fields -from apify_client._utils import catch_not_found_or_throw, encode_key_value_store_record_value, pluck_data +from apify_client._utils import ( + catch_not_found_or_throw, + encode_key_value_store_record_value, + maybe_parse_response, + pluck_data, +) from apify_client.clients.base import ResourceClient, ResourceClientAsync from apify_client.errors import ApifyApiError @@ -121,7 +126,7 @@ def get_record(self, key: str) -> dict | None: return { 'key': key, - 'value': response._maybe_parsed_body, # type: ignore[attr-defined] # noqa: SLF001 + 'value': maybe_parse_response(response), 'content_type': response.headers['content-type'], } @@ -171,7 +176,6 @@ def get_record_as_bytes(self, key: str) -> dict | None: url=self._url(f'records/{key}'), method='GET', params=self._params(), - parse_response=False, ) return { @@ -203,7 +207,6 @@ def stream_record(self, key: str) -> Iterator[dict | None]: url=self._url(f'records/{key}'), method='GET', params=self._params(), - parse_response=False, stream=True, ) @@ -364,7 +367,7 @@ async def get_record(self, key: str) -> dict | None: return { 'key': key, - 'value': response._maybe_parsed_body, # type: ignore[attr-defined] # noqa: SLF001 + 'value': maybe_parse_response(response), 'content_type': response.headers['content-type'], } @@ -414,7 +417,6 @@ async def get_record_as_bytes(self, key: str) -> dict | None: url=self._url(f'records/{key}'), method='GET', params=self._params(), - parse_response=False, ) return { @@ -446,7 +448,6 @@ async def stream_record(self, key: str) -> AsyncIterator[dict | None]: url=self._url(f'records/{key}'), method='GET', params=self._params(), - parse_response=False, stream=True, ) diff --git a/src/apify_client/clients/resource_clients/log.py b/src/apify_client/clients/resource_clients/log.py index 58e8ab39..d5ca86c9 100644 --- a/src/apify_client/clients/resource_clients/log.py +++ b/src/apify_client/clients/resource_clients/log.py @@ -76,7 +76,6 @@ def get_as_bytes(self, *, raw: bool = False) -> bytes | None: url=self.url, method='GET', params=self._params(raw=raw), - parse_response=False, ) return response.content # noqa: TRY300 @@ -105,7 +104,6 @@ def stream(self, *, raw: bool = False) -> Iterator[impit.Response | None]: method='GET', params=self._params(stream=True, raw=raw), stream=True, - parse_response=False, ) yield response @@ -166,7 +164,6 @@ async def get_as_bytes(self, *, raw: bool = False) -> bytes | None: url=self.url, method='GET', params=self._params(raw=raw), - parse_response=False, ) return response.content # noqa: TRY300 @@ -195,7 +192,6 @@ async def stream(self, *, raw: bool = False) -> AsyncIterator[impit.Response | N method='GET', params=self._params(stream=True, raw=raw), stream=True, - parse_response=False, ) yield response diff --git a/tests/unit/test_client_errors.py b/tests/unit/test_client_errors.py index 8b4a152e..19833e7b 100644 --- a/tests/unit/test_client_errors.py +++ b/tests/unit/test_client_errors.py @@ -92,7 +92,7 @@ def test_client_apify_api_error_streamed(httpserver: HTTPServer) -> None: httpserver.expect_request('/stream_error').respond_with_handler(streaming_handler) with pytest.raises(ApifyApiError) as e: - client.call(method='GET', url=httpserver.url_for('/stream_error'), stream=True, parse_response=False) + client.call(method='GET', url=httpserver.url_for('/stream_error'), stream=True) assert e.value.message == error['error']['message'] assert e.value.type == error['error']['type'] @@ -108,7 +108,7 @@ async def test_async_client_apify_api_error_streamed(httpserver: HTTPServer) -> httpserver.expect_request('/stream_error').respond_with_handler(streaming_handler) with pytest.raises(ApifyApiError) as e: - await client.call(method='GET', url=httpserver.url_for('/stream_error'), stream=True, parse_response=False) + await client.call(method='GET', url=httpserver.url_for('/stream_error'), stream=True) assert e.value.message == error['error']['message'] assert e.value.type == error['error']['type']