diff --git a/src/apify/scrapy/middlewares/apify_proxy.py b/src/apify/scrapy/middlewares/apify_proxy.py index 3e64508e..94069d57 100644 --- a/src/apify/scrapy/middlewares/apify_proxy.py +++ b/src/apify/scrapy/middlewares/apify_proxy.py @@ -93,7 +93,7 @@ def process_exception( request: Request, exception: Exception, spider: Spider, - ) -> None | Request: + ) -> None: """Process an exception that occurs during request processing. Args: @@ -102,8 +102,9 @@ def process_exception( spider: Scrapy Spider object. Returns: - If a TunnelError occurs, return the request object to halt its processing in the middleware pipeline. - Return None otherwise to allow the continuation of request processing. + Returning None, meaning Scrapy will continue processing this exception, executing any other + process_exception() methods of installed middleware, until no middleware is left and the default + exception handling kicks in. """ Actor.log.debug( f'ApifyHttpProxyMiddleware.process_exception: request={request}, exception={exception}, spider={spider}', @@ -114,9 +115,6 @@ def process_exception( f'ApifyHttpProxyMiddleware: TunnelError occurred for request="{request}", ' 'reason="{exception}", skipping...' ) - return request - - return None async def _get_new_proxy_url(self: ApifyHttpProxyMiddleware) -> ParseResult: """Get a new proxy URL. diff --git a/tests/unit/scrapy/middlewares/test_apify_proxy.py b/tests/unit/scrapy/middlewares/test_apify_proxy.py index 6bddcb74..9d089acc 100644 --- a/tests/unit/scrapy/middlewares/test_apify_proxy.py +++ b/tests/unit/scrapy/middlewares/test_apify_proxy.py @@ -129,24 +129,14 @@ async def mock_get_new_proxy_url() -> ParseResult: @pytest.mark.parametrize( - ('exception', 'none_returned_values_is_expected'), - [ - (TunnelError(), False), - (ValueError(), True), - ], + 'exception', + [TunnelError(), ValueError()], ) def test__process_exception( middleware: ApifyHttpProxyMiddleware, spider: DummySpider, dummy_request: Request, exception: Exception, - *, - none_returned_values_is_expected: bool, ) -> None: - returned_value = middleware.process_exception(dummy_request, exception, spider) - - if none_returned_values_is_expected: - assert returned_value is None - - else: - assert returned_value == dummy_request + returned_value = middleware.process_exception(dummy_request, exception, spider) # type: ignore + assert returned_value is None