update client error

docs/errors.rst

@@ -13,9 +13,13 @@ Built-in Errors
 
     在加载组件的时候抛出的异常，表示该组件未启用。
 
-.. class:: xpaw.errors.NetworkError
+.. class:: xpaw.errors.ClientError
 
-    在downloader下载过程中抛出的异常，例如超时、服务器无法访问等。
+    在downloader发起请求过程中抛出的异常，例如服务器无法访问等。
+
+.. class:: xpaw.errors.TimeoutError
+
+    请求超时。
 
 .. class:: xpaw.errors.IgnoreRequest
 

tests/test_downloadermws.py

@@ -8,7 +8,7 @@
 from xpaw.config import Config
 from xpaw.http import HttpRequest, HttpResponse
 from xpaw.downloadermws import *
-from xpaw.errors import IgnoreRequest, NetworkError, NotEnabled
+from xpaw.errors import IgnoreRequest, ClientError, NotEnabled, TimeoutError
 from xpaw.version import __version__
 from xpaw.downloader import Downloader
 from xpaw.eventbus import EventBus
@@ -124,9 +124,9 @@ def test_handle_error(self):
         req = HttpRequest("http://example.com")
         err = ValueError()
         assert mw.handle_error(req, err) is None
-        err2 = NetworkError()
-        retry_req2 = mw.handle_error(req, err2)
-        assert isinstance(retry_req2, HttpRequest) and str(retry_req2.url) == str(req.url)
+        for err in [ClientError(), TimeoutError()]:
+            retry_req = mw.handle_error(req, err)
+            assert isinstance(retry_req, HttpRequest) and str(retry_req.url) == str(req.url)
 
     def test_retry(self):
         max_retry_times = 2

xpaw/downloader.py

@@ -14,7 +14,7 @@
 
 from .middleware import MiddlewareManager
 from .http import HttpRequest, HttpResponse
-from .errors import NetworkError
+from .errors import ClientError, TimeoutError
 
 log = logging.getLogger(__name__)
 
@@ -101,9 +101,12 @@ async def download(self, downloader, request):
                     response = await downloader.download(request)
                 except CancelledError:
                     raise
+                except asyncio.TimeoutError:
+                    log.debug('Request timeout: %s', request)
+                    raise TimeoutError('Request timeout')
                 except Exception as e:
-                    log.debug("Network error: %s", e)
-                    raise NetworkError(e)
+                    log.debug("Client error: %s %s", request, e)
+                    raise ClientError(e)
                 else:
                     res = response
         except CancelledError:

xpaw/downloadermws.py

@@ -9,15 +9,15 @@
 import aiohttp
 from yarl import URL
 
-from .errors import IgnoreRequest, NetworkError, NotEnabled
+from .errors import IgnoreRequest, ClientError, NotEnabled, TimeoutError
 from .version import __version__
 from . import utils
 
 log = logging.getLogger(__name__)
 
 
 class RetryMiddleware:
-    RETRY_ERRORS = (NetworkError,)
+    RETRY_ERRORS = (ClientError, TimeoutError)
 
     def __init__(self, max_retry_times, retry_http_status=None):
         self._max_retry_times = max_retry_times
@@ -70,14 +70,13 @@ def handle_error(self, request, error):
     def retry(self, request, reason):
         retry_times = request.meta.get("retry_times", 0) + 1
         if retry_times <= self._max_retry_times:
-            log.debug("We will retry the request %s because of %s", request, reason)
+            log.debug('Retry %s (failed %s times): %s', request, retry_times, reason)
             retry_req = request.copy()
             retry_req.meta["retry_times"] = retry_times
             retry_req.dont_filter = True
             return retry_req
         else:
-            log.info("The request %s has been retried %s times,"
-                     " and it will be aborted", request, self._max_retry_times)
+            log.info('Abort %s (failed %s times): %s', request, retry_times, reason)
             raise IgnoreRequest(reason)
 
 

xpaw/errors.py

@@ -1,15 +1,19 @@
 # coding=utf-8
 
+import asyncio
+
+TimeoutError = asyncio.TimeoutError
+
 
 class NotEnabled(Exception):
     """
     Not enabled.
     """
 
 
-class NetworkError(Exception):
+class ClientError(Exception):
     """
-    Network error.
+    Downloader client error.
     """