update versions of dependencies

README.rst

@@ -56,7 +56,7 @@ http://xpaw.readthedocs.io/
 Requirements
 ============
 
-- Python >= 3.5
+- Python >= 3.5.3
 - `aiohttp`_
 - `lxml`_
 - `cssselect`_

requirements-ci.txt

@@ -3,4 +3,4 @@ pytest
 pytest-cov
 coverage
 coveralls
-pytest-aiohttp>=0.1.3
+pytest-aiohttp>=0.3.0

requirements.txt

@@ -1,3 +1,3 @@
-aiohttp>=2.3.2
-lxml>=4.1.0
-cssselect>=1.0.1
+aiohttp>=3.1.3,<4.0
+lxml>=4.1.0,<5.0
+cssselect>=1.0.3<2.0

setup.py

@@ -27,12 +27,12 @@ def run(self):
 
 
 def main():
-    if sys.version_info < (3, 5):
-        raise RuntimeError("Python 3.5+ is required")
+    if sys.version_info < (3, 5, 3):
+        raise RuntimeError("The minimal supported Python version is 3.5.3")
     install_requires = [
-        "aiohttp>=2.3.2",
-        "lxml>=4.1.0",
-        "cssselect>=1.0.1"
+        "aiohttp>=3.1.3,<4.0",
+        "lxml>=4.1.0,<5.0",
+        "cssselect>=1.0.3,<2.0"
     ]
     tests_requires = install_requires + ["pytest", "pytest-aiohttp>=0.1.3"]
     setup(

tests/test_downloader.py

@@ -10,6 +10,7 @@
 from aiohttp.helpers import BasicAuth
 from multidict import MultiDict
 from aiohttp import FormData
+import async_timeout
 
 from xpaw.http import HttpRequest
 from xpaw.downloader import Downloader, DownloaderMiddlewareManager
@@ -117,7 +118,7 @@ async def process(request):
         else:
             auth = BasicAuth.decode(auth_str)
         async with aiohttp.ClientSession(loop=loop) as session:
-            with aiohttp.Timeout(60, loop=loop):
+            with async_timeout.timeout(60, loop=loop):
                 async with session.request("GET", request.raw_path, auth=auth) as resp:
                     body = await resp.read()
                     return web.Response(status=resp.status,

tests/test_queue.py

@@ -3,7 +3,7 @@
 import asyncio
 
 import pytest
-import aiohttp
+import async_timeout
 
 from xpaw.queue import FifoQueue, LifoQueue, PriorityQueue
 
@@ -16,30 +16,30 @@ def __init__(self, loop=None):
 async def test_fifo_queue(loop):
     q = FifoQueue.from_cluster(Cluster(loop=loop))
     with pytest.raises(asyncio.TimeoutError):
-        with aiohttp.Timeout(0.1, loop=loop):
+        with async_timeout.timeout(0.1, loop=loop):
             await q.pop()
     obj_list = [1, 2, 3]
     for o in obj_list:
         await q.push(o)
     for i in range(len(obj_list)):
         assert await q.pop() == obj_list[i]
     with pytest.raises(asyncio.TimeoutError):
-        with aiohttp.Timeout(0.1, loop=loop):
+        with async_timeout.timeout(0.1, loop=loop):
             await q.pop()
 
 
 async def test_lifo_queue(loop):
     q = LifoQueue.from_cluster(Cluster(loop=loop))
     with pytest.raises(asyncio.TimeoutError):
-        with aiohttp.Timeout(0.1, loop=loop):
+        with async_timeout.timeout(0.1, loop=loop):
             await q.pop()
     obj_list = [1, 2, 3]
     for o in obj_list:
         await q.push(o)
     for i in range(len(obj_list)):
         assert await q.pop() == obj_list[len(obj_list) - i - 1]
     with pytest.raises(asyncio.TimeoutError):
-        with aiohttp.Timeout(0.1, loop=loop):
+        with async_timeout.timeout(0.1, loop=loop):
             await q.pop()
 
 
@@ -57,7 +57,7 @@ async def test_priority_queue(loop):
     item3_2 = PriorityQueueItem(3)
     q = PriorityQueue.from_cluster(Cluster(loop=loop))
     with pytest.raises(asyncio.TimeoutError):
-        with aiohttp.Timeout(0.1, loop=loop):
+        with async_timeout.timeout(0.1, loop=loop):
             await q.pop()
     await q.push(item2_1)
     await q.push(item1_1)
@@ -72,5 +72,5 @@ async def test_priority_queue(loop):
     assert await q.pop() is item1_1
     assert await q.pop() is item1_2
     with pytest.raises(asyncio.TimeoutError):
-        with aiohttp.Timeout(0.1, loop=loop):
+        with async_timeout.timeout(0.1, loop=loop):
             await q.pop()

tests/test_selector.py

@@ -65,8 +65,8 @@ def test_encoding_detection(self):
         assert s.xpath("//meta/@charset")[0].text == "gbk"
         content_type = s.xpath("//meta[@http-equiv='Content-Type']/@content")[0].text
         assert content_type == "text/html; charset=gbk"
-        mtype, stype, _, params = parse_mimetype(content_type)
-        assert mtype == "text" and stype == "html" and params.get("charset") == "gbk"
+        mimetype = parse_mimetype(content_type)
+        assert mimetype.type == "text" and mimetype.subtype == "html" and mimetype.parameters.get("charset") == "gbk"
 
     def test_wrong_arguments(self):
         html = b"<html></html>"

xpaw/downloader.py

@@ -6,6 +6,7 @@
 from asyncio import CancelledError
 
 import aiohttp
+import async_timeout
 
 from .middleware import MiddlewareManager
 from .http import HttpRequest, HttpResponse
@@ -34,7 +35,7 @@ async def download(self, request):
                                          cookies=request.cookies,
                                          cookie_jar=self._cookie_jar,
                                          loop=self._loop) as session:
-            with aiohttp.Timeout(timeout, loop=self._loop):
+            with async_timeout.timeout(timeout, loop=self._loop):
                 if isinstance(request.body, dict):
                     data, json = None, request.body
                 else:

xpaw/downloadermws.py

@@ -7,6 +7,7 @@
 from asyncio import CancelledError
 
 import aiohttp
+import async_timeout
 from yarl import URL
 
 from .errors import IgnoreRequest, NetworkError, NotEnabled
@@ -167,7 +168,7 @@ def _append_proxy(self, p):
     async def _update_proxy_list(self):
         try:
             async with aiohttp.ClientSession(loop=self._loop) as session:
-                with aiohttp.Timeout(self.TIMEOUT, loop=self._loop):
+                with async_timeout.timeout(self.TIMEOUT, loop=self._loop):
                     async with session.get(self._agent_addr) as resp:
                         body = await resp.read()
                         proxy_list = json.loads(body.decode(encoding="utf-8"))