diff --git a/README.rst b/README.rst index 94ed4b5..c2031db 100644 --- a/README.rst +++ b/README.rst @@ -56,7 +56,7 @@ http://xpaw.readthedocs.io/ Requirements ============ -- Python >= 3.5 +- Python >= 3.5.3 - `aiohttp`_ - `lxml`_ - `cssselect`_ diff --git a/requirements-ci.txt b/requirements-ci.txt index 13f1a66..ec23de0 100644 --- a/requirements-ci.txt +++ b/requirements-ci.txt @@ -3,4 +3,4 @@ pytest pytest-cov coverage coveralls -pytest-aiohttp>=0.1.3 +pytest-aiohttp>=0.3.0 diff --git a/requirements.txt b/requirements.txt index e32e9c6..920b3fd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ -aiohttp>=2.3.2 -lxml>=4.1.0 -cssselect>=1.0.1 +aiohttp>=3.1.3,<4.0 +lxml>=4.1.0,<5.0 +cssselect>=1.0.3<2.0 diff --git a/setup.py b/setup.py index a107866..553ec28 100644 --- a/setup.py +++ b/setup.py @@ -27,12 +27,12 @@ def run(self): def main(): - if sys.version_info < (3, 5): - raise RuntimeError("Python 3.5+ is required") + if sys.version_info < (3, 5, 3): + raise RuntimeError("The minimal supported Python version is 3.5.3") install_requires = [ - "aiohttp>=2.3.2", - "lxml>=4.1.0", - "cssselect>=1.0.1" + "aiohttp>=3.1.3,<4.0", + "lxml>=4.1.0,<5.0", + "cssselect>=1.0.3,<2.0" ] tests_requires = install_requires + ["pytest", "pytest-aiohttp>=0.1.3"] setup( diff --git a/tests/test_downloader.py b/tests/test_downloader.py index cdaac47..e1ee9ef 100644 --- a/tests/test_downloader.py +++ b/tests/test_downloader.py @@ -10,6 +10,7 @@ from aiohttp.helpers import BasicAuth from multidict import MultiDict from aiohttp import FormData +import async_timeout from xpaw.http import HttpRequest from xpaw.downloader import Downloader, DownloaderMiddlewareManager @@ -117,7 +118,7 @@ async def process(request): else: auth = BasicAuth.decode(auth_str) async with aiohttp.ClientSession(loop=loop) as session: - with aiohttp.Timeout(60, loop=loop): + with async_timeout.timeout(60, loop=loop): async with session.request("GET", request.raw_path, auth=auth) as resp: body = await resp.read() return web.Response(status=resp.status, diff --git a/tests/test_queue.py b/tests/test_queue.py index c8f1762..fc5de3f 100644 --- a/tests/test_queue.py +++ b/tests/test_queue.py @@ -3,7 +3,7 @@ import asyncio import pytest -import aiohttp +import async_timeout from xpaw.queue import FifoQueue, LifoQueue, PriorityQueue @@ -16,7 +16,7 @@ def __init__(self, loop=None): async def test_fifo_queue(loop): q = FifoQueue.from_cluster(Cluster(loop=loop)) with pytest.raises(asyncio.TimeoutError): - with aiohttp.Timeout(0.1, loop=loop): + with async_timeout.timeout(0.1, loop=loop): await q.pop() obj_list = [1, 2, 3] for o in obj_list: @@ -24,14 +24,14 @@ async def test_fifo_queue(loop): for i in range(len(obj_list)): assert await q.pop() == obj_list[i] with pytest.raises(asyncio.TimeoutError): - with aiohttp.Timeout(0.1, loop=loop): + with async_timeout.timeout(0.1, loop=loop): await q.pop() async def test_lifo_queue(loop): q = LifoQueue.from_cluster(Cluster(loop=loop)) with pytest.raises(asyncio.TimeoutError): - with aiohttp.Timeout(0.1, loop=loop): + with async_timeout.timeout(0.1, loop=loop): await q.pop() obj_list = [1, 2, 3] for o in obj_list: @@ -39,7 +39,7 @@ async def test_lifo_queue(loop): for i in range(len(obj_list)): assert await q.pop() == obj_list[len(obj_list) - i - 1] with pytest.raises(asyncio.TimeoutError): - with aiohttp.Timeout(0.1, loop=loop): + with async_timeout.timeout(0.1, loop=loop): await q.pop() @@ -57,7 +57,7 @@ async def test_priority_queue(loop): item3_2 = PriorityQueueItem(3) q = PriorityQueue.from_cluster(Cluster(loop=loop)) with pytest.raises(asyncio.TimeoutError): - with aiohttp.Timeout(0.1, loop=loop): + with async_timeout.timeout(0.1, loop=loop): await q.pop() await q.push(item2_1) await q.push(item1_1) @@ -72,5 +72,5 @@ async def test_priority_queue(loop): assert await q.pop() is item1_1 assert await q.pop() is item1_2 with pytest.raises(asyncio.TimeoutError): - with aiohttp.Timeout(0.1, loop=loop): + with async_timeout.timeout(0.1, loop=loop): await q.pop() diff --git a/tests/test_selector.py b/tests/test_selector.py index 8fe4316..72aebe1 100644 --- a/tests/test_selector.py +++ b/tests/test_selector.py @@ -65,8 +65,8 @@ def test_encoding_detection(self): assert s.xpath("//meta/@charset")[0].text == "gbk" content_type = s.xpath("//meta[@http-equiv='Content-Type']/@content")[0].text assert content_type == "text/html; charset=gbk" - mtype, stype, _, params = parse_mimetype(content_type) - assert mtype == "text" and stype == "html" and params.get("charset") == "gbk" + mimetype = parse_mimetype(content_type) + assert mimetype.type == "text" and mimetype.subtype == "html" and mimetype.parameters.get("charset") == "gbk" def test_wrong_arguments(self): html = b"" diff --git a/xpaw/downloader.py b/xpaw/downloader.py index 72e21bb..e29f094 100644 --- a/xpaw/downloader.py +++ b/xpaw/downloader.py @@ -6,6 +6,7 @@ from asyncio import CancelledError import aiohttp +import async_timeout from .middleware import MiddlewareManager from .http import HttpRequest, HttpResponse @@ -34,7 +35,7 @@ async def download(self, request): cookies=request.cookies, cookie_jar=self._cookie_jar, loop=self._loop) as session: - with aiohttp.Timeout(timeout, loop=self._loop): + with async_timeout.timeout(timeout, loop=self._loop): if isinstance(request.body, dict): data, json = None, request.body else: diff --git a/xpaw/downloadermws.py b/xpaw/downloadermws.py index dc47e2c..0ad2c71 100644 --- a/xpaw/downloadermws.py +++ b/xpaw/downloadermws.py @@ -7,6 +7,7 @@ from asyncio import CancelledError import aiohttp +import async_timeout from yarl import URL from .errors import IgnoreRequest, NetworkError, NotEnabled @@ -167,7 +168,7 @@ def _append_proxy(self, p): async def _update_proxy_list(self): try: async with aiohttp.ClientSession(loop=self._loop) as session: - with aiohttp.Timeout(self.TIMEOUT, loop=self._loop): + with async_timeout.timeout(self.TIMEOUT, loop=self._loop): async with session.get(self._agent_addr) as resp: body = await resp.read() proxy_list = json.loads(body.decode(encoding="utf-8"))