Merge pull request #425 from aio-libs/update-aiohttp

Update aiohttp

CHANGES.txt

@@ -1,8 +1,10 @@
 Changes
 -------
-0.4.6a0 (XXXX-XX-XX)
+0.5.0 (XXXX-XX-XX)
 ^^^^^^^^^^^^^^^^^^
-* fix #190
+* Fix error botocore error checking #190
+* Update supported botocore requirement to: >=1.7.28, <=1.7.40
+* Bump aiohttp requirement to support compressed responses correctly #298
 
 0.4.5 (2017-09-05)
 ^^^^^^^^^^^^^^^^^^

aiobotocore/endpoint.py

@@ -8,7 +8,6 @@
 import aiohttp.http_exceptions
 from aiohttp.client_proto import ResponseHandler
 from aiohttp.helpers import CeilTimeout
-from aiohttp import hdrs
 from aiohttp.client import URL
 from aiohttp.client_reqrep import ClientResponse
 from botocore.endpoint import EndpointCreator, Endpoint, DEFAULT_TIMEOUT, \
@@ -170,45 +169,6 @@ def read(self):
             return resp_msg, stream_reader
 
 
-def _aiohttp_do_redirect(session, method, url, headers, data, resp):
-    # This is the redirect code from aiohttp, remove once
-    # https://github.com/aio-libs/aiobotocore/issues/267 is supported
-
-    # For 301 and 302, mimic IE, now changed in RFC
-    # https://github.com/kennethreitz/requests/pull/269
-
-    if (resp.status == 303 and
-            resp.method != hdrs.METH_HEAD) \
-            or (resp.status in (301, 302) and
-                resp.method == hdrs.METH_POST):
-        method = hdrs.METH_GET
-        data = None
-        if headers.get(hdrs.CONTENT_LENGTH):
-            headers.pop(hdrs.CONTENT_LENGTH)
-
-    r_url = (resp.headers.get(hdrs.LOCATION) or
-             resp.headers.get(hdrs.URI))
-    if r_url is None:
-        return None
-
-    r_url = URL(
-        r_url, encoded=not session.requote_redirect_url)
-
-    scheme = r_url.scheme
-    if scheme not in ('http', 'https', ''):
-        resp.close()
-        raise ValueError(
-            'Can redirect only to http or https')
-    elif not scheme:
-        r_url = url.join(r_url)
-
-    url = r_url
-    params = None
-    resp.release()
-
-    return method, url, headers, params, data
-
-
 class AioEndpoint(Endpoint):
     def __init__(self, host,
                  endpoint_prefix, event_emitter, proxies=None, verify=True,
@@ -259,23 +219,11 @@ def __init__(self, host,
             conn_timeout=self._conn_timeout,
             skip_auto_headers={'CONTENT-TYPE'},
             response_class=ClientResponseProxy,
-            loop=self._loop)
+            loop=self._loop,
+            auto_decompress=False)
 
     @asyncio.coroutine
     def _request(self, method, url, headers, data):
-        # Note: When using aiobotocore with dynamodb, requests fail on crc32
-        # checksum computation as soon as the response data reaches ~5KB.
-        # When AWS response is gzip compressed:
-        # 1. aiohttp is automatically decompressing the data
-        # (http://aiohttp.readthedocs.io/en/stable/client.html#binary-response-content)
-        # 2. botocore computes crc32 on the uncompressed data bytes and fails
-        # cause crc32 has been computed on the compressed data
-        # The following line forces aws not to use gzip compression,
-        # if there is a way to configure aiohttp not to perform decompression,
-        # we can remove the following line and take advantage of
-        # aws gzip compression.
-        # See: https://github.com/aio-libs/aiohttp/issues/1992
-        headers['Accept-Encoding'] = 'identity'
         headers_ = MultiDict(
             (z[0], text_(z[1], encoding='utf-8')) for z in headers.items())
 
@@ -286,27 +234,11 @@ def _request(self, method, url, headers, data):
             data = _IOBaseWrapper(data)
 
         url = URL(url, encoded=True)
-
-        # See https://github.com/aio-libs/aiobotocore/issues/267 for details
-        for i in range(MAX_REDIRECTS):
-            resp = yield from self._aio_session.request(method, url=url,
-                                                        headers=headers_,
-                                                        data=data,
-                                                        proxy=proxy,
-                                                        timeout=None,
-                                                        allow_redirects=False)
-
-            if resp.status in {301, 302, 303, 307}:
-                redir_arr = _aiohttp_do_redirect(self._aio_session, method,
-                                                 url, headers, data, resp)
-
-                if redir_arr is None:
-                    break
-
-                method, url, headers, params, data = redir_arr
-            else:
-                break
-
+        resp = yield from self._aio_session.request(method, url=url,
+                                                    headers=headers_,
+                                                    data=data,
+                                                    proxy=proxy,
+                                                    timeout=None)
         return resp
 
     @asyncio.coroutine

requirements-dev.txt

@@ -6,7 +6,7 @@ moto==0.4.31
 pytest-cov==2.5.1
 pytest==3.2.3
 sphinx==1.6.5
-aiohttp==2.2.5
+aiohttp==2.3.2
 botocore==1.7.43
 multidict==3.3.2
 wrapt==1.10.11

setup.py

@@ -9,7 +9,7 @@
 # If adding requirements make sure to also add to requirements-dev.txt
 install_requires = [
     'botocore>=1.7.28, <=1.7.40',
-    'aiohttp>=2.0.4, <=2.3.0',
+    'aiohttp>=2.3.0, <3.0.0',
     'multidict>=2.1.4',
     'wrapt>=1.10.10',
     'packaging>=16.8',

tests/test_patches.py

@@ -27,20 +27,19 @@
 #    github.com/aio-libs/aiobotocore/pull/248
 _READ_TIMEOUT_DIGESTS = {
     # for our replacement of _factory
-    TCPConnector: {'a153acda1bfc891d01a56597f77b33afbd786d3d'},
+    TCPConnector: {'1e6a3c14ce703781253da5cfdc7847b4ae898234'},
 
     # for its inheritance to DataQueue
-    ResponseHandler: {'1cfdb033cb4b4d586bbebf49bed3c2930c026809'},
+    ResponseHandler: {'5f11c28d0075e36dfec4705891f1c90248202ba4'},
 
     # for our replacement of read()
     DataQueue: {'8ad4d5df1d016547daea6389707bc656630582e5'},
 
     # for our patch of _wait
-    StreamReader: {'dc5a5a33e86cedf2d2c8413d951d2274a79303f4'},
+    StreamReader: {'c0a9a31a8c3e550de5985ab642028983f709b37b'},
 
     # for digging into _protocol ( 2.1.x, 2.2.x )
-    ClientResponse: {'d1e0c16dea4fe3426caa1e9b0dc9f5f1992d838e',
-                     'bc374038ac3bfd7cc13dadb6aebbf0f67ebb7620'},
+    ClientResponse: {'1dc0008e88b3f5ec2f59f6f5f03fae601f4a011d'},
 }
 
 # These are guards to our main patches