aiobotocore bump + fixes (#744)

.flake8

@@ -0,0 +1,3 @@
+[flake8]
+# Bump by 6 to account for "await " as compared to botocore
+max-line-length = 86

CHANGES.txt

@@ -1,6 +1,13 @@
 Changes
 -------
 
+0.11.0 (2019-11-12)
+^^^^^^^^^^^^^^^^^^^
+* replace CaseInsensitiveDict with urllib3 equivalent #744
+  (thanks to inspiration from @craigmccarter and @kevchentw)
+* bump botocore to 1.13.14
+* fix for mismatched botocore method replacements
+
 0.10.4 (2019-10-24)
 ^^^^^^^^^^^^^^^^^^^
 * Make AioBaseClient.close method async #724 (thanks @bsitruk)

aiobotocore/__init__.py

@@ -1,4 +1,4 @@
 from .session import get_session, AioSession
 
 __all__ = ['get_session', 'AioSession']
-__version__ = '0.10.4'
+__version__ = '0.11.0'

aiobotocore/_endpoint_helpers.py

@@ -0,0 +1,99 @@
+import aiohttp.http_exceptions
+from aiohttp.client_reqrep import ClientResponse
+import asyncio
+import botocore.retryhandler
+import wrapt
+
+
+# Monkey patching: We need to insert the aiohttp exception equivalents
+# The only other way to do this would be to have another config file :(
+_aiohttp_retryable_exceptions = [
+    aiohttp.ClientConnectionError,
+    aiohttp.ClientPayloadError,
+    aiohttp.ServerDisconnectedError,
+    aiohttp.http_exceptions.HttpProcessingError,
+    asyncio.TimeoutError,
+]
+
+botocore.retryhandler.EXCEPTION_MAP['GENERAL_CONNECTION_ERROR'].extend(
+    _aiohttp_retryable_exceptions
+)
+
+
+def _text(s, encoding='utf-8', errors='strict'):
+    if isinstance(s, bytes):
+        return s.decode(encoding, errors)
+    return s  # pragma: no cover
+
+
+# Unfortunately aiohttp changed the behavior of streams:
+#   github.com/aio-libs/aiohttp/issues/1907
+# We need this wrapper until we have a final resolution
+class _IOBaseWrapper(wrapt.ObjectProxy):
+    def close(self):
+        # this stream should not be closed by aiohttp, like 1.x
+        pass
+
+
+# This is similar to botocore.response.StreamingBody
+class ClientResponseContentProxy(wrapt.ObjectProxy):
+    """Proxy object for content stream of http response.  This is here in case
+    you want to pass around the "Body" of the response without closing the
+    response itself."""
+
+    def __init__(self, response):
+        super().__init__(response.__wrapped__.content)
+        self._self_response = response
+
+    # Note: we don't have a __del__ method as the ClientResponse has a __del__
+    # which will warn the user if they didn't close/release the response
+    # explicitly.  A release here would mean reading all the unread data
+    # (which could be very large), and a close would mean being unable to re-
+    # use the connection, so the user MUST chose.  Default is to warn + close
+    async def __aenter__(self):
+        await self._self_response.__aenter__()
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        return await self._self_response.__aexit__(exc_type, exc_val, exc_tb)
+
+    @property
+    def url(self):
+        return self._self_response.url
+
+    def close(self):
+        self._self_response.close()
+
+
+class ClientResponseProxy(wrapt.ObjectProxy):
+    """Proxy object for http response useful for porting from
+    botocore underlying http library."""
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(ClientResponse(*args, **kwargs))
+
+        # this matches ClientResponse._body
+        self._self_body = None
+
+    @property
+    def status_code(self):
+        return self.status
+
+    @status_code.setter
+    def status_code(self, value):
+        # botocore tries to set this, see:
+        # https://github.com/aio-libs/aiobotocore/issues/190
+        # Luckily status is an attribute we can set
+        self.status = value
+
+    @property
+    def content(self):
+        return self._self_body
+
+    @property
+    def raw(self):
+        return ClientResponseContentProxy(self)
+
+    async def read(self):
+        self._self_body = await self.__wrapped__.read()
+        return self._self_body

aiobotocore/args.py

@@ -10,10 +10,8 @@
 
 
 class AioClientArgsCreator(botocore.args.ClientArgsCreator):
-    def __init__(self, event_emitter, user_agent, response_parser_factory,
-                 loader, exceptions_factory, loop=None):
-        super().__init__(event_emitter, user_agent,
-                         response_parser_factory, loader, exceptions_factory)
+    def __init__(self, *args, loop=None, **kwargs):
+        super().__init__(*args, **kwargs)
         self._loop = loop or asyncio.get_event_loop()
 
     # NOTE: we override this so we can pull out the custom AioConfig params and
@@ -32,6 +30,7 @@ def get_client_args(self, service_model, region_name, is_secure,
         config_kwargs = final_args['config_kwargs']
         s3_config = final_args['s3_config']
         partition = endpoint_config['metadata'].get('partition', None)
+        socket_options = final_args['socket_options']
 
         signing_region = endpoint_config['signing_region']
         endpoint_region_name = endpoint_config['region_name']
@@ -56,7 +55,7 @@ def get_client_args(self, service_model, region_name, is_secure,
             connector_args = None
 
         new_config = AioConfig(connector_args, **config_kwargs)
-        endpoint_creator = AioEndpointCreator(event_emitter, self._loop)
+        endpoint_creator = AioEndpointCreator(event_emitter, loop=self._loop)
 
         endpoint = endpoint_creator.create_endpoint(
             service_model, region_name=endpoint_region_name,
@@ -65,6 +64,8 @@ def get_client_args(self, service_model, region_name, is_secure,
             max_pool_connections=new_config.max_pool_connections,
             proxies=new_config.proxies,
             timeout=(new_config.connect_timeout, new_config.read_timeout),
+            socket_options=socket_options,
+            client_cert=new_config.client_cert,
             connector_args=new_config.connector_args)
 
         serializer = botocore.serialize.create_serializer(

aiobotocore/client.py

@@ -1,6 +1,6 @@
 import asyncio
 
-import botocore.client
+from botocore.client import logger, PaginatorDocstring, ClientCreator, BaseClient
 from botocore.exceptions import OperationNotPageableError
 from botocore.history import get_global_history_recorder
 from botocore.utils import get_service_module_name
@@ -14,48 +14,42 @@
 history_recorder = get_global_history_recorder()
 
 
-class AioClientCreator(botocore.client.ClientCreator):
+class AioClientCreator(ClientCreator):
 
-    def __init__(self, loader, endpoint_resolver, user_agent, event_emitter,
-                 retry_handler_factory, retry_config_translator,
-                 response_parser_factory=None, exceptions_factory=None,
-                 config_store=None, loop=None):
-        super().__init__(loader, endpoint_resolver, user_agent, event_emitter,
-                         retry_handler_factory, retry_config_translator,
-                         response_parser_factory=response_parser_factory,
-                         exceptions_factory=exceptions_factory,
-                         config_store=config_store)
-        loop = loop or asyncio.get_event_loop()
-        self._loop = loop
+    def __init__(self, *args, loop=None, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._loop = loop or asyncio.get_event_loop()
+
+    def _create_client_class(self, service_name, service_model):
+        class_attributes = self._create_methods(service_model)
+        py_name_to_operation_name = self._create_name_mapping(service_model)
+        class_attributes['_PY_TO_OP_NAME'] = py_name_to_operation_name
+        bases = [AioBaseClient]
+        service_id = service_model.service_id.hyphenize()
+        self._event_emitter.emit(
+            'creating-client-class.%s' % service_id,
+            class_attributes=class_attributes,
+            base_classes=bases)
+        class_name = get_service_module_name(service_model)
+        cls = type(str(class_name), tuple(bases), class_attributes)
+        return cls
 
     def _get_client_args(self, service_model, region_name, is_secure,
                          endpoint_url, verify, credentials,
                          scoped_config, client_config, endpoint_bridge):
-        # This is a near copy of botocore.client.ClientCreator. What's replaced
+        # This is a near copy of ClientCreator. What's replaced
         # is ClientArgsCreator->AioClientArgsCreator
         args_creator = AioClientArgsCreator(
             self._event_emitter, self._user_agent,
             self._response_parser_factory, self._loader,
-            self._exceptions_factory, loop=self._loop)
+            self._exceptions_factory, loop=self._loop,
+            config_store=self._config_store)
         return args_creator.get_client_args(
             service_model, region_name, is_secure, endpoint_url,
             verify, credentials, scoped_config, client_config, endpoint_bridge)
 
-    def _create_client_class(self, service_name, service_model):
-        class_attributes = self._create_methods(service_model)
-        py_name_to_operation_name = self._create_name_mapping(service_model)
-        class_attributes['_PY_TO_OP_NAME'] = py_name_to_operation_name
-        bases = [AioBaseClient]
-        service_id = service_model.service_id.hyphenize()
-        self._event_emitter.emit('creating-client-class.%s' % service_id,
-                                 class_attributes=class_attributes,
-                                 base_classes=bases)
-        class_name = get_service_module_name(service_model)
-        cls = type(str(class_name), tuple(bases), class_attributes)
-        return cls
-
 
-class AioBaseClient(botocore.client.BaseClient):
+class AioBaseClient(BaseClient):
     def __init__(self, *args, **kwargs):
         self._loop = kwargs.pop('loop', None) or asyncio.get_event_loop()
         super().__init__(*args, **kwargs)
@@ -66,8 +60,11 @@ async def _make_api_call(self, operation_name, api_params):
         history_recorder.record('API_CALL', {
             'service': service_name,
             'operation': operation_name,
-            'params': api_params
+            'params': api_params,
         })
+        if operation_model.deprecated:
+            logger.debug('Warning: %s.%s() is deprecated',
+                         service_name, operation_name)
         request_context = {
             'client_region': self.meta.region_name,
             'client_config': self.meta.config,
@@ -152,6 +149,14 @@ def paginate(self, **kwargs):
 
             paginator_config = self._cache['page_config'][
                 actual_operation_name]
+            # Add the docstring for the paginate method.
+            paginate.__doc__ = PaginatorDocstring(
+                paginator_name=actual_operation_name,
+                event_emitter=self.meta.events,
+                service_model=self.meta.service_model,
+                paginator_config=paginator_config,
+                include_signature=False
+            )
 
             # Rename the paginator class based on the type of paginator.
             paginator_class_name = str('%s.Paginator.%s' % (
@@ -162,13 +167,12 @@ def paginate(self, **kwargs):
             documented_paginator_cls = type(
                 paginator_class_name, (AioPaginator,), {'paginate': paginate})
 
-            operation_model = self._service_model.\
-                operation_model(actual_operation_name)
+            operation_model = self._service_model.operation_model(
+                actual_operation_name)
             paginator = documented_paginator_cls(
                 getattr(self, operation_name),
                 paginator_config,
                 operation_model)
-
             return paginator
 
     def get_waiter(self, waiter_name):
@@ -195,12 +199,12 @@ def get_waiter(self, waiter_name):
             mapping[waiter_name], model, self, loop=self._loop)
 
     async def __aenter__(self):
-        await self._endpoint._aio_session.__aenter__()
+        await self._endpoint.http_session.__aenter__()
         return self
 
     async def __aexit__(self, exc_type, exc_val, exc_tb):
-        await self._endpoint._aio_session.__aexit__(exc_type, exc_val, exc_tb)
+        await self._endpoint.http_session.__aexit__(exc_type, exc_val, exc_tb)
 
     async def close(self):
         """Close all http connections."""
-        return await self._endpoint._aio_session.close()
+        return await self._endpoint.http_session.close()