bump botocore (#823)

.travis.yml

@@ -1,12 +1,12 @@
 language: python
 
 # Needed for Python 3.7
-dist: xenial
+dist: bionic
 
 python:
-    - 3.6.10
-    - 3.7.6
-    - 3.8.1
+    - 3.6.11
+    - 3.7.8
+    - 3.8.5
 # moto is incompatible with 3.5 because it relies on the key order of its url path regex rules
 #    - 3.5.9
 

CHANGES.rst

@@ -1,5 +1,8 @@
 Changes
 -------
+1.1.0 (2020-08-18)
+^^^^^^^^^^^^^^^^^^
+* bump botocore to 1.17.44
 
 1.0.7 (2020-06-04)
 ^^^^^^^^^^^^^^^^^^

Pipfile

@@ -8,7 +8,9 @@ codecov = "*"
 coverage = "==5.0.3"
 flake8 = "==3.7.9"
 flake8-formatter-abspath = "==1.0.1"
+docker = '<4'  # fix: client version 1.39 is too new. Maximum supported API version is 1.38
 moto = {extras = ["server"],version = "==1.3.14"}
+idna = "==2.8"  # broken pipenv resolver
 pytest = "==5.3.5"
 pytest-cov = "==2.8.1"
 pytest-asyncio = "==0.10.0"

aiobotocore/__init__.py

@@ -1,4 +1,4 @@
 from .session import get_session, AioSession
 
 __all__ = ['get_session', 'AioSession']
-__version__ = '1.0.7'
+__version__ = '1.1.0'

aiobotocore/credentials.py

@@ -1,10 +1,16 @@
 import asyncio
+import datetime
 import logging
 import subprocess
+import json
 from copy import deepcopy
 from typing import Optional
+from hashlib import sha1
+
+from dateutil.tz import tzutc
 
 from botocore import UNSIGNED
+from botocore.config import Config
 import botocore.compat
 from botocore.credentials import EnvProvider, Credentials, RefreshableCredentials, \
     ReadOnlyCredentials, ContainerProvider, ContainerMetadataFetcher, \
@@ -13,15 +19,19 @@
     ProcessProvider, AssumeRoleWithWebIdentityProvider, _local_now, \
     CachedCredentialFetcher, _serialize_if_needed, BaseAssumeRoleCredentialFetcher, \
     AssumeRoleProvider, AssumeRoleCredentialFetcher, CredentialResolver, \
-    CanonicalNameCredentialSourcer, BotoProvider, OriginalEC2Provider
+    CanonicalNameCredentialSourcer, BotoProvider, OriginalEC2Provider, \
+    SSOProvider
+from botocore.exceptions import UnauthorizedSSOTokenError
 from botocore.exceptions import MetadataRetrievalError, CredentialRetrievalError, \
     InvalidConfigError, PartialCredentialsError, RefreshWithMFAUnsupportedError, \
     UnknownCredentialError
 from botocore.compat import compat_shell_split
+from botocore.utils import SSOTokenLoader
 
 from aiobotocore.utils import AioContainerMetadataFetcher, AioInstanceMetadataFetcher
 from aiobotocore.config import AioConfig
 
+
 logger = logging.getLogger(__name__)
 
 
@@ -132,6 +142,15 @@ def _create_web_identity_provider(self, profile_name, disable_env_vars):
             disable_env_vars=disable_env_vars,
         )
 
+    def _create_sso_provider(self, profile_name):
+        return AioSSOProvider(
+            load_config=lambda: self._session.full_config,
+            client_creator=self._session.create_client,
+            profile_name=profile_name,
+            cache=self._cache,
+            token_cache=self._sso_token_cache,
+        )
+
 
 async def get_credentials(session):
     resolver = create_credential_resolver(session)
@@ -795,3 +814,87 @@ async def load_credentials(self):
         # +1
         # -js
         return None
+
+
+class AioSSOCredentialFetcher(AioCachedCredentialFetcher):
+    def __init__(self, start_url, sso_region, role_name, account_id,
+                 client_creator, token_loader=None, cache=None,
+                 expiry_window_seconds=None):
+        self._client_creator = client_creator
+        self._sso_region = sso_region
+        self._role_name = role_name
+        self._account_id = account_id
+        self._start_url = start_url
+        self._token_loader = token_loader
+
+        super(AioSSOCredentialFetcher, self).__init__(
+            cache, expiry_window_seconds
+        )
+
+    def _create_cache_key(self):
+        args = {
+            'startUrl': self._start_url,
+            'roleName': self._role_name,
+            'accountId': self._account_id,
+        }
+
+        args = json.dumps(args, sort_keys=True, separators=(',', ':'))
+        argument_hash = sha1(args.encode('utf-8')).hexdigest()
+        return self._make_file_safe(argument_hash)
+
+    def _parse_timestamp(self, timestamp_ms):
+        # fromtimestamp expects seconds so: milliseconds / 1000 = seconds
+        timestamp_seconds = timestamp_ms / 1000.0
+        timestamp = datetime.datetime.fromtimestamp(timestamp_seconds, tzutc())
+        return _serialize_if_needed(timestamp)
+
+    async def _get_credentials(self):
+        """Get credentials by calling SSO get role credentials."""
+        config = Config(
+            signature_version=UNSIGNED,
+            region_name=self._sso_region,
+        )
+        async with self._client_creator('sso', config=config) as client:
+            kwargs = {
+                'roleName': self._role_name,
+                'accountId': self._account_id,
+                'accessToken': self._token_loader(self._start_url),
+            }
+            try:
+                response = await client.get_role_credentials(**kwargs)
+            except client.exceptions.UnauthorizedException:
+                raise UnauthorizedSSOTokenError()
+            credentials = response['roleCredentials']
+
+            credentials = {
+                'ProviderType': 'sso',
+                'Credentials': {
+                    'AccessKeyId': credentials['accessKeyId'],
+                    'SecretAccessKey': credentials['secretAccessKey'],
+                    'SessionToken': credentials['sessionToken'],
+                    'Expiration': self._parse_timestamp(credentials['expiration']),
+                }
+            }
+            return credentials
+
+
+class AioSSOProvider(SSOProvider):
+    async def load(self):
+        sso_config = self._load_sso_config()
+        if not sso_config:
+            return None
+
+        sso_fetcher = AioSSOCredentialFetcher(
+            sso_config['sso_start_url'],
+            sso_config['sso_region'],
+            sso_config['sso_role_name'],
+            sso_config['sso_account_id'],
+            self._client_creator,
+            token_loader=SSOTokenLoader(cache=self._token_cache),
+            cache=self.cache,
+        )
+
+        return AioDeferredRefreshableCredentials(
+            method=self.METHOD,
+            refresh_using=sso_fetcher.fetch_credentials,
+        )

aiobotocore/endpoint.py

@@ -178,6 +178,11 @@ async def _do_get_response(self, request, operation_model):
         parser = self._response_parser_factory.create_parser(protocol)
         parsed_response = parser.parse(
             response_dict, operation_model.output_shape)
+        if http_response.status_code >= 300:
+            self._add_modeled_error_fields(
+                response_dict, parsed_response,
+                operation_model, parser,
+            )
         history_recorder.record('PARSED_RESPONSE', parsed_response)
         return (http_response, parsed_response), None
 

aiobotocore/signers.py

@@ -294,6 +294,8 @@ async def generate_presigned_post(self, Bucket, Key, Fields=None, Conditions=Non
 
     if fields is None:
         fields = {}
+    else:
+        fields = fields.copy()
 
     if conditions is None:
         conditions = []

setup.py

@@ -7,7 +7,7 @@
 # NOTE: When updating botocore make sure to update awscli/boto3 versions below
 install_requires = [
     # pegged to also match items in `extras_require`
-    'botocore>=1.15.32,<1.15.33',
+    'botocore>=1.17.44,<1.17.45',
     'aiohttp>=3.3.1',
     'wrapt>=1.10.10',
     'aioitertools>=0.5.1',
@@ -19,8 +19,8 @@ def read(f):
 
 
 extras_require = {
-    'awscli': ['awscli==1.18.32'],
-    'boto3': ['boto3==1.12.32'],
+    'awscli': ['awscli==1.18.121'],
+    'boto3': ['boto3==1.14.44'],
 }