Merge branch 'release-1.12.4'

* release-1.12.4:
  Bumping version to 1.12.4
  Update to latest models
  Do not urldecode ContinuationToken
  updating changelog
  list_objects_v2 decode integration test
  generalized decode_list_object
  added decode_list_object_v2 handler and tests
  Improve error message with missing service id
  registering response decode for ListObjectsV2
  Adding s3_list_objects_encoding_type_url handler to ListObjectsV2

.changes/1.12.4.json

@@ -0,0 +1,12 @@
+[
+  {
+    "category": "s3", 
+    "description": "Adds encoding and decoding handlers for ListObjectsV2 `#1552 <https://github.com/boto/botocore/issues/1552>`__", 
+    "type": "enhancement"
+  }, 
+  {
+    "category": "``polly``", 
+    "description": "Update polly client to latest version", 
+    "type": "api-change"
+  }
+]

CHANGELOG.rst

@@ -2,6 +2,13 @@
 CHANGELOG
 =========
 
+1.12.4
+======
+
+* enhancement:s3: Adds encoding and decoding handlers for ListObjectsV2 `#1552 <https://github.com/boto/botocore/issues/1552>`__
+* api-change:``polly``: Update polly client to latest version
+
+
 1.12.3
 ======
 

botocore/__init__.py

@@ -16,7 +16,7 @@
 import re
 import logging
 
-__version__ = '1.12.3'
+__version__ = '1.12.4'
 
 
 class NullHandler(logging.Handler):

botocore/data/polly/2016-06-10/service-2.json

@@ -358,6 +358,7 @@
     "LanguageCode":{
       "type":"string",
       "enum":[
+        "cmn-CN",
         "cy-GB",
         "da-DK",
         "de-DE",
@@ -822,7 +823,7 @@
         },
         "OutputFormat":{
           "shape":"OutputFormat",
-          "documentation":"<p> The format in which the returned output will be encoded. For audio stream, this will be mp3, ogg_vorbis, or pcm. For speech marks, this will be json. </p>"
+          "documentation":"<p> The format in which the returned output will be encoded. For audio stream, this will be mp3, ogg_vorbis, or pcm. For speech marks, this will be json. </p> <p>When pcm is used, the content returned is audio/pcm in a signed 16-bit, 1 channel (mono), little-endian format. </p>"
         },
         "SampleRate":{
           "shape":"SampleRate",
@@ -1007,7 +1008,8 @@
         "Vicki",
         "Takumi",
         "Seoyeon",
-        "Aditi"
+        "Aditi",
+        "Zhiyu"
       ]
     },
     "VoiceList":{

botocore/handlers.py

@@ -274,13 +274,13 @@ def _needs_s3_sse_customization(params, sse_member_prefix):
 def register_retries_for_service(service_data, session,
                                  service_name, **kwargs):
     loader = session.get_component('data_loader')
-    service_id = service_data.get('metadata', {}).get('serviceId')
-    service_event_name = hyphenize_service_id(service_id)
     endpoint_prefix = service_data.get('metadata', {}).get('endpointPrefix')
     if endpoint_prefix is None:
         logger.debug("Not registering retry handlers, could not endpoint "
                      "prefix from model for service %s", service_name)
         return
+    service_id = service_data.get('metadata', {}).get('serviceId')
+    service_event_name = hyphenize_service_id(service_id)
     config = _load_retry_config(loader, endpoint_prefix)
     if not config:
         return
@@ -730,21 +730,38 @@ def decode_list_object(parsed, context, **kwargs):
     # Amazon S3 includes this element in the response, and returns encoded key
     # name values in the following response elements:
     # Delimiter, Marker, Prefix, NextMarker, Key.
+    _decode_list_object(
+        top_level_keys=['Delimiter', 'Marker', 'NextMarker'],
+        nested_keys=[('Contents', 'Key'), ('CommonPrefixes', 'Prefix')],
+        parsed=parsed,
+        context=context
+    )
+
+def decode_list_object_v2(parsed, context, **kwargs):
+    # From the documentation: If you specify encoding-type request parameter,
+    # Amazon S3 includes this element in the response, and returns encoded key
+    # name values in the following response elements:
+    # Delimiter, Prefix, ContinuationToken, Key, and StartAfter.
+    _decode_list_object(
+        top_level_keys=['Delimiter', 'Prefix', 'StartAfter'],
+        nested_keys=[('Contents', 'Key'), ('CommonPrefixes', 'Prefix')],
+        parsed=parsed,
+        context=context
+    )
+
+def _decode_list_object(top_level_keys, nested_keys, parsed, context):
     if parsed.get('EncodingType') == 'url' and \
                     context.get('encoding_type_auto_set'):
         # URL decode top-level keys in the response if present.
-        top_level_keys = ['Delimiter', 'Marker', 'NextMarker']
         for key in top_level_keys:
             if key in parsed:
                 parsed[key] = unquote_str(parsed[key])
         # URL decode nested keys from the response if present.
-        nested_keys = [('Contents', 'Key'), ('CommonPrefixes', 'Prefix')]
         for (top_key, child_key) in nested_keys:
             if top_key in parsed:
                 for member in parsed[top_key]:
                     member[child_key] = unquote_str(member[child_key])
 
-
 def convert_body_to_file_like_object(params, **kwargs):
     if 'Body' in params:
         if isinstance(params['Body'], six.string_types):
@@ -880,6 +897,8 @@ def remove_subscribe_to_shard(class_attributes, **kwargs):
 
     ('before-parameter-build.s3.ListObjects',
      set_list_objects_encoding_type_url),
+    ('before-parameter-build.s3.ListObjectsV2',
+     set_list_objects_encoding_type_url),
     ('before-call.s3.PutBucketTagging', calculate_md5),
     ('before-call.s3.PutBucketLifecycle', calculate_md5),
     ('before-call.s3.PutBucketLifecycleConfiguration', calculate_md5),
@@ -943,6 +962,7 @@ def remove_subscribe_to_shard(class_attributes, **kwargs):
     ('before-parameter-build.route53', fix_route53_ids),
     ('before-parameter-build.glacier', inject_account_id),
     ('after-call.s3.ListObjects', decode_list_object),
+    ('after-call.s3.ListObjectsV2', decode_list_object_v2),
 
     # Cloudsearchdomain search operation will be sent by HTTP POST
     ('request-created.cloudsearchdomain.Search',

botocore/model.py

@@ -339,6 +339,10 @@ def signature_version(self):
     def signature_version(self, value):
         self._signature_version = value
 
+    def __repr__(self):
+        return '%s(%s)' % (self.__class__.__name__, self.service_name)
+
+
 
 class OperationModel(object):
     def __init__(self, operation_model, service_model, name=None):

docs/source/conf.py

@@ -54,7 +54,7 @@
 # The short X.Y version.
 version = '1.12'
 # The full version, including alpha/beta/rc tags.
-release = '1.12.3'
+release = '1.12.4'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

tests/integration/test_s3.py

@@ -416,6 +416,21 @@ def test_unicode_system_character(self):
         self.assertEqual(len(parsed['Contents']), 1)
         self.assertEqual(parsed['Contents'][0]['Key'], 'foo%08')
 
+    def test_unicode_system_character_with_list_v2(self):
+        # Verify we can use a unicode system character which would normally
+        # break the xml parser
+        key_name = 'foo\x08'
+        self.create_object(key_name)
+        self.addCleanup(self.delete_object, key_name, self.bucket_name)
+        parsed = self.client.list_objects_v2(Bucket=self.bucket_name)
+        self.assertEqual(len(parsed['Contents']), 1)
+        self.assertEqual(parsed['Contents'][0]['Key'], key_name)
+
+        parsed = self.client.list_objects_v2(Bucket=self.bucket_name,
+                                          EncodingType='url')
+        self.assertEqual(len(parsed['Contents']), 1)
+        self.assertEqual(parsed['Contents'][0]['Key'], 'foo%08')
+
     def test_thread_safe_auth(self):
         self.auth_paths = []
         self.session.register('before-sign', self.increment_auth)

tests/unit/test_handlers.py

@@ -549,8 +549,8 @@ def test_run_instances_userdata_blob(self):
                   'UserData': b64_user_data}
         self.assertEqual(params, result)
 
-    def test_register_retry_for_handlers_with_no_endpoint_prefix(self):
-        no_endpoint_prefix = {'metadata': {'serviceId': 'foo'}}
+    def test_register_retry_for_handlers_with_no_metadata(self):
+        no_endpoint_prefix = {'metadata': {}}
         session = mock.Mock()
         handlers.register_retries_for_service(service_data=no_endpoint_prefix,
                                               session=mock.Mock(),
@@ -873,6 +873,70 @@ def test_decode_list_objects_with_delimiter(self):
         handlers.decode_list_object(parsed, context=context)
         self.assertEqual(parsed['Delimiter'], u'\xe7\xf6s% asd\x08 c')
 
+    def test_decode_list_objects_v2(self):
+        parsed = {
+            'Contents': [{'Key': "%C3%A7%C3%B6s%25asd%08"}],
+            'EncodingType': 'url',
+        }
+        context = {'encoding_type_auto_set': True}
+        handlers.decode_list_object_v2(parsed, context=context)
+        self.assertEqual(parsed['Contents'][0]['Key'], u'\xe7\xf6s%asd\x08')
+
+    def test_decode_list_objects_v2_does_not_decode_without_context(self):
+        parsed = {
+            'Contents': [{'Key': "%C3%A7%C3%B6s%25asd"}],
+            'EncodingType': 'url',
+        }
+        handlers.decode_list_object_v2(parsed, context={})
+        self.assertEqual(parsed['Contents'][0]['Key'], u'%C3%A7%C3%B6s%25asd')
+
+    def test_decode_list_objects_v2_with_delimiter(self):
+        parsed = {
+            'Delimiter': "%C3%A7%C3%B6s%25%20asd%08+c",
+            'EncodingType': 'url',
+        }
+        context = {'encoding_type_auto_set': True}
+        handlers.decode_list_object_v2(parsed, context=context)
+        self.assertEqual(parsed['Delimiter'], u'\xe7\xf6s% asd\x08 c')
+
+    def test_decode_list_objects_v2_with_prefix(self):
+        parsed = {
+            'Prefix': "%C3%A7%C3%B6s%25%20asd%08+c",
+            'EncodingType': 'url',
+        }
+        context = {'encoding_type_auto_set': True}
+        handlers.decode_list_object_v2(parsed, context=context)
+        self.assertEqual(parsed['Prefix'], u'\xe7\xf6s% asd\x08 c')
+
+    def test_decode_list_objects_v2_does_not_decode_continuationtoken(self):
+        parsed = {
+            'ContinuationToken': "%C3%A7%C3%B6s%25%20asd%08+c",
+            'EncodingType': 'url',
+        }
+        context = {'encoding_type_auto_set': True}
+        handlers.decode_list_object_v2(parsed, context=context)
+        self.assertEqual(
+            parsed['ContinuationToken'], u"%C3%A7%C3%B6s%25%20asd%08+c")
+
+    def test_decode_list_objects_v2_with_startafter(self):
+        parsed = {
+            'StartAfter': "%C3%A7%C3%B6s%25%20asd%08+c",
+            'EncodingType': 'url',
+        }
+        context = {'encoding_type_auto_set': True}
+        handlers.decode_list_object_v2(parsed, context=context)
+        self.assertEqual(parsed['StartAfter'], u'\xe7\xf6s% asd\x08 c')
+
+    def test_decode_list_objects_v2_with_common_prefixes(self):
+        parsed = {
+            'CommonPrefixes': [{'Prefix': "%C3%A7%C3%B6s%25%20asd%08+c"}],
+            'EncodingType': 'url',
+        }
+        context = {'encoding_type_auto_set': True}
+        handlers.decode_list_object_v2(parsed, context=context)
+        self.assertEqual(parsed['CommonPrefixes'][0]['Prefix'],
+                         u'\xe7\xf6s% asd\x08 c')
+
     def test_get_bucket_location_optional(self):
         # This handler should no-op if another hook (i.e. stubber) has already
         # filled in response

tests/unit/test_model.py

@@ -89,6 +89,10 @@ def test_documentation_exposed_as_property(self):
     def test_shape_names(self):
         self.assertEqual(self.service_model.shape_names, ['StringShape'])
 
+    def test_repr_has_service_name(self):
+        self.assertEqual(repr(self.service_model),
+                         'ServiceModel(endpoint-prefix)')
+
 
 class TestOperationModelFromService(unittest.TestCase):
     def setUp(self):