Merge pull request #833 from kyleknap/request-stream

Document streaming payload inputs

botocore/docs/example.py

@@ -47,7 +47,10 @@ def document_recursive_shape(self, section, shape, **kwargs):
 
     def document_shape_default(self, section, shape, history, include=None,
                                exclude=None, **kwargs):
-        py_type = py_default(shape.type_name)
+        py_type = self._get_special_py_default(shape)
+        if py_type is None:
+            py_type = py_default(shape.type_name)
+
         if self._context.get('streaming_shape') == shape:
             py_type = 'StreamingBody()'
         section.write(py_type)

botocore/docs/method.py

@@ -176,11 +176,19 @@ def document_model_driven_method(section, method_name, operation_model,
     example_section = section.add_new_section('example')
     example_section.style.new_paragraph()
     example_section.style.bold('Request Syntax')
+
+    context = {
+        'special_shape_types': {
+            'streaming_input_shape': operation_model.get_streaming_input(),
+            'streaming_output_shape': operation_model.get_streaming_output()
+        }
+    }
+
     if operation_model.input_shape:
         RequestExampleDocumenter(
             service_name=operation_model.service_model.service_name,
             operation_name=operation_model.name,
-            event_emitter=event_emitter).document_example(
+            event_emitter=event_emitter, context=context).document_example(
                 example_section, operation_model.input_shape,
                 prefix=example_prefix, include=include_input,
                 exclude=exclude_input)
@@ -195,7 +203,7 @@ def document_model_driven_method(section, method_name, operation_model,
         RequestParamsDocumenter(
             service_name=operation_model.service_model.service_name,
             operation_name=operation_model.name,
-            event_emitter=event_emitter).document_params(
+            event_emitter=event_emitter, context=context).document_params(
                 request_params_section, operation_model.input_shape,
                 include=include_input, exclude=exclude_input)
 
@@ -209,8 +217,6 @@ def document_model_driven_method(section, method_name, operation_model,
         return_section.style.indent()
         return_section.style.new_line()
 
-        context = {'streaming_shape': operation_model.get_streaming_output()}
-
         # Add an example return value
         return_example_section = return_section.add_new_section('example')
         return_example_section.style.new_line()

botocore/docs/params.py

@@ -104,6 +104,18 @@ def _add_members_to_shape(self, members, include):
                 members[param.name] = param
         return members
 
+    def _document_non_top_level_param_type(self, type_section, shape):
+        special_py_type = self._get_special_py_type_name(shape)
+        py_type = py_type_name(shape.type_name)
+
+        type_format = '(%s) -- '
+        if special_py_type is not None:
+            # Special type can reference a linked class.
+            # Italicizing it blows away the link.
+            type_section.write(type_format % special_py_type)
+        else:
+            type_section.style.italics(type_format % py_type)
+
     def _start_nested_param(self, section):
         section.style.indent()
         section.style.new_line()
@@ -120,16 +132,12 @@ class ResponseParamsDocumenter(BaseParamsDocumenter):
 
 
     def _add_member_documentation(self, section, shape, name=None, **kwargs):
-        py_type = py_type_name(shape.type_name)
         name_section = section.add_new_section('param-name')
         name_section.write('- ')
         if name is not None:
             name_section.style.bold('%s ' % name)
         type_section = section.add_new_section('param-type')
-        if self._context.get('streaming_shape') == shape:
-            type_section.write('(:class:`.StreamingBody`) -- ')
-        else:
-            type_section.style.italics('(%s) -- ' % py_type)
+        self._document_non_top_level_param_type(type_section, shape)
 
         documentation_section = section.add_new_section('param-documentation')
         if shape.documentation:
@@ -169,7 +177,9 @@ def document_shape_type_structure(self, section, shape, history,
     def _add_member_documentation(self, section, shape, name=None,
                                   is_top_level_param=False, is_required=False,
                                   **kwargs):
-        py_type = py_type_name(shape.type_name)
+        py_type = self._get_special_py_type_name(shape)
+        if py_type is None:
+            py_type = py_type_name(shape.type_name)
         if is_top_level_param:
             type_section = section.add_new_section('param-type')
             type_section.write(':type %s: %s' % (name, py_type))
@@ -184,7 +194,7 @@ def _add_member_documentation(self, section, shape, name=None,
             if name is not None:
                 name_section.style.bold('%s ' % name)
             type_section = section.add_new_section('param-type')
-            type_section.style.italics('(%s) -- ' % py_type)
+            self._document_non_top_level_param_type(type_section, shape)
 
         if is_required:
             is_required_section = section.add_new_section('is-required')

botocore/docs/shape.py

@@ -24,7 +24,11 @@ def __init__(self, service_name, operation_name, event_emitter,
         self._service_name = service_name
         self._operation_name = operation_name
         self._event_emitter = event_emitter
-        self._context = {} if context is None else context
+        self._context = context
+        if context is None:
+            self._context = {
+                'special_shape_types': {}
+            }
 
     def traverse_and_document_shape(self, section, shape, history,
                                     include=None, exclude=None, name=None,
@@ -78,3 +82,25 @@ def traverse_and_document_shape(self, section, shape, history,
                                                         self._operation_name),
                     section=section)
             history.pop()
+
+    def _get_special_py_default(self, shape):
+        special_defaults = {
+            'streaming_input_shape': 'b\'bytes\'|file',
+            'streaming_output_shape': 'StreamingBody()'
+        }
+        return self._get_value_for_special_type(shape, special_defaults)
+
+    def _get_special_py_type_name(self, shape):
+        special_type_names = {
+            'streaming_input_shape': 'bytes or seekable file-like object',
+            'streaming_output_shape': ':class:`.StreamingBody`'
+        }
+        return self._get_value_for_special_type(shape, special_type_names)
+
+    def _get_value_for_special_type(self, shape, special_type_map):
+        for special_type, marked_shape in self._context[
+                'special_shape_types'].items():
+            if special_type in special_type_map:
+                if shape == marked_shape:
+                    return special_type_map[special_type]
+        return None

botocore/model.py

@@ -407,13 +407,22 @@ def output_shape(self):
         return self._service_model.resolve_shape_ref(
             self._operation_model['output'])
 
+    @CachedProperty
+    def has_streaming_input(self):
+        return self.get_streaming_input() is not None
+
     @CachedProperty
     def has_streaming_output(self):
         return self.get_streaming_output() is not None
 
+    def get_streaming_input(self):
+        return self._get_streaming_body(self.input_shape)
+
     def get_streaming_output(self):
+        return self._get_streaming_body(self.output_shape)
+
+    def _get_streaming_body(self, shape):
         """Returns the streaming member's shape if any; or None otherwise."""
-        shape = self.output_shape
         if shape is None:
             return None
         payload = shape.serialization.get('payload')

tests/unit/docs/test_method.py

@@ -314,3 +314,21 @@ def test_streaming_body_in_output(self):
             example_prefix='response = client.foo'
         )
         self.assert_contains_line('**Body** (:class:`.StreamingBody`)')
+
+    def test_streaming_body_in_input(self):
+        del self.json_model['operations']['SampleOperation']['output']
+        self.add_shape_to_params('Body', 'Blob')
+        self.json_model['shapes']['Blob'] = {'type': 'blob'}
+        self.json_model['shapes']['SampleOperationInputOutput']['payload'] = \
+            'Body'
+        document_model_driven_method(
+            self.doc_structure, 'foo', self.operation_model,
+            event_emitter=self.event_emitter,
+            method_description='This describes the foo method.',
+            example_prefix='response = client.foo'
+        )
+        # The line in the example
+        self.assert_contains_line('Body=b\'bytes\'|file')
+        # The line in the parameter description
+        self.assert_contains_line(
+            ':type Body: bytes or seekable file-like object')

tests/unit/test_model.py

@@ -183,20 +183,26 @@ def test_operation_shape_not_required(self):
         output_shape = operation.output_shape
         self.assertIsNone(output_shape)
 
-    def test_streaming_output_for_operation(self):
+
+class TestOperationModelStreamingTypes(unittest.TestCase):
+    def setUp(self):
+        super(TestOperationModelStreamingTypes, self).setUp()
         self.model = {
             'metadata': {'protocol': 'query', 'endpointPrefix': 'foo'},
             'documentation': '',
             'operations': {
                 'OperationName': {
                     'name': 'OperationName',
+                    'input': {
+                        'shape': 'OperationRequest',
+                    },
                     'output': {
-                        'shape': 'OperationNameResponse',
+                        'shape': 'OperationResponse',
                     },
                 }
             },
             'shapes': {
-                'OperationNameResponse': {
+                'OperationRequest': {
                     'type': 'structure',
                     'members': {
                         'String': {
@@ -208,47 +214,55 @@ def test_streaming_output_for_operation(self):
                     },
                     'payload': 'Body'
                 },
-                'stringType': {
-                    'type': 'string',
-                },
-                'blobType': {
-                    'type': 'blob'
-                }
-            }
-        }
-        service_model = model.ServiceModel(self.model)
-        operation = service_model.operation_model('OperationName')
-        self.assertTrue(operation.has_streaming_output)
-
-    def test_payload_thats_not_streaming(self):
-        self.model = {
-            'metadata': {'protocol': 'query', 'endpointPrefix': 'foo'},
-            'operations': {
-                'OperationName': {
-                    'name': 'OperationName',
-                    'output': {
-                        'shape': 'OperationNameResponse',
-                    },
-                }
-            },
-            'shapes': {
-                'OperationNameResponse': {
+                'OperationResponse': {
                     'type': 'structure',
                     'members': {
                         'String': {
                             'shape': 'stringType',
                         },
+                        "Body": {
+                            'shape': 'blobType',
+                        }
                     },
-                    'payload': 'String'
+                    'payload': 'Body'
                 },
                 'stringType': {
                     'type': 'string',
                 },
+                'blobType': {
+                    'type': 'blob'
+                }
             }
         }
+
+    def remove_payload(self, type):
+        self.model['shapes']['Operation' + type].pop('payload')
+
+    def test_streaming_input_for_operation(self):
+        service_model = model.ServiceModel(self.model)
+        operation = service_model.operation_model('OperationName')
+        self.assertTrue(operation.has_streaming_input)
+        self.assertEqual(operation.get_streaming_input().name, 'blobType')
+
+    def test_not_streaming_input_for_operation(self):
+        self.remove_payload('Request')
+        service_model = model.ServiceModel(self.model)
+        operation = service_model.operation_model('OperationName')
+        self.assertFalse(operation.has_streaming_input)
+        self.assertEqual(operation.get_streaming_input(), None)
+
+    def test_streaming_output_for_operation(self):
+        service_model = model.ServiceModel(self.model)
+        operation = service_model.operation_model('OperationName')
+        self.assertTrue(operation.has_streaming_output)
+        self.assertEqual(operation.get_streaming_output().name, 'blobType')
+
+    def test_not_streaming_output_for_operation(self):
+        self.remove_payload('Response')
         service_model = model.ServiceModel(self.model)
         operation = service_model.operation_model('OperationName')
         self.assertFalse(operation.has_streaming_output)
+        self.assertEqual(operation.get_streaming_output(), None)
 
 
 class TestDeepMerge(unittest.TestCase):