Implement batch actions.

This change implements batch actions on collections:

```python
s3.Bucket('boto3').objects.delete()
```

It does the following:

* Create a `CollectionFactory` class to generate subclasses of
  `CollectionManager` and `ResourceCollection`.
* Update the `ResourceFactory` to use the new `CollectionFactory`.
* Add a public `pages()` method to collections. This returns entire
  pages of resource instances and it used by `__iter__` as well.
* Add batch actions as methods via the new `CollectionFactory`.
* Add a `BatchAction` subclass of `Action` which does the following:

    1. Get a page of results from the collection's operation
    2. Build parameters for the batch action operation
    3. Call the batch action operation
    4. Repeat until no more pages

* Makes some previously public members private on `Action` as these
  should never have been public.
* Update documentation to include collection classes.
* Add tests to cover the new functionality.

boto3/docs.py

@@ -131,7 +131,18 @@ def docs_for(service_name):
 
     print('Processing {0}-{1}'.format(service_name, service_model.api_version))
 
+    # The following creates an official name of 'Amazon Simple Storage
+    # Service (S3)' our of 'Amazon Simple Storage Service' and 'Amazon S3'.
+    # It tries to be smart, so for `Amazon DynamoDB' and 'DynamoDB' we would
+    # get an official name of 'Amazon DynamoDB'.
     official_name = service_model.metadata.get('serviceFullName')
+    short_name = service_model.metadata.get('serviceAbbreviation', '')
+    if short_name.startswith('Amazon'):
+        short_name = short_name[7:]
+    if short_name.startswith('AWS'):
+        short_name = short_name[4:]
+    if short_name and short_name.lower() not in official_name.lower():
+        official_name += ' ({0})'.format(short_name)
 
     docs = '{0}\n{1}\n\n'.format(official_name, '=' * len(official_name))
 
@@ -142,18 +153,47 @@ def docs_for(service_name):
     filename = (os.path.dirname(__file__) + '/data/resources/'
                 '{0}-{1}.resources.json').format(service_name,
                                                  service_model.api_version)
+    # We can't use a set here because dicts aren't hashable!
+    models = {}
     if os.path.exists(filename):
         data = json.load(open(filename))
         model = ResourceModel(service_name, data['service'], data['resources'])
 
+        for collection_model in model.collections:
+            collection_model.parent_name = model.name
+            models[collection_model.name] = {
+                'type': 'collection',
+                'model': collection_model
+            }
+
         docs += document_resource(service_name, official_name, model,
                                   service_model)
 
+        # First, collect all the models...
         for name, model in sorted(data['resources'].items(),
                                   key=lambda i:i[0]):
             resource_model = ResourceModel(name, model, data['resources'])
-            docs += document_resource(service_name, official_name,
-                                      resource_model, service_model)
+            if name not in models:
+                models[name] = {'type': 'resource', 'model': resource_model}
+
+            for collection_model in resource_model.collections:
+                collection_model.parent_name = xform_name(resource_model.name)
+
+                cname = collection_model.name + 'CollectionManager'
+                if cname not in models:
+                    models[cname] = {'type': 'collection',
+                                     'model': collection_model}
+
+        # Then render them out in alphabetical order.
+        for name, item in sorted(models.items()):
+            model = item['model']
+            if item['type'] == 'resource':
+                docs += document_resource(service_name, official_name,
+                                          model, service_model)
+            elif item['type'] == 'collection':
+                docs += document_collection(
+                    service_name, official_name, model,
+                    model.resource.model, service_model)
 
     return docs
 
@@ -320,16 +360,53 @@ def document_resource(service_name, official_name, resource_model,
         for collection in sorted(resource_model.collections,
                                  key=lambda i: i.name):
             docs += ('   .. py:attribute:: {0}\n\n      '
-                     '(:py:class:`~boto3.resources.collection.CollectionManager`)'
-                     ' A collection of :py:class:`{1}.{2}` instances. This'
-                     ' collection uses the :py:meth:`{3}.Client.{4}` operation'
+                     '(:py:class:`{1}.{2}CollectionManager`)'
+                     ' A collection of :py:class:`{3}.{4}` instances. This'
+                     ' collection uses the :py:meth:`{5}.Client.{6}` operation'
                      ' to get items.\n\n').format(
                         xform_name(collection.name), service_name,
+                        collection.name, service_name,
                         collection.resource.type, service_name,
                         xform_name(collection.request.operation))
 
     return docs
 
+def document_collection(service_name, official_name, collection_model,
+                        resource_model, service_model):
+    """
+    Generate reference documentation about a collection and any
+    batch actions it might have.
+    """
+    title = collection_model.name + 'Collection'
+    docs = '{0}\n{1}\n\n'.format(title, '-' * len(title))
+    docs += '.. py:class:: {0}.{1}CollectionManager()\n\n'.format(
+        service_name, collection_model.name)
+    docs += ('   A collection of :py:class:`{0}.{1}` resources for {2}. See'
+             '   the'
+             '   :py:class:`~boto3.resources.collection.CollectionManager`'
+             '   base class for additional methods.\n\n'
+             '   This collection uses the :py:meth:`{3}.Client.{4}`'
+             '   operation to get items, and its parameters can be'
+             '   used as filters::\n\n').format(
+                service_name, resource_model.name, official_name,
+                service_name, xform_name(collection_model.request.operation))
+    docs += ('       for {0} in {1}.{2}.all():\n'
+             '           print({0})\n\n').format(
+                xform_name(collection_model.resource.type),
+                collection_model.parent_name,
+                xform_name(collection_model.name),
+                xform_name(collection_model.resource.type))
+
+    if collection_model.batch_actions:
+        docs += ('   .. rst-class:: admonition-title\n\n   Batch Actions\n\n'
+                 '   Batch actions provide a way to manipulate groups of'
+                 '   resources in a single service operation call.\n\n')
+    for action in sorted(collection_model.batch_actions, key=lambda i:i.name):
+        docs += document_action(action, service_name, resource_model,
+                                service_model)
+
+    return docs
+
 def document_action(action, service_name, resource_model, service_model,
                     action_type='action'):
     """
@@ -343,7 +420,8 @@ def document_action(action, service_name, resource_model, service_model,
         print('Cannot get operation ' + action.request.operation)
         return ''
 
-    ignore_params = [p.target for p in action.request.params]
+    # Here we split because we only care about top-level parameter names
+    ignore_params = [p.target.split('.')[0] for p in action.request.params]
 
     rtype = 'dict'
     if action_type == 'action':

boto3/resources/action.py

@@ -41,34 +41,34 @@ class ServiceAction(object):
     """
     def __init__(self, action_model, factory=None, resource_defs=None,
                  service_model=None):
-        self.action_model = action_model
+        self._action_model = action_model
 
         # In the simplest case we just return the response, but if a
         # resource is defined, then we must create these before returning.
         resource = action_model.resource
         if resource:
-            self.response_handler = ResourceHandler(resource.path,
+            self._response_handler = ResourceHandler(resource.path,
                 factory, resource_defs, service_model, resource,
                 action_model.request.operation)
         else:
-            self.response_handler = RawHandler(action_model.path)
+            self._response_handler = RawHandler(action_model.path)
 
     def __call__(self, parent, *args, **kwargs):
         """
         Perform the action's request operation after building operation
         parameters and build any defined resources from the response.
 
-        :type parent: ServiceResource
+        :type parent: :py:class:`~boto3.resources.base.ServiceResource`
         :param parent: The resource instance to which this action is attached.
         :rtype: dict or ServiceResource or list(ServiceResource)
         :return: The response, either as a raw dict or resource instance(s).
         """
-        operation_name = xform_name(self.action_model.request.operation)
+        operation_name = xform_name(self._action_model.request.operation)
 
         # First, build predefined params and then update with the
         # user-supplied kwargs, which allows overriding the pre-built
         # params if needed.
-        params = create_request_parameters(parent, self.action_model.request)
+        params = create_request_parameters(parent, self._action_model.request)
         params.update(kwargs)
 
         logger.info('Calling %s:%s with %r', parent.meta['service_name'],
@@ -78,4 +78,74 @@ def __call__(self, parent, *args, **kwargs):
 
         logger.debug('Response: %r', response)
 
-        return self.response_handler(parent, params, response)
+        return self._response_handler(parent, params, response)
+
+
+class BatchAction(ServiceAction):
+    """
+    An action which operates on a batch of items in a collection, typically
+    a single page of results from the collection's underlying service
+    operation call. For example, this allows you to delete up to 999
+    S3 objects in a single operation rather than calling ``.delete()`` on
+    each one individually.
+
+    :type action_model: :py:class:`~boto3.resources.model.Action`
+    :param action_model: The action model.
+    :type factory: ResourceFactory
+    :param factory: The factory that created the resource class to which
+                    this action is attached.
+    :type resource_defs: dict
+    :param resource_defs: Service resource definitions.
+    :type service_model: :ref:`botocore.model.ServiceModel`
+    :param service_model: The Botocore service model
+    """
+    def __call__(self, parent, *args, **kwargs):
+        """
+        Perform the batch action's operation on every page of results
+        from the collection.
+
+        :type parent: :py:class:`~boto3.resources.collection.ResourceCollection`
+        :param parent: The collection iterator to which this action
+                       is attached.
+        :rtype: list(dict)
+        :return: A list of low-level response dicts from each call.
+        """
+        service_name = None
+        client = None
+        responses = []
+        operation_name = xform_name(self._action_model.request.operation)
+
+        # Unlike the simple action above, a batch action must operate
+        # on batches (or pages) of items. So we get each page, construct
+        # the necessary parameters and call the batch operation.
+        for page in parent.pages():
+            params = {}
+            for resource in page:
+                # There is no public interface to get a service name
+                # or low-level client from a collection, so we get
+                # these from the first resource in the collection.
+                if service_name is None:
+                    service_name = resource.meta['service_name']
+                if client is None:
+                    client = resource.meta['client']
+
+                create_request_parameters(
+                    resource, self._action_model.request, params=params)
+
+            if not params:
+                # There are no items, no need to make a call.
+                break
+
+            params.update(kwargs)
+
+            logger.info('Calling %s:%s with %r',
+                        service_name, operation_name, params)
+
+            response = getattr(client, operation_name)(**params)
+
+            logger.debug('Response: %r', response)
+
+            responses.append(
+                self._response_handler(parent, params, response))
+
+        return responses