Merge branch 'release-2.13.0'

* release-2.13.0: (21 commits)
  Bumping version to 2.13.0
  Updated release notes for everything that's been merged.
  Added support for modifying reserved instances in EC2.
  Added ``dry_run`` to. All. The. Things.
  Fixes #1679, #1566 - Altered DDBv2's ``batch_write`` to appropriately queue & retry unprocessed items.
  Adding API reference for boto.swf.layer2.
  Fixes #1709 - Cannot create Launch Configuration with Block Device Mappings
  Removing duplicate boto.s3.prefix entry to prevent Sphinx build errors.
  Removed the incomplete copyright notice from the README.
  More release notes for recent changes.
  Fixes #1664 - Corrected the behavior of ``dynamodb_load`` when working with sets.
  document get_all_images magic strings
  Added release notes for the recent merges.
  SNS ``publish`` now uses POST.
  Updated release notes to mention the Opsworks VPC support.
  Altered SDB to no longer cause errors when building the docs.
  Added Opsworks to the docs & cleaned up a bunch of Sphinx build errors.
  Fixed the creation of EC2 VPC instances with public IPs.
  Added dev release notes back in, as well as a template for future laziness.
  Updated Opsworks docstrings to latest.
  ...

README.rst

@@ -1,13 +1,13 @@
 ####
 boto
 ####
-boto 2.12.0
+boto 2.13.0
 
-Released: 04-September-2013
+Released: 12-September-2013
 
 .. image:: https://travis-ci.org/boto/boto.png?branch=develop
         :target: https://travis-ci.org/boto/boto
-        
+
 .. image:: https://pypip.in/d/boto/badge.png
         :target: https://crate.io/packages/boto/
 
@@ -41,6 +41,7 @@ At the moment, boto supports:
   * AWS Elastic Beanstalk
   * AWS CloudFormation
   * AWS Data Pipeline
+  * AWS Opsworks
 
 * Identity & Access
 
@@ -49,6 +50,7 @@ At the moment, boto supports:
 * Application Services
 
   * Amazon CloudSearch
+  * Amazon Elastic Transcoder
   * Amazon Simple Workflow Service (SWF)
   * Amazon Simple Queue Service (SQS)
   * Amazon Simple Notification Server (SNS)
@@ -89,9 +91,9 @@ Web Services.  In addition, boto provides support for other public
 services such as Google Storage in addition to private cloud systems
 like Eucalyptus, OpenStack and Open Nebula.
 
-Boto is developed mainly using Python 2.6.6 and Python 2.7.1 on Mac OSX
+Boto is developed mainly using Python 2.6.6 and Python 2.7.3 on Mac OSX
 and Ubuntu Maverick.  It is known to work on other Linux distributions
-and on Windows.  Boto requires no additional libraries or packages
+and on Windows.  Most of Boto requires no additional libraries or packages
 other than those that are distributed with Python.  Efforts are made
 to keep boto compatible with Python 2.5.x but no guarantees are made.
 
@@ -153,11 +155,6 @@ following environment variables to ascertain your credentials:
 Credentials and other boto-related settings can also be stored in a
 boto config file.  See `this`_ for details.
 
-Copyright (c) 2006-2012 Mitch Garnaat <mitch@garnaat.com>
-Copyright (c) 2010-2011, Eucalyptus Systems, Inc.
-Copyright (c) 2012 Amazon.com, Inc. or its affiliates.
-All rights reserved.
-
 .. _pip: http://www.pip-installer.org/
 .. _release notes: https://github.com/boto/boto/wiki
 .. _github.com: http://github.com/boto/boto

bin/dynamodb_load

@@ -66,7 +66,7 @@ def load_table(table, in_fd):
                 data[k] = set(v)
             else:
                 data[k] = v
-        table.new_item(attrs=i).put()
+        table.new_item(attrs=data).put()
 
 
 def dynamodb_load(tables, in_dir, create_tables):

boto/__init__.py

@@ -36,7 +36,7 @@
 import urlparse
 from boto.exception import InvalidUriError
 
-__version__ = '2.12.0'
+__version__ = '2.13.0'
 Version = __version__  # for backware compatibility
 
 UserAgent = 'Boto/%s Python/%s %s/%s' % (

boto/dynamodb2/layer1.py

@@ -1491,6 +1491,7 @@ def make_request(self, action, body):
 
     def _retry_handler(self, response, i, next_sleep):
         status = None
+        boto.log.debug("Saw HTTP status: %s" % response.status)
         if response.status == 400:
             response_body = response.read()
             boto.log.debug(response_body)

boto/dynamodb2/table.py

@@ -1,3 +1,4 @@
+import boto
 from boto.dynamodb2 import exceptions
 from boto.dynamodb2.fields import (HashKey, RangeKey,
                                    AllIndex, KeysOnlyIndex, IncludeIndex)
@@ -1070,17 +1071,19 @@ def __init__(self, table):
         self.table = table
         self._to_put = []
         self._to_delete = []
+        self._unprocessed = []
 
     def __enter__(self):
         return self
 
     def __exit__(self, type, value, traceback):
-        if not self._to_put and not self._to_delete:
-            return False
+        if self._to_put or self._to_delete:
+            # Flush anything that's left.
+            self.flush()
 
-        # Flush anything that's left.
-        self.flush()
-        return True
+        if self._unprocessed:
+            # Finally, handle anything that wasn't processed.
+            self.resend_unprocessed()
 
     def put_item(self, data, overwrite=False):
         self._to_put.append(data)
@@ -1122,7 +1125,43 @@ def flush(self):
                 }
             })
 
-        self.table.connection.batch_write_item(batch_data)
+        resp = self.table.connection.batch_write_item(batch_data)
+        self.handle_unprocessed(resp)
+
         self._to_put = []
         self._to_delete = []
         return True
+
+    def handle_unprocessed(self, resp):
+        if len(resp.get('UnprocessedItems', [])):
+            table_name = self.table.table_name
+            unprocessed = resp['UnprocessedItems'].get(table_name, [])
+
+            # Some items have not been processed. Stow them for now &
+            # re-attempt processing on ``__exit__``.
+            msg = "%s items were unprocessed. Storing for later."
+            boto.log.info(msg % len(unprocessed))
+            self._unprocessed.extend(unprocessed)
+
+    def resend_unprocessed(self):
+        # If there are unprocessed records (for instance, the user was over
+        # their throughput limitations), iterate over them & send until they're
+        # all there.
+        boto.log.info(
+            "Re-sending %s unprocessed items." % len(self._unprocessed)
+        )
+
+        while len(self._unprocessed):
+            # Again, do 25 at a time.
+            to_resend = self._unprocessed[:25]
+            # Remove them from the list.
+            self._unprocessed = self._unprocessed[25:]
+            batch_data = {
+                self.table.table_name: to_resend
+            }
+            boto.log.info("Sending %s items" % len(to_resend))
+            resp = self.table.connection.batch_write_item(batch_data)
+            self.handle_unprocessed(resp)
+            boto.log.info(
+                "%s unprocessed items left" % len(self._unprocessed)
+            )

boto/ec2/address.py

@@ -71,33 +71,50 @@ def endElement(self, name, value, connection):
         else:
             setattr(self, name, value)
 
-    def release(self):
+    def release(self, dry_run=False):
         """
         Free up this Elastic IP address.
         :see: :meth:`boto.ec2.connection.EC2Connection.release_address`
         """
         if self.allocation_id:
-            return self.connection.release_address(None, self.allocation_id)
+            return self.connection.release_address(
+                None,
+                self.allocation_id,
+                dry_run=dry_run)
         else:
-            return self.connection.release_address(self.public_ip)
+            return self.connection.release_address(
+                self.public_ip,
+                dry_run=dry_run
+            )
 
     delete = release
 
-    def associate(self, instance_id):
+    def associate(self, instance_id, dry_run=False):
         """
         Associate this Elastic IP address with a currently running instance.
         :see: :meth:`boto.ec2.connection.EC2Connection.associate_address`
         """
-        return self.connection.associate_address(instance_id, self.public_ip)
+        return self.connection.associate_address(
+            instance_id,
+            self.public_ip,
+            dry_run=dry_run
+        )
 
-    def disassociate(self):
+    def disassociate(self, dry_run=False):
         """
         Disassociate this Elastic IP address from a currently running instance.
         :see: :meth:`boto.ec2.connection.EC2Connection.disassociate_address`
         """
         if self.association_id:
-            return self.connection.disassociate_address(None, self.association_id)
+            return self.connection.disassociate_address(
+                None,
+                self.association_id,
+                dry_run=dry_run
+            )
         else:
-            return self.connection.disassociate_address(self.public_ip)
+            return self.connection.disassociate_address(
+                self.public_ip,
+                dry_run=dry_run
+            )
 
 

boto/ec2/blockdevicemapping.py

@@ -118,7 +118,7 @@ def endElement(self, name, value, connection):
     def build_list_params(self, params, prefix=''):
         i = 1
         for dev_name in self:
-            pre = '%sBlockDeviceMapping.%d' % (prefix, i)
+            pre = '%sBlockDeviceMappings.member.%d' % (prefix, i)
             params['%s.DeviceName' % pre] = dev_name
             block_dev = self[dev_name]
             if block_dev.ephemeral_name: