Merge branch 'master' of github.com:boto/boto

README.markdown

@@ -1,6 +1,6 @@
 # boto
-boto 2.0b4
-13-Feb-2011
+boto 2.0b5
+15-Mar-2011
 
 ## Introduction
 
@@ -61,6 +61,6 @@ AWS_SECRET_ACCESS_KEY - Your AWS Secret Access Key
 Credentials and other boto-related settings can also be stored in a boto config
 file.  See [this](http://code.google.com/p/boto/wiki/BotoConfig) for details.
 
-Copyright (c) 2006-2011 Mitch Garnaat <mitch@garnaat.org>
+Copyright (c) 2006-2011 Mitch Garnaat <mitch@garnaat.com>
 Copyright (c) 2010-2011, Eucalyptus Systems, Inc.
 All rights reserved.

boto/__init__.py

@@ -30,7 +30,7 @@
 import logging.config
 from boto.exception import InvalidUriError
 
-__version__ = '2.0b4'
+__version__ = '2.0b5'
 Version = __version__ # for backware compatibility
 
 UserAgent = 'Boto/%s (%s)' % (__version__, sys.platform)

boto/cloudfront/distribution.py

@@ -286,6 +286,7 @@ def __init__(self, connection=None, config=None, domain_name='',
         self.id = id
         self.last_modified_time = last_modified_time
         self.status = status
+        self.in_progress_invalidation_batches = 0
         self.active_signers = None
         self.etag = None
         self._bucket = None
@@ -308,6 +309,8 @@ def endElement(self, name, value, connection):
             self.last_modified_time = value
         elif name == 'Status':
             self.status = value
+        elif name == 'InProgressInvalidationBatches':
+            self.in_progress_invalidation_batches = int(value)
         elif name == 'DomainName':
             self.domain_name = value
         else:

boto/ec2/connection.py

@@ -55,7 +55,7 @@
 
 class EC2Connection(AWSQueryConnection):
 
-    APIVersion = boto.config.get('Boto', 'ec2_version', '2010-08-31')
+    APIVersion = boto.config.get('Boto', 'ec2_version', '2011-01-01')
     DefaultRegionName = boto.config.get('Boto', 'ec2_region_name', 'us-east-1')
     DefaultRegionEndpoint = boto.config.get('Boto', 'ec2_region_endpoint',
                                             'ec2.amazonaws.com')

boto/ec2/securitygroup.py

@@ -1,4 +1,5 @@
-# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/
+# Copyright (c) 2006-2011 Mitch Garnaat http://garnaat.org/
+# Copyright (c) 2011, Eucalyptus Systems, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a
 # copy of this software and associated documentation files (the
@@ -22,14 +23,15 @@
 """
 Represents an EC2 Security Group
 """
-from boto.ec2.ec2object import EC2Object
+from boto.ec2.ec2object import TaggedEC2Object
 from boto.exception import BotoClientError
 
-class SecurityGroup(EC2Object):
+class SecurityGroup(TaggedEC2Object):
 
     def __init__(self, connection=None, owner_id=None,
-                 name=None, description=None):
-        EC2Object.__init__(self, connection)
+                 name=None, description=None, id=None):
+        TaggedEC2Object.__init__(self, connection)
+        self.id = id
         self.owner_id = owner_id
         self.name = name
         self.description = description
@@ -39,6 +41,9 @@ def __repr__(self):
         return 'SecurityGroup:%s' % self.name
 
     def startElement(self, name, attrs, connection):
+        retval = TaggedEC2Object.startElement(self, name, attrs, connection)
+        if retval is not None:
+            return retval
         if name == 'item':
             self.rules.append(IPPermissions(self))
             return self.rules[-1]
@@ -48,6 +53,8 @@ def startElement(self, name, attrs, connection):
     def endElement(self, name, value, connection):
         if name == 'ownerId':
             self.owner_id = value
+        elif name == 'groupId':
+            self.id = value
         elif name == 'groupName':
             self.name = value
         elif name == 'groupDescription':
@@ -221,7 +228,7 @@ def instances(self):
                 instances.extend(reservation.instances)
         return instances
 
-class IPPermissions:
+class IPPermissions(object):
 
     def __init__(self, parent=None):
         self.parent = parent
@@ -258,7 +265,7 @@ def add_grant(self, name=None, owner_id=None, cidr_ip=None):
         self.grants.append(grant)
         return grant
 
-class GroupOrCIDR:
+class GroupOrCIDR(object):
 
     def __init__(self, parent=None):
         self.owner_id = None

boto/exception.py

@@ -390,11 +390,21 @@ class ResumableTransferDisposition(object):
     START_OVER = 'START_OVER'
 
     # WAIT_BEFORE_RETRY means the resumable transfer failed but that it can
-    # be retried after a time delay.
+    # be retried after a time delay within the current process.
     WAIT_BEFORE_RETRY = 'WAIT_BEFORE_RETRY'
 
-    # ABORT means the resumable transfer failed and that delaying/retrying
-    # within the current process will not help.
+    # ABORT_CUR_PROCESS means the resumable transfer failed and that
+    # delaying/retrying within the current process will not help. If
+    # resumable transfer included a state tracker file the upload can be
+    # retried again later, in another process (e.g., a later run of gsutil).
+    ABORT_CUR_PROCESS = 'ABORT_CUR_PROCESS'
+
+    # ABORT means the resumable transfer failed in a way that it does not
+    # make sense to continue in the current process, and further that the 
+    # current tracker ID should not be preserved (in a tracker file if one
+    # was specified at resumable upload start time). If the user tries again
+    # later (e.g., a separate run of gsutil) it will get a new resumable
+    # upload ID.
     ABORT = 'ABORT'
 
 class ResumableUploadException(Exception):

boto/gs/resumable_upload_handler.py

@@ -41,7 +41,8 @@
 
 Resumable uploads will retry failed uploads, resuming at the byte
 count completed by the last upload attempt. If too many retries happen with
-no progress (per configurable num_retries param), the upload will be aborted.
+no progress (per configurable num_retries param), the upload will be
+aborted in the current process.
 
 The caller can optionally specify a tracker_file_name param in the
 ResumableUploadHandler constructor. If you do this, that file will
@@ -260,11 +261,22 @@ def _start_new_resumable_upload(self, key, headers=None):
             'POST', key.bucket.name, key.name, post_headers)
         # Get tracker URI from response 'Location' header.
         body = resp.read()
-        # Check for '201 Created' response code.
-        if resp.status != 201:
+
+        # Check for various status conditions.
+        if resp.status == 500 or resp.status == 503:
+            # Retry status 500 and 503 errors after a delay.
+            raise ResumableUploadException(
+                'Got status %d from attempt to start resumable upload. '
+                'Will wait/retry' % resp.status,
+                ResumableTransferDisposition.WAIT_BEFORE_RETRY)
+        elif resp.status != 200 and resp.status != 201:
             raise ResumableUploadException(
-                'Got status %d from attempt to start resumable upload' %
-                resp.status, ResumableTransferDisposition.WAIT_BEFORE_RETRY)
+                'Got status %d from attempt to start resumable upload. '
+                'Aborting' % resp.status,
+                ResumableTransferDisposition.ABORT)
+
+        # Else we got 200 or 201 response code, indicating the resumable
+        # upload was created.
         tracker_uri = resp.getheader('Location')
         if not tracker_uri:
             raise ResumableUploadException(
@@ -330,10 +342,13 @@ def _upload_file_bytes(self, conn, http_conn, fp, file_length,
         if cb:
             cb(total_bytes_uploaded, file_length)
         if total_bytes_uploaded != file_length:
-            raise ResumableUploadException('File changed during upload: EOF at '
-                                           '%d bytes of %d byte file.' %
-                                           (total_bytes_uploaded, file_length),
-                                           ResumableTransferDisposition.ABORT)
+            # Abort (and delete the tracker file) so if the user retries
+            # they'll start a new resumable uplaod rather than potentially
+            # attempting to pick back up later where we left off.
+            raise ResumableUploadException(
+                'File changed during upload: EOF at %d bytes of %d byte file.' %
+                (total_bytes_uploaded, file_length),
+                ResumableTransferDisposition.ABORT)
         resp = http_conn.getresponse()
         body = resp.read()
         # Restore http connection debug level.
@@ -342,15 +357,23 @@ def _upload_file_bytes(self, conn, http_conn, fp, file_length,
         additional_note = ''
         if resp.status == 200:
             return resp.getheader('etag')  # Success
-        # Retry status 503 errors after a delay.
-        elif resp.status == 503:
+        elif resp.status == 408:
+          # Request Timeout. Try again later within the current process.
             disposition = ResumableTransferDisposition.WAIT_BEFORE_RETRY
-        elif resp.status == 500:
+        elif resp.status/100 == 4:
+            # Abort for any other 4xx errors.
             disposition = ResumableTransferDisposition.ABORT
-            additional_note = ('This can happen if you attempt to upload a '
-                               'different size file on a already partially '
-                               'uploaded resumable upload')
+            # Add some more informative note for particular 4xx error codes.
+            if resp.status == 400:
+                additional_note = ('This can happen for various reasons; one '
+                                   'common case is if you attempt to upload a '
+                                   'different size file on a already partially '
+                                   'uploaded resumable upload')
+        # Retry status 500 and 503 errors after a delay.
+        elif resp.status == 500 or resp.status == 503:
+            disposition = ResumableTransferDisposition.WAIT_BEFORE_RETRY
         else:
+            # Catch all for any other error codes.
             disposition = ResumableTransferDisposition.ABORT
         raise ResumableUploadException('Got response code %d while attempting '
                                        'upload (%s)%s' %
@@ -374,6 +397,7 @@ def _attempt_resumable_upload(self, key, fp, file_length, headers, cb,
                 (server_start, server_end) = (
                     self._query_server_state(conn, file_length))
                 self.server_has_bytes = server_start
+                key=key
                 if conn.debug >= 1:
                     print 'Resuming transfer.'
             except ResumableUploadException, e:
@@ -495,10 +519,20 @@ def send_file(self, key, fp, headers, cb=None, num_cb=10):
                 if debug >= 1:
                     print('Caught exception (%s)' % e.__repr__())
             except ResumableUploadException, e:
-                if e.disposition == ResumableTransferDisposition.ABORT:
+                if (e.disposition ==
+                    ResumableTransferDisposition.ABORT_CUR_PROCESS):
+                    if debug >= 1:
+                        print('Caught non-retryable ResumableUploadException '
+                              '(%s); aborting but retaining tracker file' %
+                              e.message)
+                    raise
+                elif (e.disposition ==
+                    ResumableTransferDisposition.ABORT):
                     if debug >= 1:
                         print('Caught non-retryable ResumableUploadException '
-                              '(%s)' % e.message)
+                              '(%s); aborting and removing tracker file' %
+                              e.message)
+                    self._remove_tracker_file()
                     raise
                 else:
                     if debug >= 1:
@@ -516,7 +550,7 @@ def send_file(self, key, fp, headers, cb=None, num_cb=10):
                 raise ResumableUploadException(
                     'Too many resumable upload attempts failed without '
                     'progress. You might try this upload again later',
-                    ResumableTransferDisposition.ABORT)
+                    ResumableTransferDisposition.ABORT_CUR_PROCESS)
 
             sleep_time_secs = 2**progress_less_iterations
             if debug >= 1:

boto/mturk/connection.py

@@ -515,9 +515,9 @@ def notify_workers(self, worker_ids, subject, message_text):
         """
         Send a text message to workers.
         """
-        params = {'WorkerId' : worker_ids,
-                  'Subject' : subject,
+        params = {'Subject' : subject,
                   'MessageText': message_text}
+        self.build_list_params(params, worker_ids, 'WorkerId')
 
         return self._process_request('NotifyWorkers', params)
 
@@ -579,7 +579,7 @@ def create_qualification_type(self,
             params['Test'] = test.get_as_xml()
 
         if test_duration is not None:
-            params['TestDuration'] = test_duration
+            params['TestDurationInSeconds'] = test_duration
 
         if answer_key is not None:
             if isinstance(answer_key, basestring):

boto/roboto/__init__.py

@@ -0,0 +1 @@
+#