Merge branch 's3-get-bucket-fix' into develop

Fixes #2082, #2078.

boto/gs/connection.py

@@ -103,3 +103,27 @@ def create_bucket(self, bucket_name, headers=None,
             raise self.provider.storage_response_error(
                 response.status, response.reason, body)
 
+    def get_bucket(self, bucket_name, validate=True, headers=None):
+        """
+        Retrieves a bucket by name.
+
+        If the bucket does not exist, an ``S3ResponseError`` will be raised. If
+        you are unsure if the bucket exists or not, you can use the
+        ``S3Connection.lookup`` method, which will either return a valid bucket
+        or ``None``.
+
+        :type bucket_name: string
+        :param bucket_name: The name of the bucket
+
+        :type headers: dict
+        :param headers: Additional headers to pass along with the request to
+            AWS.
+
+        :type validate: boolean
+        :param validate: If ``True``, it will try to fetch all keys within the
+            given bucket. (Default: ``True``)
+        """
+        bucket = self.bucket_class(self, bucket_name)
+        if validate:
+            bucket.get_all_keys(headers, maxkeys=0)
+        return bucket

boto/s3/connection.py

@@ -439,6 +439,23 @@ def get_bucket(self, bucket_name, validate=True, headers=None):
         ``S3Connection.lookup`` method, which will either return a valid bucket
         or ``None``.
 
+        If ``validate=False`` is passed, no request is made to the service (no
+        charge/communication delay). This is only safe to do if you are **sure**
+        the bucket exists.
+
+        If the default ``validate=True`` is passed, a request is made to the
+        service to ensure the bucket exists. Prior to Boto v2.25.0, this fetched
+        a list of keys (but with a max limit set to ``0``, always returning an empty
+        list) in the bucket (& included better error messages), at an
+        increased expense. As of Boto v2.25.0, this now performs a HEAD request
+        (less expensive but worse error messages).
+
+        If you were relying on parsing the error message before, you should call
+        something like::
+
+            bucket = conn.get_bucket('<bucket_name>', validate=False)
+            bucket.get_all_keys(maxkeys=0)
+
         :type bucket_name: string
         :param bucket_name: The name of the bucket
 
@@ -447,13 +464,58 @@ def get_bucket(self, bucket_name, validate=True, headers=None):
             AWS.
 
         :type validate: boolean
-        :param validate: If ``True``, it will try to fetch all keys within the
-            given bucket. (Default: ``True``)
+        :param validate: If ``True``, it will try to verify the bucket exists
+            on the service-side. (Default: ``True``)
         """
-        bucket = self.bucket_class(self, bucket_name)
         if validate:
-            bucket.get_all_keys(headers, maxkeys=0)
-        return bucket
+            return self.head_bucket(bucket_name, headers=headers)
+        else:
+            return self.bucket_class(self, bucket_name)
+
+    def head_bucket(self, bucket_name, headers=None):
+        """
+        Determines if a bucket exists by name.
+
+        If the bucket does not exist, an ``S3ResponseError`` will be raised.
+
+        :type bucket_name: string
+        :param bucket_name: The name of the bucket
+
+        :type headers: dict
+        :param headers: Additional headers to pass along with the request to
+            AWS.
+
+        :returns: A <Bucket> object
+        """
+        response = self.make_request('HEAD', bucket_name, headers=headers)
+        body = response.read()
+        if response.status == 200:
+            return self.bucket_class(self, bucket_name)
+        elif response.status == 403:
+            # For backward-compatibility, we'll populate part of the exception
+            # with the most-common default.
+            err = self.provider.storage_response_error(
+                response.status,
+                response.reason,
+                body
+            )
+            err.error_code = 'AccessDenied'
+            err.error_message = 'Access Denied'
+            raise err
+        elif response.status == 404:
+            # For backward-compatibility, we'll populate part of the exception
+            # with the most-common default.
+            err = self.provider.storage_response_error(
+                response.status,
+                response.reason,
+                body
+            )
+            err.error_code = 'NoSuchBucket'
+            err.error_message = 'The specified bucket does not exist'
+            raise err
+        else:
+            raise self.provider.storage_response_error(
+                response.status, response.reason, body)
 
     def lookup(self, bucket_name, validate=True, headers=None):
         """

docs/source/s3_tut.rst

@@ -175,6 +175,26 @@ override this behavior by passing ``validate=False``.::
 
     >>> nonexistent = conn.get_bucket('i-dont-exist-at-all', validate=False)
 
+.. versionchanged:: 2.25.0
+.. warning::
+
+    If ``validate=False`` is passed, no request is made to the service (no
+    charge/communication delay). This is only safe to do if you are **sure**
+    the bucket exists.
+
+    If the default ``validate=True`` is passed, a request is made to the
+    service to ensure the bucket exists. Prior to Boto v2.25.0, this fetched
+    a list of keys (but with a max limit set to ``0``, always returning an empty
+    list) in the bucket (& included better error messages), at an
+    increased expense. As of Boto v2.25.0, this now performs a HEAD request
+    (less expensive but worse error messages).
+
+    If you were relying on parsing the error message before, you should call
+    something like::
+
+        bucket = conn.get_bucket('<bucket_name>', validate=False)
+        bucket.get_all_keys(maxkeys=0)
+
 If the bucket does not exist, a ``S3ResponseError`` will commonly be thrown. If
 you'd rather not deal with any exceptions, you can use the ``lookup`` method.::
 
@@ -184,6 +204,7 @@ you'd rather not deal with any exceptions, you can use the ``lookup`` method.::
     ...
     No such bucket!
 
+
 Deleting A Bucket
 -----------------
 

tests/unit/s3/test_bucket.py

@@ -113,23 +113,23 @@ def test__get_all_query_args(self):
             'initial=1&bar=%E2%98%83&max-keys=0&foo=true&some-other=thing'
         )
 
-    @patch.object(Bucket, 'get_all_keys')
-    def test_bucket_copy_key_no_validate(self, mock_get_all_keys):
+    @patch.object(S3Connection, 'head_bucket')
+    def test_bucket_copy_key_no_validate(self, mock_head_bucket):
         self.set_http_response(status_code=200)
         bucket = self.service_connection.create_bucket('mybucket')
 
-        self.assertFalse(mock_get_all_keys.called)
+        self.assertFalse(mock_head_bucket.called)
         self.service_connection.get_bucket('mybucket', validate=True)
-        self.assertTrue(mock_get_all_keys.called)
+        self.assertTrue(mock_head_bucket.called)
 
-        mock_get_all_keys.reset_mock()
-        self.assertFalse(mock_get_all_keys.called)
+        mock_head_bucket.reset_mock()
+        self.assertFalse(mock_head_bucket.called)
         try:
             bucket.copy_key('newkey', 'srcbucket', 'srckey', preserve_acl=True)
         except:
             # Will throw because of empty response.
             pass
-        self.assertFalse(mock_get_all_keys.called)
+        self.assertFalse(mock_head_bucket.called)
 
     @patch.object(Bucket, '_get_all')
     def test_bucket_encoding(self, mock_get_all):

tests/unit/s3/test_connection.py

@@ -20,12 +20,14 @@
 # IN THE SOFTWARE.
 #
 import mock
+import time
 
 from tests.unit import unittest
 from tests.unit import AWSMockServiceTestCase
 from tests.unit import MockServiceWithConfigTestCase
 
 from boto.s3.connection import S3Connection, HostRequiredError
+from boto.s3.connection import S3ResponseError, Bucket
 
 
 class TestSignatureAlteration(AWSMockServiceTestCase):
@@ -124,5 +126,53 @@ def test_unicode_calling_format(self):
         self.service_connection.get_all_buckets()
 
 
+class TestHeadBucket(AWSMockServiceTestCase):
+    connection_class = S3Connection
+
+    def default_body(self):
+        # HEAD requests always have an empty body.
+        return ""
+
+    def test_head_bucket_success(self):
+        self.set_http_response(status_code=200)
+        buck = self.service_connection.head_bucket('my-test-bucket')
+        self.assertTrue(isinstance(buck, Bucket))
+        self.assertEqual(buck.name, 'my-test-bucket')
+
+    def test_head_bucket_forbidden(self):
+        self.set_http_response(status_code=403)
+
+        with self.assertRaises(S3ResponseError) as cm:
+            self.service_connection.head_bucket('cant-touch-this')
+
+        err = cm.exception
+        self.assertEqual(err.status, 403)
+        self.assertEqual(err.error_code, 'AccessDenied')
+        self.assertEqual(err.message, 'Access Denied')
+
+    def test_head_bucket_notfound(self):
+        self.set_http_response(status_code=404)
+
+        with self.assertRaises(S3ResponseError) as cm:
+            self.service_connection.head_bucket('totally-doesnt-exist')
+
+        err = cm.exception
+        self.assertEqual(err.status, 404)
+        self.assertEqual(err.error_code, 'NoSuchBucket')
+        self.assertEqual(err.message, 'The specified bucket does not exist')
+
+    def test_head_bucket_other(self):
+        self.set_http_response(status_code=405)
+
+        with self.assertRaises(S3ResponseError) as cm:
+            self.service_connection.head_bucket('you-broke-it')
+
+        err = cm.exception
+        self.assertEqual(err.status, 405)
+        # We don't have special-cases for this error status.
+        self.assertEqual(err.error_code, None)
+        self.assertEqual(err.message, '')
+
+
 if __name__ == "__main__":
     unittest.main()