Permalink
Browse files

Merge pull request #1206 from jterrace/gs-test-fixes

Refactored GCS tests and added tests for GCS versioning feature.
  • Loading branch information...
2 parents cef0558 + cf6ec35 commit 01de444accb346ccea49d5db33f9e927aa3620f9 @mfschwartz mfschwartz committed Jan 2, 2013
View
@@ -98,6 +98,8 @@ def copy_key(self, new_key_name, src_bucket_name, src_key_name,
preserve_acl=False, encrypt_key=False, headers=None,
query_args=None, src_generation=None):
if src_generation:
+ if headers is None:
+ headers = {}
headers['x-goog-copy-source-generation'] = src_generation
super(Bucket, self).copy_key(new_key_name, src_bucket_name,
src_key_name, metadata=metadata,
@@ -161,7 +163,7 @@ def set_acl(self, acl_or_str, key_name='', headers=None, version_id=None,
key_name = key_name or ''
query_args = STANDARD_ACL
if generation:
- query_args += '&generation=%d' % generation
+ query_args += '&generation=%s' % str(generation)
if isinstance(acl_or_str, Policy):
raise InvalidAclError('Attempt to set S3 Policy on GS ACL')
elif isinstance(acl_or_str, ACL):
@@ -204,7 +206,7 @@ def get_acl(self, key_name='', headers=None, version_id=None,
and is therefore ignored here."""
query_args = STANDARD_ACL
if generation:
- query_args += '&generation=%d' % generation
+ query_args += '&generation=%s' % str(generation)
return self.get_acl_helper(key_name, headers, query_args)
def get_def_acl(self, key_name='', headers=None):
@@ -237,7 +239,7 @@ def set_canned_acl(self, acl_str, key_name='', headers=None,
Google Cloud Storage buckets and is therefore ignored here."""
query_args = STANDARD_ACL
if generation:
- query_args += '&generation=%d' % generation
+ query_args += '&generation=%s' % str(generation)
return self.set_canned_acl_helper(acl_str, key_name, headers,
query_args=query_args)
No changes.
@@ -20,17 +20,17 @@
# IN THE SOFTWARE.
"""
-Test harnass that allows us to raise exceptions, change file content,
+Test harness that allows us to raise exceptions, change file content,
and record the byte transfer callback sequence, to test various resumable
-upload and download cases. The 'call' method of this harnass can be passed
+upload and download cases. The 'call' method of this harness can be passed
as the 'cb' parameter to boto.s3.Key.send_file() and boto.s3.Key.get_file(),
allowing testing of various file upload/download conditions.
"""
import socket
-class CallbackTestHarnass(object):
+class CallbackTestHarness(object):
def __init__(self, fail_after_n_bytes=0, num_times_to_fail=1,
exception=socket.error('mock socket error', 0),
@@ -48,7 +48,7 @@ def __init__(self, fail_after_n_bytes=0, num_times_to_fail=1,
def call(self, total_bytes_transferred, unused_total_size):
"""
- To use this test harnass, pass the 'call' method of the instantiated
+ To use this test harness, pass the 'call' method of the instantiated
object as the cb param to the set_contents_from_file() or
get_contents_to_file() call.
"""
@@ -37,25 +37,10 @@
from boto.gs.cors import Cors
from boto import handler
from boto import storage_uri
-from boto.provider import Provider
+from tests.integration.gs.util import has_google_credentials
from tests.unit import unittest
-_HAS_GOOGLE_CREDENTIALS = None
-
-
-def has_google_credentials():
- global _HAS_GOOGLE_CREDENTIALS
- if _HAS_GOOGLE_CREDENTIALS is None:
- provider = Provider('google')
- if provider.access_key is None or provider.secret_key is None:
- _HAS_GOOGLE_CREDENTIALS = False
- else:
- _HAS_GOOGLE_CREDENTIALS = True
- return _HAS_GOOGLE_CREDENTIALS
-
-
-
@unittest.skipUnless(has_google_credentials(),
"Google credentials are required to run the Google "
"Cloud Storage tests. Update your boto.cfg to run "
@@ -45,7 +45,8 @@
from boto.exception import ResumableTransferDisposition
from boto.exception import ResumableDownloadException
from boto.exception import StorageResponseError
-from cb_test_harnass import CallbackTestHarnass
+from cb_test_harness import CallbackTestHarness
+from tests.integration.gs.util import has_google_credentials
# We don't use the OAuth2 authentication plugin directly; importing it here
# ensures that it's loaded and available by default.
@@ -57,6 +58,10 @@
pass
+@unittest.skipUnless(has_google_credentials(),
+ "Google credentials are required to run the Google "
+ "Cloud Storage tests. Update your boto.cfg to run "
+ "these tests.")
class ResumableDownloadTests(unittest.TestCase):
"""
Resumable download test suite.
@@ -182,17 +187,17 @@ def test_failed_download_with_persistent_tracker(self):
"""
Tests that failed resumable download leaves a correct tracker file
"""
- harnass = CallbackTestHarnass()
+ harness = CallbackTestHarness()
res_download_handler = ResumableDownloadHandler(
tracker_file_name=self.tracker_file_name, num_retries=0)
try:
self.small_src_key.get_contents_to_file(
- self.dst_fp, cb=harnass.call,
+ self.dst_fp, cb=harness.call,
res_download_handler=res_download_handler)
self.fail('Did not get expected ResumableDownloadException')
except ResumableDownloadException, e:
# We'll get a ResumableDownloadException at this point because
- # of CallbackTestHarnass (above). Check that the tracker file was
+ # of CallbackTestHarness (above). Check that the tracker file was
# created correctly.
self.assertEqual(e.disposition,
ResumableTransferDisposition.ABORT_CUR_PROCESS)
@@ -209,10 +214,10 @@ def test_retryable_exception_recovery(self):
"""
# Test one of the RETRYABLE_EXCEPTIONS.
exception = ResumableDownloadHandler.RETRYABLE_EXCEPTIONS[0]
- harnass = CallbackTestHarnass(exception=exception)
+ harness = CallbackTestHarness(exception=exception)
res_download_handler = ResumableDownloadHandler(num_retries=1)
self.small_src_key.get_contents_to_file(
- self.dst_fp, cb=harnass.call,
+ self.dst_fp, cb=harness.call,
res_download_handler=res_download_handler)
# Ensure downloaded object has correct content.
self.assertEqual(self.small_src_key_size,
@@ -225,10 +230,10 @@ def test_broken_pipe_recovery(self):
Tests handling of a Broken Pipe (which interacts with an httplib bug)
"""
exception = IOError(errno.EPIPE, "Broken pipe")
- harnass = CallbackTestHarnass(exception=exception)
+ harness = CallbackTestHarness(exception=exception)
res_download_handler = ResumableDownloadHandler(num_retries=1)
self.small_src_key.get_contents_to_file(
- self.dst_fp, cb=harnass.call,
+ self.dst_fp, cb=harness.call,
res_download_handler=res_download_handler)
# Ensure downloaded object has correct content.
self.assertEqual(self.small_src_key_size,
@@ -240,12 +245,12 @@ def test_non_retryable_exception_handling(self):
"""
Tests resumable download that fails with a non-retryable exception
"""
- harnass = CallbackTestHarnass(
+ harness = CallbackTestHarness(
exception=OSError(errno.EACCES, 'Permission denied'))
res_download_handler = ResumableDownloadHandler(num_retries=1)
try:
self.small_src_key.get_contents_to_file(
- self.dst_fp, cb=harnass.call,
+ self.dst_fp, cb=harness.call,
res_download_handler=res_download_handler)
self.fail('Did not get expected OSError')
except OSError, e:
@@ -257,11 +262,11 @@ def test_failed_and_restarted_download_with_persistent_tracker(self):
Tests resumable download that fails once and then completes,
with tracker file
"""
- harnass = CallbackTestHarnass()
+ harness = CallbackTestHarness()
res_download_handler = ResumableDownloadHandler(
tracker_file_name=self.tracker_file_name, num_retries=1)
self.small_src_key.get_contents_to_file(
- self.dst_fp, cb=harnass.call,
+ self.dst_fp, cb=harness.call,
res_download_handler=res_download_handler)
# Ensure downloaded object has correct content.
self.assertEqual(self.small_src_key_size,
@@ -289,16 +294,16 @@ def test_multiple_in_process_failures_then_succeed_with_tracker_file(self):
Tests resumable download that fails completely in one process,
then when restarted completes, using a tracker file
"""
- # Set up test harnass that causes more failures than a single
+ # Set up test harness that causes more failures than a single
# ResumableDownloadHandler instance will handle, writing enough data
# before the first failure that some of it survives that process run.
- harnass = CallbackTestHarnass(
+ harness = CallbackTestHarness(
fail_after_n_bytes=self.larger_src_key_size/2, num_times_to_fail=2)
res_download_handler = ResumableDownloadHandler(
tracker_file_name=self.tracker_file_name, num_retries=0)
try:
self.larger_src_key.get_contents_to_file(
- self.dst_fp, cb=harnass.call,
+ self.dst_fp, cb=harness.call,
res_download_handler=res_download_handler)
self.fail('Did not get expected ResumableDownloadException')
except ResumableDownloadException, e:
@@ -308,7 +313,7 @@ def test_multiple_in_process_failures_then_succeed_with_tracker_file(self):
self.assertTrue(os.path.exists(self.tracker_file_name))
# Try it one more time; this time should succeed.
self.larger_src_key.get_contents_to_file(
- self.dst_fp, cb=harnass.call,
+ self.dst_fp, cb=harness.call,
res_download_handler=res_download_handler)
self.assertEqual(self.larger_src_key_size,
get_cur_file_size(self.dst_fp))
@@ -317,21 +322,21 @@ def test_multiple_in_process_failures_then_succeed_with_tracker_file(self):
self.assertFalse(os.path.exists(self.tracker_file_name))
# Ensure some of the file was downloaded both before and after failure.
self.assertTrue(
- len(harnass.transferred_seq_before_first_failure) > 1 and
- len(harnass.transferred_seq_after_first_failure) > 1)
+ len(harness.transferred_seq_before_first_failure) > 1 and
+ len(harness.transferred_seq_after_first_failure) > 1)
def test_download_with_inital_partial_download_before_failure(self):
"""
Tests resumable download that successfully downloads some content
before it fails, then restarts and completes
"""
- # Set up harnass to fail download after several hundred KB so download
+ # Set up harness to fail download after several hundred KB so download
# server will have saved something before we retry.
- harnass = CallbackTestHarnass(
+ harness = CallbackTestHarness(
fail_after_n_bytes=self.larger_src_key_size/2)
res_download_handler = ResumableDownloadHandler(num_retries=1)
self.larger_src_key.get_contents_to_file(
- self.dst_fp, cb=harnass.call,
+ self.dst_fp, cb=harness.call,
res_download_handler=res_download_handler)
# Ensure downloaded object has correct content.
self.assertEqual(self.larger_src_key_size,
@@ -340,8 +345,8 @@ def test_download_with_inital_partial_download_before_failure(self):
self.larger_src_key.get_contents_as_string())
# Ensure some of the file was downloaded both before and after failure.
self.assertTrue(
- len(harnass.transferred_seq_before_first_failure) > 1 and
- len(harnass.transferred_seq_after_first_failure) > 1)
+ len(harness.transferred_seq_before_first_failure) > 1 and
+ len(harness.transferred_seq_after_first_failure) > 1)
def test_zero_length_object_download(self):
"""
Oops, something went wrong.

0 comments on commit 01de444

Please sign in to comment.