Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Bug Fixes/Python 2.4/2.5 Support

- Exposed the size parameter for chunked connection
- Fixed issue with using chunked connection while knowing the size.
- Fixed issue with chunked connection not working with https
- Json is now being imported with legacy fallbacks
- No more with statements :(
- storage_object.send() now uses chunked_upload.
- Added md5 hash calculation/validation to send()
- send() now updates the 'model' representation based on the response.
- Python 2.4/2.5 Support tested with listing/detailing containers/objects
  uploading/downloading objects. Requires httplib2 and simplejson.
  • Loading branch information...
commit 8088180c262548a9a4e4f0606f00dbd6946d5ab2 1 parent 04dace9
@sudorandom sudorandom authored
View
11 object_storage/client.py
@@ -3,10 +3,7 @@
See COPYING for license information.
"""
-try:
- import simplejson as json
-except ImportError:
- import json
+from object_storage.utils import json
from object_storage.container import Container
from object_storage.storage_object import StorageObject
@@ -230,7 +227,7 @@ def set_metadata(self, meta, headers={}):
for k, v in headers.iteritems():
meta_headers[k] = v
for k, v in meta.iteritems():
- meta_headers["x-account-meta-{0}".format(k)] = v
+ meta_headers["x-account-meta-%s" % (k, )] = v
self.make_request('POST', headers=meta_headers)
def create_container(self, name):
@@ -351,7 +348,7 @@ def chunk_download(self, path, chunk_size=10 * 1024, headers=None):
url = self.get_url(path)
return self.conn.chunk_download(url, chunk_size=chunk_size)
- def chunk_upload(self, path, headers=None):
+ def chunk_upload(self, path, size=None, headers=None):
""" Returns a chunkable connection object at the given path
@param path: path
@@ -359,7 +356,7 @@ def chunk_upload(self, path, headers=None):
@raises ResponseError
"""
url = self.get_url(path)
- return self.conn.chunk_upload('PUT', url, headers)
+ return self.conn.chunk_upload('PUT', url, size=size, headers=headers)
def __getitem__(self, name):
""" Returns a container object with the given name """
View
6 object_storage/container.py
@@ -3,8 +3,8 @@
See COPYING for license information
"""
-import json
import os
+from object_storage.utils import json
import UserDict
from object_storage import errors
from object_storage.storage_object import StorageObject
@@ -150,7 +150,7 @@ def set_metadata(self, meta):
"""
meta_headers = {}
for k, v in meta.iteritems():
- meta_headers["x-container-meta-{0}".format(k)] = v
+ meta_headers["x-container-meta-%s" % (k, )] = v
return self.make_request('POST', headers=meta_headers)
def create(self):
@@ -315,7 +315,7 @@ def __str__(self):
return self.name
def __repr__(self):
- return 'Container({0})'.format(self.name.encode("utf-8"))
+ return 'Container(%s)' % (self.name.encode("utf-8"), )
def __iter__(self):
""" Returns an interator based on results of self.objects() """
View
53 object_storage/storage_object.py
@@ -3,10 +3,15 @@
See COPYING for license information
"""
-import json
+from object_storage.utils import json
import mimetypes
import os
+import StringIO
import UserDict
+try:
+ from hashlib import md5
+except ImportError:
+ from md5 import md5
from object_storage import errors
from object_storage.utils import get_path
@@ -91,7 +96,7 @@ def _formatter(res):
self.model = StorageObjectModel(self, self.container, self.name, res.headers)
return True
try:
- return self.make_request('HEAD', headers={'X-Context': 'cdn'}, formatter=_formatter)
+ return self.make_request('HEAD', formatter=_formatter)
except errors.NotFound:
return False
@@ -192,7 +197,7 @@ def set_metadata(self, meta):
"""
meta_headers = {}
for k, v in meta.iteritems():
- meta_headers["X-Object-Meta-{0}".format(k)] = v
+ meta_headers["X-Object-Meta-%s" % (k, )] = v
return self.make_request('POST', headers=meta_headers)
def create(self):
@@ -262,19 +267,20 @@ def chunk_download(self, chunk_size=None):
iter_content = chunk_download
__iter__ = chunk_download
- def chunk_upload(self, headers=None):
+ def chunk_upload(self, size=None, headers=None):
""" Returns a chunkable upload instance.
This is needed for transient data uploads
+ @param headers: size in bytes, if known
@param headers: extra headers to use to initialize the request
@raises: ResponseError
@return: object that responds to o.send('data') to send data
and o.finish() to finish the upload.
"""
- chunkable = self.client.chunk_upload([self.container, self.name], headers=headers)
+ chunkable = self.client.chunk_upload([self.container, self.name], size=size, headers=headers)
return chunkable
- def send(self, data):
+ def send(self, data, check_md5=True):
""" Uploads object data
@param data: either a file-like object or a string.
@@ -292,6 +298,9 @@ def send(self, data):
if hasattr(data, '__len__'):
size = len(data)
+ if isinstance(data, basestring):
+ data = StringIO.StringIO(data)
+
headers = {}
content_type = self.content_type
if not content_type:
@@ -301,12 +310,25 @@ def send(self, data):
content_type = _type or mimetypes.guess_type(self.name)[0] or 'application/octet-stream'
headers['Content-Type'] = content_type
- if size or size == 0:
- headers['Content-Length'] = str(size)
- else:
- headers['Transfer-Encoding'] = 'chunked'
+ checksum = md5()
+ transfered = 0
+ conn = self.chunk_upload(size=size, headers=headers)
+ buff = data.read(4096)
+ while len(buff) > 0:
+ conn.send(buff)
+ if check_md5:
+ checksum.update(buff)
+ transfered += len(buff)
+ buff = data.read(4096)
+ res = conn.finish()
+
+ if check_md5:
+ assert checksum.hexdigest() == res.headers['etag'], 'md5 hashes do not match'
+ res.headers['content-length'] = transfered
+ self.model = StorageObjectModel(self, self.container, self.name, res.headers)
+ headers['Content-Type'] = content_type
+ return self
- return self.make_request('PUT', data=data, headers=headers, formatter=lambda r: self)
write = send
def upload_directory(self, directory):
@@ -341,7 +363,9 @@ def load_from_filename(self, filename):
if os.path.isdir(filename):
self.upload_directory(filename)
else:
- with open(filename, 'rb') as _file:
+ try:
+ _file = open(filename, 'rb')
+ finally:
return self.send(_file)
def copy_from(self, old_obj, *args, **kwargs):
@@ -382,7 +406,8 @@ def _delete(res):
return self.delete()
def _copy_to(res):
- return new_obj.copy_from(self, *args, formatter=_delete, **kwargs)
+ kwargs['formatter'] = _delete
+ return new_obj.copy_from(self, *args, **kwargs)
return new_obj.make_request('PUT', headers={'Content-Length': '0'}, formatter=_copy_to)
def search(self, q, options=None, **kwargs):
@@ -422,7 +447,7 @@ def __str__(self):
size = 'Unknown'
if self.model:
size = self.model.get('size', 0)
- return 'StorageObject({0}, {1}, {2}B)'.format(self.container.encode("utf-8"), self.name.encode("utf-8"), size)
+ return 'StorageObject(%s, %s, %sB)' % (self.container.encode("utf-8"), self.name.encode("utf-8"), size)
__repr__ = __str__
def __enter__(self):
View
45 object_storage/transport/__init__.py
@@ -11,6 +11,7 @@
import urllib
import urllib2
+import re
class Response(object):
@@ -43,11 +44,11 @@ def get_headers(self):
""" Get default headers for this connection """
return dict([('User-Agent', consts.USER_AGENT)] + self.auth_headers.items())
- def chunk_upload(self, method, url, headers=None):
+ def chunk_upload(self, method, url, size=None, headers=None):
""" Returns new ChunkedConnection """
headers = headers or {}
headers.update(self.get_headers())
- return ChunkedUploadConnection(self, method, url, headers)
+ return ChunkedUploadConnection(self, method, url, size=size, headers=headers)
def chunk_download(self, url, chunk_size=10 * 1024):
""" Returns new ChunkedConnection """
@@ -112,26 +113,42 @@ class ChunkedUploadConnection:
send_chunk() will send more data.
finish() will end the request.
"""
- def __init__(self, conn, method, url, headers=None, size=None):
+ def __init__(self, conn, method, url, size=None, headers=None):
self.conn = conn
self.method = method
self.req = None
+ self._chunked_encoding = True
headers = headers or {}
if size is None:
- if 'Content-Length' in headers:
- del headers['Content-Length']
headers['Transfer-Encoding'] = 'chunked'
else:
+ self._chunked_encoding = False
headers['Content-Length'] = str(size)
if 'ETag' in headers:
del headers['ETag']
- url_parts = urlparse(url)
- self.req = httplib.HTTPConnection(url_parts.hostname, url_parts.port)
+ scheme, netloc, path, params, query, fragment = urlparse(url)
+ match = re.match('([a-zA-Z0-9\-\.]+):?([0-9]{2,5})?', netloc)
- path = requote_path(url_parts.path)
+ if match:
+ (host, port) = match.groups()
+ else:
+ ValueError('Invalid URL')
+
+ if not port:
+ if scheme == 'https':
+ port = 443
+ else:
+ port = 80
+
+ if scheme == 'https':
+ self.req = httplib.HTTPSConnection(host, port)
+ else:
+ self.req = httplib.HTTPConnection(host, port)
+
+ path = requote_path(path)
try:
self.req.putrequest('PUT', path)
for key, value in headers.iteritems():
@@ -143,9 +160,12 @@ def __init__(self, conn, method, url, headers=None, size=None):
def send(self, chunk):
""" Sends a chunk of data. """
try:
- self.req.send("%X\r\n" % len(chunk))
- self.req.send(chunk)
- self.req.send("\r\n")
+ if self._chunked_encoding:
+ self.req.send("%X\r\n" % len(chunk))
+ self.req.send(chunk)
+ self.req.send("\r\n")
+ else:
+ self.req.send(chunk)
except timeout, err:
raise err
except Exception, err:
@@ -154,7 +174,8 @@ def send(self, chunk):
def finish(self):
""" Finished the request out and receives a response. """
try:
- self.req.send("0\r\n\r\n")
+ if self._chunked_encoding:
+ self.req.send("0\r\n\r\n")
except timeout, err:
raise err
View
5 object_storage/transport/httplib2conn.py
@@ -8,10 +8,7 @@
from object_storage.transport import BaseAuthentication, BaseAuthenticatedConnection, Response
import httplib2
-try:
- import simplejson as json
-except ImportError:
- import json
+from object_storage.utils import json
import logging
logger = logging.getLogger(__name__)
View
6 object_storage/transport/requestsconn.py
@@ -6,11 +6,7 @@
import requests
from object_storage.transport import BaseAuthentication, BaseAuthenticatedConnection
from object_storage import errors
-
-try:
- import simplejson as json
-except ImportError:
- import json
+from object_storage.utils import json
import logging
logger = logging.getLogger(__name__)
View
2  object_storage/transport/twist.py
@@ -21,7 +21,7 @@
import urlparse
import urllib
-import json
+from object_storage.utils import json
def complete_request(resp, callback=None, load_body=True):
View
11 object_storage/utils.py
@@ -6,6 +6,17 @@
import urllib
+try:
+ import json
+except ImportError:
+ try:
+ import simplejson as json
+ except ImportError:
+ try:
+ import django.utils.simplejson as json
+ except ImportError:
+ ImportError("Requires a json parsing library")
+
def unicode_quote(s):
""" Solves an issue with url-quoting unicode strings"""
View
31 tests/unit/client.py
@@ -2,6 +2,7 @@
from mock import Mock
from object_storage.client import Client
+
class ClientTest(unittest.TestCase):
def test_instance_setup(self):
self.assert_(self.client.username == 'username', "username set")
@@ -10,68 +11,68 @@ def test_instance_setup(self):
self.assert_(self.client.object_class == self.object_class, "object_class set")
self.assert_(self.client.conn == self.connection, "connection set")
self.assert_(self.client.delimiter == '/', "default delimiter set")
-
+
def test_set_delimiter(self):
delimiter = Mock()
self.client.set_delimiter(delimiter)
self.assert_(self.client.delimiter == delimiter, "set_delimiter sets the delimiter")
-
+
def test_container(self):
self.client.container('container')
self.container_class.assert_called_once_with('container', client=self.client, headers=None)
-
+
def test_container_with_props(self):
self.client.container('container', {'properties': 'property'})
self.container_class.assert_called_once_with('container', client=self.client, headers={'properties': 'property'})
-
+
def test_get_container(self):
loaded_item = Mock()
self.container_class().load.return_value = loaded_item
result = self.client.get_container('container_name')
self.assert_(loaded_item == result)
self.container_class.assert_called_with('container_name', client=self.client, headers=None)
-
+
def test_create_container(self):
created_item = Mock()
self.container_class().create.return_value = created_item
result = self.client.create_container('container_name')
self.assert_(created_item == result, 'returns the container itself')
self.container_class.assert_called_with('container_name', client=self.client, headers=None)
-
+
def test_list_containers(self):
self.connection.storage_url = 'storage_url'
self.connection.make_request().content = '[{"name":"container_name","count":10,"bytes":100}]'
- containers = self.client.containers()
- #self.connection.make_request.assert_called_with('GET',
+ #containers = self.client.containers()
+ #self.connection.make_request.assert_called_with('GET',
# 'storage_url',
# headers=None,
# params={'format': 'json'})
-
+
def test_object(self):
self.client.storage_object('container', 'name')
self.object_class.assert_called_once_with('container', 'name', client=self.client, headers=None)
-
+
def test_object_with_props(self):
self.client.storage_object('container', 'name', {'properties': 'property'})
self.object_class.assert_called_once_with('container', 'name', client=self.client, headers={'properties': 'property'})
-
+
def test_get_object(self):
loaded_item = Mock()
self.object_class().load.return_value = loaded_item
result = self.client.get_object('object_name', 'container_name')
self.assert_(loaded_item == result, "Returns the correct object")
self.object_class.assert_called_with('object_name', 'container_name', client=self.client, headers=None)
-
+
def test_make_request(self):
self.connection.storage_url = 'storage_url'
self.client.make_request('METHOD', 'PATH')
self.connection.make_request.assert_called_once_with('METHOD', "storage_url/PATH")
-
+
def test_make_request_listpath(self):
self.connection.storage_url = 'storage_url'
self.client.make_request('METHOD', ['PATH', 'PATH2'])
self.connection.make_request.assert_called_once_with('METHOD', "storage_url/PATH/PATH2")
-
+
def test_is_dir(self):
self.assert_(self.client.is_dir() == True, "Client itself is a directory")
@@ -118,7 +119,7 @@ def setUp(self):
self.container_class = Mock()
self.object_class = Mock()
self.connection_class = Mock()
- self.client = Client( 'username', 'api_key',
+ self.client = Client('username', 'api_key',
container_class=self.container_class,
object_class=self.object_class,
connection=self.connection,
Please sign in to comment.
Something went wrong with that request. Please try again.