Skip to content

Commit

Permalink
crc
Browse files Browse the repository at this point in the history
  • Loading branch information
@baiyubin2020
baiyubin2020 committed Aug 25, 2016
1 parent 276a9f6 commit a6e3e99
Show file tree
Hide file tree
Showing 5 changed files with 64 additions and 55 deletions.
6 changes: 3 additions & 3 deletions oss2/api.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -348,7 +348,7 @@ def put_object(self, key, data,
result = PutObjectResult(resp)

if self.enable_crc:
utils.check_crc('put', data.get_crc(), result.crc)
utils.check_crc('put', data.crc, result.crc)

return result

Expand Down Expand Up @@ -411,7 +411,7 @@ def append_object(self, key, position, data,
result = AppendObjectResult(resp)

if self.enable_crc and init_crc is not None:
utils.check_crc('append', data.get_crc(), result.crc)
utils.check_crc('append', data.crc, result.crc)

return result

Expand Down Expand Up @@ -644,7 +644,7 @@ def upload_part(self, key, upload_id, part_number, data, progress_callback=None)
result = PutObjectResult(resp)

if self.enable_crc:
utils.check_crc('put', data.get_crc(), result.crc)
utils.check_crc('put', data.crc, result.crc)

return result

Expand Down
20 changes: 11 additions & 9 deletions oss2/models.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,17 +77,17 @@ def __init__(self, resp):


class GetObjectResult(HeadObjectResult):
def __init__(self, resp, progress_callback=None, crc_callback=None):
def __init__(self, resp, progress_callback=None, crc_enabled=False):
super(GetObjectResult, self).__init__(resp)
self.crc_callback = crc_callback
self.__crc_enabled = crc_enabled

if progress_callback:
self.stream = make_progress_adapter(self.resp, progress_callback, self.content_length)
else:
self.stream = self.resp

self.crc = _hget(self.headers, 'x-oss-hash-crc64ecma', int)
if self.crc_callback:
self.__crc = _hget(self.headers, 'x-oss-hash-crc64ecma', int)
if self.__crc_enabled:
self.stream = make_crc_adapter(self.stream)

def read(self, amt=None):
Expand All @@ -96,14 +96,16 @@ def read(self, amt=None):
def __iter__(self):
return iter(self.stream)

def get_client_crc(self):
if self.crc_callback:
return self.stream.get_crc()
@property
def client_crc(self):
if self.__crc_enabled:
return self.stream.crc
else:
return None

def get_oss_crc(self):
return self.crc
@property
def oss_crc(self):
return self.__crc


class PutObjectResult(RequestResult):
Expand Down
73 changes: 40 additions & 33 deletions oss2/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -205,54 +205,62 @@ def make_crc_adapter(data, init_crc=0):
if hasattr(data, '__len__') or (hasattr(data, 'seek') and hasattr(data, 'tell')):
return _BytesAndFileAdapter(data,
size=_get_data_size(data),
crc_callback=crc_callback,
crc_callback=_crc_callback,
init_crc=init_crc)
# file-like object
elif hasattr(data, 'read'):
return _FileLikeAdapter(data, crc_callback=crc_callback, init_crc=init_crc)
return _FileLikeAdapter(data, crc_callback=_crc_callback, init_crc=init_crc)
# iterator
elif hasattr(data, '__iter__'):
return _IterableAdapter(data, crc_callback=crc_callback, init_crc=init_crc)
return _IterableAdapter(data, crc_callback=_crc_callback, init_crc=init_crc)
else:
raise ClientError('{0} is not a file object, nor an iterator'.format(data.__class__.__name__))

def crc_callback(crc, data):
def _crc_callback(crc, data):
crc.update(data)

def check_crc(operation, client_crc, oss_crc):
if client_crc != oss_crc:
raise CrcError('the crc of {0} between client and oss is not inconsistent'.format(operation))

def _invoke_crc_updater(crc_callback, __crc_calculator, content):
if crc_callback:
crc_callback(__crc_calculator, content)

def _invoke_progress_callback(progress_callback, consumed_bytes, total_bytes):
if progress_callback:
progress_callback(consumed_bytes, total_bytes)

class _IterableAdapter(object):
def __init__(self, data, progress_callback=None, crc_callback=None, init_crc=0):
self.iter = iter(data)
self.progress_callback = progress_callback
self.offset = 0

self.crc_callback = crc_callback
self.__crc_calculator = None
if crc_callback:
self.crc = Crc64(init_crc)
self.__crc_calculator = Crc64(init_crc)

def __iter__(self):
return self

def __next__(self):
return self.next()

def next(self):
if self.progress_callback:
self.progress_callback(self.offset, None)
def next(self):
_invoke_progress_callback(self.progress_callback, self.offset, None)

content = next(self.iter)
self.offset += len(content)

if self.crc_callback:
self.crc_callback(self.crc, content)

_invoke_crc_updater(self.crc_callback, self.__crc_calculator, content)

return content

def get_crc(self):
return self.crc.get_crc_value()
@property
def crc(self):
return self.__crc_calculator.crc_value()

class _FileLikeAdapter(object):
"""通过这个适配器，可以给无法确定内容长度的 `fileobj` 加上进度监控。
Expand All @@ -266,8 +274,9 @@ def __init__(self, fileobj, progress_callback=None, crc_callback=None, init_crc=
self.offset = 0

self.crc_callback = crc_callback
self.__crc_calculator = None
if crc_callback:
self.crc = Crc64(init_crc)
self.__crc_calculator = Crc64(init_crc)

def __iter__(self):
return self
Expand All @@ -286,21 +295,19 @@ def next(self):
def read(self, amt=None):
content = self.fileobj.read(amt)
if not content:
if self.progress_callback:
self.progress_callback(self.offset, None)
_invoke_progress_callback(self.progress_callback, self.offset, None)
else:
if self.progress_callback:
self.progress_callback(self.offset, None)
_invoke_progress_callback(self.progress_callback, self.offset, None)

self.offset += len(content)

if self.crc_callback:
self.crc_callback(self.crc, content)

_invoke_crc_updater(self.crc_callback, self.__crc_calculator, content)

return content

def get_crc(self):
return self.crc.get_crc_value()
@property
def crc(self):
return self.__crc_calculator.crc_value()

class _BytesAndFileAdapter(object):
"""通过这个适配器，可以给 `data` 加上进度监控。
Expand All @@ -317,8 +324,9 @@ def __init__(self, data, progress_callback=None, size=None, crc_callback=None, i
self.offset = 0

self.crc_callback = crc_callback
self.__crc_calculator = None
if crc_callback:
self.crc = Crc64(init_crc)
self.__crc_calculator = Crc64(init_crc)

def __len__(self):
return self.size
Expand Down Expand Up @@ -358,17 +366,16 @@ def read(self, amt=None):
content = self.data.read(bytes_to_read)

self.offset += bytes_to_read

if self.progress_callback:
self.progress_callback(min(self.offset, self.size), self.size)

if self.crc_callback:
self.crc_callback(self.crc, content)

_invoke_progress_callback(self.progress_callback, min(self.offset, self.size), self.size)

_invoke_crc_updater(self.crc_callback, self.__crc_calculator, content)

return content

def get_crc(self):
return self.crc.get_crc_value()
@property
def crc(self):
return self.__crc_calculator.crc_value()

class Crc64(object):

Expand All @@ -381,7 +388,7 @@ def __init__(self, init_crc=0):
def update(self, data):
self.crc64.update(data)

def get_crc_value(self):
def crc_value(self):
return self.crc64.crcValue


Expand Down
18 changes: 9 additions & 9 deletions tests/test_object.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,9 +38,9 @@ def assert_result(result):
get_result = self.bucket.get_object(key)
self.assertEqual(get_result.read(), content)
assert_result(get_result)
self.assertTrue(get_result.get_client_crc() is not None)
self.assertTrue(get_result.get_oss_crc() is not None)
self.assertTrue(get_result.get_client_crc() == get_result.get_oss_crc())
self.assertTrue(get_result.client_crc is not None)
self.assertTrue(get_result.oss_crc is not None)
self.assertTrue(get_result.client_crc == get_result.oss_crc)

head_result = self.bucket.head_object(key)
assert_result(head_result)
Expand Down Expand Up @@ -103,10 +103,10 @@ def test_streaming(self):
result = self.bucket.put_object(dst_key, src)

# verify
self.assertTrue(src.get_client_crc() is not None)
self.assertTrue(src.get_oss_crc() is not None)
self.assertEqual(src.get_client_crc(), src.get_oss_crc())
self.assertEqual(result.crc, src.get_oss_crc())
self.assertTrue(src.client_crc is not None)
self.assertTrue(src.oss_crc is not None)
self.assertEqual(src.client_crc, src.oss_crc)
self.assertEqual(result.crc, src.oss_crc)
self.assertEqual(self.bucket.get_object(src_key).read(), self.bucket.get_object(dst_key).read())

def make_generator(self, content, chunk_size):
Expand Down Expand Up @@ -442,8 +442,8 @@ def test_disable_crc(self):
# get
get_result = bucket.get_object(key)
self.assertEqual(get_result.read(), content)
self.assertTrue(get_result.get_client_crc() is None)
self.assertTrue(get_result.get_oss_crc())
self.assertTrue(get_result.client_crc is None)
self.assertTrue(get_result.oss_crc)

bucket.delete_object(key)

Expand Down
2 changes: 1 addition & 1 deletion unittests/common.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -276,7 +276,7 @@ def get_length(data):
def calc_crc(data):
crc = oss2.utils.Crc64()
crc.update(data)
return crc.get_crc_value()
return crc.crc_value()


class MockSocket(object):
Expand Down

0 comments on commit a6e3e99

Please sign in to comment.