From f1848c6b96c636461d8ddce1ac3f541b6a24c72d Mon Sep 17 00:00:00 2001 From: tiedu Date: Thu, 26 Apr 2018 00:45:15 +0800 Subject: [PATCH 1/3] Modify lifecycle --- qcloud_cos/cos_client.py | 39 +++++++++++++++++++++++++++------------ ut/test.py | 23 ++++++++++++++++++++--- 2 files changed, 47 insertions(+), 15 deletions(-) diff --git a/qcloud_cos/cos_client.py b/qcloud_cos/cos_client.py index b33ef236..45e41c07 100644 --- a/qcloud_cos/cos_client.py +++ b/qcloud_cos/cos_client.py @@ -391,7 +391,7 @@ def delete_objects(self, Bucket, Delete={}, **kwargs): auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key), headers=headers) data = xml_to_dict(rt.text) - data = format_dict(data, ['Deleted', 'Error']) + format_dict(data, ['Deleted', 'Error']) return data def head_object(self, Bucket, Key, **kwargs): @@ -723,9 +723,9 @@ def list_parts(self, Bucket, Key, UploadId, EncodingType='', MaxParts=1000, Part headers=headers, params=params) data = xml_to_dict(rt.text) - data = format_dict(data, ['Part']) + format_dict(data, ['Part']) if decodeflag: - data = decode_result(data, ['Key'], []) + decode_result(data, ['Key'], []) return data def put_object_acl(self, Bucket, Key, AccessControlPolicy={}, **kwargs): @@ -938,9 +938,9 @@ def list_objects(self, Bucket, Prefix="", Delimiter="", Marker="", MaxKeys=1000, headers=headers, auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key)) data = xml_to_dict(rt.text) - data = format_dict(data, ['Contents', 'CommonPrefixes']) + format_dict(data, ['Contents', 'CommonPrefixes']) if decodeflag: - data = decode_result( + decode_result( data, [ 'Prefix', @@ -1006,9 +1006,9 @@ def list_objects_versions(self, Bucket, Prefix="", Delimiter="", KeyMarker="", V headers=headers, auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key)) data = xml_to_dict(rt.text) - data = format_dict(data, ['Version', 'DeleteMarker', 'CommonPrefixes']) + format_dict(data, ['Version', 'DeleteMarker', 'CommonPrefixes']) if decodeflag: - data = decode_result( + decode_result( data, [ 'Prefix', @@ -1078,9 +1078,9 @@ def list_multipart_uploads(self, Bucket, Prefix="", Delimiter="", KeyMarker="", auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key)) data = xml_to_dict(rt.text) - data = format_dict(data, ['Upload', 'CommonPrefixes']) + format_dict(data, ['Upload', 'CommonPrefixes']) if decodeflag: - data = decode_result( + decode_result( data, [ 'Prefix', @@ -1349,7 +1349,17 @@ def put_bucket_lifecycle(self, Bucket, LifecycleConfiguration={}, **kwargs): LifecycleConfiguration=lifecycle_config ) """ - lst = ['', '', '', ''] # 类型为list的标签 + # 类型为list的标签 + lst = [ + '', + '', + '', + '', + '', + '', + '', + '' + ] xml_config = format_xml(data=LifecycleConfiguration, root='LifecycleConfiguration', lst=lst) headers = mapped(kwargs) headers['Content-MD5'] = get_md5(xml_config) @@ -1393,7 +1403,12 @@ def get_bucket_lifecycle(self, Bucket, **kwargs): auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key), headers=headers) data = xml_to_dict(rt.text) - data = format_dict(data, ['Rule']) + format_dict(data, ['Rule']) + if 'Rule' in data.keys(): + for rule in data['Rule']: + format_dict(rule, ['Transition', 'NoncurrentVersionTransition']) + if 'Filter' in rule.keys(): + format_dict(rule['Filter'], ['Tag']) return data def delete_bucket_lifecycle(self, Bucket, **kwargs): @@ -1596,7 +1611,7 @@ def get_bucket_replication(self, Bucket, **kwargs): auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key), headers=headers) data = xml_to_dict(rt.text) - data = format_dict(data, ['Rule']) + format_dict(data, ['Rule']) return data def delete_bucket_replication(self, Bucket, **kwargs): diff --git a/ut/test.py b/ut/test.py index a4315913..b835c13d 100644 --- a/ut/test.py +++ b/ut/test.py @@ -430,10 +430,27 @@ def test_put_get_delete_lifecycle(): lifecycle_config = { 'Rule': [ { - 'Expiration': {'Date': get_date(2030, 5, 1)}, - 'ID': '123', - 'Filter': {'Prefix': ''}, 'Status': 'Enabled', + 'Filter': { + # 作用于带标签键 datalevel 和值 backup 的标签的对象 + 'Tag': [ + { + 'Key': 'datalevel', + 'Value': 'backup' + } + ] + }, + 'Transation': [ + { + # 30天后转换为Standard_IA + 'Days': 30, + 'StorageClass': 'Standard_IA' + } + ], + 'Expiration': { + # 3650天后过期删除 + 'Days': 3650 + } } ] } From c856672ecea71391909dcf87bdd481ca781e8a3c Mon Sep 17 00:00:00 2001 From: tiedu Date: Fri, 4 May 2018 11:19:59 +0800 Subject: [PATCH 2/3] fix some bugs and modify ut --- qcloud_cos/cos_auth.py | 3 +- qcloud_cos/cos_client.py | 5 +-- ut/test.py | 74 ++++++++++++++++++++++++++++------------ 3 files changed, 58 insertions(+), 24 deletions(-) diff --git a/qcloud_cos/cos_auth.py b/qcloud_cos/cos_auth.py index 4a010443..de6af792 100644 --- a/qcloud_cos/cos_auth.py +++ b/qcloud_cos/cos_auth.py @@ -54,12 +54,13 @@ def __call__(self, r): path = self._path uri_params = self._params headers = filter_headers(r.headers) + uri_params = dict([(k.lower(), v.lower()) for k, v in uri_params.items()]) # reserved keywords in headers urlencode are -_.~, notice that / should be encoded and space should not be encoded to plus sign(+) headers = dict([(k.lower(), quote(v, '-_.~')) for k, v in headers.items()]) # headers中的key转换为小写，value进行encode format_str = "{method}\n{host}\n{params}\n{headers}\n".format( method=r.method.lower(), host=path, - params=urllib.urlencode(sorted(uri_params.items())), + params=urllib.urlencode(sorted(uri_params.items())).replace('+', '%20'), headers='&'.join(map(lambda (x, y): "%s=%s" % (x, y), sorted(headers.items()))) ) logger.debug("format str: " + format_str) diff --git a/qcloud_cos/cos_client.py b/qcloud_cos/cos_client.py index 45e41c07..0aa4c402 100644 --- a/qcloud_cos/cos_client.py +++ b/qcloud_cos/cos_client.py @@ -1829,11 +1829,12 @@ def _check_all_upload_parts(self, bucket, key, uploadid, local_path, parts_num, UploadId=uploadid, PartNumberMarker=part_number_marker ) - parts_info.extend(response['Part']) + if 'Part' in response: + parts_info.extend(response['Part']) if response['IsTruncated'] == 'false': list_over_status = True else: - part_number_marker = int(response['NextMarker']) + part_number_marker = int(response['NextPartNumberMarker']) for part in parts_info: part_num = int(part['PartNumber']) # 如果分块数量大于本地计算出的最大数量,校验失败 diff --git a/ut/test.py b/ut/test.py index b835c13d..937496b2 100644 --- a/ut/test.py +++ b/ut/test.py @@ -12,7 +12,8 @@ SECRET_ID = os.environ["SECRET_ID"] SECRET_KEY = os.environ["SECRET_KEY"] -test_bucket = "test01-1252448703" +TRAVIS_FLAG = os.environ["TRAVIS_FLAG"] +test_bucket = 'cos-python-v5-test-' + str(sys.version_info[0]) + '-' + str(sys.version_info[1]) + '-' + '1252448703' test_object = "test.txt" special_file_name = "中文" + "→↓←→↖↗↙↘! \"#$%&'()*+,-./0123456789:;<=>@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~" conf = CosConfig( @@ -23,6 +24,19 @@ client = CosS3Client(conf) +def _create_test_bucket(test_bucket): + try: + response = client.create_bucket( + Bucket=test_bucket, + ) + except Exception as e: + if e.get_error_code() == 'BucketAlreadyOwnedByYou': + print('BucketAlreadyOwnedByYou') + else: + raise e + return None + + def get_raw_md5(data): m2 = hashlib.md5(data) etag = '"' + str(m2.hexdigest()) + '"' @@ -37,22 +51,24 @@ def gen_file(path, size): def print_error_msg(e): - print e.get_origin_msg() - print e.get_digest_msg() - print e.get_status_code() - print e.get_error_code() - print e.get_error_msg() - print e.get_resource_location() - print e.get_trace_id() - print e.get_request_id() + print (e.get_origin_msg()) + print (e.get_digest_msg()) + print (e.get_status_code()) + print (e.get_error_code()) + print (e.get_error_msg()) + print (e.get_resource_location()) + print (e.get_trace_id()) + print (e.get_request_id()) def setUp(): - print "start test..." + print ("start test...") + print ("start create bucket " + test_bucket) + _create_test_bucket(test_bucket) def tearDown(): - print "function teardown" + print ("function teardown") def test_put_get_delete_object_10MB(): @@ -146,6 +162,11 @@ def test_put_object_non_exist_bucket(): def test_put_object_acl(): """设置object acl""" + response = client.put_object( + Bucket=test_bucket, + Key=test_object, + Body='test acl' + ) response = client.put_object_acl( Bucket=test_bucket, Key=test_object, @@ -160,6 +181,10 @@ def test_get_object_acl(): Key=test_object ) assert response + response = client.delete_object( + Bucket=test_bucket, + Key=test_object + ) def test_copy_object_diff_bucket(): @@ -352,7 +377,7 @@ def test_get_bucket_acl_normal(): def test_list_objects(): """列出bucket下的objects""" response = client.list_objects( - Bucket=test_bucket, + Bucket='test01-1252448703', MaxKeys=100, Prefix='中文', Delimiter='/' @@ -376,7 +401,7 @@ def test_get_presigned_url(): Key='中文.txt' ) assert url - print url + print (url) def test_get_bucket_location(): @@ -496,7 +521,7 @@ def test_put_get_delete_replication(): { 'ID': '123', 'Status': 'Enabled', - 'Prefix': 'replication', + 'Prefix': '中文', 'Destination': { 'Bucket': 'qcs:id/0:cos:cn-south:appid/1252448703:replicationsouth' } @@ -516,7 +541,7 @@ def test_put_get_delete_replication(): Bucket=test_bucket ) assert response - # delete lifecycle + # delete replication response = client.delete_bucket_replication( Bucket=test_bucket ) @@ -564,7 +589,10 @@ def test_list_multipart_uploads(): def test_upload_file_multithreading(): """根据文件大小自动选择分块大小,多线程并发上传提高上传速度""" file_name = "thread_1GB" - gen_file(file_name, 5) # set 5MB beacuse travis too slow + file_size = 1024 + if TRAVIS_FLAG == 'true': + file_size = 5 # set 5MB beacuse travis too slow + gen_file(file_name, file_size) st = time.time() # 记录开始时间 response = client.upload_file( Bucket=test_bucket, @@ -577,7 +605,7 @@ def test_upload_file_multithreading(): ed = time.time() # 记录结束时间 if os.path.exists(file_name): os.remove(file_name) - print ed - st + print (ed - st) def test_copy_file_automatically(): @@ -625,7 +653,8 @@ def test_use_get_auth(): Key='test.txt', Params={'acl': '', 'unsed': '123'} ) - response = requests.get('http://test01-1252448703.cos.ap-beijing-1.myqcloud.com/test.txt?acl&unsed=123', headers={'Authorization': auth}) + url = 'http://' + test_bucket + '.cos.ap-beijing-1.myqcloud.com/test.txt?acl&unsed=123' + response = requests.get(url, headers={'Authorization': auth}) assert response.status_code == 200 @@ -663,16 +692,15 @@ def test_put_get_bucket_logging(): response = logging_client.get_bucket_logging( Bucket=logging_bucket ) - print response + print (response) assert response['LoggingEnabled']['TargetBucket'] == logging_bucket assert response['LoggingEnabled']['TargetPrefix'] == 'test' def test_put_object_enable_md5(): """上传文件,SDK计算content-md5头部""" - file_size = 10 file_name = 'test_object_sdk_caculate_md5.file' - gen_file(file_name, 10) + gen_file(file_name, 1) with open(file_name, 'rb') as f: etag = get_raw_md5(f.read()) with open(file_name, 'rb') as fp: @@ -703,6 +731,10 @@ def test_put_object_from_local_file(): Key=file_name ) assert put_response['ETag'] == etag + response = client.delete_object( + Bucket=test_bucket, + Key=file_name + ) if os.path.exists(file_name): os.remove(file_name) From 7db4cc6dd6333d332cc7f178b9a6bee46cfa3fd0 Mon Sep 17 00:00:00 2001 From: tiedu Date: Fri, 4 May 2018 11:20:54 +0800 Subject: [PATCH 3/3] modify UA --- qcloud_cos/cos_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qcloud_cos/cos_client.py b/qcloud_cos/cos_client.py index 0aa4c402..c3297550 100644 --- a/qcloud_cos/cos_client.py +++ b/qcloud_cos/cos_client.py @@ -153,7 +153,7 @@ def send_request(self, method, url, timeout=30, **kwargs): timeout = self._conf._timeout if self._conf._token is not None: kwargs['headers']['x-cos-security-token'] = self._conf._token - kwargs['headers']['User-Agent'] = 'cos-python-sdk-v5.1.4.1' + kwargs['headers']['User-Agent'] = 'cos-python-sdk-v5.1.4.2' try: for j in range(self._retry): if method == 'POST':