From 8bf3acab58fa0e3f6c5c05145b5da12cb62d7f94 Mon Sep 17 00:00:00 2001 From: tiedu Date: Wed, 27 Sep 2017 14:13:41 +0800 Subject: [PATCH 01/40] Modify Copy Object --- qcloud_cos/cos_client.py | 21 ++++++++++++++++++--- qcloud_cos/test.py | 2 +- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/qcloud_cos/cos_client.py b/qcloud_cos/cos_client.py index 7358dd5f..944bf14b 100644 --- a/qcloud_cos/cos_client.py +++ b/qcloud_cos/cos_client.py @@ -319,16 +319,31 @@ def head_object(self, Bucket, Key, **kwargs): def gen_copy_source_url(self, CopySource): """拼接拷贝源url""" + if 'Appid' in CopySource.keys(): + appid = CopySource['Appid'] + else: + raise CosClientError('CopySource Need Parameter Appid') if 'Bucket' in CopySource.keys(): bucket = CopySource['Bucket'] else: raise CosClientError('CopySource Need Parameter Bucket') + if 'Region' in CopySource.keys(): + region = CopySource['Region'] + region = format_region(region) + else: + raise CosClientError('CopySource Need Parameter Region') if 'Key' in CopySource.keys(): - key = CopySource['Key'] + path = CopySource['Key'] + if path and path[0] == '/': + path = path[1:] else: raise CosClientError('CopySource Need Parameter Key') - url = self._conf.uri(bucket=bucket, path=key).encode('utf8') - url = url[7:] # copysource不支持http://开头，去除 + url = "{bucket}-{uid}.{region}.myqcloud.com/{path}".format( + bucket=bucket, + uid=appid, + region=region, + path=path + ) return url def copy_object(self, Bucket, Key, CopySource, CopyStatus='Copy', **kwargs): diff --git a/qcloud_cos/test.py b/qcloud_cos/test.py index 5382c9d7..2926e1c3 100644 --- a/qcloud_cos/test.py +++ b/qcloud_cos/test.py @@ -60,7 +60,7 @@ def Test(): print "Test List Buckets" response = client.list_buckets() - copy_source = {'Bucket': 'test01', 'Key': '/test.txt'} + copy_source = {'Appid': '1252408340', 'Bucket': 'test02', 'Key': '/test.txt', 'Region': 'ap-guangzhou'} print "Test Copy Object From Other Bucket " response = client.copy_object( From eb0e7cef4e30a05467974dfaab45b8944405c1e3 Mon Sep 17 00:00:00 2001 From: tiedu Date: Thu, 28 Sep 2017 19:41:52 +0800 Subject: [PATCH 02/40] Add Cors And Lifecycle --- qcloud_cos/2q | 319 +++++++++++++++++++++++++++++++++++++++ qcloud_cos/cos_client.py | 144 +++++++++++++++++- qcloud_cos/test.py | 57 ++++++- 3 files changed, 517 insertions(+), 3 deletions(-) create mode 100644 qcloud_cos/2q diff --git a/qcloud_cos/2q b/qcloud_cos/2q new file mode 100644 index 00000000..f9105e3e --- /dev/null +++ b/qcloud_cos/2q @@ -0,0 +1,319 @@ +# -*- coding=utf-8 +import random +import sys +import os +import time +import datetime +from cos_client import CosS3Client +from cos_client import CosConfig +from cos_exception import CosServiceError + +ACCESS_ID = 'AKID15IsskiBQKTZbAo6WhgcBqVls9SmuG00' +# os.environ["ACCESS_ID"] +ACCESS_KEY = 'ciivKvnnrMvSvQpMAWuIz12pThGGlWRW' +# os.environ["ACCESS_KEY"] + + +def gen_file(path, size): + _file = open(path, 'w') + _file.seek(1024*1024*size) + _file.write('cos') + _file.close() + + +def print_error_msg(e): + print e.get_origin_msg() + print e.get_digest_msg() + print e.get_status_code() + print e.get_error_code() + print e.get_error_msg() + print e.get_resource_location() + print e.get_trace_id() + print e.get_request_id() + + +def setUp(): + print "start test" + + +def tearDown(): + print "function teardown" + + +def Test(): + conf = CosConfig( + Appid="1252448703", + Region="ap-beijing-1", + Access_id=ACCESS_ID, + Access_key=ACCESS_KEY + ) + client = CosS3Client(conf) + + test_bucket = 'test01' + file_size = 2 # 方便CI通过 + file_id = str(random.randint(0, 1000)) + str(random.randint(0, 1000)) + file_name = "tmp" + file_id + "_" + str(file_size) + "MB" + + print "test put bucket cors " + test_bucket + cors_config = { + 'CORSRule':[ + {'ID':'1234', + 'AllowedOrigin':['http://www.qq.com'], + 'AllowedMethod':['GET', 'PUT'], + 'AllowedHeader':['x-cos-meta-test'], + 'ExposeHeader':['x-cos-meta-test1'], + 'MaxAgeSeconds':500}]} + response = client.put_bucket_cors( + Bucket = test_bucket, + CORSConfiguration = cors_config + ) + + print "test get bucket cors " + test_bucket + response = client.get_bucket_cors( + Bucket = test_bucket + ) + print response + + print "test delete bucket cors " + test_bucket + response = client.delete_bucket_cors( + Bucket = test_bucket + ) + + print "test put bucket lifecycle " + test_bucket + response = client.put_bucket_lifecycle( + Bucket = test_bucket, + LifecycleConfiguration={ + 'Rule': [ + { + 'Expiration': {'Days': 100}, + 'ID': '123', + 'Filter':{'Prefix': '456'}, + 'Status': 'Enabled', + } + ] + } + ) + + print "test get bucket lifecycle " + test_bucket + response = client.get_bucket_lifecycle( + Bucket = test_bucket + ) + print response + + print "test delete bucket lifecycle " + test_bucket + response = client.delete_bucket_lifecycle( + Bucket = test_bucket + ) + ''' + print "Test Get Presigned Download URL " + url = client.get_presigned_download_url( + Bucket=test_bucket, + Key='中文.txt' + ) + print url + + print "Test List Buckets" + response = client.list_buckets() + + copy_source = {'Appid': '1252408340', 'Bucket': 'test02', 'Key': '/test.txt', 'Region': 'ap-guangzhou'} + print "Test Copy Object From Other Bucket " + + response = client.copy_object( + Bucket='test04', + Key='test.txt', + CopySource=copy_source + ) + + print "Test Put Object That Bucket Not Exist " + file_name + try: + response = client.put_object( + Bucket='test0xx', + Body='T'*1024*1024, + Key=file_name, + CacheControl='no-cache', + ContentDisposition='download.txt' + ) + except CosServiceError as e: + print_error_msg(e) + + special_file_name = "中文" + "→↓←→↖↗↙↘! \"#$%&'()*+,-./0123456789:;<=>@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~" + print "Test Put Object Contains Special Characters " + special_file_name + response = client.put_object( + Bucket=test_bucket, + Body='S'*1024*1024, + Key=special_file_name, + CacheControl='no-cache', + ContentDisposition='download.txt' + ) + + print "Test Get Object Contains Special Characters " + special_file_name + response = client.get_object( + Bucket=test_bucket, + Key=special_file_name, + ) + + print "Test Delete Object Contains Special Characters " + special_file_name + response = client.delete_object( + Bucket=test_bucket, + Key=special_file_name + ) + + print "Test Put Object " + file_name + gen_file(file_name, file_size) + fp = open(file_name, 'rb') + response = client.put_object( + Bucket=test_bucket, + Body=fp, + Key=file_name, + CacheControl='no-cache', + ContentDisposition='download.txt', + Metadata={ + "x-cos-meta-tiedu": "value1" + } + ) + fp.close() + os.remove(file_name) + + print "Test Get Object " + file_name + response = client.get_object( + Bucket=test_bucket, + Key=file_name, + ) + # 返回一个raw stream + # fp = response['Body'].get_raw_stream() + # 返回一个generator + # stream_generator = response['Body'].get_stream(stream_size=1024*512) + response['Body'].get_stream_to_file('cos.txt') + if os.path.exists('cos.txt'): + os.remove('cos.txt') + + print "Test Head Object " + file_name + response = client.head_object( + Bucket=test_bucket, + Key=file_name + ) + + print "Test Head Object " + file_name + "123" + try: + response = client.head_object( + Bucket=test_bucket, + Key=file_name+"123" + ) + except CosServiceError as e: + print_error_msg(e) + + print "Test Put Object ACL " + file_name + response = client.put_object_acl( + Bucket=test_bucket, + Key=file_name, + ACL='public-read-write' + ) + + print "Test Get Object ACL" + file_name + response = client.get_object_acl( + Bucket=test_bucket, + Key=file_name + ) + + print "Test Delete Object " + file_name + response = client.delete_object( + Bucket=test_bucket, + Key=file_name + ) + + print "Test List Objects" + response = client.list_objects( + Bucket=test_bucket + ) + + print "Test Create Bucket" + response = client.create_bucket( + Bucket='test'+file_id, + ACL='public-read' + ) + + print "Test PUT Bucket ACL" + try: + response = client.put_bucket_acl( + Bucket='test'+file_id, + ACL='public-read-writea' + ) + except CosServiceError as e: + print_error_msg(e) + + print "Test GET Bucket ACL" + response = client.get_bucket_acl( + Bucket='test'+file_id, + ) + + print "Test Delete Bucket" + response = client.delete_bucket( + Bucket='test'+file_id + ) + + print "Test Head Bucket" + try: + response = client.head_bucket( + Bucket='test'+file_id + ) + except CosServiceError as e: + print_error_msg(e) + + print "Test Create MultipartUpload" + response = client.create_multipart_upload( + Bucket=test_bucket, + Key='multipartfile.txt', + ) + uploadid = response['UploadId'] + + print "Test Abort MultipartUpload" + response = client.abort_multipart_upload( + Bucket=test_bucket, + Key='multipartfile.txt', + UploadId=uploadid + ) + + print "Test Create MultipartUpload" + response = client.create_multipart_upload( + Bucket=test_bucket, + Key='multipartfile.txt', + ) + uploadid = response['UploadId'] + + print "Test Upload Part1" + response = client.upload_part( + Bucket=test_bucket, + Key='multipartfile.txt', + UploadId=uploadid, + PartNumber=1, + Body='A'*1024*1024*2 + ) + + print "Test Upload Part2" + response = client.upload_part( + Bucket=test_bucket, + Key='multipartfile.txt', + UploadId=uploadid, + PartNumber=2, + Body='B'*1024*1024*2 + ) + + print "List Upload Parts" + response = client.list_parts( + Bucket=test_bucket, + Key='multipartfile.txt', + UploadId=uploadid + ) + lst = response['Part'] + + print "Test Complete MultipartUpload" + response = client.complete_multipart_upload( + Bucket=test_bucket, + Key='multipartfile.txt', + UploadId=uploadid, + MultipartUpload={'Part': lst} + ) + ''' +if __name__ == "__main__": + setUp() + Test() diff --git a/qcloud_cos/cos_client.py b/qcloud_cos/cos_client.py index 944bf14b..d9011760 100644 --- a/qcloud_cos/cos_client.py +++ b/qcloud_cos/cos_client.py @@ -3,6 +3,9 @@ import requests import urllib import logging +import hashlib +import base64 +import os import sys import copy import xml.dom.minidom @@ -10,6 +13,7 @@ from requests import Request, Session from streambody import StreamBody from xml2dict import Xml2Dict +from dicttoxml import dicttoxml from cos_auth import CosS3Auth from cos_exception import CosClientError from cos_exception import CosServiceError @@ -60,6 +64,12 @@ def to_unicode(s): return s.decode('utf-8') +def get_md5(data): + m2 = hashlib.md5(data) + MD5 = base64.standard_b64encode(m2.digest()) + return MD5 + + def dict_to_xml(data): """V5使用xml格式，将输入的dict转换为xml""" doc = xml.dom.minidom.Document() @@ -121,6 +131,14 @@ def mapped(headers): return _headers +def format_xml(data, root, lst): + """将dict转换为xml""" + xml_config = dicttoxml(data, item_func=lambda x: x, custom_root=root, attr_type=False) + for i in lst: + xml_config = xml_config.replace(i+i, i) + return xml_config + + def format_region(region): if region.find('cos.') != -1: return region # 传入cos.ap-beijing-1这样显示加上cos.的region @@ -590,12 +608,134 @@ def get_bucket_acl(self, Bucket, **kwargs): auth=CosS3Auth(self._conf._access_id, self._conf._access_key), headers=headers) data = xml_to_dict(rt.text) - if data['AccessControlList'] is not None and isinstance(data['AccessControlList']['Grant'], dict): + if data['AccessControlList'] is not None and not isinstance(data['AccessControlList']['Grant'], list): lst = [] lst.append(data['AccessControlList']['Grant']) data['AccessControlList']['Grant'] = lst return data + def put_bucket_cors(self, Bucket, CORSConfiguration={}, **kwargs): + """设置bucket CORS""" + lst = [ # 类型为list的标签 + '', + '', + '', + '', + '', + '', + '', + '', + '', + ''] + xml_config = format_xml(data=CORSConfiguration, root='CORSConfiguration', lst=lst) + headers = mapped(kwargs) + headers['Content-MD5'] = get_md5(xml_config) + headers['Content-Type'] = 'application/xml' + url = self._conf.uri(bucket=Bucket, path="?cors") + logger.info("put bucket cors, url=:{url} ,headers=:{headers}".format( + url=url, + headers=headers)) + rt = self.send_request( + method='PUT', + url=url, + data=xml_config, + auth=CosS3Auth(self._conf._access_id, self._conf._access_key), + headers=headers) + return None + + def get_bucket_cors(self, Bucket, **kwargs): + """获取bucket CORS""" + headers = mapped(kwargs) + url = self._conf.uri(bucket=Bucket, path="?cors") + logger.info("get bucket cors, url=:{url} ,headers=:{headers}".format( + url=url, + headers=headers)) + rt = self.send_request( + method='GET', + url=url, + auth=CosS3Auth(self._conf._access_id, self._conf._access_key), + headers=headers) + data = xml_to_dict(rt.text) + if 'CORSRule' in data.keys() and not isinstance(data['CORSRule'], list): + lst = [] + lst.append(data['CORSRule']) + data['CORSRule'] = lst + if 'CORSRule' in data.keys(): + allow_lst = ['AllowedOrigin', 'AllowedMethod', 'AllowedHeader', 'ExposeHeader'] + for rule in data['CORSRule']: + for text in allow_lst: + if text in rule.keys() and not isinstance(rule[text], list): + lst = [] + lst.append(rule[text]) + rule[text] = lst + return data + + def delete_bucket_cors(self, Bucket, **kwargs): + """删除bucket CORS""" + headers = mapped(kwargs) + url = self._conf.uri(bucket=Bucket, path="?cors") + logger.info("delete bucket cors, url=:{url} ,headers=:{headers}".format( + url=url, + headers=headers)) + rt = self.send_request( + method='DELETE', + url=url, + auth=CosS3Auth(self._conf._access_id, self._conf._access_key), + headers=headers) + return None + + def put_bucket_lifecycle(self, Bucket, LifecycleConfiguration={}, **kwargs): + """设置bucket LifeCycle""" + lst = ['', '', '', ''] # 类型为list的标签 + xml_config = format_xml(data=LifecycleConfiguration, root='LifecycleConfiguration', lst=lst) + headers = mapped(kwargs) + headers['Content-MD5'] = get_md5(xml_config) + headers['Content-Type'] = 'application/xml' + url = self._conf.uri(bucket=Bucket, path="?lifecycle") + logger.info("put bucket lifecycle, url=:{url} ,headers=:{headers}".format( + url=url, + headers=headers)) + rt = self.send_request( + method='PUT', + url=url, + data=xml_config, + auth=CosS3Auth(self._conf._access_id, self._conf._access_key), + headers=headers) + return None + + def get_bucket_lifecycle(self, Bucket, **kwargs): + """获取bucket LifeCycle""" + headers = mapped(kwargs) + url = self._conf.uri(bucket=Bucket, path="?lifecycle") + logger.info("get bucket cors, url=:{url} ,headers=:{headers}".format( + url=url, + headers=headers)) + rt = self.send_request( + method='GET', + url=url, + auth=CosS3Auth(self._conf._access_id, self._conf._access_key), + headers=headers) + data = xml_to_dict(rt.text) + if 'Rule' in data.keys() and not isinstance(data['Rule'], list): + lst = [] + lst.append(data['Rule']) + data['Rule'] = lst + return data + + def delete_bucket_lifecycle(self, Bucket, **kwargs): + """删除bucket LifeCycle""" + headers = mapped(kwargs) + url = self._conf.uri(bucket=Bucket, path="?lifecycle") + logger.info("delete bucket cors, url=:{url} ,headers=:{headers}".format( + url=url, + headers=headers)) + rt = self.send_request( + method='DELETE', + url=url, + auth=CosS3Auth(self._conf._access_id, self._conf._access_key), + headers=headers) + return None + # service interface begin def list_buckets(self, **kwargs): """列出所有bucket""" @@ -608,7 +748,7 @@ def list_buckets(self, **kwargs): auth=CosS3Auth(self._conf._access_id, self._conf._access_key), ) data = xml_to_dict(rt.text) - if data['Buckets'] is not None and isinstance(data['Buckets']['Bucket'], dict): + if data['Buckets'] is not None and not isinstance(data['Buckets']['Bucket'], list): lst = [] lst.append(data['Buckets']['Bucket']) data['Buckets']['Bucket'] = lst diff --git a/qcloud_cos/test.py b/qcloud_cos/test.py index 2926e1c3..06ae1400 100644 --- a/qcloud_cos/test.py +++ b/qcloud_cos/test.py @@ -50,6 +50,61 @@ def Test(): file_id = str(random.randint(0, 1000)) + str(random.randint(0, 1000)) file_name = "tmp" + file_id + "_" + str(file_size) + "MB" + print "test put bucket cors " + test_bucket + cors_config = { + 'CORSRule': [ + { + 'ID': '1234', + 'AllowedOrigin': ['http://www.qq.com'], + 'AllowedMethod': ['GET', 'PUT'], + 'AllowedHeader': ['x-cos-meta-test'], + 'ExposeHeader': ['x-cos-meta-test1'], + 'MaxAgeSeconds': 500 + }] + } + response = client.put_bucket_cors( + Bucket=test_bucket, + CORSConfiguration=cors_config + ) + + print "test get bucket cors " + test_bucket + response = client.get_bucket_cors( + Bucket=test_bucket + ) + print response + + print "test delete bucket cors " + test_bucket + response = client.delete_bucket_cors( + Bucket=test_bucket + ) + + print "test put bucket lifecycle " + test_bucket + life_config = { + 'Rule': [ + { + 'Expiration': {'Days': 100}, + 'ID': '123', + 'Filter': {'Prefix': '456'}, + 'Status': 'Enabled', + } + ] + } + response = client.put_bucket_lifecycle( + Bucket=test_bucket, + LifecycleConfiguration=life_config + ) + + print "test get bucket lifecycle " + test_bucket + response = client.get_bucket_lifecycle( + Bucket=test_bucket + ) + print response + + print "test delete bucket lifecycle " + test_bucket + response = client.delete_bucket_lifecycle( + Bucket=test_bucket + ) + ''' print "Test Get Presigned Download URL " url = client.get_presigned_download_url( Bucket=test_bucket, @@ -258,7 +313,7 @@ def Test(): UploadId=uploadid, MultipartUpload={'Part': lst} ) - + ''' if __name__ == "__main__": setUp() Test() From 4dbae21b10eacbde46ee4e99119bab582bd850eb Mon Sep 17 00:00:00 2001 From: tiedu Date: Thu, 28 Sep 2017 20:02:15 +0800 Subject: [PATCH 03/40] Add dicttoxml --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index bc29d3d8..2331919a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ requests==2.12.4 argparse==1.4.0 +dicttoxml==1.7.4 From 585f094bcab1bc7618acefc0c38704d74ac47cf4 Mon Sep 17 00:00:00 2001 From: tiedu Date: Thu, 28 Sep 2017 20:16:25 +0800 Subject: [PATCH 04/40] change travis --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 0deba634..d18a973f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,6 +9,7 @@ install: - pip install requests - pip install nose - pip install pep8 + - pip install dicttoxml script: - pep8 --max-line-length=180 qcloud_cos/. From 09274a240202a157b8026dda02be989f0b671d87 Mon Sep 17 00:00:00 2001 From: tiedu Date: Thu, 28 Sep 2017 20:34:08 +0800 Subject: [PATCH 05/40] remove 2q --- qcloud_cos/2q | 319 -------------------------------------------------- 1 file changed, 319 deletions(-) delete mode 100644 qcloud_cos/2q diff --git a/qcloud_cos/2q b/qcloud_cos/2q deleted file mode 100644 index f9105e3e..00000000 --- a/qcloud_cos/2q +++ /dev/null @@ -1,319 +0,0 @@ -# -*- coding=utf-8 -import random -import sys -import os -import time -import datetime -from cos_client import CosS3Client -from cos_client import CosConfig -from cos_exception import CosServiceError - -ACCESS_ID = 'AKID15IsskiBQKTZbAo6WhgcBqVls9SmuG00' -# os.environ["ACCESS_ID"] -ACCESS_KEY = 'ciivKvnnrMvSvQpMAWuIz12pThGGlWRW' -# os.environ["ACCESS_KEY"] - - -def gen_file(path, size): - _file = open(path, 'w') - _file.seek(1024*1024*size) - _file.write('cos') - _file.close() - - -def print_error_msg(e): - print e.get_origin_msg() - print e.get_digest_msg() - print e.get_status_code() - print e.get_error_code() - print e.get_error_msg() - print e.get_resource_location() - print e.get_trace_id() - print e.get_request_id() - - -def setUp(): - print "start test" - - -def tearDown(): - print "function teardown" - - -def Test(): - conf = CosConfig( - Appid="1252448703", - Region="ap-beijing-1", - Access_id=ACCESS_ID, - Access_key=ACCESS_KEY - ) - client = CosS3Client(conf) - - test_bucket = 'test01' - file_size = 2 # 方便CI通过 - file_id = str(random.randint(0, 1000)) + str(random.randint(0, 1000)) - file_name = "tmp" + file_id + "_" + str(file_size) + "MB" - - print "test put bucket cors " + test_bucket - cors_config = { - 'CORSRule':[ - {'ID':'1234', - 'AllowedOrigin':['http://www.qq.com'], - 'AllowedMethod':['GET', 'PUT'], - 'AllowedHeader':['x-cos-meta-test'], - 'ExposeHeader':['x-cos-meta-test1'], - 'MaxAgeSeconds':500}]} - response = client.put_bucket_cors( - Bucket = test_bucket, - CORSConfiguration = cors_config - ) - - print "test get bucket cors " + test_bucket - response = client.get_bucket_cors( - Bucket = test_bucket - ) - print response - - print "test delete bucket cors " + test_bucket - response = client.delete_bucket_cors( - Bucket = test_bucket - ) - - print "test put bucket lifecycle " + test_bucket - response = client.put_bucket_lifecycle( - Bucket = test_bucket, - LifecycleConfiguration={ - 'Rule': [ - { - 'Expiration': {'Days': 100}, - 'ID': '123', - 'Filter':{'Prefix': '456'}, - 'Status': 'Enabled', - } - ] - } - ) - - print "test get bucket lifecycle " + test_bucket - response = client.get_bucket_lifecycle( - Bucket = test_bucket - ) - print response - - print "test delete bucket lifecycle " + test_bucket - response = client.delete_bucket_lifecycle( - Bucket = test_bucket - ) - ''' - print "Test Get Presigned Download URL " - url = client.get_presigned_download_url( - Bucket=test_bucket, - Key='中文.txt' - ) - print url - - print "Test List Buckets" - response = client.list_buckets() - - copy_source = {'Appid': '1252408340', 'Bucket': 'test02', 'Key': '/test.txt', 'Region': 'ap-guangzhou'} - print "Test Copy Object From Other Bucket " - - response = client.copy_object( - Bucket='test04', - Key='test.txt', - CopySource=copy_source - ) - - print "Test Put Object That Bucket Not Exist " + file_name - try: - response = client.put_object( - Bucket='test0xx', - Body='T'*1024*1024, - Key=file_name, - CacheControl='no-cache', - ContentDisposition='download.txt' - ) - except CosServiceError as e: - print_error_msg(e) - - special_file_name = "中文" + "→↓←→↖↗↙↘! \"#$%&'()*+,-./0123456789:;<=>@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~" - print "Test Put Object Contains Special Characters " + special_file_name - response = client.put_object( - Bucket=test_bucket, - Body='S'*1024*1024, - Key=special_file_name, - CacheControl='no-cache', - ContentDisposition='download.txt' - ) - - print "Test Get Object Contains Special Characters " + special_file_name - response = client.get_object( - Bucket=test_bucket, - Key=special_file_name, - ) - - print "Test Delete Object Contains Special Characters " + special_file_name - response = client.delete_object( - Bucket=test_bucket, - Key=special_file_name - ) - - print "Test Put Object " + file_name - gen_file(file_name, file_size) - fp = open(file_name, 'rb') - response = client.put_object( - Bucket=test_bucket, - Body=fp, - Key=file_name, - CacheControl='no-cache', - ContentDisposition='download.txt', - Metadata={ - "x-cos-meta-tiedu": "value1" - } - ) - fp.close() - os.remove(file_name) - - print "Test Get Object " + file_name - response = client.get_object( - Bucket=test_bucket, - Key=file_name, - ) - # 返回一个raw stream - # fp = response['Body'].get_raw_stream() - # 返回一个generator - # stream_generator = response['Body'].get_stream(stream_size=1024*512) - response['Body'].get_stream_to_file('cos.txt') - if os.path.exists('cos.txt'): - os.remove('cos.txt') - - print "Test Head Object " + file_name - response = client.head_object( - Bucket=test_bucket, - Key=file_name - ) - - print "Test Head Object " + file_name + "123" - try: - response = client.head_object( - Bucket=test_bucket, - Key=file_name+"123" - ) - except CosServiceError as e: - print_error_msg(e) - - print "Test Put Object ACL " + file_name - response = client.put_object_acl( - Bucket=test_bucket, - Key=file_name, - ACL='public-read-write' - ) - - print "Test Get Object ACL" + file_name - response = client.get_object_acl( - Bucket=test_bucket, - Key=file_name - ) - - print "Test Delete Object " + file_name - response = client.delete_object( - Bucket=test_bucket, - Key=file_name - ) - - print "Test List Objects" - response = client.list_objects( - Bucket=test_bucket - ) - - print "Test Create Bucket" - response = client.create_bucket( - Bucket='test'+file_id, - ACL='public-read' - ) - - print "Test PUT Bucket ACL" - try: - response = client.put_bucket_acl( - Bucket='test'+file_id, - ACL='public-read-writea' - ) - except CosServiceError as e: - print_error_msg(e) - - print "Test GET Bucket ACL" - response = client.get_bucket_acl( - Bucket='test'+file_id, - ) - - print "Test Delete Bucket" - response = client.delete_bucket( - Bucket='test'+file_id - ) - - print "Test Head Bucket" - try: - response = client.head_bucket( - Bucket='test'+file_id - ) - except CosServiceError as e: - print_error_msg(e) - - print "Test Create MultipartUpload" - response = client.create_multipart_upload( - Bucket=test_bucket, - Key='multipartfile.txt', - ) - uploadid = response['UploadId'] - - print "Test Abort MultipartUpload" - response = client.abort_multipart_upload( - Bucket=test_bucket, - Key='multipartfile.txt', - UploadId=uploadid - ) - - print "Test Create MultipartUpload" - response = client.create_multipart_upload( - Bucket=test_bucket, - Key='multipartfile.txt', - ) - uploadid = response['UploadId'] - - print "Test Upload Part1" - response = client.upload_part( - Bucket=test_bucket, - Key='multipartfile.txt', - UploadId=uploadid, - PartNumber=1, - Body='A'*1024*1024*2 - ) - - print "Test Upload Part2" - response = client.upload_part( - Bucket=test_bucket, - Key='multipartfile.txt', - UploadId=uploadid, - PartNumber=2, - Body='B'*1024*1024*2 - ) - - print "List Upload Parts" - response = client.list_parts( - Bucket=test_bucket, - Key='multipartfile.txt', - UploadId=uploadid - ) - lst = response['Part'] - - print "Test Complete MultipartUpload" - response = client.complete_multipart_upload( - Bucket=test_bucket, - Key='multipartfile.txt', - UploadId=uploadid, - MultipartUpload={'Part': lst} - ) - ''' -if __name__ == "__main__": - setUp() - Test() From e754b2581dc52a8cd7d3f601b4f57861a4ee9f87 Mon Sep 17 00:00:00 2001 From: tiedu Date: Thu, 28 Sep 2017 20:35:09 +0800 Subject: [PATCH 06/40] Modify test.py --- qcloud_cos/test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qcloud_cos/test.py b/qcloud_cos/test.py index 06ae1400..dead89ff 100644 --- a/qcloud_cos/test.py +++ b/qcloud_cos/test.py @@ -104,7 +104,7 @@ def Test(): response = client.delete_bucket_lifecycle( Bucket=test_bucket ) - ''' + print "Test Get Presigned Download URL " url = client.get_presigned_download_url( Bucket=test_bucket, @@ -313,7 +313,7 @@ def Test(): UploadId=uploadid, MultipartUpload={'Part': lst} ) - ''' + if __name__ == "__main__": setUp() Test() From cc372f9b2a73c3e0e3045819a9c55fb2abcb233f Mon Sep 17 00:00:00 2001 From: tiedu Date: Fri, 29 Sep 2017 13:05:37 +0800 Subject: [PATCH 07/40] Modify ACL --- qcloud_cos/cos_client.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/qcloud_cos/cos_client.py b/qcloud_cos/cos_client.py index d9011760..dcd1d760 100644 --- a/qcloud_cos/cos_client.py +++ b/qcloud_cos/cos_client.py @@ -478,8 +478,14 @@ def list_parts(self, Bucket, Key, UploadId, EncodingType='url', MaxParts=1000, P data['Part'] = lst return data - def put_object_acl(self, Bucket, Key, **kwargs): + def put_object_acl(self, Bucket, Key, AccessControlPolicy={}, **kwargs): """设置object ACL""" + lst = [ # 类型为list的标签 + '', + ''] + xml_config = "" + if AccessControlPolicy: + xml_config = format_xml(data=AccessControlPolicy, root='AccessControlPolicy', lst=lst) headers = mapped(kwargs) url = self._conf.uri(bucket=Bucket, path=Key+"?acl") logger.info("put object acl, url=:{url} ,headers=:{headers}".format( @@ -488,6 +494,7 @@ def put_object_acl(self, Bucket, Key, **kwargs): rt = self.send_request( method='PUT', url=url, + data=xml_config, auth=CosS3Auth(self._conf._access_id, self._conf._access_key), headers=headers) return None @@ -581,16 +588,24 @@ def head_bucket(self, Bucket, **kwargs): headers=headers) return None - def put_bucket_acl(self, Bucket, **kwargs): + def put_bucket_acl(self, Bucket, AccessControlPolicy={}, **kwargs): """设置bucket ACL""" + lst = [ # 类型为list的标签 + '', + ''] + xml_config = "" + if AccessControlPolicy: + xml_config = format_xml(data=AccessControlPolicy, root='AccessControlPolicy', lst=lst) headers = mapped(kwargs) url = self._conf.uri(bucket=Bucket, path="?acl") logger.info("put bucket acl, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) + print xml_config rt = self.send_request( method='PUT', url=url, + data=xml_config, auth=CosS3Auth(self._conf._access_id, self._conf._access_key), headers=headers) return None From a57e6a9be95fc01996ecaa64fcf0b80afe7419be Mon Sep 17 00:00:00 2001 From: tiedu Date: Fri, 29 Sep 2017 19:59:44 +0800 Subject: [PATCH 08/40] Modify the Auth Module --- qcloud_cos/cos_auth.py | 33 +++++++++++++++++++++++---------- qcloud_cos/cos_client.py | 2 +- qcloud_cos/test.py | 3 +-- 3 files changed, 25 insertions(+), 13 deletions(-) diff --git a/qcloud_cos/cos_auth.py b/qcloud_cos/cos_auth.py index 6cadc4e2..491d1581 100644 --- a/qcloud_cos/cos_auth.py +++ b/qcloud_cos/cos_auth.py @@ -5,12 +5,28 @@ import urllib import hashlib import logging -from urllib import quote +from urllib import quote_plus from urlparse import urlparse from requests.auth import AuthBase logger = logging.getLogger(__name__) +def filter_headers(data): + """只设置host content-type 还有x开头的头部""" + headers = {} + for i in data.keys(): + if i == 'Content-Type' or i == 'Host' or i[0] == 'x' or i[0] == 'X': + headers[i] = data[i] + return headers + + +def cos_quote(value): + """对头部进行encode""" + data = quote_plus(value, '-_.~') # 保留字为-_.~,并且要对/进行Encode + data = data.replace('+', '%20') # 对于空格，需要Encode成%20,而不是+ + return data + + class CosS3Auth(AuthBase): def __init__(self, access_id, secret_key, expire=10000): @@ -19,12 +35,10 @@ def __init__(self, access_id, secret_key, expire=10000): self._expire = expire def __call__(self, r): - method = r.method.lower() + method = r.method.lower() # 获取小写method uri = urllib.unquote(r.url) uri = uri.split('?')[0] - http_header = r.headers - r.headers = {} - rt = urlparse(uri) + rt = urlparse(uri) # 解析host以及params logger.debug("url parse: " + str(rt)) if rt.query != "" and ("&" in rt.query or '=' in rt.query): uri_params = dict(map(lambda s: s.lower().split('='), rt.query.split('&'))) @@ -32,7 +46,8 @@ def __call__(self, r): uri_params = {rt.query: ""} else: uri_params = {} - headers = dict([(k.lower(), quote(v).lower()) for k, v in r.headers.items()]) + headers = filter_headers(r.headers) + headers = dict([(k.lower(), cos_quote(v)) for k, v in headers.items()]) # headers中的key转换为小写，value进行encode format_str = "{method}\n{host}\n{params}\n{headers}\n".format( method=method.lower(), host=rt.path, @@ -42,7 +57,7 @@ def __call__(self, r): logger.debug("format str: " + format_str) start_sign_time = int(time.time()) - sign_time = "{bg_time};{ed_time}".format(bg_time=start_sign_time-60, ed_time=start_sign_time + self._expire) + sign_time = "{bg_time};{ed_time}".format(bg_time=start_sign_time-60, ed_time=start_sign_time+self._expire) sha1 = hashlib.sha1() sha1.update(format_str) @@ -54,7 +69,7 @@ def __call__(self, r): logger.debug('sign: ' + str(sign)) sign_tpl = "q-sign-algorithm=sha1&q-ak={ak}&q-sign-time={sign_time}&q-key-time={key_time}&q-header-list={headers}&q-url-param-list={params}&q-signature={sign}" - http_header['Authorization'] = sign_tpl.format( + r.headers['Authorization'] = sign_tpl.format( ak=self._access_id, sign_time=sign_time, key_time=sign_time, @@ -62,10 +77,8 @@ def __call__(self, r): headers=';'.join(sorted(headers.keys())), sign=sign ) - r.headers = http_header logger.debug("sign_key" + str(sign_key)) logger.debug(r.headers['Authorization']) - logger.debug("request headers: " + str(r.headers)) return r diff --git a/qcloud_cos/cos_client.py b/qcloud_cos/cos_client.py index dcd1d760..98ecc942 100644 --- a/qcloud_cos/cos_client.py +++ b/qcloud_cos/cos_client.py @@ -22,7 +22,7 @@ level=logging.INFO, format='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s', datefmt='%a, %d %b %Y %H:%M:%S', - filename='cos_s3.log', + filename='cos_v5.log', filemode='w') logger = logging.getLogger(__name__) reload(sys) diff --git a/qcloud_cos/test.py b/qcloud_cos/test.py index dead89ff..5263e53b 100644 --- a/qcloud_cos/test.py +++ b/qcloud_cos/test.py @@ -116,8 +116,7 @@ def Test(): response = client.list_buckets() copy_source = {'Appid': '1252408340', 'Bucket': 'test02', 'Key': '/test.txt', 'Region': 'ap-guangzhou'} - print "Test Copy Object From Other Bucket " - + print "Test Copy Object From Otopy Object response = client.copy_object( Bucket='test04', Key='test.txt', From cfe079fee67ed3603a176cd92ee18601da31a9b5 Mon Sep 17 00:00:00 2001 From: tiedu Date: Fri, 29 Sep 2017 20:04:35 +0800 Subject: [PATCH 09/40] Modify Auth Moudle --- qcloud_cos/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qcloud_cos/test.py b/qcloud_cos/test.py index 5263e53b..691cf468 100644 --- a/qcloud_cos/test.py +++ b/qcloud_cos/test.py @@ -116,7 +116,7 @@ def Test(): response = client.list_buckets() copy_source = {'Appid': '1252408340', 'Bucket': 'test02', 'Key': '/test.txt', 'Region': 'ap-guangzhou'} - print "Test Copy Object From Otopy Object + print "Test Copy Object From Otopy Object" response = client.copy_object( Bucket='test04', Key='test.txt', From 936f0fbf9e6ef6aa70c30fec51d451ef7271a805 Mon Sep 17 00:00:00 2001 From: tiedu Date: Fri, 29 Sep 2017 22:11:50 +0800 Subject: [PATCH 10/40] Add params to auth --- qcloud_cos/cos_auth.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/qcloud_cos/cos_auth.py b/qcloud_cos/cos_auth.py index 491d1581..862c75e0 100644 --- a/qcloud_cos/cos_auth.py +++ b/qcloud_cos/cos_auth.py @@ -37,7 +37,6 @@ def __init__(self, access_id, secret_key, expire=10000): def __call__(self, r): method = r.method.lower() # 获取小写method uri = urllib.unquote(r.url) - uri = uri.split('?')[0] rt = urlparse(uri) # 解析host以及params logger.debug("url parse: " + str(rt)) if rt.query != "" and ("&" in rt.query or '=' in rt.query): @@ -51,7 +50,7 @@ def __call__(self, r): format_str = "{method}\n{host}\n{params}\n{headers}\n".format( method=method.lower(), host=rt.path, - params=urllib.urlencode(uri_params), + params=urllib.urlencode(sorted(uri_params.items())), headers='&'.join(map(lambda (x, y): "%s=%s" % (x, y), sorted(headers.items()))) ) logger.debug("format str: " + format_str) From 2b764046bc40f1c772317957ae5145206afb5908 Mon Sep 17 00:00:00 2001 From: tiedu Date: Fri, 29 Sep 2017 22:52:20 +0800 Subject: [PATCH 11/40] Modify Reverse Character --- qcloud_cos/cos_auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qcloud_cos/cos_auth.py b/qcloud_cos/cos_auth.py index 862c75e0..e5a5491b 100644 --- a/qcloud_cos/cos_auth.py +++ b/qcloud_cos/cos_auth.py @@ -50,7 +50,7 @@ def __call__(self, r): format_str = "{method}\n{host}\n{params}\n{headers}\n".format( method=method.lower(), host=rt.path, - params=urllib.urlencode(sorted(uri_params.items())), + params=urllib.urlencode(sorted(uri_params.items())).replace('+', '%20').replace('%7E', '~'), headers='&'.join(map(lambda (x, y): "%s=%s" % (x, y), sorted(headers.items()))) ) logger.debug("format str: " + format_str) From a81d0f83ccd54ac2f6fb36874762d35ecb1a872f Mon Sep 17 00:00:00 2001 From: tiedu Date: Sat, 30 Sep 2017 11:02:00 +0800 Subject: [PATCH 12/40] reserved keywords in headers urlencode are -_.~, notice that / should be encoded and space should not be encoded to plus sign(+) --- qcloud_cos/cos_auth.py | 17 ++++++----------- qcloud_cos/cos_client.py | 1 - 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/qcloud_cos/cos_auth.py b/qcloud_cos/cos_auth.py index e5a5491b..99850801 100644 --- a/qcloud_cos/cos_auth.py +++ b/qcloud_cos/cos_auth.py @@ -5,7 +5,7 @@ import urllib import hashlib import logging -from urllib import quote_plus +from urllib import quote from urlparse import urlparse from requests.auth import AuthBase logger = logging.getLogger(__name__) @@ -20,13 +20,6 @@ def filter_headers(data): return headers -def cos_quote(value): - """对头部进行encode""" - data = quote_plus(value, '-_.~') # 保留字为-_.~,并且要对/进行Encode - data = data.replace('+', '%20') # 对于空格，需要Encode成%20,而不是+ - return data - - class CosS3Auth(AuthBase): def __init__(self, access_id, secret_key, expire=10000): @@ -36,7 +29,7 @@ def __init__(self, access_id, secret_key, expire=10000): def __call__(self, r): method = r.method.lower() # 获取小写method - uri = urllib.unquote(r.url) + uri = urllib.unquote(r.url) # r.url encode ~ but reserverd for cos.fix may be r.url = r.url.replace('%7E','~') rt = urlparse(uri) # 解析host以及params logger.debug("url parse: " + str(rt)) if rt.query != "" and ("&" in rt.query or '=' in rt.query): @@ -46,11 +39,13 @@ def __call__(self, r): else: uri_params = {} headers = filter_headers(r.headers) - headers = dict([(k.lower(), cos_quote(v)) for k, v in headers.items()]) # headers中的key转换为小写，value进行encode + headers = r.headers + # reserved keywords in headers urlencode are -_.~, notice that / should be encoded and space should not be encoded to plus sign(+) + headers = dict([(k.lower(), quote(v, '-_.~')) for k, v in headers.items()]) # headers中的key转换为小写，value进行encode format_str = "{method}\n{host}\n{params}\n{headers}\n".format( method=method.lower(), host=rt.path, - params=urllib.urlencode(sorted(uri_params.items())).replace('+', '%20').replace('%7E', '~'), + params=urllib.urlencode(sorted(uri_params.items())).replace('+', '%20').replace('%7E', '~'), # use quote_plus to encode, handle that headers='&'.join(map(lambda (x, y): "%s=%s" % (x, y), sorted(headers.items()))) ) logger.debug("format str: " + format_str) diff --git a/qcloud_cos/cos_client.py b/qcloud_cos/cos_client.py index 98ecc942..d4e70af4 100644 --- a/qcloud_cos/cos_client.py +++ b/qcloud_cos/cos_client.py @@ -601,7 +601,6 @@ def put_bucket_acl(self, Bucket, AccessControlPolicy={}, **kwargs): logger.info("put bucket acl, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) - print xml_config rt = self.send_request( method='PUT', url=url, From 223b512d888252bab7e7122d1183e23a17a3abe4 Mon Sep 17 00:00:00 2001 From: tiedu Date: Sat, 30 Sep 2017 11:05:09 +0800 Subject: [PATCH 13/40] Filter headers to auth --- qcloud_cos/cos_auth.py | 1 - 1 file changed, 1 deletion(-) diff --git a/qcloud_cos/cos_auth.py b/qcloud_cos/cos_auth.py index 99850801..f4541c97 100644 --- a/qcloud_cos/cos_auth.py +++ b/qcloud_cos/cos_auth.py @@ -39,7 +39,6 @@ def __call__(self, r): else: uri_params = {} headers = filter_headers(r.headers) - headers = r.headers # reserved keywords in headers urlencode are -_.~, notice that / should be encoded and space should not be encoded to plus sign(+) headers = dict([(k.lower(), quote(v, '-_.~')) for k, v in headers.items()]) # headers中的key转换为小写，value进行encode format_str = "{method}\n{host}\n{params}\n{headers}\n".format( From 91aa5a470a3f9f6438c09159c052a83e1d8241c1 Mon Sep 17 00:00:00 2001 From: tiedu Date: Sat, 30 Sep 2017 15:23:13 +0800 Subject: [PATCH 14/40] Only Auth the ?acl format params --- qcloud_cos/cos_auth.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/qcloud_cos/cos_auth.py b/qcloud_cos/cos_auth.py index f4541c97..1ebfa9ad 100644 --- a/qcloud_cos/cos_auth.py +++ b/qcloud_cos/cos_auth.py @@ -29,11 +29,12 @@ def __init__(self, access_id, secret_key, expire=10000): def __call__(self, r): method = r.method.lower() # 获取小写method - uri = urllib.unquote(r.url) # r.url encode ~ but reserverd for cos.fix may be r.url = r.url.replace('%7E','~') + uri = urllib.unquote(r.url) rt = urlparse(uri) # 解析host以及params logger.debug("url parse: " + str(rt)) if rt.query != "" and ("&" in rt.query or '=' in rt.query): uri_params = dict(map(lambda s: s.lower().split('='), rt.query.split('&'))) + uri_params = {} elif rt.query != "": uri_params = {rt.query: ""} else: @@ -44,7 +45,7 @@ def __call__(self, r): format_str = "{method}\n{host}\n{params}\n{headers}\n".format( method=method.lower(), host=rt.path, - params=urllib.urlencode(sorted(uri_params.items())).replace('+', '%20').replace('%7E', '~'), # use quote_plus to encode, handle that + params=urllib.urlencode(sorted(uri_params.items())) headers='&'.join(map(lambda (x, y): "%s=%s" % (x, y), sorted(headers.items()))) ) logger.debug("format str: " + format_str) From 5439078c625d176a7af9a4b945b27fa6e097ca97 Mon Sep 17 00:00:00 2001 From: tiedu Date: Sat, 30 Sep 2017 15:25:44 +0800 Subject: [PATCH 15/40] only auth the ?acl format params --- qcloud_cos/cos_auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qcloud_cos/cos_auth.py b/qcloud_cos/cos_auth.py index 1ebfa9ad..e976156c 100644 --- a/qcloud_cos/cos_auth.py +++ b/qcloud_cos/cos_auth.py @@ -45,7 +45,7 @@ def __call__(self, r): format_str = "{method}\n{host}\n{params}\n{headers}\n".format( method=method.lower(), host=rt.path, - params=urllib.urlencode(sorted(uri_params.items())) + params=urllib.urlencode(sorted(uri_params.items())), headers='&'.join(map(lambda (x, y): "%s=%s" % (x, y), sorted(headers.items()))) ) logger.debug("format str: " + format_str) From 1a0832fc7ebffcff8385115c59ff51eca49a576f Mon Sep 17 00:00:00 2001 From: tiedu Date: Sat, 30 Sep 2017 17:25:57 +0800 Subject: [PATCH 16/40] Update README.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 限制文件命名 --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index b9304788..dc7acbb1 100644 --- a/README.rst +++ b/README.rst @@ -40,6 +40,7 @@ cos最新可用地域,参照https://www.qcloud.com/document/product/436/6224 # 文件操作 # ############################################################################ # 1. 上传单个文件 + NOTICE:Key中不可包含#%;?这四个字符 response = client.put_object( Bucket='test01', Body='TY'*1024*512*file_size, From 6e7f2f9e6ce6cd31e342ecbb82a7dd28ecbf2b8d Mon Sep 17 00:00:00 2001 From: tiedu Date: Mon, 9 Oct 2017 10:38:24 +0800 Subject: [PATCH 17/40] Modify Copy Object --- qcloud_cos/cos_client.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/qcloud_cos/cos_client.py b/qcloud_cos/cos_client.py index d4e70af4..aa2074a8 100644 --- a/qcloud_cos/cos_client.py +++ b/qcloud_cos/cos_client.py @@ -368,6 +368,8 @@ def copy_object(self, Bucket, Key, CopySource, CopyStatus='Copy', **kwargs): """文件拷贝，文件信息修改""" headers = mapped(kwargs) headers['x-cos-copy-source'] = self.gen_copy_source_url(CopySource) + if CopyStatus != 'Copy' and CopyStatus != 'Replaced': + raise CosClientError('CopyStatus must be Copy or Replaced') headers['x-cos-metadata-directive'] = CopyStatus url = self._conf.uri(bucket=Bucket, path=Key) logger.info("copy object, url=:{url} ,headers=:{headers}".format( From 8f3721c46a4a872f7a75da30a0afafba47628977 Mon Sep 17 00:00:00 2001 From: tiedu Date: Mon, 9 Oct 2017 17:18:03 +0800 Subject: [PATCH 18/40] Modify demo --- README.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/README.rst b/README.rst index dc7acbb1..b9304788 100644 --- a/README.rst +++ b/README.rst @@ -40,7 +40,6 @@ cos最新可用地域,参照https://www.qcloud.com/document/product/436/6224 # 文件操作 # ############################################################################ # 1. 上传单个文件 - NOTICE:Key中不可包含#%;?这四个字符 response = client.put_object( Bucket='test01', Body='TY'*1024*512*file_size, From 0cd71c70fc84035de8a36f3fb89f9ad567922d44 Mon Sep 17 00:00:00 2001 From: tiedu Date: Mon, 9 Oct 2017 20:48:25 +0800 Subject: [PATCH 19/40] Modify list objects --- qcloud_cos/cos_client.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/qcloud_cos/cos_client.py b/qcloud_cos/cos_client.py index aa2074a8..316a5ccf 100644 --- a/qcloud_cos/cos_client.py +++ b/qcloud_cos/cos_client.py @@ -549,7 +549,7 @@ def delete_bucket(self, Bucket, **kwargs): headers=headers) return None - def list_objects(self, Bucket, Delimiter="", Marker="", MaxKeys=1000, Prefix="", EncodingType="url", **kwargs): + def list_objects(self, Bucket, Delimiter="", Marker="", MaxKeys=1000, Prefix="", EncodingType="", **kwargs): """获取文件列表""" headers = mapped(kwargs) url = self._conf.uri(bucket=Bucket) @@ -560,8 +560,12 @@ def list_objects(self, Bucket, Delimiter="", Marker="", MaxKeys=1000, Prefix="", 'delimiter': Delimiter, 'marker': Marker, 'max-keys': MaxKeys, - 'prefix': Prefix, - 'encoding-type': EncodingType} + 'prefix': Prefix + } + if EncodingType: + if EncodingType != 'url': + raise CosClientError('EncodingType must be url') + params['encoding-type'] = EncodingType rt = self.send_request( method='GET', url=url, From 99a8bbf48c22f3692678e8b6bf9a5d2912f6ab83 Mon Sep 17 00:00:00 2001 From: tiedu Date: Tue, 10 Oct 2017 19:43:10 +0800 Subject: [PATCH 20/40] Modify Acl type to Type --- qcloud_cos/cos_client.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/qcloud_cos/cos_client.py b/qcloud_cos/cos_client.py index 316a5ccf..5ba41da1 100644 --- a/qcloud_cos/cos_client.py +++ b/qcloud_cos/cos_client.py @@ -100,13 +100,15 @@ def dict_to_xml(data): return doc.toxml('utf-8') -def xml_to_dict(data): +def xml_to_dict(data, origin_str="", replace_str=""): """V5使用xml格式，将response中的xml转换为dict""" root = xml.etree.ElementTree.fromstring(data) xmldict = Xml2Dict(root) xmlstr = str(xmldict) xmlstr = xmlstr.replace("{http://www.qcloud.com/document/product/436/7751}", "") xmlstr = xmlstr.replace("{http://www.w3.org/2001/XMLSchema-instance}", "") + if origin_str: + xmlstr = xmlstr.replace(origin_str, replace_str) xmldict = eval(xmlstr) return xmldict @@ -513,7 +515,7 @@ def get_object_acl(self, Bucket, Key, **kwargs): url=url, auth=CosS3Auth(self._conf._access_id, self._conf._access_key), headers=headers) - data = xml_to_dict(rt.text) + data = xml_to_dict(rt.text, "type", "Type") if data['AccessControlList'] is not None and isinstance(data['AccessControlList']['Grant'], dict): lst = [] lst.append(data['AccessControlList']['Grant']) @@ -627,7 +629,7 @@ def get_bucket_acl(self, Bucket, **kwargs): url=url, auth=CosS3Auth(self._conf._access_id, self._conf._access_key), headers=headers) - data = xml_to_dict(rt.text) + data = xml_to_dict(rt.text, "type", "Type") if data['AccessControlList'] is not None and not isinstance(data['AccessControlList']['Grant'], list): lst = [] lst.append(data['AccessControlList']['Grant']) From f83e3cc011d3970c92048d50dfbb0cd1cb2af7fb Mon Sep 17 00:00:00 2001 From: tiedu Date: Wed, 11 Oct 2017 12:02:25 +0800 Subject: [PATCH 21/40] Support all special characters --- qcloud_cos/cos_auth.py | 29 +++++++++---------- qcloud_cos/cos_client.py | 61 ++++++++++++++++++++-------------------- 2 files changed, 45 insertions(+), 45 deletions(-) diff --git a/qcloud_cos/cos_auth.py b/qcloud_cos/cos_auth.py index e976156c..dc0fe3e7 100644 --- a/qcloud_cos/cos_auth.py +++ b/qcloud_cos/cos_auth.py @@ -22,29 +22,28 @@ def filter_headers(data): class CosS3Auth(AuthBase): - def __init__(self, access_id, secret_key, expire=10000): + def __init__(self, access_id, secret_key, key='', params={}, expire=10000): self._access_id = access_id self._secret_key = secret_key self._expire = expire - - def __call__(self, r): - method = r.method.lower() # 获取小写method - uri = urllib.unquote(r.url) - rt = urlparse(uri) # 解析host以及params - logger.debug("url parse: " + str(rt)) - if rt.query != "" and ("&" in rt.query or '=' in rt.query): - uri_params = dict(map(lambda s: s.lower().split('='), rt.query.split('&'))) - uri_params = {} - elif rt.query != "": - uri_params = {rt.query: ""} + self._params = params + if key: + if key[0] == '/': + self._path = key + else: + self._path = '/' + key else: - uri_params = {} + self._path = '/' + + def __call__(self, r): + path = self._path + uri_params = self._params headers = filter_headers(r.headers) # reserved keywords in headers urlencode are -_.~, notice that / should be encoded and space should not be encoded to plus sign(+) headers = dict([(k.lower(), quote(v, '-_.~')) for k, v in headers.items()]) # headers中的key转换为小写，value进行encode format_str = "{method}\n{host}\n{params}\n{headers}\n".format( - method=method.lower(), - host=rt.path, + method=r.method.lower(), + host=path, params=urllib.urlencode(sorted(uri_params.items())), headers='&'.join(map(lambda (x, y): "%s=%s" % (x, y), sorted(headers.items()))) ) diff --git a/qcloud_cos/cos_client.py b/qcloud_cos/cos_client.py index 5ba41da1..b41557d3 100644 --- a/qcloud_cos/cos_client.py +++ b/qcloud_cos/cos_client.py @@ -11,6 +11,7 @@ import xml.dom.minidom import xml.etree.ElementTree from requests import Request, Session +from urllib import quote from streambody import StreamBody from xml2dict import Xml2Dict from dicttoxml import dicttoxml @@ -193,7 +194,7 @@ def uri(self, bucket, path=None): path=to_unicode(path) ) else: - url = u"http://{bucket}-{uid}.{region}.myqcloud.com".format( + url = u"http://{bucket}-{uid}.{region}.myqcloud.com/".format( bucket=to_unicode(bucket), uid=self._appid, region=self._region @@ -211,11 +212,11 @@ def __init__(self, conf, retry=1, session=None): else: self._session = session - def get_auth(self, Method, Bucket, Key=None, Expired=300, headers={}, params={}): + def get_auth(self, Method, Bucket, Key='', Expired=300, headers={}, params={}): """获取签名""" - url = self._conf.uri(bucket=Bucket, path=Key) + url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')) r = Request(Method, url, headers=headers, params=params) - auth = CosS3Auth(self._conf._access_id, self._conf._access_key, Expired) + auth = CosS3Auth(self._conf._access_id, self._conf._access_key, Key, params, Expired) return auth(r).headers['Authorization'] def send_request(self, method, url, timeout=30, **kwargs): @@ -266,14 +267,14 @@ def put_object(self, Bucket, Body, Key, **kwargs): headers[i] = headers['Metadata'][i] headers.pop('Metadata') - url = self._conf.uri(bucket=Bucket, path=Key) + url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')) # 提前对key做encode logger.info("put object, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) rt = self.send_request( method='PUT', url=url, - auth=CosS3Auth(self._conf._access_id, self._conf._access_key), + auth=CosS3Auth(self._conf._access_id, self._conf._access_key, Key), data=Body, headers=headers) @@ -284,7 +285,7 @@ def put_object(self, Bucket, Body, Key, **kwargs): def get_object(self, Bucket, Key, **kwargs): """单文件下载接口""" headers = mapped(kwargs) - url = self._conf.uri(bucket=Bucket, path=Key) + url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')) logger.info("get object, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) @@ -292,7 +293,7 @@ def get_object(self, Bucket, Key, **kwargs): method='GET', url=url, stream=True, - auth=CosS3Auth(self._conf._access_id, self._conf._access_key), + auth=CosS3Auth(self._conf._access_id, self._conf._access_key, Key), headers=headers) response = dict() @@ -304,36 +305,36 @@ def get_object(self, Bucket, Key, **kwargs): def get_presigned_download_url(self, Bucket, Key, Expired=300): """生成预签名的下载url""" - url = self._conf.uri(bucket=Bucket, path=Key) + url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')) sign = self.get_auth(Method='GET', Bucket=Bucket, Key=Key, Expired=300) - url = urllib.quote(url.encode('utf8'), ':/') + '?sign=' + urllib.quote(sign) + url = url + '?sign=' + urllib.quote(sign) return url def delete_object(self, Bucket, Key, **kwargs): """单文件删除接口""" headers = mapped(kwargs) - url = self._conf.uri(bucket=Bucket, path=Key) + url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')) logger.info("delete object, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) rt = self.send_request( method='DELETE', url=url, - auth=CosS3Auth(self._conf._access_id, self._conf._access_key), + auth=CosS3Auth(self._conf._access_id, self._conf._access_key, Key), headers=headers) return None def head_object(self, Bucket, Key, **kwargs): """获取文件信息""" headers = mapped(kwargs) - url = self._conf.uri(bucket=Bucket, path=Key) + url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')) logger.info("head object, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) rt = self.send_request( method='HEAD', url=url, - auth=CosS3Auth(self._conf._access_id, self._conf._access_key), + auth=CosS3Auth(self._conf._access_id, self._conf._access_key, Key), headers=headers) return rt.headers @@ -373,14 +374,14 @@ def copy_object(self, Bucket, Key, CopySource, CopyStatus='Copy', **kwargs): if CopyStatus != 'Copy' and CopyStatus != 'Replaced': raise CosClientError('CopyStatus must be Copy or Replaced') headers['x-cos-metadata-directive'] = CopyStatus - url = self._conf.uri(bucket=Bucket, path=Key) + url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')) logger.info("copy object, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) rt = self.send_request( method='PUT', url=url, - auth=CosS3Auth(self._conf._access_id, self._conf._access_key), + auth=CosS3Auth(self._conf._access_id, self._conf._access_key, Key), headers=headers) data = xml_to_dict(rt.text) return data @@ -393,14 +394,14 @@ def create_multipart_upload(self, Bucket, Key, **kwargs): headers[i] = headers['Metadata'][i] headers.pop('Metadata') - url = self._conf.uri(bucket=Bucket, path=Key+"?uploads") + url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')+"?uploads") logger.info("create multipart upload, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) rt = self.send_request( method='POST', url=url, - auth=CosS3Auth(self._conf._access_id, self._conf._access_key), + auth=CosS3Auth(self._conf._access_id, self._conf._access_key, Key), headers=headers) data = xml_to_dict(rt.text) @@ -409,7 +410,7 @@ def create_multipart_upload(self, Bucket, Key, **kwargs): def upload_part(self, Bucket, Key, Body, PartNumber, UploadId, **kwargs): """上传分片，单个大小不得超过5GB""" headers = mapped(kwargs) - url = self._conf.uri(bucket=Bucket, path=Key+"?partNumber={PartNumber}&uploadId={UploadId}".format( + url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')+"?partNumber={PartNumber}&uploadId={UploadId}".format( PartNumber=PartNumber, UploadId=UploadId)) logger.info("put object, url=:{url} ,headers=:{headers}".format( @@ -419,7 +420,7 @@ def upload_part(self, Bucket, Key, Body, PartNumber, UploadId, **kwargs): method='PUT', url=url, headers=headers, - auth=CosS3Auth(self._conf._access_id, self._conf._access_key), + auth=CosS3Auth(self._conf._access_id, self._conf._access_key, Key), data=Body) response = dict() response['ETag'] = rt.headers['ETag'] @@ -428,14 +429,14 @@ def upload_part(self, Bucket, Key, Body, PartNumber, UploadId, **kwargs): def complete_multipart_upload(self, Bucket, Key, UploadId, MultipartUpload={}, **kwargs): """完成分片上传，组装后的文件不得小于1MB,否则会返回错误""" headers = mapped(kwargs) - url = self._conf.uri(bucket=Bucket, path=Key+"?uploadId={UploadId}".format(UploadId=UploadId)) + url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')+"?uploadId={UploadId}".format(UploadId=UploadId)) logger.info("complete multipart upload, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) rt = self.send_request( method='POST', url=url, - auth=CosS3Auth(self._conf._access_id, self._conf._access_key), + auth=CosS3Auth(self._conf._access_id, self._conf._access_key, Key), data=dict_to_xml(MultipartUpload), timeout=1200, # 分片上传大文件的时间比较长，设置为20min headers=headers) @@ -445,14 +446,14 @@ def complete_multipart_upload(self, Bucket, Key, UploadId, MultipartUpload={}, * def abort_multipart_upload(self, Bucket, Key, UploadId, **kwargs): """放弃一个已经存在的分片上传任务，删除所有已经存在的分片""" headers = mapped(kwargs) - url = self._conf.uri(bucket=Bucket, path=Key+"?uploadId={UploadId}".format(UploadId=UploadId)) + url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')+"?uploadId={UploadId}".format(UploadId=UploadId)) logger.info("abort multipart upload, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) rt = self.send_request( method='DELETE', url=url, - auth=CosS3Auth(self._conf._access_id, self._conf._access_key), + auth=CosS3Auth(self._conf._access_id, self._conf._access_key, Key), headers=headers) return None @@ -465,14 +466,14 @@ def list_parts(self, Bucket, Key, UploadId, EncodingType='url', MaxParts=1000, P 'max-parts': MaxParts, 'encoding-type': EncodingType} - url = self._conf.uri(bucket=Bucket, path=Key) + url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')) logger.info("list multipart upload, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) rt = self.send_request( method='GET', url=url, - auth=CosS3Auth(self._conf._access_id, self._conf._access_key), + auth=CosS3Auth(self._conf._access_id, self._conf._access_key, Key), headers=headers, params=params) data = xml_to_dict(rt.text) @@ -491,7 +492,7 @@ def put_object_acl(self, Bucket, Key, AccessControlPolicy={}, **kwargs): if AccessControlPolicy: xml_config = format_xml(data=AccessControlPolicy, root='AccessControlPolicy', lst=lst) headers = mapped(kwargs) - url = self._conf.uri(bucket=Bucket, path=Key+"?acl") + url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')+"?acl") logger.info("put object acl, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) @@ -499,21 +500,21 @@ def put_object_acl(self, Bucket, Key, AccessControlPolicy={}, **kwargs): method='PUT', url=url, data=xml_config, - auth=CosS3Auth(self._conf._access_id, self._conf._access_key), + auth=CosS3Auth(self._conf._access_id, self._conf._access_key, Key), headers=headers) return None def get_object_acl(self, Bucket, Key, **kwargs): """获取object ACL""" headers = mapped(kwargs) - url = self._conf.uri(bucket=Bucket, path=Key+"?acl") + url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')+"?acl") logger.info("get object acl, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) rt = self.send_request( method='GET', url=url, - auth=CosS3Auth(self._conf._access_id, self._conf._access_key), + auth=CosS3Auth(self._conf._access_id, self._conf._access_key, Key), headers=headers) data = xml_to_dict(rt.text, "type", "Type") if data['AccessControlList'] is not None and isinstance(data['AccessControlList']['Grant'], dict): From c049a2f522028abf36a10a07fc65f156c584a292 Mon Sep 17 00:00:00 2001 From: tiedu Date: Wed, 11 Oct 2017 12:07:25 +0800 Subject: [PATCH 22/40] use cos style to encode path --- qcloud_cos/cos_auth.py | 2 +- qcloud_cos/cos_client.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/qcloud_cos/cos_auth.py b/qcloud_cos/cos_auth.py index dc0fe3e7..4aa8a11c 100644 --- a/qcloud_cos/cos_auth.py +++ b/qcloud_cos/cos_auth.py @@ -34,7 +34,7 @@ def __init__(self, access_id, secret_key, key='', params={}, expire=10000): self._path = '/' + key else: self._path = '/' - + def __call__(self, r): path = self._path uri_params = self._params diff --git a/qcloud_cos/cos_client.py b/qcloud_cos/cos_client.py index b41557d3..63e0f811 100644 --- a/qcloud_cos/cos_client.py +++ b/qcloud_cos/cos_client.py @@ -267,7 +267,7 @@ def put_object(self, Bucket, Body, Key, **kwargs): headers[i] = headers['Metadata'][i] headers.pop('Metadata') - url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')) # 提前对key做encode + url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')) # 提前对key做encode logger.info("put object, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) From effc466d7848550e6e5af152ac429f26057366e8 Mon Sep 17 00:00:00 2001 From: tiedu Date: Fri, 13 Oct 2017 11:31:18 +0800 Subject: [PATCH 23/40] copy object support metadata --- qcloud_cos/cos_client.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/qcloud_cos/cos_client.py b/qcloud_cos/cos_client.py index 63e0f811..c75fcfcc 100644 --- a/qcloud_cos/cos_client.py +++ b/qcloud_cos/cos_client.py @@ -370,6 +370,10 @@ def gen_copy_source_url(self, CopySource): def copy_object(self, Bucket, Key, CopySource, CopyStatus='Copy', **kwargs): """文件拷贝，文件信息修改""" headers = mapped(kwargs) + if 'Metadata' in headers.keys(): + for i in headers['Metadata'].keys(): + headers[i] = headers['Metadata'][i] + headers.pop('Metadata') headers['x-cos-copy-source'] = self.gen_copy_source_url(CopySource) if CopyStatus != 'Copy' and CopyStatus != 'Replaced': raise CosClientError('CopyStatus must be Copy or Replaced') From 29e74d176358b42eea0e465c0c4c9d5f668910c5 Mon Sep 17 00:00:00 2001 From: tiedu Date: Tue, 17 Oct 2017 16:19:40 +0800 Subject: [PATCH 24/40] Modify default params --- qcloud_cos/cos_client.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/qcloud_cos/cos_client.py b/qcloud_cos/cos_client.py index c75fcfcc..f868900c 100644 --- a/qcloud_cos/cos_client.py +++ b/qcloud_cos/cos_client.py @@ -461,14 +461,17 @@ def abort_multipart_upload(self, Bucket, Key, UploadId, **kwargs): headers=headers) return None - def list_parts(self, Bucket, Key, UploadId, EncodingType='url', MaxParts=1000, PartNumberMarker=0, **kwargs): + def list_parts(self, Bucket, Key, UploadId, EncodingType='', MaxParts=1000, PartNumberMarker=0, **kwargs): """列出已上传的分片""" headers = mapped(kwargs) params = { 'uploadId': UploadId, 'part-number-marker': PartNumberMarker, - 'max-parts': MaxParts, - 'encoding-type': EncodingType} + 'max-parts': MaxParts} + if EncodingType: + if EncodingType != 'url': + raise CosClientError('EncodingType must be url') + params['encoding-type'] = EncodingType url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')) logger.info("list multipart upload, url=:{url} ,headers=:{headers}".format( From 99563181253e4859379aa4975cf9c6f09bf3f04b Mon Sep 17 00:00:00 2001 From: tiedu Date: Wed, 18 Oct 2017 16:18:12 +0800 Subject: [PATCH 25/40] Modify put/get object return --- qcloud_cos/cos_client.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/qcloud_cos/cos_client.py b/qcloud_cos/cos_client.py index f868900c..ece82c87 100644 --- a/qcloud_cos/cos_client.py +++ b/qcloud_cos/cos_client.py @@ -278,8 +278,7 @@ def put_object(self, Bucket, Body, Key, **kwargs): data=Body, headers=headers) - response = dict() - response['ETag'] = rt.headers['ETag'] + response = rt.headers return response def get_object(self, Bucket, Key, **kwargs): @@ -296,11 +295,9 @@ def get_object(self, Bucket, Key, **kwargs): auth=CosS3Auth(self._conf._access_id, self._conf._access_key, Key), headers=headers) - response = dict() + response = rt.headers response['Body'] = StreamBody(rt) - for k in rt.headers.keys(): - response[k] = rt.headers[k] return response def get_presigned_download_url(self, Bucket, Key, Expired=300): From 1ca764a1a96a7b8fd138fc7849ac7998946ff09c Mon Sep 17 00:00:00 2001 From: tiedu Date: Wed, 18 Oct 2017 20:20:26 +0800 Subject: [PATCH 26/40] Add if assert for copy object --- qcloud_cos/cos_client.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/qcloud_cos/cos_client.py b/qcloud_cos/cos_client.py index ece82c87..3c21fe53 100644 --- a/qcloud_cos/cos_client.py +++ b/qcloud_cos/cos_client.py @@ -54,6 +54,10 @@ 'IfNoneMatch': 'If-None-Match', 'IfModifiedSince': 'If-Modified-Since', 'IfUnmodifiedSince': 'If-Unmodified-Since', + 'CopySourceIfMatch': 'x-cos-copy-source-If-Match', + 'CopySourceIfNoneMatch': 'x-cos-copy-source-If-None-Match', + 'CopySourceIfModifiedSince': 'x-cos-copy-source-If-Modified-Since', + 'CopySourceIfUnmodifiedSince': 'x-cos-copy-source-If-Unmodified-Since', 'VersionId': 'x-cos-version-id', } From 6fd2e7e276eab652107d6eb4b799219b5b1c7721 Mon Sep 17 00:00:00 2001 From: tiedu Date: Thu, 19 Oct 2017 19:15:35 +0800 Subject: [PATCH 27/40] Add Bucket versioning,location,head --- qcloud_cos/cos_client.py | 64 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/qcloud_cos/cos_client.py b/qcloud_cos/cos_client.py index 3c21fe53..1286e8b1 100644 --- a/qcloud_cos/cos_client.py +++ b/qcloud_cos/cos_client.py @@ -767,6 +767,70 @@ def delete_bucket_lifecycle(self, Bucket, **kwargs): headers=headers) return None + def put_bucket_versioning(self, Bucket, Status, **kwargs): + """设置bucket版本控制""" + headers = mapped(kwargs) + url = self._conf.uri(bucket=Bucket, path="?versioning") + logger.info("put bucket versioning, url=:{url} ,headers=:{headers}".format( + url=url, + headers=headers)) + if Status != 'Enabled' and Status != 'Disabled': + raise CosClientError('versioning status must be set to Enabled or Disabled!') + config = dict() + config['Status'] = Status + xml_config = format_xml(data=config, root='VersioningConfiguration') + rt = self.send_request( + method='PUT', + url=url, + data=xml_config, + auth=CosS3Auth(self._conf._access_id, self._conf._access_key), + headers=headers) + return None + + def get_bucket_versioning(self, Bucket, **kwargs): + """查询bucket版本控制""" + headers = mapped(kwargs) + url = self._conf.uri(bucket=Bucket, path="?versioning") + logger.info("get bucket versioning, url=:{url} ,headers=:{headers}".format( + url=url, + headers=headers)) + rt = self.send_request( + method='GET', + url=url, + auth=CosS3Auth(self._conf._access_id, self._conf._access_key), + headers=headers) + data = xml_to_dict(rt.text) + return data + + def get_bucket_location(self, Bucket, **kwargs): + """查询bucket所属地域""" + headers = mapped(kwargs) + url = self._conf.uri(bucket=Bucket, path="?location") + logger.info("get bucket location, url=:{url} ,headers=:{headers}".format( + url=url, + headers=headers)) + rt = self.send_request( + method='GET', + url=url, + auth=CosS3Auth(self._conf._access_id, self._conf._access_key), + headers=headers) + data = xml_to_dict(rt.text) + return data + + def head_bucket(self, Bucket, **kwargs): + """确认Bucket是否存在""" + headers = mapped(kwargs) + url = self._conf.uri(bucket=Bucket) + logger.info("head bucket, url=:{url} ,headers=:{headers}".format( + url=url, + headers=headers)) + rt = self.send_request( + method='HEAD', + url=url, + auth=CosS3Auth(self._conf._access_id, self._conf._access_key), + headers=headers) + return None + # service interface begin def list_buckets(self, **kwargs): """列出所有bucket""" From 4953c3c85fed555624aea56e0ba54b76ad29b9ea Mon Sep 17 00:00:00 2001 From: tiedu Date: Thu, 19 Oct 2017 21:01:57 +0800 Subject: [PATCH 28/40] Add bucket versioning,location,head --- qcloud_cos/cos_client.py | 6 ++++-- qcloud_cos/test.py | 23 +++++++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/qcloud_cos/cos_client.py b/qcloud_cos/cos_client.py index 1286e8b1..91911d72 100644 --- a/qcloud_cos/cos_client.py +++ b/qcloud_cos/cos_client.py @@ -138,7 +138,7 @@ def mapped(headers): return _headers -def format_xml(data, root, lst): +def format_xml(data, root, lst=list()): """将dict转换为xml""" xml_config = dicttoxml(data, item_func=lambda x: x, custom_root=root, attr_type=False) for i in lst: @@ -814,7 +814,9 @@ def get_bucket_location(self, Bucket, **kwargs): url=url, auth=CosS3Auth(self._conf._access_id, self._conf._access_key), headers=headers) - data = xml_to_dict(rt.text) + root = xml.etree.ElementTree.fromstring(rt.text) + data = dict() + data['LocationConstraint'] = root.text return data def head_bucket(self, Bucket, **kwargs): diff --git a/qcloud_cos/test.py b/qcloud_cos/test.py index 691cf468..daa9d1f9 100644 --- a/qcloud_cos/test.py +++ b/qcloud_cos/test.py @@ -105,6 +105,29 @@ def Test(): Bucket=test_bucket ) + print "test put bucket versioning " + test_bucket + response = client.put_bucket_versioning( + Bucket=test_bucket, + Status='Enabled' + ) + + print "test get bucket versioning " + test_bucket + response = client.get_bucket_versioning( + Bucket=test_bucket + ) + print response + + print "test get bucket location " + test_bucket + response = client.get_bucket_location( + Bucket=test_bucket + ) + print response + + print "test head bucket " + test_bucket + response = client.head_bucket( + Bucket=test_bucket + ) + print "Test Get Presigned Download URL " url = client.get_presigned_download_url( Bucket=test_bucket, From 4767d64c6a950aff2ee2e637361ee8abef9432b8 Mon Sep 17 00:00:00 2001 From: tiedu Date: Mon, 23 Oct 2017 10:27:27 +0800 Subject: [PATCH 29/40] Modify Readme Config --- README.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index b9304788..2eeb0450 100644 --- a/README.rst +++ b/README.rst @@ -27,10 +27,10 @@ cos最新可用地域,参照https://www.qcloud.com/document/product/436/6224 .. code:: python # 设置用户属性, 包括appid, secret_id, secret_key, region - appid = 100000 # 替换为用户的appid - secret_id = u'xxxxxxxx' # 替换为用户的secret_id - secret_key = u'xxxxxxx' # 替换为用户的secret_key -   region = "ap-beiging-1"    # 替换为用户的region + appid = '100000' # 替换为用户的appid + secret_id = 'xxxxxxxx' # 替换为用户的secret_id + secret_key = 'xxxxxxx' # 替换为用户的secret_key +   region = 'ap-beiging-1'    # 替换为用户的region token = '' # 使用临时秘钥需要传入Token，默认为空,可不填 config = CosConfig(Appid=appid, Region=region, Access_id=secret_id, Access_key=secret_key, Token=token) #获取配置对象 client = CosS3Client(config) #获取客户端对象 From 7055f045f19ba9d9945edd75441037841ae0a091 Mon Sep 17 00:00:00 2001 From: tiedu Date: Mon, 23 Oct 2017 10:31:36 +0800 Subject: [PATCH 30/40] Modify Readme --- README.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 2eeb0450..520b0f70 100644 --- a/README.rst +++ b/README.rst @@ -27,11 +27,11 @@ cos最新可用地域,参照https://www.qcloud.com/document/product/436/6224 .. code:: python # 设置用户属性, 包括appid, secret_id, secret_key, region - appid = '100000' # 替换为用户的appid + appid = '100000' # 替换为用户的appid secret_id = 'xxxxxxxx' # 替换为用户的secret_id secret_key = 'xxxxxxx' # 替换为用户的secret_key -   region = 'ap-beiging-1'    # 替换为用户的region - token = '' # 使用临时秘钥需要传入Token，默认为空,可不填 +   region = 'ap-beiging-1'   # 替换为用户的region + token = '' # 使用临时秘钥需要传入Token，默认为空,可不填 config = CosConfig(Appid=appid, Region=region, Access_id=secret_id, Access_key=secret_key, Token=token) #获取配置对象 client = CosS3Client(config) #获取客户端对象 From 755878fee9956c8ac9fb1b453f4477aa92b3ddb8 Mon Sep 17 00:00:00 2001 From: tiedu Date: Mon, 23 Oct 2017 20:03:31 +0800 Subject: [PATCH 31/40] Add standard annotation --- qcloud_cos/cos_auth.py | 6 +- qcloud_cos/cos_client.py | 284 ++++++++++++++++++++++++++++++++------- qcloud_cos/test.py | 2 +- 3 files changed, 245 insertions(+), 47 deletions(-) diff --git a/qcloud_cos/cos_auth.py b/qcloud_cos/cos_auth.py index 4aa8a11c..25ec0bdd 100644 --- a/qcloud_cos/cos_auth.py +++ b/qcloud_cos/cos_auth.py @@ -12,7 +12,11 @@ def filter_headers(data): - """只设置host content-type 还有x开头的头部""" + """只设置host content-type 还有x开头的头部. + + :param data(dict): 所有的头部信息. + :return(dict): 计算进签名的头部. + """ headers = {} for i in data.keys(): if i == 'Content-Type' or i == 'Host' or i[0] == 'x' or i[0] == 'X': diff --git a/qcloud_cos/cos_client.py b/qcloud_cos/cos_client.py index 91911d72..bcba7617 100644 --- a/qcloud_cos/cos_client.py +++ b/qcloud_cos/cos_client.py @@ -28,6 +28,8 @@ logger = logging.getLogger(__name__) reload(sys) sys.setdefaultencoding('utf-8') + +# kwargs中params到http headers的映射 maplist = { 'ContentLength': 'Content-Length', 'ContentMD5': 'Content-MD5', @@ -147,6 +149,7 @@ def format_xml(data, root, lst=list()): def format_region(region): + """格式化地域""" if region.find('cos.') != -1: return region # 传入cos.ap-beijing-1这样显示加上cos.的region if region == 'cn-north' or region == 'cn-south' or region == 'cn-east' or region == 'cn-south-2' or region == 'cn-southwest' or region == 'sg': @@ -177,6 +180,14 @@ def format_region(region): class CosConfig(object): """config类，保存用户相关信息""" def __init__(self, Appid, Region, Access_id, Access_key, Token=None): + """初始化，保存用户的信息 + + :param Appid(string): 用户APPID. + :param Region(string): 地域信息. + :param Access_id(string): 秘钥SecretId. + :param Access_key(string): 秘钥SecretKey. + :param Token(string): 临时秘钥使用的token. + """ self._appid = Appid self._region = format_region(Region) self._access_id = Access_id @@ -187,7 +198,12 @@ def __init__(self, Appid, Region, Access_id, Access_key, Token=None): region=Region)) def uri(self, bucket, path=None): - """拼接url""" + """拼接url + + :param bucket(string): 存储桶名称. + :param path(string): 请求COS的路径. + :return(string): 请求COS的URL地址. + """ if path: if path[0] == '/': path = path[1:] @@ -209,6 +225,12 @@ def uri(self, bucket, path=None): class CosS3Client(object): """cos客户端类，封装相应请求""" def __init__(self, conf, retry=1, session=None): + """初始化client对象 + + :param conf(CosConfig): 用户的配置. + :param retry(int): 失败重试的次数. + :param session(object): http session. + """ self._conf = conf self._retry = retry # 重试的次数，分片上传时可适当增大 if session is None: @@ -217,13 +239,23 @@ def __init__(self, conf, retry=1, session=None): self._session = session def get_auth(self, Method, Bucket, Key='', Expired=300, headers={}, params={}): - """获取签名""" + """获取签名 + + :param Method(string): http method,如'PUT','GET'. + :param Bucket(string): 存储桶名称. + :param Key(string): 请求COS的路径. + :param Expired(int): 签名有效时间,单位为s. + :param headers(dict): 签名中的http headers. + :param params(dict): 签名中的http params. + :return (string): 计算出的V5签名. + """ url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')) r = Request(Method, url, headers=headers, params=params) auth = CosS3Auth(self._conf._access_id, self._conf._access_key, Key, params, Expired) return auth(r).headers['Authorization'] def send_request(self, method, url, timeout=30, **kwargs): + """封装request库发起http请求""" if self._conf._token is not None: kwargs['headers']['x-cos-security-token'] = self._conf._token kwargs['headers']['User-Agent'] = 'cos-python-sdk-v5' @@ -264,7 +296,14 @@ def send_request(self, method, url, timeout=30, **kwargs): # s3 object interface begin def put_object(self, Bucket, Body, Key, **kwargs): - """单文件上传接口，适用于小文件，最大不得超过5GB""" + """单文件上传接口，适用于小文件，最大不得超过5GB + + :param Bucket(string): 存储桶名称. + :param Body(file|string): 上传的文件内容，类型为文件流或字节流. + :param Key(string): COS路径. + :kwargs(dict): 设置上传的headers. + :return(dict): 上传成功返回的结果，包含ETag等信息. + """ headers = mapped(kwargs) if 'Metadata' in headers.keys(): for i in headers['Metadata'].keys(): @@ -286,7 +325,13 @@ def put_object(self, Bucket, Body, Key, **kwargs): return response def get_object(self, Bucket, Key, **kwargs): - """单文件下载接口""" + """单文件下载接口 + + :param Bucket(string): 存储桶名称. + :param Key(string): COS路径. + :param kwargs(dict): 设置下载的headers. + :return(dict): 下载成功返回的结果,包含Body对应的StreamBody,可以获取文件流或下载文件到本地. + """ headers = mapped(kwargs) url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')) logger.info("get object, url=:{url} ,headers=:{headers}".format( @@ -305,14 +350,26 @@ def get_object(self, Bucket, Key, **kwargs): return response def get_presigned_download_url(self, Bucket, Key, Expired=300): - """生成预签名的下载url""" + """生成预签名的下载url + + :param Bucket(string): 存储桶名称. + :param Key(string): COS路径. + :param Expired(int): 签名过期时间. + :return(string): 预先签名的下载URL. + """ url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')) sign = self.get_auth(Method='GET', Bucket=Bucket, Key=Key, Expired=300) url = url + '?sign=' + urllib.quote(sign) return url def delete_object(self, Bucket, Key, **kwargs): - """单文件删除接口""" + """单文件删除接口 + + :param Bucket(string): 存储桶名称. + :param Key(string): COS路径. + :param kwargs(dict): 设置请求headers. + :return: None. + """ headers = mapped(kwargs) url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')) logger.info("delete object, url=:{url} ,headers=:{headers}".format( @@ -326,7 +383,13 @@ def delete_object(self, Bucket, Key, **kwargs): return None def head_object(self, Bucket, Key, **kwargs): - """获取文件信息""" + """获取文件信息 + + :param Bucket(string): 存储桶名称. + :param Key(string): COS路径. + :param kwargs(dict): 设置请求headers. + :return(dict): 文件的metadata信息. + """ headers = mapped(kwargs) url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')) logger.info("head object, url=:{url} ,headers=:{headers}".format( @@ -369,7 +432,15 @@ def gen_copy_source_url(self, CopySource): return url def copy_object(self, Bucket, Key, CopySource, CopyStatus='Copy', **kwargs): - """文件拷贝，文件信息修改""" + """文件拷贝，文件信息修改 + + :param Bucket(string): 存储桶名称. + :param Key(string): 上传COS路径. + :param CopySource(dict): 拷贝源,包含Appid,Bucket,Region,Key. + :param CopyStatus(string): 拷贝状态,可选值'Copy'|'Replaced'. + :param kwargs(dict): 设置请求headers. + :return(dict): 拷贝成功的结果. + """ headers = mapped(kwargs) if 'Metadata' in headers.keys(): for i in headers['Metadata'].keys(): @@ -392,7 +463,13 @@ def copy_object(self, Bucket, Key, CopySource, CopyStatus='Copy', **kwargs): return data def create_multipart_upload(self, Bucket, Key, **kwargs): - """创建分片上传，适用于大文件上传""" + """创建分片上传，适用于大文件上传 + + :param Bucket(string): 存储桶名称. + :param Key(string): COS路径. + :param kwargs(dict): 设置请求headers. + :return(dict): 初始化分块上传返回的结果，包含UploadId等信息. + """ headers = mapped(kwargs) if 'Metadata' in headers.keys(): for i in headers['Metadata'].keys(): @@ -413,7 +490,16 @@ def create_multipart_upload(self, Bucket, Key, **kwargs): return data def upload_part(self, Bucket, Key, Body, PartNumber, UploadId, **kwargs): - """上传分片，单个大小不得超过5GB""" + """上传分片，单个大小不得超过5GB + + :param Bucket(string): 存储桶名称. + :param Key(string): COS路径. + :param Body(file|string): 上传分块的内容,可以为文件流或者字节流. + :param PartNumber(int): 上传分块的编号. + :param UploadId(string): 分块上传创建的UploadId. + :param kwargs(dict): 设置请求headers. + :return(dict): 上传成功返回的结果，包含单个分块ETag等信息. + """ headers = mapped(kwargs) url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')+"?partNumber={PartNumber}&uploadId={UploadId}".format( PartNumber=PartNumber, @@ -432,7 +518,15 @@ def upload_part(self, Bucket, Key, Body, PartNumber, UploadId, **kwargs): return response def complete_multipart_upload(self, Bucket, Key, UploadId, MultipartUpload={}, **kwargs): - """完成分片上传，组装后的文件不得小于1MB,否则会返回错误""" + """完成分片上传,除最后一块分块块大小必须大于等于1MB,否则会返回错误. + + :param Bucket(string): 存储桶名称. + :param Key(string): COS路径. + :param UploadId(string): 分块上传创建的UploadId. + :param MultipartUpload(dict): 所有分块的信息,包含Etag和PartNumber. + :param kwargs(dict): 设置请求headers. + :return(dict): 上传成功返回的结果，包含整个文件的ETag等信息. + """ headers = mapped(kwargs) url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')+"?uploadId={UploadId}".format(UploadId=UploadId)) logger.info("complete multipart upload, url=:{url} ,headers=:{headers}".format( @@ -449,7 +543,14 @@ def complete_multipart_upload(self, Bucket, Key, UploadId, MultipartUpload={}, * return data def abort_multipart_upload(self, Bucket, Key, UploadId, **kwargs): - """放弃一个已经存在的分片上传任务，删除所有已经存在的分片""" + """放弃一个已经存在的分片上传任务，删除所有已经存在的分片. + + :param Bucket(string): 存储桶名称. + :param Key(string): COS路径. + :param UploadId(string): 分块上传创建的UploadId. + :param kwargs(dict): 设置请求headers. + :return: None. + """ headers = mapped(kwargs) url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')+"?uploadId={UploadId}".format(UploadId=UploadId)) logger.info("abort multipart upload, url=:{url} ,headers=:{headers}".format( @@ -463,7 +564,17 @@ def abort_multipart_upload(self, Bucket, Key, UploadId, **kwargs): return None def list_parts(self, Bucket, Key, UploadId, EncodingType='', MaxParts=1000, PartNumberMarker=0, **kwargs): - """列出已上传的分片""" + """列出已上传的分片. + + :param Bucket(string): 存储桶名称. + :param Key(string): COS路径. + :param UploadId(string): 分块上传创建的UploadId. + :param EncodingType(string): 设置返回结果编码方式,只能设置为url. + :param MaxParts(int): 设置单次返回最大的分块数量,最大为1000. + :param PartNumberMarker(int): 设置返回的开始处,从PartNumberMarker下一个分块开始列出. + :param kwargs(dict): 设置请求headers. + :return(dict): 分块的相关信息，包括Etag和PartNumber等信息. + """ headers = mapped(kwargs) params = { 'uploadId': UploadId, @@ -492,7 +603,14 @@ def list_parts(self, Bucket, Key, UploadId, EncodingType='', MaxParts=1000, Part return data def put_object_acl(self, Bucket, Key, AccessControlPolicy={}, **kwargs): - """设置object ACL""" + """设置object ACL + + :param Bucket(string): 存储桶名称. + :param Key(string): COS路径. + :param AccessControlPolicy(dict): 设置object ACL规则. + :param kwargs(dict): 通过headers来设置ACL. + :return: None. + """ lst = [ # 类型为list的标签 '', ''] @@ -513,7 +631,13 @@ def put_object_acl(self, Bucket, Key, AccessControlPolicy={}, **kwargs): return None def get_object_acl(self, Bucket, Key, **kwargs): - """获取object ACL""" + """获取object ACL + + :param Bucket(string): 存储桶名称. + :param Key(string): COS路径. + :param kwargs(dict): 设置请求headers. + :return(dict): Object对应的ACL信息. + """ headers = mapped(kwargs) url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')+"?acl") logger.info("get object acl, url=:{url} ,headers=:{headers}".format( @@ -533,7 +657,12 @@ def get_object_acl(self, Bucket, Key, **kwargs): # s3 bucket interface begin def create_bucket(self, Bucket, **kwargs): - """创建一个bucket""" + """创建一个bucket + + :param Bucket(string): 存储桶名称. + :param kwargs(dict): 设置请求headers. + :return: None. + """ headers = mapped(kwargs) url = self._conf.uri(bucket=Bucket) logger.info("create bucket, url=:{url} ,headers=:{headers}".format( @@ -547,7 +676,12 @@ def create_bucket(self, Bucket, **kwargs): return None def delete_bucket(self, Bucket, **kwargs): - """删除一个bucket，bucket必须为空""" + """删除一个bucket，bucket必须为空 + + :param Bucket(string): 存储桶名称. + :param kwargs(dict): 设置请求headers. + :return: None. + """ headers = mapped(kwargs) url = self._conf.uri(bucket=Bucket) logger.info("delete bucket, url=:{url} ,headers=:{headers}".format( @@ -561,7 +695,17 @@ def delete_bucket(self, Bucket, **kwargs): return None def list_objects(self, Bucket, Delimiter="", Marker="", MaxKeys=1000, Prefix="", EncodingType="", **kwargs): - """获取文件列表""" + """获取文件列表 + + :param Bucket(string): 存储桶名称. + :param Delimiter(string): 分隔符. + :param Marker(string): 从marker开始列出条目. + :param MaxKeys(int): 设置单次返回最大的数量,最大为1000. + :param Prefix(string): 设置匹配文件的前缀. + :param EncodingType(string): 设置返回结果编码方式,只能设置为url. + :param kwargs(dict): 设置请求headers. + :return(dict): 文件的相关信息，包括Etag等信息. + """ headers = mapped(kwargs) url = self._conf.uri(bucket=Bucket) logger.info("list objects, url=:{url} ,headers=:{headers}".format( @@ -592,7 +736,12 @@ def list_objects(self, Bucket, Delimiter="", Marker="", MaxKeys=1000, Prefix="", return data def head_bucket(self, Bucket, **kwargs): - """获取bucket信息""" + """确认bucket是否存在 + + :param Bucket(string): 存储桶名称. + :param kwargs(dict): 设置请求headers. + :return: None. + """ headers = mapped(kwargs) url = self._conf.uri(bucket=Bucket) logger.info("head bucket, url=:{url} ,headers=:{headers}".format( @@ -606,7 +755,13 @@ def head_bucket(self, Bucket, **kwargs): return None def put_bucket_acl(self, Bucket, AccessControlPolicy={}, **kwargs): - """设置bucket ACL""" + """设置bucket ACL + + :param Bucket(string): 存储桶名称. + :param AccessControlPolicy(dict): 设置bucket ACL规则. + :param kwargs(dict): 通过headers来设置ACL. + :return: None. + """ lst = [ # 类型为list的标签 '', ''] @@ -627,7 +782,12 @@ def put_bucket_acl(self, Bucket, AccessControlPolicy={}, **kwargs): return None def get_bucket_acl(self, Bucket, **kwargs): - """获取bucket ACL""" + """获取bucket ACL + + :param Bucket(string): 存储桶名称. + :param kwargs(dict): 设置headers. + :return(dict): Bucket对应的ACL信息. + """ headers = mapped(kwargs) url = self._conf.uri(bucket=Bucket, path="?acl") logger.info("get bucket acl, url=:{url} ,headers=:{headers}".format( @@ -646,7 +806,13 @@ def get_bucket_acl(self, Bucket, **kwargs): return data def put_bucket_cors(self, Bucket, CORSConfiguration={}, **kwargs): - """设置bucket CORS""" + """设置bucket CORS + + :param Bucket(string): 存储桶名称. + :param CORSConfiguration(dict): 设置Bucket跨域规则. + :param kwargs(dict): 设置请求headers. + :return: None. + """ lst = [ # 类型为list的标签 '', '', @@ -675,7 +841,11 @@ def put_bucket_cors(self, Bucket, CORSConfiguration={}, **kwargs): return None def get_bucket_cors(self, Bucket, **kwargs): - """获取bucket CORS""" + """获取bucket CORS + :param Bucket(string): 存储桶名称. + :param kwargs(dict): 设置请求headers. + :return(dict): 获取Bucket对应的跨域配置. + """ headers = mapped(kwargs) url = self._conf.uri(bucket=Bucket, path="?cors") logger.info("get bucket cors, url=:{url} ,headers=:{headers}".format( @@ -702,7 +872,12 @@ def get_bucket_cors(self, Bucket, **kwargs): return data def delete_bucket_cors(self, Bucket, **kwargs): - """删除bucket CORS""" + """删除bucket CORS + + :param Bucket(string): 存储桶名称. + :param kwargs(dict): 设置请求headers. + :return: None. + """ headers = mapped(kwargs) url = self._conf.uri(bucket=Bucket, path="?cors") logger.info("delete bucket cors, url=:{url} ,headers=:{headers}".format( @@ -716,7 +891,12 @@ def delete_bucket_cors(self, Bucket, **kwargs): return None def put_bucket_lifecycle(self, Bucket, LifecycleConfiguration={}, **kwargs): - """设置bucket LifeCycle""" + """设置bucket LifeCycle + :param Bucket(string): 存储桶名称. + :param LifecycleConfiguration(dict): 设置Bucket的生命周期规则. + :param kwargs(dict): 设置请求headers. + :return: None. + """ lst = ['', '', '', ''] # 类型为list的标签 xml_config = format_xml(data=LifecycleConfiguration, root='LifecycleConfiguration', lst=lst) headers = mapped(kwargs) @@ -735,7 +915,12 @@ def put_bucket_lifecycle(self, Bucket, LifecycleConfiguration={}, **kwargs): return None def get_bucket_lifecycle(self, Bucket, **kwargs): - """获取bucket LifeCycle""" + """获取bucket LifeCycle + + :param Bucket(string): 存储桶名称. + :param kwargs(dict): 设置请求headers. + :return(dict): Bucket对应的生命周期配置. + """ headers = mapped(kwargs) url = self._conf.uri(bucket=Bucket, path="?lifecycle") logger.info("get bucket cors, url=:{url} ,headers=:{headers}".format( @@ -754,7 +939,12 @@ def get_bucket_lifecycle(self, Bucket, **kwargs): return data def delete_bucket_lifecycle(self, Bucket, **kwargs): - """删除bucket LifeCycle""" + """删除bucket LifeCycle + + :param Bucket(string): 存储桶名称. + :param kwargs(dict): 设置请求headers. + :return: None. + """ headers = mapped(kwargs) url = self._conf.uri(bucket=Bucket, path="?lifecycle") logger.info("delete bucket cors, url=:{url} ,headers=:{headers}".format( @@ -768,7 +958,12 @@ def delete_bucket_lifecycle(self, Bucket, **kwargs): return None def put_bucket_versioning(self, Bucket, Status, **kwargs): - """设置bucket版本控制""" + """设置bucket版本控制 + :param Bucket(string): 存储桶名称. + :param Status(string): 设置Bucket版本控制的状态，可选值为'Enabled'|'Disabled'. + :param kwargs(dict): 设置请求headers. + :return: None. + """ headers = mapped(kwargs) url = self._conf.uri(bucket=Bucket, path="?versioning") logger.info("put bucket versioning, url=:{url} ,headers=:{headers}".format( @@ -788,7 +983,12 @@ def put_bucket_versioning(self, Bucket, Status, **kwargs): return None def get_bucket_versioning(self, Bucket, **kwargs): - """查询bucket版本控制""" + """查询bucket版本控制 + + :param Bucket(string): 存储桶名称. + :param kwargs(dict): 设置请求headers. + :return(dict): 获取Bucket版本控制的配置. + """ headers = mapped(kwargs) url = self._conf.uri(bucket=Bucket, path="?versioning") logger.info("get bucket versioning, url=:{url} ,headers=:{headers}".format( @@ -803,7 +1003,12 @@ def get_bucket_versioning(self, Bucket, **kwargs): return data def get_bucket_location(self, Bucket, **kwargs): - """查询bucket所属地域""" + """查询bucket所属地域 + + :param Bucket(string): 存储桶名称. + :param kwargs(dict): 设置请求headers. + :return(dict): 存储桶的地域信息. + """ headers = mapped(kwargs) url = self._conf.uri(bucket=Bucket, path="?location") logger.info("get bucket location, url=:{url} ,headers=:{headers}".format( @@ -819,23 +1024,12 @@ def get_bucket_location(self, Bucket, **kwargs): data['LocationConstraint'] = root.text return data - def head_bucket(self, Bucket, **kwargs): - """确认Bucket是否存在""" - headers = mapped(kwargs) - url = self._conf.uri(bucket=Bucket) - logger.info("head bucket, url=:{url} ,headers=:{headers}".format( - url=url, - headers=headers)) - rt = self.send_request( - method='HEAD', - url=url, - auth=CosS3Auth(self._conf._access_id, self._conf._access_key), - headers=headers) - return None - # service interface begin def list_buckets(self, **kwargs): - """列出所有bucket""" + """列出所有bucket + + :return(dict): 账号下bucket相关信息. + """ headers = mapped(kwargs) url = 'http://service.cos.myqcloud.com/' rt = self.send_request( diff --git a/qcloud_cos/test.py b/qcloud_cos/test.py index daa9d1f9..842469b2 100644 --- a/qcloud_cos/test.py +++ b/qcloud_cos/test.py @@ -138,7 +138,7 @@ def Test(): print "Test List Buckets" response = client.list_buckets() - copy_source = {'Appid': '1252408340', 'Bucket': 'test02', 'Key': '/test.txt', 'Region': 'ap-guangzhou'} + copy_source = {'Appid': '1252448703', 'Bucket': 'test04', 'Key': '/test.txt', 'Region': 'ap-beijing-1'} print "Test Copy Object From Otopy Object" response = client.copy_object( Bucket='test04', From abb4eaaf469ea4dd96d0b2c2d79b603a218d68dd Mon Sep 17 00:00:00 2001 From: tiedu Date: Mon, 23 Oct 2017 20:06:10 +0800 Subject: [PATCH 32/40] Modify Copy Demo --- qcloud_cos/test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qcloud_cos/test.py b/qcloud_cos/test.py index 842469b2..ade34355 100644 --- a/qcloud_cos/test.py +++ b/qcloud_cos/test.py @@ -138,8 +138,8 @@ def Test(): print "Test List Buckets" response = client.list_buckets() - copy_source = {'Appid': '1252448703', 'Bucket': 'test04', 'Key': '/test.txt', 'Region': 'ap-beijing-1'} - print "Test Copy Object From Otopy Object" + copy_source = {'Appid': '1252448703', 'Bucket': 'test01', 'Key': '/test.txt', 'Region': 'ap-beijing-1'} + print "Test Copy Object From Other Object" response = client.copy_object( Bucket='test04', Key='test.txt', From fe566742642210e8cc4e15daeea23d595eb5a305 Mon Sep 17 00:00:00 2001 From: tiedu Date: Mon, 23 Oct 2017 20:50:30 +0800 Subject: [PATCH 33/40] Add demo for upload/download --- qcloud_cos/demo.py | 57 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 qcloud_cos/demo.py diff --git a/qcloud_cos/demo.py b/qcloud_cos/demo.py new file mode 100644 index 00000000..b759a410 --- /dev/null +++ b/qcloud_cos/demo.py @@ -0,0 +1,57 @@ +# -*- coding=utf-8 +from qcloud_cos import CosConfig +from qcloud_cos import CosS3Client + +# 腾讯云COSV5Python SDK, 目前可以支持Python2.6与Python2.7 + +# pip安装指南:pip install -U cos-python-sdk-v5 + +# cos最新可用地域,参照https://www.qcloud.com/document/product/436/6224 + +# 设置用户属性, 包括appid, secret_id, secret_key, region +appid = '1242338703' # 替换为用户的appid +secret_id = 'AKID15IsskiBQACGbAo6WhgcQbVls7HmuG00' # 替换为用户的secret_id +secret_key = 'csivKvxxrMvSvQpMWHuIz12pThQQlWRW' # 替换为用户的secret_key +region = 'ap-beijing-1' # 替换为用户的region +token = '' # 使用临时秘钥需要传入Token，默认为空,可不填 +config = CosConfig(Appid=appid, Region=region, Access_id=secret_id, Access_key=secret_key, Token=token) # 获取配置对象 +client = CosS3Client(config) + +# 文件流 简单上传 +fp = open('test.txt', 'rb') +file_name = 'test.txt' +response = client.put_object( + Bucket='test04', + Body=fp, + Key=file_name, + StorageClass='STANDARD', + CacheControl='no-cache', + ContentDisposition='download.txt' +) +fp.close() +print response['ETag'] + +# 字节流 简单上传 +response = client.put_object( + Bucket='test04', + Body='abcdefg', + Key=file_name, + CacheControl='no-cache', + ContentDisposition='download.txt' +) +print response['ETag'] + +# 文件下载 获取文件到本地 +response = client.get_object( + Bucket='test04', + Key=file_name, +) +response['Body'].get_stream_to_file('output.txt') + +# 文件下载 获取文件流 +response = client.get_object( + Bucket='test04', + Key=file_name, +) +fp = response['Body'].get_raw_stream() +print fp.read(2) From 167728102c9fb44236215cd947a637069a5767f6 Mon Sep 17 00:00:00 2001 From: tiedu Date: Mon, 23 Oct 2017 21:01:14 +0800 Subject: [PATCH 34/40] Modify demo.py --- qcloud_cos/demo.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/qcloud_cos/demo.py b/qcloud_cos/demo.py index b759a410..260cca72 100644 --- a/qcloud_cos/demo.py +++ b/qcloud_cos/demo.py @@ -1,6 +1,8 @@ # -*- coding=utf-8 from qcloud_cos import CosConfig from qcloud_cos import CosS3Client +from qcloud_cos import CosServiceError +from qcloud_cos import CosClientError # 腾讯云COSV5Python SDK, 目前可以支持Python2.6与Python2.7 @@ -55,3 +57,22 @@ ) fp = response['Body'].get_raw_stream() print fp.read(2) + +# 文件下载 捕获异常 +try: + response = client.get_object( + Bucket='test04', + Key='not_exist.txt', + ) + fp = response['Body'].get_raw_stream() + print fp.read(2) +except CosServiceError as e: + print e.get_origin_msg() + print e.get_digest_msg() + print e.get_status_code() + print e.get_error_code() + print e.get_error_msg() + print e.get_resource_location() + print e.get_trace_id() + print e.get_request_id() + From a66dfe0d61d32545ee2f4ab958582ec3ce5aafeb Mon Sep 17 00:00:00 2001 From: tiedu Date: Mon, 23 Oct 2017 21:06:49 +0800 Subject: [PATCH 35/40] fix pep8 --- qcloud_cos/demo.py | 1 - 1 file changed, 1 deletion(-) diff --git a/qcloud_cos/demo.py b/qcloud_cos/demo.py index 260cca72..7720cf3f 100644 --- a/qcloud_cos/demo.py +++ b/qcloud_cos/demo.py @@ -75,4 +75,3 @@ print e.get_resource_location() print e.get_trace_id() print e.get_request_id() - From 35254fa2b0852cb002fc3d28c155fe4ca0505013 Mon Sep 17 00:00:00 2001 From: tiedu Date: Thu, 26 Oct 2017 10:43:30 +0800 Subject: [PATCH 36/40] Modify requirements --- requirements.txt | 5 ++--- setup.py | 5 ----- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/requirements.txt b/requirements.txt index 2331919a..1cbfa8ca 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,2 @@ -requests==2.12.4 -argparse==1.4.0 -dicttoxml==1.7.4 +requests +dicttoxml diff --git a/setup.py b/setup.py index 042d9e74..38bb8dbf 100644 --- a/setup.py +++ b/setup.py @@ -6,11 +6,6 @@ def requirements(): with open('requirements.txt', 'r') as fileobj: requirements = [line.strip() for line in fileobj] - - version = python_version_tuple() - - if version[0] == 2 and version[1] == 6: - requirements.append("argparse==1.4.0") return requirements From 0a5e08214bf592267bc6944d2a288c031d881e5f Mon Sep 17 00:00:00 2001 From: tiedu Date: Thu, 26 Oct 2017 10:48:57 +0800 Subject: [PATCH 37/40] Modify versioning status --- qcloud_cos/cos_client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/qcloud_cos/cos_client.py b/qcloud_cos/cos_client.py index bcba7617..9ea8e3bd 100644 --- a/qcloud_cos/cos_client.py +++ b/qcloud_cos/cos_client.py @@ -960,7 +960,7 @@ def delete_bucket_lifecycle(self, Bucket, **kwargs): def put_bucket_versioning(self, Bucket, Status, **kwargs): """设置bucket版本控制 :param Bucket(string): 存储桶名称. - :param Status(string): 设置Bucket版本控制的状态，可选值为'Enabled'|'Disabled'. + :param Status(string): 设置Bucket版本控制的状态，可选值为'Enabled'|'Suspended'. :param kwargs(dict): 设置请求headers. :return: None. """ @@ -969,8 +969,8 @@ def put_bucket_versioning(self, Bucket, Status, **kwargs): logger.info("put bucket versioning, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) - if Status != 'Enabled' and Status != 'Disabled': - raise CosClientError('versioning status must be set to Enabled or Disabled!') + if Status != 'Enabled' and Status != 'Suspended': + raise CosClientError('versioning status must be set to Enabled or Suspended!') config = dict() config['Status'] = Status xml_config = format_xml(data=config, root='VersioningConfiguration') From 5d04504eb2cd5d0526566808d6713e914e559d98 Mon Sep 17 00:00:00 2001 From: tiedu Date: Thu, 26 Oct 2017 11:02:49 +0800 Subject: [PATCH 38/40] add license --- LICENSE | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..275d2e42 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2017 腾讯云 + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. From 3a6766ef4aa30193806b5c4a2b766bf5947591e9 Mon Sep 17 00:00:00 2001 From: tiedu Date: Thu, 26 Oct 2017 16:10:54 +0800 Subject: [PATCH 39/40] Fix up hmac param can not be unicode --- qcloud_cos/cos_auth.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/qcloud_cos/cos_auth.py b/qcloud_cos/cos_auth.py index 25ec0bdd..4a010443 100644 --- a/qcloud_cos/cos_auth.py +++ b/qcloud_cos/cos_auth.py @@ -24,11 +24,22 @@ def filter_headers(data): return headers +def to_string(data): + """转换unicode为string. + + :param data(unicode|string): 待转换的unicode|string. + :return(string): 转换后的string. + """ + if isinstance(data, unicode): + return data.encode('utf8') + return data + + class CosS3Auth(AuthBase): - def __init__(self, access_id, secret_key, key='', params={}, expire=10000): - self._access_id = access_id - self._secret_key = secret_key + def __init__(self, secret_id, secret_key, key='', params={}, expire=10000): + self._secret_id = to_string(secret_id) + self._secret_key = to_string(secret_key) self._expire = expire self._params = params if key: @@ -67,7 +78,7 @@ def __call__(self, r): sign_tpl = "q-sign-algorithm=sha1&q-ak={ak}&q-sign-time={sign_time}&q-key-time={key_time}&q-header-list={headers}&q-url-param-list={params}&q-signature={sign}" r.headers['Authorization'] = sign_tpl.format( - ak=self._access_id, + ak=self._secret_id, sign_time=sign_time, key_time=sign_time, params=';'.join(sorted(map(lambda k: k.lower(), uri_params.keys()))), From a031974d515c4a9aec8e874ca114e9ee7ca49414 Mon Sep 17 00:00:00 2001 From: tiedu Date: Mon, 30 Oct 2017 12:00:49 +0800 Subject: [PATCH 40/40] Add Travis PyPI deployment --- .travis.yml | 38 ++++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/.travis.yml b/.travis.yml index d18a973f..7fd7c717 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,16 +1,22 @@ -sudo: false -language: python - -python: - - "2.6" - - "2.7" - -install: - - pip install requests - - pip install nose - - pip install pep8 - - pip install dicttoxml - -script: - - pep8 --max-line-length=180 qcloud_cos/. - - nosetests -s -v +sudo: false +language: python +python: +- '2.6' +- '2.7' +install: +- pip install requests +- pip install nose +- pip install pep8 +- pip install dicttoxml +script: +- pep8 --max-line-length=180 qcloud_cos/. +- nosetests -s -v +deploy: + provider: pypi + distributions: "sdist bdist bdist_wheel" + user: dt3310321 + password: + secure: LPiyxt41FBOXUM4WyzMjQvFz9ctQMcJQl7fTKkZ58fgD15XIWb/JXG2+tpCF/hFe6Fa8Kk1qL3cD+FxnQMWeTftebeWz35cn3EqWpd43/5BoSXjf0n2XTg2Ere6USCmTlosmW6A3GF18+DtsD2xzWTf6uWzdpJCHpBdNaxuph4poLfd4A/SCmggfEi7MN8d99sdJ8ENa3iUo51TWqCt5o57N7QLMII2nW8UpntP2XJ5aHZD1KDd8J8RMuDjtGBOvKbWjKCdFSZXT2Rcj/BDsacCUyACnDXdUAQ+Ic55eTtgpQFc7vm9yk5ApNrLC0xtf3WspO0HZO0ZgzsT1a5eGYYckeh2uVudCeC3VMgp3NQE7bLsO/ZLAhg8mQKr16bUNF+rfs59CL1/8/CeLxuIgKm6MnT1NRMsltdDqPjPsQTeEXpZQZ7urgMrXPCte8HgcCsgfZe39v0TrzZ4jES3orB68MnIsusDzoDfQzN2Z4UkkEEktihEwRjel6vzzMs1XGhC0ksvV+Tb6YUpdpnx6FO9ioZJS4ml4kDk3y8iowRcXBOvtMqrA7QwGmwpBARqts6bSa6ziXNqzFiiGWwbY7eeFscsX3uy84+5UIkABRylSl5C+C2B6jEXFD2T5r7YBQ+30eoPucLELyPxVgMkWTZTg8BVOfM2do4fkZQ+H0Rk= + on: + tags: true + branch: master