From a5e0540b44dd1b04a2828705754d2d3d4041d370 Mon Sep 17 00:00:00 2001 From: Itamar Ostricher Date: Wed, 15 Nov 2017 15:42:22 +0200 Subject: [PATCH 01/21] Working on Tencent COS Storage Driver implementation - hacky phase --- libcloud/common/base.py | 18 + libcloud/storage/base.py | 6 +- libcloud/storage/drivers/tencent_cos.py | 456 ++++++++++++++++++++++++ libcloud/storage/providers.py | 2 + libcloud/storage/types.py | 2 + 5 files changed, 481 insertions(+), 3 deletions(-) create mode 100644 libcloud/storage/drivers/tencent_cos.py diff --git a/libcloud/common/base.py b/libcloud/common/base.py index fbcdd9255c..ac7a103748 100644 --- a/libcloud/common/base.py +++ b/libcloud/common/base.py @@ -912,6 +912,24 @@ def __init__(self, user_id, key, secure=True, host=None, port=None, self.user_id = user_id +class ConnectionAppIdAndUserAndKey(ConnectionKey): + """ + Base connection class which accepts an ``app_id``, ``user_id`` and + ``key`` argument. + """ + + user_id = None + + def __init__(self, app_id, user_id, key, secure=True, host=None, port=None, + url=None, timeout=None, proxy_url=None, + backoff=None, retry_delay=None): + super(ConnectionAppIdAndUserAndKey, self).__init__( + user_id, key, secure=secure, host=host, port=port, url=url, + timeout=timeout, backoff=backoff, retry_delay=retry_delay, + proxy_url=proxy_url) + self.app_id = app_id + + class BaseDriver(object): """ Base driver class from which other classes can inherit from. diff --git a/libcloud/storage/base.py b/libcloud/storage/base.py index b3de63d18a..ec3b6638d4 100644 --- a/libcloud/storage/base.py +++ b/libcloud/storage/base.py @@ -65,15 +65,15 @@ def __init__(self, name, size, hash, extra, meta_data, container, :param hash: Object hash. :type hash: ``str`` - :param container: Object container. - :type container: :class:`Container` - :param extra: Extra attributes. :type extra: ``dict`` :param meta_data: Optional object meta data. :type meta_data: ``dict`` + :param container: Object container. + :type container: :class:`Container` + :param driver: StorageDriver instance. :type driver: :class:`StorageDriver` """ diff --git a/libcloud/storage/drivers/tencent_cos.py b/libcloud/storage/drivers/tencent_cos.py new file mode 100644 index 0000000000..f987779cfe --- /dev/null +++ b/libcloud/storage/drivers/tencent_cos.py @@ -0,0 +1,456 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +import json +from pprint import pprint + +import email.utils + +from qcloud_cos import ListFolderRequest + +from libcloud.common.base import ConnectionAppIdAndUserAndKey +from libcloud.common.google import GoogleAuthType +from libcloud.common.google import GoogleOAuth2Credential +from libcloud.common.google import GoogleResponse +from libcloud.common.types import ProviderError +from libcloud.storage.base import StorageDriver, Container, Object +from libcloud.storage.drivers.s3 import BaseS3Connection +from libcloud.storage.drivers.s3 import S3RawResponse +from libcloud.storage.drivers.s3 import S3Response +from libcloud.utils.py3 import httplib +from libcloud.utils.py3 import urlquote + +# Docs are a lie. Actual namespace returned is different that the one listed +# in the docs. +SIGNATURE_IDENTIFIER = 'GOOG1' + + +def _clean_object_name(name): + """ + Return the URL encoded name. name=None returns None. Useful for input + checking without having to check for None first. + + :param name: The object name + :type name: ``str`` or ``None`` + + :return: The url-encoded object name or None if name=None. + :rtype ``str`` or ``None`` + """ + return urlquote(name, safe='') if name else None + + +class ContainerPermissions(object): + values = ['NONE', 'READER', 'WRITER', 'OWNER'] + NONE = 0 + READER = 1 + WRITER = 2 + OWNER = 3 + + +class ObjectPermissions(object): + values = ['NONE', 'READER', 'OWNER'] + NONE = 0 + READER = 1 + OWNER = 2 + + +class TencentCosConnection(ConnectionAppIdAndUserAndKey): + """ + Represents a single connection to the Tencent COS API endpoint. + """ + + host = 'storage.googleapis.com' + responseCls = S3Response + rawResponseCls = S3RawResponse + PROJECT_ID_HEADER = 'x-goog-project-id' + + # def __init__(self, app_id, user_id, key, secure=True, + # credential_file=None, **kwargs): + # super(GoogleStorageConnection, self).__init__(user_id, key, secure, + # **kwargs) + + def add_default_headers(self, headers): + date = email.utils.formatdate(usegmt=True) + headers['Date'] = date + project = self.get_project() + if project: + headers[self.PROJECT_ID_HEADER] = project + return headers + + def get_project(self): + return getattr(self.driver, 'project', None) + + def pre_connect_hook(self, params, headers): + if self.auth_type == GoogleAuthType.GCS_S3: + signature = self._get_s3_auth_signature(params, headers) + headers['Authorization'] = '%s %s:%s' % (SIGNATURE_IDENTIFIER, + self.user_id, signature) + else: + headers['Authorization'] = ('Bearer ' + + self.oauth2_credential.access_token) + return params, headers + + def _get_s3_auth_signature(self, params, headers): + """Hacky wrapper to work with S3's get_auth_signature.""" + headers_copy = {} + params_copy = copy.deepcopy(params) + + # Lowercase all headers except 'date' and Google header values + for k, v in headers.items(): + k_lower = k.lower() + if (k_lower == 'date' or k_lower.startswith( + GoogleStorageDriver.http_vendor_prefix) or + not isinstance(v, str)): + headers_copy[k_lower] = v + else: + headers_copy[k_lower] = v.lower() + + return BaseS3Connection.get_auth_signature( + method=self.method, + headers=headers_copy, + params=params_copy, + expires=None, + secret_key=self.key, + path=self.action, + vendor_prefix=GoogleStorageDriver.http_vendor_prefix) + + +class GCSResponse(GoogleResponse): + pass + + +# class GoogleStorageJSONConnection(GoogleStorageConnection): +# """ +# Represents a single connection to the Google storage JSON API endpoint. + +# This can either authenticate via the Google OAuth2 methods or via +# the S3 HMAC interoperability method. +# """ +# host = 'www.googleapis.com' +# responseCls = GCSResponse +# rawResponseCls = None + +# def add_default_headers(self, headers): +# headers = super(GoogleStorageJSONConnection, self).add_default_headers( +# headers) +# headers['Content-Type'] = 'application/json' +# return headers + + +class TencentCosDriver(StorageDriver): + """ + Driver for Tencent Cloud Object Storage (COS). + + Can authenticate via standard Google Cloud methods (Service Accounts, + Installed App credentials, and GCE instance service accounts) + + Examples: + + Service Accounts:: + + driver = TencentCosDriver(key=client_email, secret=private_key, ...) + + Installed Application:: + + driver = TencentCosDriver(key=client_id, secret=client_secret, ...) + + From GCE instance:: + + driver = TencentCosDriver(key=foo, secret=bar, ...) + + Can also authenticate via Google Cloud Storage's S3 HMAC interoperability + API. S3 user keys are 20 alphanumeric characters, starting with GOOG. + + Example:: + + driver = TencentCosDriver(key='GOOG0123456789ABCXYZ', + secret=key_secret) + """ + name = 'Tencent COS' + website = 'https://cloud.tencent.com/product/cos' + # connectionCls = GoogleStorageConnection + # jsonConnectionCls = GoogleStorageJSONConnection + hash_type = 'md5' + supports_chunked_encoding = False + supports_s3_multipart_upload = False + + def __init__(self, key, secret=None, app_id=None, region=None, **kwargs): + super(TencentCosDriver, self).__init__(key, secret, **kwargs) + from qcloud_cos import CosClient + self.cos_client = CosClient(app_id, key, secret, region) + + # self.json_connection = GoogleStorageJSONConnection( + # key, secret, **kwargs) + + def _to_containers(self, obj_list): + for obj in obj_list: + yield self._to_container(obj) + + def _to_container(self, obj): + extra = { + 'creation_date': obj['ctime'], + 'modified_data': obj['mtime'], + } + return Container(obj['name'].rstrip('/'), extra, self) + + def _walk_container_folder(self, container, folder): + # TODO: handle paging + response = self.cos_client.list_folder( + ListFolderRequest(container.name, folder)) + if response['message'] == 'SUCCESS': + for obj in response['data']['infos']: + if obj['name'].endswith('/'): + # need to recurse into folder + yield from self._walk_container_folder( + container, folder + obj['name']) + else: + yield self._to_obj(obj, folder, container) + + def _to_obj(self, obj, folder, container): + extra = { + 'creation_date': obj['ctime'], + 'modified_data': obj['mtime'], + 'access_url': obj['access_url'], + 'source_url': obj['source_url'], + } + meta_data = {} + return Object(folder + obj['name'], obj['filesize'], obj['sha'], + extra, meta_data, container, self) + + def iterate_containers(self): + """ + Return a generator of containers for the given account + + :return: A generator of Container instances. + :rtype: ``generator`` of :class:`Container` + """ + # TODO: handle paging + response = self.cos_client.list_folder(ListFolderRequest('', '/')) + if response['message'] == 'SUCCESS': + return self._to_containers(response['data']['infos']) + + def iterate_container_objects(self, container): + """ + Return a generator of objects for the given container. + + :param container: Container instance + :type container: :class:`Container` + + :return: A generator of Object instances. + :rtype: ``generator`` of :class:`Object` + """ + return self._walk_container_folder(container, '/') + + # def _get_container_permissions(self, container_name): + # """ + # Return the container permissions for the current authenticated user. + + # :param container_name: The container name. + # :param container_name: ``str`` + + # :return: The permissions on the container. + # :rtype: ``int`` from ContainerPermissions + # """ + # # Try OWNER permissions first: try listing the bucket ACL. + # # FORBIDDEN -> exists, but not an OWNER. + # # NOT_FOUND -> bucket DNE, return NONE. + # try: + # self.json_connection.request( + # '/storage/v1/b/%s/acl' % container_name) + # return ContainerPermissions.OWNER + # except ProviderError as e: + # if e.http_code == httplib.FORBIDDEN: + # pass + # elif e.http_code == httplib.NOT_FOUND: + # return ContainerPermissions.NONE + # else: + # raise + + # # Try WRITER permissions with a noop request: try delete with an + # # impossible precondition. Authorization is checked before file + # # existence or preconditions. So, if we get a NOT_FOUND or a + # # PRECONDITION_FAILED, then we must be authorized. + # try: + # self.json_connection.request( + # '/storage/v1/b/%s/o/writecheck' % container_name, + # headers={'x-goog-if-generation-match': '0'}, method='DELETE') + # except ProviderError as e: + # if e.http_code in [httplib.NOT_FOUND, httplib.PRECONDITION_FAILED]: + # return ContainerPermissions.WRITER + # elif e.http_code != httplib.FORBIDDEN: + # raise + + # # Last, try READER permissions: try getting container metadata. + # try: + # self.json_connection.request('/storage/v1/b/%s' % container_name) + # return ContainerPermissions.READER + # except ProviderError as e: + # if e.http_code not in [httplib.FORBIDDEN, httplib.NOT_FOUND]: + # raise + + # return ContainerPermissions.NONE + + # def _get_user(self): + # """Gets this drivers' authenticated user, if any.""" + # oauth2_creds = getattr(self.connection, 'oauth2_credential') + # if oauth2_creds: + # return oauth2_creds.user_id + # else: + # return None + + # def _get_object_permissions(self, container_name, object_name): + # """ + # Return the object permissions for the current authenticated user. + # If the object does not exist, or no object_name is given, return the + # default object permissions. + + # :param container_name: The container name. + # :type container_name: ``str`` + + # :param object_name: The object name. + # :type object_name: ``str`` + + # :return: The permissions on the object or default object permissions. + # :rtype: ``int`` from ObjectPermissions + # """ + # # Try OWNER permissions first: try listing the object ACL. + # try: + # self.json_connection.request( + # '/storage/v1/b/%s/o/%s/acl' % (container_name, object_name)) + # return ObjectPermissions.OWNER + # except ProviderError as e: + # if e.http_code not in [httplib.FORBIDDEN, httplib.NOT_FOUND]: + # raise + + # # Try READER permissions: try getting the object. + # try: + # self.json_connection.request( + # '/storage/v1/b/%s/o/%s' % (container_name, object_name)) + # return ObjectPermissions.READER + # except ProviderError as e: + # if e.http_code not in [httplib.FORBIDDEN, httplib.NOT_FOUND]: + # raise + + # return ObjectPermissions.NONE + + # def ex_delete_permissions(self, container_name, object_name=None, + # entity=None): + # """ + # Delete permissions for an ACL entity on a container or object. + + # :param container_name: The container name. + # :type container_name: ``str`` + + # :param object_name: The object name. Optional. Not providing an object + # will delete a container permission. + # :type object_name: ``str`` + + # :param entity: The entity to whose permission will be deleted. + # Optional. If not provided, the role will be applied to the + # authenticated user, if using an OAuth2 authentication scheme. + # :type entity: ``str`` or ``None`` + # """ + # object_name = _clean_object_name(object_name) + # if not entity: + # user_id = self._get_user() + # if not user_id: + # raise ValueError( + # 'Must provide an entity. Driver is not using an ' + # 'authenticated user.') + # else: + # entity = 'user-%s' % user_id + + # if object_name: + # url = ('/storage/v1/b/%s/o/%s/acl/%s' % + # (container_name, object_name, entity)) + # else: + # url = '/storage/v1/b/%s/acl/%s' % (container_name, entity) + + # self.json_connection.request(url, method='DELETE') + + # def ex_get_permissions(self, container_name, object_name=None): + # """ + # Return the permissions for the currently authenticated user. + + # :param container_name: The container name. + # :type container_name: ``str`` + + # :param object_name: The object name. Optional. Not providing an object + # will return only container permissions. + # :type object_name: ``str`` or ``None`` + + # :return: A tuple of container and object permissions. + # :rtype: ``tuple`` of (``int``, ``int`` or ``None``) from + # ContainerPermissions and ObjectPermissions, respectively. + # """ + # object_name = _clean_object_name(object_name) + # obj_perms = self._get_object_permissions( + # container_name, object_name) if object_name else None + # return self._get_container_permissions(container_name), obj_perms + + # def ex_set_permissions(self, container_name, object_name=None, + # entity=None, role=None): + # """ + # Set the permissions for an ACL entity on a container or an object. + + # :param container_name: The container name. + # :type container_name: ``str`` + + # :param object_name: The object name. Optional. Not providing an object + # will apply the acl to the container. + # :type object_name: ``str`` + + # :param entity: The entity to which apply the role. Optional. If not + # provided, the role will be applied to the authenticated user, if + # using an OAuth2 authentication scheme. + # :type entity: ``str`` + + # :param role: The permission/role to set on the entity. + # :type role: ``int`` from ContainerPermissions or ObjectPermissions + # or ``str``. + + # :raises ValueError: If no entity was given, but was required. Or if + # the role isn't valid for the bucket or object. + # """ + # object_name = _clean_object_name(object_name) + # if isinstance(role, int): + # perms = ObjectPermissions if object_name else ContainerPermissions + # try: + # role = perms.values[role] + # except IndexError: + # raise ValueError( + # '%s is not a valid role level for container=%s object=%s' % + # (role, container_name, object_name)) + # elif not isinstance(role, str): + # raise ValueError('%s is not a valid permission.' % role) + + # if not entity: + # user_id = self._get_user() + # if not user_id: + # raise ValueError( + # 'Must provide an entity. Driver is not using an ' + # 'authenticated user.') + # else: + # entity = 'user-%s' % user_id + + # if object_name: + # url = '/storage/v1/b/%s/o/%s/acl' % (container_name, object_name) + # else: + # url = '/storage/v1/b/%s/acl' % container_name + + # self.json_connection.request( + # url, method='POST', + # data=json.dumps({'role': role, 'entity': entity})) diff --git a/libcloud/storage/providers.py b/libcloud/storage/providers.py index 183189ebb8..177187c2e3 100644 --- a/libcloud/storage/providers.py +++ b/libcloud/storage/providers.py @@ -84,6 +84,8 @@ Provider.DIGITALOCEAN_SPACES: ('libcloud.storage.drivers.digitalocean_spaces', 'DigitalOceanSpacesStorageDriver'), + Provider.TENCENT_COS: + ('libcloud.storage.drivers.tencent_cos', 'TencentCosDriver'), } diff --git a/libcloud/storage/types.py b/libcloud/storage/types.py index c9e7451413..cb3d39dedb 100644 --- a/libcloud/storage/types.py +++ b/libcloud/storage/types.py @@ -55,6 +55,7 @@ class Provider(object): :cvar S3_US_WEST_OREGON: Amazon S3 US West 2 (Oregon) :cvar S3_RGW: S3 RGW :cvar S3_RGW_OUTSCALE: OUTSCALE S3 RGW + :cvar TENCENT_COS Tencent COS """ DUMMY = 'dummy' ALIYUN_OSS = 'aliyun_oss' @@ -88,6 +89,7 @@ class Provider(object): S3_US_GOV_WEST = 's3_us_gov_west' S3_RGW = 's3_rgw' S3_RGW_OUTSCALE = 's3_rgw_outscale' + TENCENT_COS = 'tencent_cos' # Deperecated CLOUDFILES_US = 'cloudfiles_us' From 1f2e496a1102eeebae2ffca8e21fa034ffd5174c Mon Sep 17 00:00:00 2001 From: Itamar Ostricher Date: Mon, 20 Nov 2017 15:21:31 +0200 Subject: [PATCH 02/21] Handle paging correctly when iterating containers and container-objects --- libcloud/storage/drivers/tencent_cos.py | 27 +++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/libcloud/storage/drivers/tencent_cos.py b/libcloud/storage/drivers/tencent_cos.py index f987779cfe..e83efd96e6 100644 --- a/libcloud/storage/drivers/tencent_cos.py +++ b/libcloud/storage/drivers/tencent_cos.py @@ -207,10 +207,15 @@ def _to_container(self, obj): return Container(obj['name'].rstrip('/'), extra, self) def _walk_container_folder(self, container, folder): - # TODO: handle paging - response = self.cos_client.list_folder( - ListFolderRequest(container.name, folder)) - if response['message'] == 'SUCCESS': + exhausted = False + context = '' + while not exhausted: + response = self.cos_client.list_folder( + ListFolderRequest(container.name, folder, context=context)) + if response.get('message') != 'SUCCESS': + return + exhausted = response['data']['listover'] + context = response['data']['context'] for obj in response['data']['infos']: if obj['name'].endswith('/'): # need to recurse into folder @@ -237,10 +242,16 @@ def iterate_containers(self): :return: A generator of Container instances. :rtype: ``generator`` of :class:`Container` """ - # TODO: handle paging - response = self.cos_client.list_folder(ListFolderRequest('', '/')) - if response['message'] == 'SUCCESS': - return self._to_containers(response['data']['infos']) + exhausted = False + context = '' + while not exhausted: + response = self.cos_client.list_folder( + ListFolderRequest('', '/', context=context)) + if response.get('message') != 'SUCCESS': + return + exhausted = response['data']['listover'] + context = response['data']['context'] + yield from self._to_containers(response['data']['infos']) def iterate_container_objects(self, container): """ From 46d580b07ba57591fee216e7095770ae14d29ab1 Mon Sep 17 00:00:00 2001 From: Itamar Ostricher Date: Tue, 21 Nov 2017 08:48:08 +0200 Subject: [PATCH 03/21] Implement `get_container` method --- libcloud/storage/drivers/tencent_cos.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/libcloud/storage/drivers/tencent_cos.py b/libcloud/storage/drivers/tencent_cos.py index e83efd96e6..020e4e9511 100644 --- a/libcloud/storage/drivers/tencent_cos.py +++ b/libcloud/storage/drivers/tencent_cos.py @@ -20,6 +20,7 @@ import email.utils from qcloud_cos import ListFolderRequest +from qcloud_cos import StatFolderRequest from libcloud.common.base import ConnectionAppIdAndUserAndKey from libcloud.common.google import GoogleAuthType @@ -265,6 +266,24 @@ def iterate_container_objects(self, container): """ return self._walk_container_folder(container, '/') + def get_container(self, container_name): + """ + Return a container instance. + + :param container_name: Container name. + :type container_name: ``str`` + + :return: :class:`Container` instance. + :rtype: :class:`Container` + """ + response = self.cos_client.stat_folder( + StatFolderRequest(container_name, '/')) + if response.get('message') != 'SUCCESS': + return None + # "inject" the container name to the dictionary for `_to_container` + response['data']['name'] = container_name + return self._to_container(response['data']) + # def _get_container_permissions(self, container_name): # """ # Return the container permissions for the current authenticated user. From 00cdbb7d511f498231487e4a808d31638a51689b Mon Sep 17 00:00:00 2001 From: Itamar Ostricher Date: Tue, 21 Nov 2017 10:38:26 +0200 Subject: [PATCH 04/21] Implement `get_object` method --- libcloud/storage/drivers/tencent_cos.py | 36 ++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/libcloud/storage/drivers/tencent_cos.py b/libcloud/storage/drivers/tencent_cos.py index 020e4e9511..c0297bbe00 100644 --- a/libcloud/storage/drivers/tencent_cos.py +++ b/libcloud/storage/drivers/tencent_cos.py @@ -19,8 +19,11 @@ import email.utils -from qcloud_cos import ListFolderRequest -from qcloud_cos import StatFolderRequest +from qcloud_cos import ( + ListFolderRequest, + StatFileRequest, + StatFolderRequest, +) from libcloud.common.base import ConnectionAppIdAndUserAndKey from libcloud.common.google import GoogleAuthType @@ -31,6 +34,8 @@ from libcloud.storage.drivers.s3 import BaseS3Connection from libcloud.storage.drivers.s3 import S3RawResponse from libcloud.storage.drivers.s3 import S3Response +from libcloud.storage.types import ContainerDoesNotExistError +from libcloud.storage.types import ObjectDoesNotExistError from libcloud.utils.py3 import httplib from libcloud.utils.py3 import urlquote @@ -279,11 +284,36 @@ def get_container(self, container_name): response = self.cos_client.stat_folder( StatFolderRequest(container_name, '/')) if response.get('message') != 'SUCCESS': - return None + raise ContainerDoesNotExistError(value=None, driver=self, + container_name=container_name) # "inject" the container name to the dictionary for `_to_container` response['data']['name'] = container_name return self._to_container(response['data']) + def get_object(self, container_name, object_name): + """ + Return an object instance. + + :param container_name: Container name. + :type container_name: ``str`` + + :param object_name: Object name. + :type object_name: ``str`` + + :return: :class:`Object` instance. + :rtype: :class:`Object` + """ + response = self.cos_client.stat_file( + StatFileRequest(container_name, object_name)) + if response.get('message') != 'SUCCESS': + raise ObjectDoesNotExistError(value=None, driver=self, + object_name=object_name) + # "inject" the object name to the dictionary for `_to_obj` + response['data']['name'] = object_name + return self._to_obj(response['data'], '', + self.get_container(container_name)) + + # def _get_container_permissions(self, container_name): # """ # Return the container permissions for the current authenticated user. From 62725d7e1b120c754308b68359a6a8197037ec0f Mon Sep 17 00:00:00 2001 From: Itamar Ostricher Date: Tue, 21 Nov 2017 17:35:23 +0200 Subject: [PATCH 05/21] Remove prepending `/` from object names --- libcloud/storage/drivers/tencent_cos.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/libcloud/storage/drivers/tencent_cos.py b/libcloud/storage/drivers/tencent_cos.py index c0297bbe00..ded0df7166 100644 --- a/libcloud/storage/drivers/tencent_cos.py +++ b/libcloud/storage/drivers/tencent_cos.py @@ -20,6 +20,7 @@ import email.utils from qcloud_cos import ( + DownloadFileRequest, ListFolderRequest, StatFileRequest, StatFolderRequest, @@ -29,6 +30,7 @@ from libcloud.common.google import GoogleAuthType from libcloud.common.google import GoogleOAuth2Credential from libcloud.common.google import GoogleResponse +from libcloud.common.types import LibcloudError from libcloud.common.types import ProviderError from libcloud.storage.base import StorageDriver, Container, Object from libcloud.storage.drivers.s3 import BaseS3Connection @@ -238,7 +240,7 @@ def _to_obj(self, obj, folder, container): 'source_url': obj['source_url'], } meta_data = {} - return Object(folder + obj['name'], obj['filesize'], obj['sha'], + return Object(folder[1:] + obj['name'], obj['filesize'], obj['sha'], extra, meta_data, container, self) def iterate_containers(self): @@ -304,7 +306,7 @@ def get_object(self, container_name, object_name): :rtype: :class:`Object` """ response = self.cos_client.stat_file( - StatFileRequest(container_name, object_name)) + StatFileRequest(container_name, '/' + object_name)) if response.get('message') != 'SUCCESS': raise ObjectDoesNotExistError(value=None, driver=self, object_name=object_name) @@ -313,7 +315,6 @@ def get_object(self, container_name, object_name): return self._to_obj(response['data'], '', self.get_container(container_name)) - # def _get_container_permissions(self, container_name): # """ # Return the container permissions for the current authenticated user. From 5034ba98d9cd235ecfa36a8fdfa17e2c44237503 Mon Sep 17 00:00:00 2001 From: Itamar Ostricher Date: Wed, 22 Nov 2017 13:03:02 +0200 Subject: [PATCH 06/21] a little cleanup --- libcloud/storage/drivers/tencent_cos.py | 31 ++++++++++++++----------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/libcloud/storage/drivers/tencent_cos.py b/libcloud/storage/drivers/tencent_cos.py index ded0df7166..5a457a3012 100644 --- a/libcloud/storage/drivers/tencent_cos.py +++ b/libcloud/storage/drivers/tencent_cos.py @@ -203,6 +203,10 @@ def __init__(self, key, secret=None, app_id=None, region=None, **kwargs): # self.json_connection = GoogleStorageJSONConnection( # key, secret, **kwargs) + @staticmethod + def _is_ok(response): + return response['code'] == 0 + def _to_containers(self, obj_list): for obj in obj_list: yield self._to_container(obj) @@ -218,9 +222,9 @@ def _walk_container_folder(self, container, folder): exhausted = False context = '' while not exhausted: - response = self.cos_client.list_folder( - ListFolderRequest(container.name, folder, context=context)) - if response.get('message') != 'SUCCESS': + req = ListFolderRequest(container.name, folder, context=context) + response = self.cos_client.list_folder(req) + if not self._is_ok(response): return exhausted = response['data']['listover'] context = response['data']['context'] @@ -240,7 +244,8 @@ def _to_obj(self, obj, folder, container): 'source_url': obj['source_url'], } meta_data = {} - return Object(folder[1:] + obj['name'], obj['filesize'], obj['sha'], + return Object(folder.lstrip('/') + obj['name'], + obj['filesize'], obj['sha'], extra, meta_data, container, self) def iterate_containers(self): @@ -253,9 +258,9 @@ def iterate_containers(self): exhausted = False context = '' while not exhausted: - response = self.cos_client.list_folder( - ListFolderRequest('', '/', context=context)) - if response.get('message') != 'SUCCESS': + req = ListFolderRequest('', '/', context=context) + response = self.cos_client.list_folder(req) + if not self._is_ok(response): return exhausted = response['data']['listover'] context = response['data']['context'] @@ -283,9 +288,9 @@ def get_container(self, container_name): :return: :class:`Container` instance. :rtype: :class:`Container` """ - response = self.cos_client.stat_folder( - StatFolderRequest(container_name, '/')) - if response.get('message') != 'SUCCESS': + req = StatFolderRequest(container_name, '/') + response = self.cos_client.stat_folder(req) + if not self._is_ok(response): raise ContainerDoesNotExistError(value=None, driver=self, container_name=container_name) # "inject" the container name to the dictionary for `_to_container` @@ -305,9 +310,9 @@ def get_object(self, container_name, object_name): :return: :class:`Object` instance. :rtype: :class:`Object` """ - response = self.cos_client.stat_file( - StatFileRequest(container_name, '/' + object_name)) - if response.get('message') != 'SUCCESS': + req = StatFileRequest(container_name, '/' + object_name) + response = self.cos_client.stat_file(req) + if not self._is_ok(response): raise ObjectDoesNotExistError(value=None, driver=self, object_name=object_name) # "inject" the object name to the dictionary for `_to_obj` From c970c7d1051aeb7256a7475cefefa437d8ebbca8 Mon Sep 17 00:00:00 2001 From: Itamar Ostricher Date: Wed, 22 Nov 2017 13:03:52 +0200 Subject: [PATCH 07/21] Implement object download methods --- libcloud/storage/drivers/tencent_cos.py | 55 +++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/libcloud/storage/drivers/tencent_cos.py b/libcloud/storage/drivers/tencent_cos.py index 5a457a3012..04bbbe8dfb 100644 --- a/libcloud/storage/drivers/tencent_cos.py +++ b/libcloud/storage/drivers/tencent_cos.py @@ -15,12 +15,16 @@ import copy import json +import os from pprint import pprint +import shutil +import tempfile import email.utils from qcloud_cos import ( DownloadFileRequest, + DownloadObjectRequest, ListFolderRequest, StatFileRequest, StatFolderRequest, @@ -38,6 +42,7 @@ from libcloud.storage.drivers.s3 import S3Response from libcloud.storage.types import ContainerDoesNotExistError from libcloud.storage.types import ObjectDoesNotExistError +from libcloud.utils.files import read_in_chunks from libcloud.utils.py3 import httplib from libcloud.utils.py3 import urlquote @@ -320,6 +325,56 @@ def get_object(self, container_name, object_name): return self._to_obj(response['data'], '', self.get_container(container_name)) + def download_object(self, obj, destination_path, overwrite_existing=False, + delete_on_failure=True): + """ + Download an object to the specified destination path. + + :param obj: Object instance. + :type obj: :class:`Object` + + :param destination_path: Full path to a file or a directory where the + incoming file will be saved. + :type destination_path: ``str`` + + :param overwrite_existing: True to overwrite an existing file, + defaults to False. + :type overwrite_existing: ``bool`` + + :param delete_on_failure: True to delete a partially downloaded file if + the download was not successful (hash + mismatch / file size). + :type delete_on_failure: ``bool`` + + :return: True if an object has been successfully downloaded, False + otherwise. + :rtype: ``bool`` + """ + if os.path.exists(destination_path) and not overwrite_existing: + return False + req = DownloadFileRequest(obj.container.name, '/' + obj.name, + destination_path) + response = self.cos_client.download_file(req) + if self._is_ok(response): + return True + if delete_on_failure and os.path.exists(destination_path): + os.remove(destination_path) + return False + + def download_object_as_stream(self, obj, chunk_size=None): + """ + Return a generator which yields object data. + + :param obj: Object instance + :type obj: :class:`Object` + + :param chunk_size: Optional chunk size (in bytes). + :type chunk_size: ``int`` + """ + req = DownloadObjectRequest(obj.container.name, '/' + obj.name) + response = self.cos_client.download_object(req) + return read_in_chunks(response, chunk_size) + # def _get_container_permissions(self, container_name): # """ # Return the container permissions for the current authenticated user. From 24b556c07530f3789edae3815972b637d5a4db66 Mon Sep 17 00:00:00 2001 From: Itamar Ostricher Date: Wed, 22 Nov 2017 13:59:12 +0200 Subject: [PATCH 08/21] Implement upload methods --- libcloud/storage/drivers/tencent_cos.py | 111 +++++++++++++++++++++++- 1 file changed, 109 insertions(+), 2 deletions(-) diff --git a/libcloud/storage/drivers/tencent_cos.py b/libcloud/storage/drivers/tencent_cos.py index 04bbbe8dfb..36c810e55d 100644 --- a/libcloud/storage/drivers/tencent_cos.py +++ b/libcloud/storage/drivers/tencent_cos.py @@ -28,6 +28,7 @@ ListFolderRequest, StatFileRequest, StatFolderRequest, + UploadFileRequest, ) from libcloud.common.base import ConnectionAppIdAndUserAndKey @@ -42,7 +43,7 @@ from libcloud.storage.drivers.s3 import S3Response from libcloud.storage.types import ContainerDoesNotExistError from libcloud.storage.types import ObjectDoesNotExistError -from libcloud.utils.files import read_in_chunks +from libcloud.utils.files import exhaust_iterator, read_in_chunks from libcloud.utils.py3 import httplib from libcloud.utils.py3 import urlquote @@ -196,7 +197,7 @@ class TencentCosDriver(StorageDriver): website = 'https://cloud.tencent.com/product/cos' # connectionCls = GoogleStorageConnection # jsonConnectionCls = GoogleStorageJSONConnection - hash_type = 'md5' + hash_type = 'sha1' supports_chunked_encoding = False supports_s3_multipart_upload = False @@ -375,6 +376,112 @@ def download_object_as_stream(self, obj, chunk_size=None): response = self.cos_client.download_object(req) return read_in_chunks(response, chunk_size) + def upload_object(self, file_path, container, object_name, extra=None, + verify_hash=True, headers=None): + """ + Upload an object currently located on a disk. + + :param file_path: Path to the object on disk. + :type file_path: ``str`` + + :param container: Destination container. + :type container: :class:`Container` + + :param object_name: Object name. + :type object_name: ``str`` + + :param verify_hash: Verify hash + :type verify_hash: ``bool`` + + :param extra: Extra attributes (driver specific). (optional) + :type extra: ``dict`` + + :param headers: (optional) Additional request headers, + such as CORS headers. For example: + headers = {'Access-Control-Allow-Origin': 'http://mozilla.com'} + :type headers: ``dict`` + + :rtype: :class:`Object` + """ + if file_path and not os.path.exists(file_path): + raise OSError('File %s does not exist' % (file_path)) + req = UploadFileRequest(container.name, '/' + object_name, file_path) + + def set_extra(field_name): + if field_name in extra: + set_func = req.getattr('set_%s' % (field_name)) + set_func(extra[field_name]) + + set_extra('authority') + set_extra('biz_attr') + set_extra('cache_control') + set_extra('content_type') + set_extra('content_language') + set_extra('content_encoding') + set_extra('x_cos_meta') + response = self.cos_client.upload_file(req) + if not self._is_ok(response): + raise LibcloudError( + 'Error in uploading object: %s' % (response['message']), + driver=self) + obj = self.get_object(container.name, object_name) + if verify_hash: + hasher = self._get_hash_function() + with open(file_path, 'rb') as src_file: + hasher.update(src_file.read()) + data_hash = hasher.hexdigest() + if data_hash != obj.hash: + raise ObjectHashMismatchError( + value='SHA1 hash {0} checksum does not match {1}'.format( + obj.hash, data_hash), + object_name=object_name, driver=self) + return obj + + def upload_object_via_stream(self, iterator, container, object_name, + extra=None, headers=None): + """ + Upload an object using an iterator. + + If a provider supports it, chunked transfer encoding is used and you + don't need to know in advance the amount of data to be uploaded. + + Otherwise if a provider doesn't support it, iterator will be exhausted + so a total size for data to be uploaded can be determined. + + Note: Exhausting the iterator means that the whole data must be + buffered in memory which might result in memory exhausting when + uploading a very large object. + + If a file is located on a disk you are advised to use upload_object + function which uses fs.stat function to determine the file size and it + doesn't need to buffer whole object in the memory. + + :param iterator: An object which implements the iterator interface. + :type iterator: :class:`object` + + :param container: Destination container. + :type container: :class:`Container` + + :param object_name: Object name. + :type object_name: ``str`` + + :param extra: (optional) Extra attributes (driver specific). Note: + This dictionary must contain a 'content_type' key which represents + a content type of the stored object. + :type extra: ``dict`` + + :param headers: (optional) Additional request headers, + such as CORS headers. For example: + headers = {'Access-Control-Allow-Origin': 'http://mozilla.com'} + :type headers: ``dict`` + + :rtype: ``object`` + """ + with tempfile.NamedTemporaryFile() as tmp_file: + tmp_file.write(exhaust_iterator(iterator)) + return self.upload_object( + tmp_file.name, container, object_name, extra, headers) + # def _get_container_permissions(self, container_name): # """ # Return the container permissions for the current authenticated user. From 5406fe8259e9aa83d8853dc1f38256e6bbe8e404 Mon Sep 17 00:00:00 2001 From: Itamar Ostricher Date: Wed, 22 Nov 2017 14:13:34 +0200 Subject: [PATCH 09/21] Fix `extra` handling when `None` --- libcloud/storage/drivers/tencent_cos.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libcloud/storage/drivers/tencent_cos.py b/libcloud/storage/drivers/tencent_cos.py index 36c810e55d..ad4b213948 100644 --- a/libcloud/storage/drivers/tencent_cos.py +++ b/libcloud/storage/drivers/tencent_cos.py @@ -408,7 +408,7 @@ def upload_object(self, file_path, container, object_name, extra=None, req = UploadFileRequest(container.name, '/' + object_name, file_path) def set_extra(field_name): - if field_name in extra: + if extra and field_name in extra: set_func = req.getattr('set_%s' % (field_name)) set_func(extra[field_name]) From 8700c21851729fcd4ce270166cd12c5dde278a3a Mon Sep 17 00:00:00 2001 From: Itamar Ostricher Date: Wed, 22 Nov 2017 14:15:10 +0200 Subject: [PATCH 10/21] Implement `delete_object` method --- libcloud/storage/drivers/tencent_cos.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/libcloud/storage/drivers/tencent_cos.py b/libcloud/storage/drivers/tencent_cos.py index ad4b213948..c635ced48d 100644 --- a/libcloud/storage/drivers/tencent_cos.py +++ b/libcloud/storage/drivers/tencent_cos.py @@ -23,6 +23,7 @@ import email.utils from qcloud_cos import ( + DelFileRequest, DownloadFileRequest, DownloadObjectRequest, ListFolderRequest, @@ -482,6 +483,20 @@ def upload_object_via_stream(self, iterator, container, object_name, return self.upload_object( tmp_file.name, container, object_name, extra, headers) + def delete_object(self, obj): + """ + Delete an object. + + :param obj: Object instance. + :type obj: :class:`Object` + + :return: ``bool`` True on success. + :rtype: ``bool`` + """ + req = DelFileRequest(obj.container.name, '/' + obj.name) + response = self.cos_client.del_file(req) + return self._is_ok(response) + # def _get_container_permissions(self, container_name): # """ # Return the container permissions for the current authenticated user. From a570eb000768e38600dbfed86262a440bf3d5f02 Mon Sep 17 00:00:00 2001 From: Itamar Ostricher Date: Wed, 22 Nov 2017 16:27:20 +0200 Subject: [PATCH 11/21] Fix upload object from stream - need to close temp file before calling upload object from file --- libcloud/storage/drivers/tencent_cos.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/libcloud/storage/drivers/tencent_cos.py b/libcloud/storage/drivers/tencent_cos.py index c635ced48d..6979d74ece 100644 --- a/libcloud/storage/drivers/tencent_cos.py +++ b/libcloud/storage/drivers/tencent_cos.py @@ -478,10 +478,13 @@ def upload_object_via_stream(self, iterator, container, object_name, :rtype: ``object`` """ - with tempfile.NamedTemporaryFile() as tmp_file: + with tempfile.NamedTemporaryFile(delete=False) as tmp_file: tmp_file.write(exhaust_iterator(iterator)) + try: return self.upload_object( tmp_file.name, container, object_name, extra, headers) + finally: + os.remove(tmp_file.name) def delete_object(self, obj): """ From c445f600d8f0500c8e4f66aab283cf925c266945 Mon Sep 17 00:00:00 2001 From: Itamar Ostricher Date: Wed, 22 Nov 2017 16:27:50 +0200 Subject: [PATCH 12/21] Fix download object from stream when object is empty - need to allow yielding empty string --- libcloud/storage/drivers/tencent_cos.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libcloud/storage/drivers/tencent_cos.py b/libcloud/storage/drivers/tencent_cos.py index 6979d74ece..28362af3a7 100644 --- a/libcloud/storage/drivers/tencent_cos.py +++ b/libcloud/storage/drivers/tencent_cos.py @@ -375,7 +375,7 @@ def download_object_as_stream(self, obj, chunk_size=None): """ req = DownloadObjectRequest(obj.container.name, '/' + obj.name) response = self.cos_client.download_object(req) - return read_in_chunks(response, chunk_size) + return read_in_chunks(response, chunk_size, yield_empty=True) def upload_object(self, file_path, container, object_name, extra=None, verify_hash=True, headers=None): From 3b754292fd5d987ca8052cc1d56cb28a9423d704 Mon Sep 17 00:00:00 2001 From: Itamar Ostricher Date: Thu, 23 Nov 2017 11:01:33 +0200 Subject: [PATCH 13/21] Implement `get_object_cdn_url` method --- libcloud/storage/drivers/tencent_cos.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/libcloud/storage/drivers/tencent_cos.py b/libcloud/storage/drivers/tencent_cos.py index 28362af3a7..82b2b377e3 100644 --- a/libcloud/storage/drivers/tencent_cos.py +++ b/libcloud/storage/drivers/tencent_cos.py @@ -327,6 +327,18 @@ def get_object(self, container_name, object_name): return self._to_obj(response['data'], '', self.get_container(container_name)) + def get_object_cdn_url(self, obj): + """ + Return an object CDN URL. + + :param obj: Object instance + :type obj: :class:`Object` + + :return: A CDN URL for this object. + :rtype: ``str`` + """ + return obj.extra['access_url'] + def download_object(self, obj, destination_path, overwrite_existing=False, delete_on_failure=True): """ From 0d51b704d3e02397128bc262d8e21aa4b60225b4 Mon Sep 17 00:00:00 2001 From: Itamar Ostricher Date: Thu, 23 Nov 2017 12:03:27 +0200 Subject: [PATCH 14/21] Cleanup --- libcloud/common/base.py | 18 -- libcloud/storage/drivers/tencent_cos.py | 362 +----------------------- 2 files changed, 4 insertions(+), 376 deletions(-) diff --git a/libcloud/common/base.py b/libcloud/common/base.py index ac7a103748..fbcdd9255c 100644 --- a/libcloud/common/base.py +++ b/libcloud/common/base.py @@ -912,24 +912,6 @@ def __init__(self, user_id, key, secure=True, host=None, port=None, self.user_id = user_id -class ConnectionAppIdAndUserAndKey(ConnectionKey): - """ - Base connection class which accepts an ``app_id``, ``user_id`` and - ``key`` argument. - """ - - user_id = None - - def __init__(self, app_id, user_id, key, secure=True, host=None, port=None, - url=None, timeout=None, proxy_url=None, - backoff=None, retry_delay=None): - super(ConnectionAppIdAndUserAndKey, self).__init__( - user_id, key, secure=secure, host=host, port=port, url=url, - timeout=timeout, backoff=backoff, retry_delay=retry_delay, - proxy_url=proxy_url) - self.app_id = app_id - - class BaseDriver(object): """ Base driver class from which other classes can inherit from. diff --git a/libcloud/storage/drivers/tencent_cos.py b/libcloud/storage/drivers/tencent_cos.py index 82b2b377e3..bfe918dc7e 100644 --- a/libcloud/storage/drivers/tencent_cos.py +++ b/libcloud/storage/drivers/tencent_cos.py @@ -13,15 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import copy -import json import os -from pprint import pprint import shutil import tempfile -import email.utils - from qcloud_cos import ( DelFileRequest, DownloadFileRequest, @@ -32,184 +27,36 @@ UploadFileRequest, ) -from libcloud.common.base import ConnectionAppIdAndUserAndKey -from libcloud.common.google import GoogleAuthType -from libcloud.common.google import GoogleOAuth2Credential -from libcloud.common.google import GoogleResponse from libcloud.common.types import LibcloudError -from libcloud.common.types import ProviderError from libcloud.storage.base import StorageDriver, Container, Object -from libcloud.storage.drivers.s3 import BaseS3Connection -from libcloud.storage.drivers.s3 import S3RawResponse -from libcloud.storage.drivers.s3 import S3Response from libcloud.storage.types import ContainerDoesNotExistError from libcloud.storage.types import ObjectDoesNotExistError from libcloud.utils.files import exhaust_iterator, read_in_chunks -from libcloud.utils.py3 import httplib -from libcloud.utils.py3 import urlquote - -# Docs are a lie. Actual namespace returned is different that the one listed -# in the docs. -SIGNATURE_IDENTIFIER = 'GOOG1' - - -def _clean_object_name(name): - """ - Return the URL encoded name. name=None returns None. Useful for input - checking without having to check for None first. - - :param name: The object name - :type name: ``str`` or ``None`` - - :return: The url-encoded object name or None if name=None. - :rtype ``str`` or ``None`` - """ - return urlquote(name, safe='') if name else None - - -class ContainerPermissions(object): - values = ['NONE', 'READER', 'WRITER', 'OWNER'] - NONE = 0 - READER = 1 - WRITER = 2 - OWNER = 3 - - -class ObjectPermissions(object): - values = ['NONE', 'READER', 'OWNER'] - NONE = 0 - READER = 1 - OWNER = 2 - - -class TencentCosConnection(ConnectionAppIdAndUserAndKey): - """ - Represents a single connection to the Tencent COS API endpoint. - """ - - host = 'storage.googleapis.com' - responseCls = S3Response - rawResponseCls = S3RawResponse - PROJECT_ID_HEADER = 'x-goog-project-id' - - # def __init__(self, app_id, user_id, key, secure=True, - # credential_file=None, **kwargs): - # super(GoogleStorageConnection, self).__init__(user_id, key, secure, - # **kwargs) - - def add_default_headers(self, headers): - date = email.utils.formatdate(usegmt=True) - headers['Date'] = date - project = self.get_project() - if project: - headers[self.PROJECT_ID_HEADER] = project - return headers - - def get_project(self): - return getattr(self.driver, 'project', None) - - def pre_connect_hook(self, params, headers): - if self.auth_type == GoogleAuthType.GCS_S3: - signature = self._get_s3_auth_signature(params, headers) - headers['Authorization'] = '%s %s:%s' % (SIGNATURE_IDENTIFIER, - self.user_id, signature) - else: - headers['Authorization'] = ('Bearer ' + - self.oauth2_credential.access_token) - return params, headers - - def _get_s3_auth_signature(self, params, headers): - """Hacky wrapper to work with S3's get_auth_signature.""" - headers_copy = {} - params_copy = copy.deepcopy(params) - - # Lowercase all headers except 'date' and Google header values - for k, v in headers.items(): - k_lower = k.lower() - if (k_lower == 'date' or k_lower.startswith( - GoogleStorageDriver.http_vendor_prefix) or - not isinstance(v, str)): - headers_copy[k_lower] = v - else: - headers_copy[k_lower] = v.lower() - - return BaseS3Connection.get_auth_signature( - method=self.method, - headers=headers_copy, - params=params_copy, - expires=None, - secret_key=self.key, - path=self.action, - vendor_prefix=GoogleStorageDriver.http_vendor_prefix) - - -class GCSResponse(GoogleResponse): - pass - - -# class GoogleStorageJSONConnection(GoogleStorageConnection): -# """ -# Represents a single connection to the Google storage JSON API endpoint. - -# This can either authenticate via the Google OAuth2 methods or via -# the S3 HMAC interoperability method. -# """ -# host = 'www.googleapis.com' -# responseCls = GCSResponse -# rawResponseCls = None - -# def add_default_headers(self, headers): -# headers = super(GoogleStorageJSONConnection, self).add_default_headers( -# headers) -# headers['Content-Type'] = 'application/json' -# return headers class TencentCosDriver(StorageDriver): """ Driver for Tencent Cloud Object Storage (COS). - Can authenticate via standard Google Cloud methods (Service Accounts, - Installed App credentials, and GCE instance service accounts) + Can authenticate via API key & secret - requires App ID and region as well. Examples: - Service Accounts:: - - driver = TencentCosDriver(key=client_email, secret=private_key, ...) - - Installed Application:: - - driver = TencentCosDriver(key=client_id, secret=client_secret, ...) + API key, secret & app ID:: - From GCE instance:: - - driver = TencentCosDriver(key=foo, secret=bar, ...) - - Can also authenticate via Google Cloud Storage's S3 HMAC interoperability - API. S3 user keys are 20 alphanumeric characters, starting with GOOG. - - Example:: - - driver = TencentCosDriver(key='GOOG0123456789ABCXYZ', - secret=key_secret) + driver = TencentCosDriver(key=api_key_id, secret=api_secret_key + region=region, app_id=app_id) """ name = 'Tencent COS' website = 'https://cloud.tencent.com/product/cos' - # connectionCls = GoogleStorageConnection - # jsonConnectionCls = GoogleStorageJSONConnection hash_type = 'sha1' supports_chunked_encoding = False - supports_s3_multipart_upload = False def __init__(self, key, secret=None, app_id=None, region=None, **kwargs): super(TencentCosDriver, self).__init__(key, secret, **kwargs) from qcloud_cos import CosClient self.cos_client = CosClient(app_id, key, secret, region) - # self.json_connection = GoogleStorageJSONConnection( - # key, secret, **kwargs) - @staticmethod def _is_ok(response): return response['code'] == 0 @@ -511,204 +358,3 @@ def delete_object(self, obj): req = DelFileRequest(obj.container.name, '/' + obj.name) response = self.cos_client.del_file(req) return self._is_ok(response) - - # def _get_container_permissions(self, container_name): - # """ - # Return the container permissions for the current authenticated user. - - # :param container_name: The container name. - # :param container_name: ``str`` - - # :return: The permissions on the container. - # :rtype: ``int`` from ContainerPermissions - # """ - # # Try OWNER permissions first: try listing the bucket ACL. - # # FORBIDDEN -> exists, but not an OWNER. - # # NOT_FOUND -> bucket DNE, return NONE. - # try: - # self.json_connection.request( - # '/storage/v1/b/%s/acl' % container_name) - # return ContainerPermissions.OWNER - # except ProviderError as e: - # if e.http_code == httplib.FORBIDDEN: - # pass - # elif e.http_code == httplib.NOT_FOUND: - # return ContainerPermissions.NONE - # else: - # raise - - # # Try WRITER permissions with a noop request: try delete with an - # # impossible precondition. Authorization is checked before file - # # existence or preconditions. So, if we get a NOT_FOUND or a - # # PRECONDITION_FAILED, then we must be authorized. - # try: - # self.json_connection.request( - # '/storage/v1/b/%s/o/writecheck' % container_name, - # headers={'x-goog-if-generation-match': '0'}, method='DELETE') - # except ProviderError as e: - # if e.http_code in [httplib.NOT_FOUND, httplib.PRECONDITION_FAILED]: - # return ContainerPermissions.WRITER - # elif e.http_code != httplib.FORBIDDEN: - # raise - - # # Last, try READER permissions: try getting container metadata. - # try: - # self.json_connection.request('/storage/v1/b/%s' % container_name) - # return ContainerPermissions.READER - # except ProviderError as e: - # if e.http_code not in [httplib.FORBIDDEN, httplib.NOT_FOUND]: - # raise - - # return ContainerPermissions.NONE - - # def _get_user(self): - # """Gets this drivers' authenticated user, if any.""" - # oauth2_creds = getattr(self.connection, 'oauth2_credential') - # if oauth2_creds: - # return oauth2_creds.user_id - # else: - # return None - - # def _get_object_permissions(self, container_name, object_name): - # """ - # Return the object permissions for the current authenticated user. - # If the object does not exist, or no object_name is given, return the - # default object permissions. - - # :param container_name: The container name. - # :type container_name: ``str`` - - # :param object_name: The object name. - # :type object_name: ``str`` - - # :return: The permissions on the object or default object permissions. - # :rtype: ``int`` from ObjectPermissions - # """ - # # Try OWNER permissions first: try listing the object ACL. - # try: - # self.json_connection.request( - # '/storage/v1/b/%s/o/%s/acl' % (container_name, object_name)) - # return ObjectPermissions.OWNER - # except ProviderError as e: - # if e.http_code not in [httplib.FORBIDDEN, httplib.NOT_FOUND]: - # raise - - # # Try READER permissions: try getting the object. - # try: - # self.json_connection.request( - # '/storage/v1/b/%s/o/%s' % (container_name, object_name)) - # return ObjectPermissions.READER - # except ProviderError as e: - # if e.http_code not in [httplib.FORBIDDEN, httplib.NOT_FOUND]: - # raise - - # return ObjectPermissions.NONE - - # def ex_delete_permissions(self, container_name, object_name=None, - # entity=None): - # """ - # Delete permissions for an ACL entity on a container or object. - - # :param container_name: The container name. - # :type container_name: ``str`` - - # :param object_name: The object name. Optional. Not providing an object - # will delete a container permission. - # :type object_name: ``str`` - - # :param entity: The entity to whose permission will be deleted. - # Optional. If not provided, the role will be applied to the - # authenticated user, if using an OAuth2 authentication scheme. - # :type entity: ``str`` or ``None`` - # """ - # object_name = _clean_object_name(object_name) - # if not entity: - # user_id = self._get_user() - # if not user_id: - # raise ValueError( - # 'Must provide an entity. Driver is not using an ' - # 'authenticated user.') - # else: - # entity = 'user-%s' % user_id - - # if object_name: - # url = ('/storage/v1/b/%s/o/%s/acl/%s' % - # (container_name, object_name, entity)) - # else: - # url = '/storage/v1/b/%s/acl/%s' % (container_name, entity) - - # self.json_connection.request(url, method='DELETE') - - # def ex_get_permissions(self, container_name, object_name=None): - # """ - # Return the permissions for the currently authenticated user. - - # :param container_name: The container name. - # :type container_name: ``str`` - - # :param object_name: The object name. Optional. Not providing an object - # will return only container permissions. - # :type object_name: ``str`` or ``None`` - - # :return: A tuple of container and object permissions. - # :rtype: ``tuple`` of (``int``, ``int`` or ``None``) from - # ContainerPermissions and ObjectPermissions, respectively. - # """ - # object_name = _clean_object_name(object_name) - # obj_perms = self._get_object_permissions( - # container_name, object_name) if object_name else None - # return self._get_container_permissions(container_name), obj_perms - - # def ex_set_permissions(self, container_name, object_name=None, - # entity=None, role=None): - # """ - # Set the permissions for an ACL entity on a container or an object. - - # :param container_name: The container name. - # :type container_name: ``str`` - - # :param object_name: The object name. Optional. Not providing an object - # will apply the acl to the container. - # :type object_name: ``str`` - - # :param entity: The entity to which apply the role. Optional. If not - # provided, the role will be applied to the authenticated user, if - # using an OAuth2 authentication scheme. - # :type entity: ``str`` - - # :param role: The permission/role to set on the entity. - # :type role: ``int`` from ContainerPermissions or ObjectPermissions - # or ``str``. - - # :raises ValueError: If no entity was given, but was required. Or if - # the role isn't valid for the bucket or object. - # """ - # object_name = _clean_object_name(object_name) - # if isinstance(role, int): - # perms = ObjectPermissions if object_name else ContainerPermissions - # try: - # role = perms.values[role] - # except IndexError: - # raise ValueError( - # '%s is not a valid role level for container=%s object=%s' % - # (role, container_name, object_name)) - # elif not isinstance(role, str): - # raise ValueError('%s is not a valid permission.' % role) - - # if not entity: - # user_id = self._get_user() - # if not user_id: - # raise ValueError( - # 'Must provide an entity. Driver is not using an ' - # 'authenticated user.') - # else: - # entity = 'user-%s' % user_id - - # if object_name: - # url = '/storage/v1/b/%s/o/%s/acl' % (container_name, object_name) - # else: - # url = '/storage/v1/b/%s/acl' % container_name - - # self.json_connection.request( - # url, method='POST', - # data=json.dumps({'role': role, 'entity': entity})) From 4ab189fd45fec3cec3ee6b6dba26f68a169edca2 Mon Sep 17 00:00:00 2001 From: Itamar Ostricher Date: Thu, 23 Nov 2017 12:05:05 +0200 Subject: [PATCH 15/21] Move import --- libcloud/storage/drivers/tencent_cos.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libcloud/storage/drivers/tencent_cos.py b/libcloud/storage/drivers/tencent_cos.py index bfe918dc7e..c030beff53 100644 --- a/libcloud/storage/drivers/tencent_cos.py +++ b/libcloud/storage/drivers/tencent_cos.py @@ -18,6 +18,7 @@ import tempfile from qcloud_cos import ( + CosClient, DelFileRequest, DownloadFileRequest, DownloadObjectRequest, @@ -54,7 +55,6 @@ class TencentCosDriver(StorageDriver): def __init__(self, key, secret=None, app_id=None, region=None, **kwargs): super(TencentCosDriver, self).__init__(key, secret, **kwargs) - from qcloud_cos import CosClient self.cos_client = CosClient(app_id, key, secret, region) @staticmethod From 91ccb48e7091aca8f3f3ffcc4c61b15a2664f956 Mon Sep 17 00:00:00 2001 From: Itamar Ostricher Date: Thu, 23 Nov 2017 12:31:51 +0200 Subject: [PATCH 16/21] Refactor all API requests to common class method --- libcloud/storage/drivers/tencent_cos.py | 74 +++++++++++++++---------- 1 file changed, 46 insertions(+), 28 deletions(-) diff --git a/libcloud/storage/drivers/tencent_cos.py b/libcloud/storage/drivers/tencent_cos.py index c030beff53..2c094cc3f1 100644 --- a/libcloud/storage/drivers/tencent_cos.py +++ b/libcloud/storage/drivers/tencent_cos.py @@ -61,6 +61,26 @@ def __init__(self, key, secret=None, app_id=None, region=None, **kwargs): def _is_ok(response): return response['code'] == 0 + @classmethod + def _make_request(cls, method, req): + """Make a COS API request. + + :param method: COS client method to use for request. + :type method: ``callable`` + + :param req: COS client method to use for request. + :type req: ``qcloud_cos.BaseRequest`` + + :return: :class:Tuple with (result, error). + :rtype: :class:`tuple(dict, str)` + On success: result is the response data, error is None. + On error: result is None, error is the API error message. + """ + response = method(req) + if cls._is_ok(response): + return response.get('data', {}), None + return None, response['message'] + def _to_containers(self, obj_list): for obj in obj_list: yield self._to_container(obj) @@ -77,12 +97,12 @@ def _walk_container_folder(self, container, folder): context = '' while not exhausted: req = ListFolderRequest(container.name, folder, context=context) - response = self.cos_client.list_folder(req) - if not self._is_ok(response): + result, err = self._make_request(self.cos_client.list_folder, req) + if err is not None: return - exhausted = response['data']['listover'] - context = response['data']['context'] - for obj in response['data']['infos']: + exhausted = result['listover'] + context = result['context'] + for obj in result['infos']: if obj['name'].endswith('/'): # need to recurse into folder yield from self._walk_container_folder( @@ -113,12 +133,12 @@ def iterate_containers(self): context = '' while not exhausted: req = ListFolderRequest('', '/', context=context) - response = self.cos_client.list_folder(req) - if not self._is_ok(response): + result, err = self._make_request(self.cos_client.list_folder, req) + if err is not None: return - exhausted = response['data']['listover'] - context = response['data']['context'] - yield from self._to_containers(response['data']['infos']) + exhausted = result['listover'] + context = result['context'] + yield from self._to_containers(result['infos']) def iterate_container_objects(self, container): """ @@ -143,13 +163,13 @@ def get_container(self, container_name): :rtype: :class:`Container` """ req = StatFolderRequest(container_name, '/') - response = self.cos_client.stat_folder(req) - if not self._is_ok(response): + result, err = self._make_request(self.cos_client.stat_folder, req) + if err is not None: raise ContainerDoesNotExistError(value=None, driver=self, container_name=container_name) # "inject" the container name to the dictionary for `_to_container` - response['data']['name'] = container_name - return self._to_container(response['data']) + result['name'] = container_name + return self._to_container(result) def get_object(self, container_name, object_name): """ @@ -165,14 +185,13 @@ def get_object(self, container_name, object_name): :rtype: :class:`Object` """ req = StatFileRequest(container_name, '/' + object_name) - response = self.cos_client.stat_file(req) - if not self._is_ok(response): + result, err = self._make_request(self.cos_client.stat_file, req) + if err is not None: raise ObjectDoesNotExistError(value=None, driver=self, object_name=object_name) # "inject" the object name to the dictionary for `_to_obj` - response['data']['name'] = object_name - return self._to_obj(response['data'], '', - self.get_container(container_name)) + result['name'] = object_name + return self._to_obj(result, '', self.get_container(container_name)) def get_object_cdn_url(self, obj): """ @@ -215,8 +234,8 @@ def download_object(self, obj, destination_path, overwrite_existing=False, return False req = DownloadFileRequest(obj.container.name, '/' + obj.name, destination_path) - response = self.cos_client.download_file(req) - if self._is_ok(response): + result, err = self._make_request(self.cos_client.download_file, req) + if err is None: return True if delete_on_failure and os.path.exists(destination_path): os.remove(destination_path) @@ -279,11 +298,10 @@ def set_extra(field_name): set_extra('content_language') set_extra('content_encoding') set_extra('x_cos_meta') - response = self.cos_client.upload_file(req) - if not self._is_ok(response): - raise LibcloudError( - 'Error in uploading object: %s' % (response['message']), - driver=self) + _, err = self._make_request(self.cos_client.upload_file, req) + if err is not None: + raise LibcloudError('Error in uploading object: %s' % (err), + driver=self) obj = self.get_object(container.name, object_name) if verify_hash: hasher = self._get_hash_function() @@ -356,5 +374,5 @@ def delete_object(self, obj): :rtype: ``bool`` """ req = DelFileRequest(obj.container.name, '/' + obj.name) - response = self.cos_client.del_file(req) - return self._is_ok(response) + _, err = self._make_request(self.cos_client.del_file, req) + return err is None From 6711f0f996810721204c0fc6c8c19e6867b70082 Mon Sep 17 00:00:00 2001 From: Itamar Ostricher Date: Thu, 23 Nov 2017 12:34:33 +0200 Subject: [PATCH 17/21] yield-in-loop instead of `yield from` to support Python 2 --- libcloud/storage/drivers/tencent_cos.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/libcloud/storage/drivers/tencent_cos.py b/libcloud/storage/drivers/tencent_cos.py index 2c094cc3f1..45bf05a5cd 100644 --- a/libcloud/storage/drivers/tencent_cos.py +++ b/libcloud/storage/drivers/tencent_cos.py @@ -105,8 +105,9 @@ def _walk_container_folder(self, container, folder): for obj in result['infos']: if obj['name'].endswith('/'): # need to recurse into folder - yield from self._walk_container_folder( - container, folder + obj['name']) + for obj in self._walk_container_folder( + container, folder + obj['name']): + yield obj else: yield self._to_obj(obj, folder, container) @@ -138,7 +139,8 @@ def iterate_containers(self): return exhausted = result['listover'] context = result['context'] - yield from self._to_containers(result['infos']) + for container in self._to_containers(result['infos']): + yield container def iterate_container_objects(self, container): """ From f168792ce0c1a2a25717e6dabb862c1443e5c653 Mon Sep 17 00:00:00 2001 From: Itamar Ostricher Date: Thu, 23 Nov 2017 14:35:49 +0200 Subject: [PATCH 18/21] Missing `:` in docstring --- libcloud/storage/types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libcloud/storage/types.py b/libcloud/storage/types.py index cb3d39dedb..a8de8604d0 100644 --- a/libcloud/storage/types.py +++ b/libcloud/storage/types.py @@ -55,7 +55,7 @@ class Provider(object): :cvar S3_US_WEST_OREGON: Amazon S3 US West 2 (Oregon) :cvar S3_RGW: S3 RGW :cvar S3_RGW_OUTSCALE: OUTSCALE S3 RGW - :cvar TENCENT_COS Tencent COS + :cvar TENCENT_COS: Tencent COS """ DUMMY = 'dummy' ALIYUN_OSS = 'aliyun_oss' From 0118a52730aa6dbb68129f56cc37ba4aa904373c Mon Sep 17 00:00:00 2001 From: Itamar Ostricher Date: Tue, 28 Nov 2017 12:42:20 +0200 Subject: [PATCH 19/21] Add tests for Tencent COS storage driver (partial) --- libcloud/storage/drivers/tencent_cos.py | 9 +- libcloud/test/storage/test_tencent_cos.py | 330 ++++++++++++++++++++++ 2 files changed, 337 insertions(+), 2 deletions(-) create mode 100644 libcloud/test/storage/test_tencent_cos.py diff --git a/libcloud/storage/drivers/tencent_cos.py b/libcloud/storage/drivers/tencent_cos.py index 45bf05a5cd..7b9cb61362 100644 --- a/libcloud/storage/drivers/tencent_cos.py +++ b/libcloud/storage/drivers/tencent_cos.py @@ -55,7 +55,11 @@ class TencentCosDriver(StorageDriver): def __init__(self, key, secret=None, app_id=None, region=None, **kwargs): super(TencentCosDriver, self).__init__(key, secret, **kwargs) - self.cos_client = CosClient(app_id, key, secret, region) + self.cos_client = self._get_client(app_id, key, secret, region) + + @staticmethod + def _get_client(app_id, key, secret, region): + return CosClient(app_id, key, secret, region) @staticmethod def _is_ok(response): @@ -136,7 +140,8 @@ def iterate_containers(self): req = ListFolderRequest('', '/', context=context) result, err = self._make_request(self.cos_client.list_folder, req) if err is not None: - return + raise LibcloudError('Error in ListFolder: %s' % (err), + driver=self) exhausted = result['listover'] context = result['context'] for container in self._to_containers(result['infos']): diff --git a/libcloud/test/storage/test_tencent_cos.py b/libcloud/test/storage/test_tencent_cos.py new file mode 100644 index 0000000000..3e03fdd85e --- /dev/null +++ b/libcloud/test/storage/test_tencent_cos.py @@ -0,0 +1,330 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +import json +import mock +import re +import sys +import unittest + +from io import BytesIO + +import email.utils +import pytest + +from libcloud.common.types import InvalidCredsError +from libcloud.storage.base import Container, Object +from libcloud.storage.drivers import tencent_cos +from libcloud.storage.types import ContainerDoesNotExistError +from libcloud.storage.types import ObjectDoesNotExistError + + +def _qcloud_access_url(path): + return 'http://foobar.file.myqcloud.com/{}'.format(path) + + +def _qcloud_source_url(path): + return 'http://foobar.cosgz.myqcloud.com/{}'.format(path) + + +_MOCK_CONTAINERS = [ + { + 'code': 0, + 'message': 'SUCCESS', + 'request_id': 'request-id-1==', + 'data': { + 'context': 'public_read_1', + 'infos': [ + { + 'authority': 'eWRPrivate', + 'biz_attr': '', + 'ctime': 1504236956, + 'mtime': 1504236956, + 'name': 'private_1/', + }, + { + 'authority': 'eWPrivateRPublic', + 'biz_attr': '', + 'ctime': 1482718269, + 'mtime': 1482718269, + 'name': 'public_read_1/', + }, + ], + 'listover': False, + }, + }, + { + 'code': 0, + 'message': 'SUCCESS', + 'request_id': 'request-id-2==', + 'data': { + 'context': '', + 'infos': [ + { + 'authority': 'eWRPrivate', + 'biz_attr': '', + 'ctime': 1503648523, + 'mtime': 1503648523, + 'name': 'private_2/', + }, + { + 'authority': 'eWPrivateRPublic', + 'biz_attr': '', + 'ctime': 1484183162, + 'mtime': 1484183162, + 'name': 'public_read_2/' + }, + ], + 'listover': True, + }, + }, +] +_MOCK_OBJECTS = [ + { + 'code': 0, + 'message': 'SUCCESS', + 'request_id': 'request-id-1==', + 'data': { + 'context': 'deep/tree/object/1', + 'infos': [ + { + 'access_url': _qcloud_access_url('top-level-1'), + 'authority': 'eInvalid', + 'biz_attr': '', + 'ctime': 1511428772, + 'filelen': 4328, + 'filesize': 4328, + 'mtime': 1511428772, + 'name': 'top-level-1', + 'sha': 'ec318800b567dde4f6cdd82aecd5b8385eef1f79', + 'source_url': _qcloud_source_url('top-level-1'), + }, + { + 'access_url': _qcloud_access_url('top-level-2'), + 'authority': 'eInvalid', + 'biz_attr': '', + 'ctime': 1511428743, + 'filelen': 282585, + 'filesize': 282585, + 'mtime': 1511428743, + 'name': 'top-level-2', + 'sha': 'afb21075914256cddfc9459535d679ae0a6c5731', + 'source_url': _qcloud_source_url('top-level-2'), + }, + { + 'access_url': _qcloud_access_url('deep/tree/object/1'), + 'authority': 'eInvalid', + 'biz_attr': '', + 'ctime': 1511438556, + 'filelen': 464364, + 'filesize': 464364, + 'mtime': 1511438556, + 'name': 'deep/tree/object/1', + 'sha': '7fa357010b6581b8f4cb0e06948fae352e1e8458', + 'source_url': _qcloud_source_url('deep/tree/object/1'), + }, + ], + 'listover': False, + }, + }, + { + 'code': 0, + 'message': 'SUCCESS', + 'request_id': 'request-id-2==', + 'data': { + 'context': '', + 'infos': [ + { + 'access_url': _qcloud_access_url('deep/tree/object/2'), + 'authority': 'eInvalid', + 'biz_attr': '', + 'ctime': 1511428772, + 'filelen': 4328, + 'filesize': 4328, + 'mtime': 1511428772, + 'name': 'deep/tree/object/2', + 'sha': 'ec318800b567dde4f6cdd82aecd5b8385eef1f79', + 'source_url': _qcloud_source_url('deep/tree/object/2'), + }, + { + 'access_url': _qcloud_access_url('deep/tree/object/a'), + 'authority': 'eInvalid', + 'biz_attr': '', + 'ctime': 1511428743, + 'filelen': 282585, + 'filesize': 282585, + 'mtime': 1511428743, + 'name': 'deep/tree/object/a', + 'sha': 'afb21075914256cddfc9459535d679ae0a6c5731', + 'source_url': _qcloud_source_url('deep/tree/object/a'), + }, + { + 'access_url': _qcloud_access_url('deep/tree/object/b'), + 'authority': 'eInvalid', + 'biz_attr': '', + 'ctime': 1511438556, + 'filelen': 464364, + 'filesize': 464364, + 'mtime': 1511438556, + 'name': 'deep/tree/object/b', + 'sha': '7fa357010b6581b8f4cb0e06948fae352e1e8458', + 'source_url': _qcloud_source_url('deep/tree/object/b'), + }, + ], + 'listover': True, + }, + }, +] +_MOCK_STAT = { + '/public_read_1/': { + 'code': 0, + 'message': 'SUCCESS', + 'request_id': 'NWExZDJiYThfNTIyNWI2NF9hMDBlXzUzMmY5Nw==', + 'data': { + 'CORSConfiguration': {'CORSRule': [], 'NeedCORS': False}, + 'authority': 'eWPrivateRPublic', + 'biz_attr': '', + 'blackrefers': [], + 'brower_exec': '0', + 'cnames': [], + 'ctime': 1504236956, + 'forbid': 0, + 'mtime': 1504236956, + 'refers': [], + }, + }, + '/public_read_1/deep/tree/object/1': { + 'code': 0, + 'message': 'SUCCESS', + 'request_id': 'NWExZDJiYWFfNTIyNWI2NF9hMDA5XzUzOTFiMA==', + 'data': { + 'access_url': _qcloud_access_url('deep/tree/object/1'), + 'authority': 'eInvalid', + 'biz_attr': '', + 'ctime': 1511438556, + 'custom_headers': {}, + 'filelen': 464364, + 'filesize': 464364, + 'forbid': 0, + 'mtime': 1511438556, + 'name': 'deep/tree/object/1', + 'sha': '7fa357010b6581b8f4cb0e06948fae352e1e8458', + 'slicesize': 464364, + 'source_url': _qcloud_source_url('deep/tree/object/1'), + }, + }, +} + + +class MockCosClient(object): + + def list_folder(self, req): + cos_path = req.get_cos_path() + bucket_name = req.get_bucket_name() + context = req.get_context() + if bucket_name == '': + if context == '': + return _MOCK_CONTAINERS[0] + elif context == _MOCK_CONTAINERS[0]['data']['context']: + return _MOCK_CONTAINERS[1] + else: + raise ValueError( + 'Bad context for list containers: %s' % (context)) + elif bucket_name == 'public_read_1': + if context == '': + return _MOCK_OBJECTS[0] + elif context == _MOCK_OBJECTS[0]['data']['context']: + return _MOCK_OBJECTS[1] + else: + raise ValueError( + 'Bad context for list container objects: %s' % (context)) + + def stat_folder(self, req): + return _MOCK_STAT['/{}/'.format(req.get_bucket_name())] + + def stat_file(self, req): + return _MOCK_STAT['/{}{}'.format( + req.get_bucket_name(), req.get_cos_path())] + + +class TencentStorageTests(unittest.TestCase): + + def _get_driver(self): + return tencent_cos.TencentCosDriver( + 'api-key-id', 'api-secret-key', region='gz', app_id=1111111) + + @mock.patch('libcloud.storage.drivers.tencent_cos' + '.TencentCosDriver._get_client') + def test_create_driver(self, mock_get_client): + mock_get_client.return_value = MockCosClient() + driver = self._get_driver() + mock_get_client.assert_called_with( + 1111111, 'api-key-id', 'api-secret-key', 'gz') + + @mock.patch('libcloud.storage.drivers.tencent_cos' + '.TencentCosDriver._get_client') + def test_list_containers(self, mock_get_client): + mock_get_client.return_value = MockCosClient() + driver = self._get_driver() + containers = driver.list_containers() + self.assertEqual(4, len(containers)) + self.assertTrue( + all(isinstance(container, Container) for container in containers)) + self.assertSetEqual( + set(('private_1', 'public_read_1', 'private_2', 'public_read_2')), + set(container.name for container in containers)) + + @mock.patch('libcloud.storage.drivers.tencent_cos' + '.TencentCosDriver._get_client') + def test_get_and_list_container_objects(self, mock_get_client): + mock_get_client.return_value = MockCosClient() + driver = self._get_driver() + container = driver.get_container('public_read_1') + self.assertIsInstance(container, Container) + self.assertEqual('public_read_1', container.name) + objects = container.list_objects() + self.assertEqual(6, len(objects)) + self.assertTrue( + all(isinstance(obj, Object) for obj in objects)) + self.assertSetEqual( + set(('top-level-1', 'top-level-2', + 'deep/tree/object/1', 'deep/tree/object/2', + 'deep/tree/object/a', 'deep/tree/object/b')), + set(obj.name for obj in objects)) + + @mock.patch('libcloud.storage.drivers.tencent_cos' + '.TencentCosDriver._get_client') + def test_get_object_and_cdn_url(self, mock_get_client): + mock_get_client.return_value = MockCosClient() + driver = self._get_driver() + obj = driver.get_object('public_read_1', 'deep/tree/object/1') + exp_extra = { + 'creation_date': 1511438556, + 'modified_data': 1511438556, + 'access_url': 'http://foobar.file.myqcloud.com/deep/tree/object/1', + 'source_url': 'http://foobar.cosgz.myqcloud.com/deep/tree/object/1', + } + self.assertIsInstance(obj, Object) + self.assertEqual('deep/tree/object/1', obj.name) + self.assertEqual(464364, obj.size) + self.assertEqual('7fa357010b6581b8f4cb0e06948fae352e1e8458', obj.hash) + self.assertDictEqual(exp_extra, obj.extra) + self.assertEqual('http://foobar.file.myqcloud.com/deep/tree/object/1', + obj.get_cdn_url()) + + +if __name__ == '__main__': + sys.exit(unittest.main()) From fed1cd3f0f7bb73dc554d1972769afcb67006ef1 Mon Sep 17 00:00:00 2001 From: Itamar Ostricher Date: Tue, 28 Nov 2017 12:47:27 +0200 Subject: [PATCH 20/21] flake8 --- libcloud/storage/drivers/tencent_cos.py | 2 +- libcloud/test/storage/test_tencent_cos.py | 24 ++++++----------------- 2 files changed, 7 insertions(+), 19 deletions(-) diff --git a/libcloud/storage/drivers/tencent_cos.py b/libcloud/storage/drivers/tencent_cos.py index 7b9cb61362..b170ecf652 100644 --- a/libcloud/storage/drivers/tencent_cos.py +++ b/libcloud/storage/drivers/tencent_cos.py @@ -14,7 +14,6 @@ # limitations under the License. import os -import shutil import tempfile from qcloud_cos import ( @@ -32,6 +31,7 @@ from libcloud.storage.base import StorageDriver, Container, Object from libcloud.storage.types import ContainerDoesNotExistError from libcloud.storage.types import ObjectDoesNotExistError +from libcloud.storage.types import ObjectHashMismatchError from libcloud.utils.files import exhaust_iterator, read_in_chunks diff --git a/libcloud/test/storage/test_tencent_cos.py b/libcloud/test/storage/test_tencent_cos.py index 3e03fdd85e..e9de3d32b4 100644 --- a/libcloud/test/storage/test_tencent_cos.py +++ b/libcloud/test/storage/test_tencent_cos.py @@ -13,31 +13,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -import copy -import json import mock -import re import sys import unittest -from io import BytesIO - -import email.utils -import pytest - -from libcloud.common.types import InvalidCredsError from libcloud.storage.base import Container, Object from libcloud.storage.drivers import tencent_cos -from libcloud.storage.types import ContainerDoesNotExistError -from libcloud.storage.types import ObjectDoesNotExistError def _qcloud_access_url(path): - return 'http://foobar.file.myqcloud.com/{}'.format(path) + return 'http://foo.file.myqcloud.com/{}'.format(path) def _qcloud_source_url(path): - return 'http://foobar.cosgz.myqcloud.com/{}'.format(path) + return 'http://foo.cosgz.myqcloud.com/{}'.format(path) _MOCK_CONTAINERS = [ @@ -232,7 +221,6 @@ def _qcloud_source_url(path): class MockCosClient(object): def list_folder(self, req): - cos_path = req.get_cos_path() bucket_name = req.get_bucket_name() context = req.get_context() if bucket_name == '': @@ -270,7 +258,7 @@ def _get_driver(self): '.TencentCosDriver._get_client') def test_create_driver(self, mock_get_client): mock_get_client.return_value = MockCosClient() - driver = self._get_driver() + self._get_driver() mock_get_client.assert_called_with( 1111111, 'api-key-id', 'api-secret-key', 'gz') @@ -314,15 +302,15 @@ def test_get_object_and_cdn_url(self, mock_get_client): exp_extra = { 'creation_date': 1511438556, 'modified_data': 1511438556, - 'access_url': 'http://foobar.file.myqcloud.com/deep/tree/object/1', - 'source_url': 'http://foobar.cosgz.myqcloud.com/deep/tree/object/1', + 'access_url': 'http://foo.file.myqcloud.com/deep/tree/object/1', + 'source_url': 'http://foo.cosgz.myqcloud.com/deep/tree/object/1', } self.assertIsInstance(obj, Object) self.assertEqual('deep/tree/object/1', obj.name) self.assertEqual(464364, obj.size) self.assertEqual('7fa357010b6581b8f4cb0e06948fae352e1e8458', obj.hash) self.assertDictEqual(exp_extra, obj.extra) - self.assertEqual('http://foobar.file.myqcloud.com/deep/tree/object/1', + self.assertEqual('http://foo.file.myqcloud.com/deep/tree/object/1', obj.get_cdn_url()) From edaa7171b4604447cd0ea7c2f4b625e36d5d7017 Mon Sep 17 00:00:00 2001 From: Itamar Ostricher Date: Wed, 29 Nov 2017 16:58:25 +0200 Subject: [PATCH 21/21] Use libcloud.test.unittest --- libcloud/test/storage/test_tencent_cos.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libcloud/test/storage/test_tencent_cos.py b/libcloud/test/storage/test_tencent_cos.py index e9de3d32b4..435ba22937 100644 --- a/libcloud/test/storage/test_tencent_cos.py +++ b/libcloud/test/storage/test_tencent_cos.py @@ -15,7 +15,7 @@ import mock import sys -import unittest +from libcloud.test import unittest from libcloud.storage.base import Container, Object from libcloud.storage.drivers import tencent_cos