From 03e52b83aa75d18e1e86802a8202d4ccfb4147c8 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sat, 19 Dec 2015 17:59:44 +0100 Subject: [PATCH 01/13] Initial commit of W.I.P. implementation of Backblaze B2 storage driver. --- libcloud/storage/drivers/backblaze_b2.py | 324 +++++++++++++++++++++++ libcloud/storage/providers.py | 2 + libcloud/storage/types.py | 1 + 3 files changed, 327 insertions(+) create mode 100644 libcloud/storage/drivers/backblaze_b2.py diff --git a/libcloud/storage/drivers/backblaze_b2.py b/libcloud/storage/drivers/backblaze_b2.py new file mode 100644 index 0000000000..2be6ffcc7f --- /dev/null +++ b/libcloud/storage/drivers/backblaze_b2.py @@ -0,0 +1,324 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Driver for Backblaze B2 service. +""" + +import base64 + +try: + import simplejson as json +except ImportError: + import json + +from libcloud.utils.py3 import b +from libcloud.utils.py3 import httplib +from libcloud.utils.py3 import urlparse + +from libcloud.common.base import ConnectionUserAndKey +from libcloud.common.base import JsonResponse +from libcloud.common.types import InvalidCredsError +from libcloud.storage.providers import Provider +from libcloud.storage.base import Object, Container, StorageDriver + +__all__ = [ + 'BackblazeB2StorageDriver', + + 'BackblazeB2Connection', + 'BackblazeB2AuthConnection' +] + +AUTH_API_HOST = 'api.backblaze.com' +API_PATH = '/b2api/v1/' + + +class BackblazeB2Response(JsonResponse): + def success(self): + return self.status in [httplib.OK, httplib.CREATED, httplib.ACCEPTED] + + def parse_error(self): + status = int(self.status) + body = self.parse_body() + + if status == httplib.UNAUTHORIZED: + raise InvalidCredsError(body['message']) + + return self.body + + +class BackblazeB2AuthConnection(ConnectionUserAndKey): + host = AUTH_API_HOST + secure = True + responseCls = BackblazeB2Response + + def __init__(self, *args, **kwargs): + super(BackblazeB2AuthConnection, self).__init__(*args, **kwargs) + + # Those attributes are populated after authentication + self.account_id = None + self.api_url = None + self.api_host = None + self.download_url = None + self.download_host = None + self.auth_token = None + + def authenticate(self, force=False): + """ + :param force: Force authentication if if we have already obtained the + token. + :type force: ``bool`` + """ + if not self._is_authentication_needed(force=force): + return self + + headers = {} + action = 'b2_authorize_account' + auth_b64 = base64.b64encode(b('%s:%s' % (self.user_id, self.key))) + headers['Authorization'] = 'Basic %s' % (auth_b64.decode('utf-8')) + + action = API_PATH + 'b2_authorize_account' + resp = self.request(action=action, headers=headers, method='GET') + + if resp.status == httplib.OK: + self._parse_and_set_auth_info(data=resp.object) + else: + raise Exception('Failed to authenticate: %s' % (str(resp.object))) + + return self + + def _parse_and_set_auth_info(self, data): + result = {} + self.account_id = data['accountId'] + self.api_url = data['apiUrl'] + self.download_url = data['downloadUrl'] + self.auth_token = data['authorizationToken'] + + parsed_api_url = urlparse.urlparse(self.api_url) + self.api_host = parsed_api_url.netloc + + parsed_download_url = urlparse.urlparse(self.download_url) + self.download_host = parsed_download_url.netloc + + return result + + def _is_authentication_needed(self, force=False): + if not self.auth_token or force: + return True + + return False + + +class BackblazeB2Connection(ConnectionUserAndKey): + host = None # Note: host is set after authentication + secure = True + responseCls = BackblazeB2Response + + def __init__(self, *args, **kwargs): + super(BackblazeB2Connection, self).__init__(*args, **kwargs) + + # Stores info retrieved after authnetication (auth token, api url, + # dowload url). + self._auth_conn = BackblazeB2AuthConnection(*args, **kwargs) + + def request(self, action, params=None, data=None, headers=None, + method='GET', raw=False, include_account_id=False): + params = params or {} + headers = headers or {} + + # Lazily perform authentication + auth_conn = self._auth_conn.authenticate() + + # Set host + if raw: + # File download or upload request: + self.host = auth_conn.download_host + else: + self.host = auth_conn.api_host + + # Provide auth token + headers['Authorization'] = '%s' % (auth_conn.auth_token) + + # Include Content-Type + if not raw and data: + headers['Content-Type'] = 'application/json' + + # Include account id + if include_account_id: + if method == 'GET': + params['accountId'] = auth_conn.account_id + elif method == 'POST': + data = data or {} + data['accountId'] = auth_conn.account_id + + if not raw: + action = API_PATH + action + else: + action = '/file/' + action + + if data: + data = json.dumps(data) + + response = super(BackblazeB2Connection, self).request(action=action, + params=params, + data=data, + method=method, + headers=headers, + raw=raw) + return response + + +class BackblazeB2StorageDriver(StorageDriver): + connectionCls = BackblazeB2Connection + name = 'Backblaze B2' + website = 'https://www.backblaze.com/b2/' + type = Provider.BACKBLAZE_B2 + hash_type = 'sha1' + + def iterate_containers(self): + resp = self.connection.request(action='b2_list_buckets', + method='GET', + include_account_id=True) + containers = self._to_containers(data=resp.object) + return containers + + def iterate_container_objects(self, container): + # TODO: Support pagination + params = {'bucketId': container.extra['id']} + resp = self.connection.request(action='b2_list_file_names', + method='GET', + params=params) + objects = self._to_objects(data=resp.object, container=container) + return objects + + def create_container(self, container_name, ex_type='allPrivate'): + data = {} + data['bucketName'] = container_name + data['bucketType'] = ex_type + resp = self.connection.request(action='b2_create_bucket', + data=data, method='POST', + include_account_id=True) + container = self._to_container(item=resp.object) + return container + + def delete_container(self, container): + data = {} + data['bucketId'] = container.extra['id'] + resp = self.connection.request(action='b2_delete_bucket', + data=data, method='POST', + include_account_id=True) + return resp.status == httplib.OK + + def upload_object(self, file_path, container, object_name, extra=None, + verify_hash=True): + pass + + def download_object(self, obj, destination_path, overwrite_existing=False, + delete_on_failure=True): + # TODO: Escape name + action = obj.container.name + '/' + obj.name + response = self.connection.request(action=action, method='GET', raw=True) + + # TODO: Include metadata from response headers + return self._get_object(obj=obj, callback=self._save_object, + response=response, + callback_kwargs={ + 'obj': obj, + 'response': response.response, + 'destination_path': destination_path, + 'overwrite_existing': overwrite_existing, + 'delete_on_failure': delete_on_failure + }, + success_status_code=httplib.OK) + + def delete_object(self, obj): + data = {} + data['fileName'] = obj.name + data['fileId'] = obj.extra['fileId'] + resp = self.connection.request(action='b2_delete_file_version', + data=data, method='POST') + return resp.status == httplib.OK + + def ex_get_object(self, object_id): + params = {} + params['fileId'] = object_id + resp = self.connection.request(action='b2_get_file_info', + method='GET', + params=params) + obj = self._to_object(item=resp.object, container=None) + return obj + + def ex_hide_object(self, container_id, object_name): + data = {} + data['bucketId'] = container_id + data['fileName'] = object_name + resp = self.connection.request(action='b2_hide_file', + data=data, method='POST') + obj = self._to_object(item=resp.object, container=None) + return obj + + def ex_list_object_versions(self, container_id, ex_start_file_name=None, + ex_start_file_id=None, ex_max_file_count=None): + params = {} + params['bucketId'] = container_id + + if ex_start_file_name: + params['startFileName'] = ex_start_file_name + + if ex_start_file_id: + params['startFileId'] = ex_start_file_id + + if ex_max_file_count: + params['maxFileCount'] = ex_max_file_count + + resp = self.connection.request(action='b2_list_file_versions', + params=params, method='GET') + objects = self._to_objects(data=resp.object, container=None) + return objects + + def _to_containers(self, data): + result = [] + for item in data['buckets']: + container = self._to_container(item=item) + result.append(container) + + return result + + def _to_container(self, item): + extra = {} + extra['id'] = item['bucketId'] + extra['bucketType'] = item['bucketType'] + container = Container(name=item['bucketName'], extra=extra, + driver=self) + return container + + def _to_objects(self, data, container): + result = [] + for item in data['files']: + obj = self._to_object(item=item, container=container) + result.append(obj) + + return result + + def _to_object(self, item, container=None): + extra = {} + extra['fileId'] = item['fileId'] + extra['uploadTimestamp'] = item.get('uploadTimestamp', None) + size = item.get('size', item.get('contentLength', None)) + hash = item.get('contentSha1', None) + meta_data = item.get('fileInfo', {}) + obj = Object(name=item['fileName'], size=size, hash=hash, extra=extra, + meta_data=meta_data, container=container, driver=self) + return obj diff --git a/libcloud/storage/providers.py b/libcloud/storage/providers.py index cff34c80c6..1e79acf397 100644 --- a/libcloud/storage/providers.py +++ b/libcloud/storage/providers.py @@ -52,6 +52,8 @@ ('libcloud.storage.drivers.ktucloud', 'KTUCloudStorageDriver'), Provider.AURORAOBJECTS: ('libcloud.storage.drivers.auroraobjects', 'AuroraObjectsStorageDriver'), + Provider.BACKBLAZE_B2: + ('libcloud.storage.drivers.backblaze_b2', 'BackblazeB2StorageDriver'), # Deprecated Provider.CLOUDFILES_US: diff --git a/libcloud/storage/types.py b/libcloud/storage/types.py index 53d5973685..0ac45f19e1 100644 --- a/libcloud/storage/types.py +++ b/libcloud/storage/types.py @@ -61,6 +61,7 @@ class Provider(object): AZURE_BLOBS = 'azure_blobs' KTUCLOUD = 'ktucloud' AURORAOBJECTS = 'auroraobjects' + BACKBLAZE_B2 = 'backblaze_b2' # Deperecated CLOUDFILES_US = 'cloudfiles_us' From f63f0bba776a6e463f057dcfd382c6556265bfcd Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sun, 20 Dec 2015 13:00:08 +0100 Subject: [PATCH 02/13] Implement download_object_as_stream. --- libcloud/storage/drivers/backblaze_b2.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/libcloud/storage/drivers/backblaze_b2.py b/libcloud/storage/drivers/backblaze_b2.py index 2be6ffcc7f..e27ed72f49 100644 --- a/libcloud/storage/drivers/backblaze_b2.py +++ b/libcloud/storage/drivers/backblaze_b2.py @@ -243,6 +243,16 @@ def download_object(self, obj, destination_path, overwrite_existing=False, }, success_status_code=httplib.OK) + def download_object_as_stream(self, obj, chunk_size=None): + action = obj.container.name + '/' + obj.name + response = self.connection.request(action=action, method='GET', raw=True) + + return self._get_object(obj=obj, callback=read_in_chunks, + response=response, + callback_kwargs={'iterator': response.response, + 'chunk_size': chunk_size}, + success_status_code=httplib.OK) + def delete_object(self, obj): data = {} data['fileName'] = obj.name From d6da648589f88d2536261f78b4e62781c3f015e3 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sun, 20 Dec 2015 13:02:17 +0100 Subject: [PATCH 03/13] Use utility method to get the object download path. --- libcloud/storage/drivers/backblaze_b2.py | 26 ++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/libcloud/storage/drivers/backblaze_b2.py b/libcloud/storage/drivers/backblaze_b2.py index e27ed72f49..eb57555718 100644 --- a/libcloud/storage/drivers/backblaze_b2.py +++ b/libcloud/storage/drivers/backblaze_b2.py @@ -27,6 +27,7 @@ from libcloud.utils.py3 import b from libcloud.utils.py3 import httplib from libcloud.utils.py3 import urlparse +from libcloud.utils.files import read_in_chunks from libcloud.common.base import ConnectionUserAndKey from libcloud.common.base import JsonResponse @@ -134,7 +135,8 @@ def __init__(self, *args, **kwargs): self._auth_conn = BackblazeB2AuthConnection(*args, **kwargs) def request(self, action, params=None, data=None, headers=None, - method='GET', raw=False, include_account_id=False): + method='GET', raw=False, include_account_id=False, + download_request=False): params = params or {} headers = headers or {} @@ -227,9 +229,10 @@ def upload_object(self, file_path, container, object_name, extra=None, def download_object(self, obj, destination_path, overwrite_existing=False, delete_on_failure=True): - # TODO: Escape name - action = obj.container.name + '/' + obj.name - response = self.connection.request(action=action, method='GET', raw=True) + action = self._get_object_download_path(container=obj.container, + obj=obj) + response = self.connection.request(action=action, method='GET', + raw=True) # TODO: Include metadata from response headers return self._get_object(obj=obj, callback=self._save_object, @@ -244,8 +247,10 @@ def download_object(self, obj, destination_path, overwrite_existing=False, success_status_code=httplib.OK) def download_object_as_stream(self, obj, chunk_size=None): - action = obj.container.name + '/' + obj.name - response = self.connection.request(action=action, method='GET', raw=True) + action = self._get_object_download_path(container=obj.container, + obj=obj) + response = self.connection.request(action=action, method='GET', + raw=True) return self._get_object(obj=obj, callback=read_in_chunks, response=response, @@ -332,3 +337,12 @@ def _to_object(self, item, container=None): obj = Object(name=item['fileName'], size=size, hash=hash, extra=extra, meta_data=meta_data, container=container, driver=self) return obj + + def _get_object_download_path(self, container, obj): + """ + Return a path used in the download requests. + + :rtype: ``str`` + """ + path = container.name + '/' + obj.name + return path From 02317ce6736eaf18b70f3af5ae61334925386e5e Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sun, 20 Dec 2015 14:59:16 +0100 Subject: [PATCH 04/13] Implement upload_object method (wip). Note: The whole code is a mess and still needs a lot of refactoring love. --- libcloud/storage/drivers/backblaze_b2.py | 121 +++++++++++++++++++++-- 1 file changed, 111 insertions(+), 10 deletions(-) diff --git a/libcloud/storage/drivers/backblaze_b2.py b/libcloud/storage/drivers/backblaze_b2.py index eb57555718..afb9ea6c75 100644 --- a/libcloud/storage/drivers/backblaze_b2.py +++ b/libcloud/storage/drivers/backblaze_b2.py @@ -18,6 +18,7 @@ """ import base64 +import hashlib try: import simplejson as json @@ -28,10 +29,12 @@ from libcloud.utils.py3 import httplib from libcloud.utils.py3 import urlparse from libcloud.utils.files import read_in_chunks +from libcloud.utils.files import exhaust_iterator from libcloud.common.base import ConnectionUserAndKey from libcloud.common.base import JsonResponse from libcloud.common.types import InvalidCredsError +from libcloud.common.types import LibcloudError from libcloud.storage.providers import Provider from libcloud.storage.base import Object, Container, StorageDriver @@ -134,9 +137,17 @@ def __init__(self, *args, **kwargs): # dowload url). self._auth_conn = BackblazeB2AuthConnection(*args, **kwargs) + def download_request(self): + # TODO + pass + + def upload_request(self): + # TODO + pass + def request(self, action, params=None, data=None, headers=None, method='GET', raw=False, include_account_id=False, - download_request=False): + download_request=False, upload_host=None, auth_token=None): params = params or {} headers = headers or {} @@ -145,13 +156,25 @@ def request(self, action, params=None, data=None, headers=None, # Set host if raw: + # TODO: Refactor this mess. # File download or upload request: - self.host = auth_conn.download_host + if method == 'GET': + # Download + self.host = auth_conn.download_host + elif method == 'POST': + self.host = upload_host else: self.host = auth_conn.api_host + if upload_host: + self.host = upload_host + # Provide auth token - headers['Authorization'] = '%s' % (auth_conn.auth_token) + # TODO: Refactor + if not auth_token: + auth_token = auth_conn.auth_token + + headers['Authorization'] = '%s' % (auth_token) # Include Content-Type if not raw and data: @@ -165,12 +188,13 @@ def request(self, action, params=None, data=None, headers=None, data = data or {} data['accountId'] = auth_conn.account_id - if not raw: + if not raw and not upload_host: action = API_PATH + action - else: + elif method == 'GET': + # Download action = '/file/' + action - if data: + if data and not upload_host: data = json.dumps(data) response = super(BackblazeB2Connection, self).request(action=action, @@ -188,6 +212,7 @@ class BackblazeB2StorageDriver(StorageDriver): website = 'https://www.backblaze.com/b2/' type = Provider.BACKBLAZE_B2 hash_type = 'sha1' + supports_chunked_encoding = False def iterate_containers(self): resp = self.connection.request(action='b2_list_buckets', @@ -223,10 +248,6 @@ def delete_container(self, container): include_account_id=True) return resp.status == httplib.OK - def upload_object(self, file_path, container, object_name, extra=None, - verify_hash=True): - pass - def download_object(self, obj, destination_path, overwrite_existing=False, delete_on_failure=True): action = self._get_object_download_path(container=obj.container, @@ -258,6 +279,61 @@ def download_object_as_stream(self, obj, chunk_size=None): 'chunk_size': chunk_size}, success_status_code=httplib.OK) + def upload_object(self, file_path, container, object_name, extra=None, + verify_hash=True, headers=None): + """ + Upload an object. + + Note: This will override file with a same name if it already exists. + """ + # Note: We don't use any of the base driver functions since Backblaze + # API requires you to provide SHA1 has upfront and the base methods + # don't support that + fh = open(file_path, 'rb') + iterator = iter(fh) + iterator = read_in_chunks(iterator=iterator) + data = exhaust_iterator(iterator=iterator) + + extra = extra or {} + content_type = extra.get('content_type', 'b2/x-auto') + meta_data = extra.get('meta_data', {}) + + # Note: Backblaze API doesn't support chunked encoding and we need to + # provide Content-Length up front (this is one inside _upload_object):/ + headers = headers or {} + headers['X-Bz-File-Name'] = object_name + headers['Content-Type'] = content_type + + sha1 = hashlib.sha1() + sha1.update(data.encode()) + headers['X-Bz-Content-Sha1'] = sha1.hexdigest() + + # Include optional meta-data (up to 10 items) + for key, value in meta_data: + # TODO: Encode / escape key + headers['X-Bz-Info-%s' % (key)] = value + + upload_data = self.ex_get_upload_data(container_id=container.extra['id']) + upload_token = upload_data['authorizationToken'] + parsed_url = urlparse.urlparse(upload_data['uploadUrl']) + + upload_host = parsed_url.netloc + request_path = parsed_url.path + + response = self.connection.request(action=request_path, method='POST', + headers=headers, + upload_host=upload_host, + auth_token=upload_token, + data=data) + + if response.status == httplib.OK: + obj = self._to_object(item=response.object, container=container) + return obj + else: + body = response.read() + raise LibcloudError('Upload failed. status_code=%s, body=%s' % + (response.status, body), driver=self) + def delete_object(self, obj): data = {} data['fileName'] = obj.name @@ -303,6 +379,31 @@ def ex_list_object_versions(self, container_id, ex_start_file_name=None, objects = self._to_objects(data=resp.object, container=None) return objects + def ex_get_upload_data(self, container_id): + """ + Retrieve information used for uploading files (upload url, auth token, + etc). + + :rype: ``dict`` + """ + # TODO: This is static (AFAIK) so it could be cached + params = {} + params['bucketId'] = container_id + response = self.connection.request(action='b2_get_upload_url', + method='GET', + params=params) + return response.object + + def ex_get_upload_url(self, container_id): + """ + Retrieve URL used for file uploads. + + :rtype: ``str`` + """ + result = self.ex_get_upload_data(container_id=container_id) + upload_url = result['uploadUrl'] + return upload_url + def _to_containers(self, data): result = [] for item in data['buckets']: From aa1e41ed1b91e6ff1da3d15d130e591fee18b445 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sun, 10 Jan 2016 17:23:41 +0100 Subject: [PATCH 05/13] Refactor connection class, make code more readable and maintainable. --- libcloud/storage/drivers/backblaze_b2.py | 114 +++++++++++++---------- 1 file changed, 63 insertions(+), 51 deletions(-) diff --git a/libcloud/storage/drivers/backblaze_b2.py b/libcloud/storage/drivers/backblaze_b2.py index afb9ea6c75..63eb28b17a 100644 --- a/libcloud/storage/drivers/backblaze_b2.py +++ b/libcloud/storage/drivers/backblaze_b2.py @@ -133,21 +133,42 @@ class BackblazeB2Connection(ConnectionUserAndKey): def __init__(self, *args, **kwargs): super(BackblazeB2Connection, self).__init__(*args, **kwargs) - # Stores info retrieved after authnetication (auth token, api url, + # Stores info retrieved after authentication (auth token, api url, # dowload url). self._auth_conn = BackblazeB2AuthConnection(*args, **kwargs) - def download_request(self): - # TODO - pass + def download_request(self, action, params=None): + # Lazily perform authentication + auth_conn = self._auth_conn.authenticate() + + # Set host to the download server + self.host = auth_conn.download_host + + action = '/file/' + action + method = 'GET' + raw = True + response = self._request(auth_conn=auth_conn, action=action, + params=params, method=method, + raw=raw) + return response + + def upload_request(self, action, headers, upload_host, auth_token, data): + # Lazily perform authentication + auth_conn = self._auth_conn.authenticate() + + # Upload host is dynamically retrieved for each upload request + self.host = upload_host - def upload_request(self): - # TODO - pass + method = 'POST' + raw = True + response = self._request(auth_conn=auth_conn, action=action, + params=None, data=data, + headers=headers, method=method, + raw=raw, auth_token=auth_token) + return response def request(self, action, params=None, data=None, headers=None, - method='GET', raw=False, include_account_id=False, - download_request=False, upload_host=None, auth_token=None): + method='GET', raw=False, include_account_id=False): params = params or {} headers = headers or {} @@ -155,26 +176,7 @@ def request(self, action, params=None, data=None, headers=None, auth_conn = self._auth_conn.authenticate() # Set host - if raw: - # TODO: Refactor this mess. - # File download or upload request: - if method == 'GET': - # Download - self.host = auth_conn.download_host - elif method == 'POST': - self.host = upload_host - else: - self.host = auth_conn.api_host - - if upload_host: - self.host = upload_host - - # Provide auth token - # TODO: Refactor - if not auth_token: - auth_token = auth_conn.auth_token - - headers['Authorization'] = '%s' % (auth_token) + self.host = auth_conn.api_host # Include Content-Type if not raw and data: @@ -188,15 +190,26 @@ def request(self, action, params=None, data=None, headers=None, data = data or {} data['accountId'] = auth_conn.account_id - if not raw and not upload_host: - action = API_PATH + action - elif method == 'GET': - # Download - action = '/file/' + action - - if data and not upload_host: + action = API_PATH + action + if data: data = json.dumps(data) + response = self._request(auth_conn=self._auth_conn, action=action, + params=params, data=data, + method=method, headers=headers, raw=raw) + return response + + def _request(self, auth_conn, action, params=None, data=None, headers=None, + method='GET', raw=False, auth_token=None): + params = params or {} + headers = headers or {} + + if not auth_token: + # If auth token is not explicitly provided, use the default one + auth_token = self._auth_conn.auth_token + + # Include auth token + headers['Authorization'] = '%s' % (auth_token) response = super(BackblazeB2Connection, self).request(action=action, params=params, data=data, @@ -252,8 +265,7 @@ def download_object(self, obj, destination_path, overwrite_existing=False, delete_on_failure=True): action = self._get_object_download_path(container=obj.container, obj=obj) - response = self.connection.request(action=action, method='GET', - raw=True) + response = self.connection.download_request(action=action) # TODO: Include metadata from response headers return self._get_object(obj=obj, callback=self._save_object, @@ -270,8 +282,7 @@ def download_object(self, obj, destination_path, overwrite_existing=False, def download_object_as_stream(self, obj, chunk_size=None): action = self._get_object_download_path(container=obj.container, obj=obj) - response = self.connection.request(action=action, method='GET', - raw=True) + response = self.connection.download_request(action=action) return self._get_object(obj=obj, callback=read_in_chunks, response=response, @@ -289,10 +300,10 @@ def upload_object(self, file_path, container, object_name, extra=None, # Note: We don't use any of the base driver functions since Backblaze # API requires you to provide SHA1 has upfront and the base methods # don't support that - fh = open(file_path, 'rb') - iterator = iter(fh) - iterator = read_in_chunks(iterator=iterator) - data = exhaust_iterator(iterator=iterator) + with open(file_path, 'rb') as fp: + iterator = iter(fp) + iterator = read_in_chunks(iterator=iterator) + data = exhaust_iterator(iterator=iterator) extra = extra or {} content_type = extra.get('content_type', 'b2/x-auto') @@ -313,24 +324,25 @@ def upload_object(self, file_path, container, object_name, extra=None, # TODO: Encode / escape key headers['X-Bz-Info-%s' % (key)] = value - upload_data = self.ex_get_upload_data(container_id=container.extra['id']) + upload_data = self.ex_get_upload_data( + container_id=container.extra['id']) upload_token = upload_data['authorizationToken'] parsed_url = urlparse.urlparse(upload_data['uploadUrl']) upload_host = parsed_url.netloc request_path = parsed_url.path - response = self.connection.request(action=request_path, method='POST', - headers=headers, - upload_host=upload_host, - auth_token=upload_token, - data=data) + response = self.connection.upload_request(action=request_path, + headers=headers, + upload_host=upload_host, + auth_token=upload_token, + data=data) if response.status == httplib.OK: obj = self._to_object(item=response.object, container=container) return obj else: - body = response.read() + body = response.response.read() raise LibcloudError('Upload failed. status_code=%s, body=%s' % (response.status, body), driver=self) From b035d8ab849cc1845fcaa527f91240d8c84a961c Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Mon, 11 Jan 2016 07:30:54 +0100 Subject: [PATCH 06/13] Don't use raw requests when uploading data - this is not needed and not supported. --- libcloud/storage/drivers/backblaze_b2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libcloud/storage/drivers/backblaze_b2.py b/libcloud/storage/drivers/backblaze_b2.py index 63eb28b17a..08cc63e308 100644 --- a/libcloud/storage/drivers/backblaze_b2.py +++ b/libcloud/storage/drivers/backblaze_b2.py @@ -160,7 +160,7 @@ def upload_request(self, action, headers, upload_host, auth_token, data): self.host = upload_host method = 'POST' - raw = True + raw = False response = self._request(auth_conn=auth_conn, action=action, params=None, data=data, headers=headers, method=method, From 88c22780e1d546b89a1386133e5f81f9f488139e Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sun, 17 Jan 2016 19:03:33 +0100 Subject: [PATCH 07/13] Add some initial basic tests for Backblaze B2 storage driver. --- .../backblaze_b2/b2_create_bucket.json | 6 + .../backblaze_b2/b2_delete_bucket.json | 6 + .../backblaze_b2/b2_delete_file_version.json | 4 + .../backblaze_b2/b2_get_upload_url.json | 5 + .../backblaze_b2/b2_list_buckets.json | 22 +++ .../backblaze_b2/b2_list_file_names.json | 34 ++++ .../fixtures/backblaze_b2/b2_upload_file.json | 10 + libcloud/test/storage/test_backblaze_b2.py | 172 ++++++++++++++++++ 8 files changed, 259 insertions(+) create mode 100644 libcloud/test/storage/fixtures/backblaze_b2/b2_create_bucket.json create mode 100644 libcloud/test/storage/fixtures/backblaze_b2/b2_delete_bucket.json create mode 100644 libcloud/test/storage/fixtures/backblaze_b2/b2_delete_file_version.json create mode 100644 libcloud/test/storage/fixtures/backblaze_b2/b2_get_upload_url.json create mode 100644 libcloud/test/storage/fixtures/backblaze_b2/b2_list_buckets.json create mode 100644 libcloud/test/storage/fixtures/backblaze_b2/b2_list_file_names.json create mode 100644 libcloud/test/storage/fixtures/backblaze_b2/b2_upload_file.json create mode 100644 libcloud/test/storage/test_backblaze_b2.py diff --git a/libcloud/test/storage/fixtures/backblaze_b2/b2_create_bucket.json b/libcloud/test/storage/fixtures/backblaze_b2/b2_create_bucket.json new file mode 100644 index 0000000000..326a1d40dc --- /dev/null +++ b/libcloud/test/storage/fixtures/backblaze_b2/b2_create_bucket.json @@ -0,0 +1,6 @@ +{ + "accountId": "8c7eea3fe570", + "bucketId": "681c87aebeaa530f5e250710", + "bucketName": "test0005", + "bucketType": "allPrivate" +} diff --git a/libcloud/test/storage/fixtures/backblaze_b2/b2_delete_bucket.json b/libcloud/test/storage/fixtures/backblaze_b2/b2_delete_bucket.json new file mode 100644 index 0000000000..326a1d40dc --- /dev/null +++ b/libcloud/test/storage/fixtures/backblaze_b2/b2_delete_bucket.json @@ -0,0 +1,6 @@ +{ + "accountId": "8c7eea3fe570", + "bucketId": "681c87aebeaa530f5e250710", + "bucketName": "test0005", + "bucketType": "allPrivate" +} diff --git a/libcloud/test/storage/fixtures/backblaze_b2/b2_delete_file_version.json b/libcloud/test/storage/fixtures/backblaze_b2/b2_delete_file_version.json new file mode 100644 index 0000000000..c0ac2f831d --- /dev/null +++ b/libcloud/test/storage/fixtures/backblaze_b2/b2_delete_file_version.json @@ -0,0 +1,4 @@ +{ + "fileId": "4_z481c37de2e1ab3bf5e150710_f1060bc60382f42a2_d20160111_m063042_c001_v0001012_t0008", + "fileName": "test8.txt" +} diff --git a/libcloud/test/storage/fixtures/backblaze_b2/b2_get_upload_url.json b/libcloud/test/storage/fixtures/backblaze_b2/b2_get_upload_url.json new file mode 100644 index 0000000000..2ee28de0c4 --- /dev/null +++ b/libcloud/test/storage/fixtures/backblaze_b2/b2_get_upload_url.json @@ -0,0 +1,5 @@ +{ + "authorizationToken": "nope", + "bucketId": "481c37de2e1ab3bf5e150710", + "uploadUrl": "https://podxxx.backblaze.com/b2api/v1/b2_upload_file/abcd/defg" +} diff --git a/libcloud/test/storage/fixtures/backblaze_b2/b2_list_buckets.json b/libcloud/test/storage/fixtures/backblaze_b2/b2_list_buckets.json new file mode 100644 index 0000000000..5ccefac98e --- /dev/null +++ b/libcloud/test/storage/fixtures/backblaze_b2/b2_list_buckets.json @@ -0,0 +1,22 @@ +{ + "buckets": [ + { + "accountId": "8c7eea3fe570", + "bucketId": "481c37de2e1ab3bf5e150710", + "bucketName": "test00001", + "bucketType": "allPrivate" + }, + { + "accountId": "8c7eea3fe570", + "bucketId": "886c573e2e1ab3bf5e150710", + "bucketName": "test00002", + "bucketType": "allPublic" + }, + { + "accountId": "8c7eea3fe570", + "bucketId": "c85c97ee2eeab3bf5e150710", + "bucketName": "test00003", + "bucketType": "allPrivate" + } + ] +} diff --git a/libcloud/test/storage/fixtures/backblaze_b2/b2_list_file_names.json b/libcloud/test/storage/fixtures/backblaze_b2/b2_list_file_names.json new file mode 100644 index 0000000000..a8e3264086 --- /dev/null +++ b/libcloud/test/storage/fixtures/backblaze_b2/b2_list_file_names.json @@ -0,0 +1,34 @@ + +{ + "files": [ + { + "action": "upload", + "fileId": "abcd", + "fileName": "2.txt", + "size": 2, + "uploadTimestamp": 1450545966000 + }, + { + "action": "upload", + "fileId": "4_z481c37de2e1ab3bf5e150710_f108012807a71d788_d20160110_m155541_c001_v0001001_t0016", + "fileName": "test5.txt", + "size": 24, + "uploadTimestamp": 1452441341000 + }, + { + "action": "upload", + "fileId": "4_z481c37de2e1ab3bf5e150710_f1071961a62d3426d_d20160110_m155708_c001_v0001013_t0042", + "fileName": "test6.txt", + "size": 24, + "uploadTimestamp": 1452441428000 + }, + { + "action": "upload", + "fileId": "4_z481c37de2e1ab3bf5e150710_f1060bc60382f42a2_d20160111_m063042_c001_v0001012_t0008", + "fileName": "test8.txt", + "size": 24, + "uploadTimestamp": 1452493842000 + } + ], + "nextFileName": null +} diff --git a/libcloud/test/storage/fixtures/backblaze_b2/b2_upload_file.json b/libcloud/test/storage/fixtures/backblaze_b2/b2_upload_file.json new file mode 100644 index 0000000000..2f3cdb0df5 --- /dev/null +++ b/libcloud/test/storage/fixtures/backblaze_b2/b2_upload_file.json @@ -0,0 +1,10 @@ +{ + "accountId": "8c7eea3fe570", + "bucketId": "481c37de2e1ab3bf5e150710", + "contentLength": 24, + "contentSha1": "23d23d43d4ecad793c049c81c4bc436ba1e8531e", + "contentType": "text/plain", + "fileId": "abcde", + "fileInfo": {}, + "fileName": "test0007.txt" +} diff --git a/libcloud/test/storage/test_backblaze_b2.py b/libcloud/test/storage/test_backblaze_b2.py new file mode 100644 index 0000000000..6ce01e8dce --- /dev/null +++ b/libcloud/test/storage/test_backblaze_b2.py @@ -0,0 +1,172 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys +import tempfile + +import mock + +from libcloud.storage.drivers.backblaze_b2 import BackblazeB2StorageDriver +from libcloud.utils.py3 import httplib +from libcloud.test import unittest +from libcloud.test import StorageMockHttp +from libcloud.test import MockRawResponse +from libcloud.test import MockHttpTestCase +from libcloud.test.file_fixtures import StorageFileFixtures + + +class MockAuthConn(mock.Mock): + account_id = 'abcdefgh' + + +class BackblazeB2StorageDriverTestCase(unittest.TestCase): + driver_klass = BackblazeB2StorageDriver + driver_args = ('a', 'b') + + def setUp(self): + self.driver_klass.connectionCls.authCls = MockAuthConn() + self.driver_klass.connectionCls.conn_classes = ( + None, BackblazeB2MockHttp) + self.driver_klass.connectionCls.rawResponseCls = \ + BackblazeB2MockRawResponse + BackblazeB2MockHttp.type = None + BackblazeB2MockRawResponse.type = None + self.driver = self.driver_klass(*self.driver_args) + + def test_list_containers(self): + containers = self.driver.list_containers() + self.assertEqual(len(containers), 3) + self.assertEqual(containers[0].name, 'test00001') + self.assertEqual(containers[0].extra['id'], '481c37de2e1ab3bf5e150710') + self.assertEqual(containers[0].extra['bucketType'], 'allPrivate') + + def test_list_container_objects(self): + container = self.driver.list_containers()[0] + objects = self.driver.list_container_objects(container=container) + self.assertEqual(len(objects), 4) + self.assertEqual(objects[0].name, '2.txt') + self.assertEqual(objects[0].size, 2) + self.assertEqual(objects[0].extra['fileId'], 'abcd') + self.assertEqual(objects[0].extra['uploadTimestamp'], 1450545966000) + + def test_create_container(self): + container = self.driver.create_container(container_name='test0005') + self.assertEqual(container.name, 'test0005') + self.assertEqual(container.extra['id'], '681c87aebeaa530f5e250710') + self.assertEqual(container.extra['bucketType'], 'allPrivate') + + def test_delete_container(self): + container = self.driver.list_containers()[0] + result = self.driver.delete_container(container=container) + self.assertTrue(result) + + def test_download_object(self): + container = self.driver.list_containers()[0] + obj = self.driver.list_container_objects(container=container)[0] + _, destination_path = tempfile.mkstemp() + result = self.driver.download_object(obj=obj, destination_path=destination_path, + overwrite_existing=True) + self.assertTrue(result) + + def test_download_object_as_stream(self): + container = self.driver.list_containers()[0] + obj = self.driver.list_container_objects(container=container)[0] + result = self.driver.download_object_as_stream(obj=obj) + result = ''.join(list(result)) + self.assertEqual(result, 'ab') + + def test_upload_object(self): + file_path = os.path.abspath(__file__) + container = self.driver.list_containers()[0] + obj = self.driver.upload_object(file_path=file_path, container=container, + object_name='test0007.txt') + self.assertEqual(obj.name, 'test0007.txt') + self.assertEqual(obj.size, 24) + self.assertEqual(obj.extra['fileId'], 'abcde') + + def test_delete_object(self): + container = self.driver.list_containers()[0] + obj = self.driver.list_container_objects(container=container)[0] + result = self.driver.delete_object(obj=obj) + self.assertTrue(result) + + +class BackblazeB2MockHttp(StorageMockHttp, MockHttpTestCase): + fixtures = StorageFileFixtures('backblaze_b2') + + def _b2api_v1_b2_list_buckets(self, method, url, body, headers): + if method == 'GET': + body = self.fixtures.load('b2_list_buckets.json') + else: + raise AssertionError('Unsupported method') + return (httplib.OK, body, {}, httplib.responses[httplib.OK]) + + def _b2api_v1_b2_list_file_names(self, method, url, body, headers): + if method == 'GET': + body = self.fixtures.load('b2_list_file_names.json') + else: + raise AssertionError('Unsupported method') + return (httplib.OK, body, {}, httplib.responses[httplib.OK]) + + def _b2api_v1_b2_create_bucket(self, method, url, body, headers): + if method == 'POST': + body = self.fixtures.load('b2_create_bucket.json') + else: + raise AssertionError('Unsupported method') + return (httplib.OK, body, {}, httplib.responses[httplib.OK]) + + def _b2api_v1_b2_delete_bucket(self, method, url, body, headers): + if method == 'POST': + body = self.fixtures.load('b2_delete_bucket.json') + else: + raise AssertionError('Unsupported method') + return (httplib.OK, body, {}, httplib.responses[httplib.OK]) + + def _b2api_v1_b2_delete_file_version(self, method, url, body, headers): + if method == 'POST': + body = self.fixtures.load('b2_delete_file_version.json') + else: + raise AssertionError('Unsupported method') + return (httplib.OK, body, {}, httplib.responses[httplib.OK]) + + def _b2api_v1_b2_get_upload_url(self, method, url, body, headers): + # test_upload_object + if method == 'GET': + body = self.fixtures.load('b2_get_upload_url.json') + else: + raise AssertionError('Unsupported method') + return (httplib.OK, body, {}, httplib.responses[httplib.OK]) + + def _b2api_v1_b2_upload_file_abcd_defg(self, method, url, body, headers): + # test_upload_object + if method == 'POST': + body = self.fixtures.load('b2_upload_file.json') + else: + raise AssertionError('Unsupported method') + return (httplib.OK, body, {}, httplib.responses[httplib.OK]) + + +class BackblazeB2MockRawResponse(MockRawResponse): + def _file_test00001_2_txt(self, method, url, body, headers): + # test_download_object + if method == 'GET': + body = 'ab' + else: + raise AssertionError('Unsupported method') + return (httplib.OK, body, {}, httplib.responses[httplib.OK]) + +if __name__ == '__main__': + sys.exit(unittest.main()) From ea7409e9da9b1c8d10acfdc61a85da8f8c51b62a Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sun, 17 Jan 2016 19:21:20 +0100 Subject: [PATCH 08/13] Make it easier to mock auth class. --- libcloud/storage/drivers/backblaze_b2.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libcloud/storage/drivers/backblaze_b2.py b/libcloud/storage/drivers/backblaze_b2.py index 08cc63e308..2481e516fa 100644 --- a/libcloud/storage/drivers/backblaze_b2.py +++ b/libcloud/storage/drivers/backblaze_b2.py @@ -129,13 +129,14 @@ class BackblazeB2Connection(ConnectionUserAndKey): host = None # Note: host is set after authentication secure = True responseCls = BackblazeB2Response + authCls = BackblazeB2AuthConnection def __init__(self, *args, **kwargs): super(BackblazeB2Connection, self).__init__(*args, **kwargs) # Stores info retrieved after authentication (auth token, api url, # dowload url). - self._auth_conn = BackblazeB2AuthConnection(*args, **kwargs) + self._auth_conn = self.authCls(*args, **kwargs) def download_request(self, action, params=None): # Lazily perform authentication From 4be89d24b3f8cb5376f476437bda2342baf42043 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sun, 17 Jan 2016 22:55:37 +0100 Subject: [PATCH 09/13] Fix Python 3 related issues. --- libcloud/storage/drivers/backblaze_b2.py | 2 +- libcloud/test/storage/test_backblaze_b2.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libcloud/storage/drivers/backblaze_b2.py b/libcloud/storage/drivers/backblaze_b2.py index 2481e516fa..bb37025097 100644 --- a/libcloud/storage/drivers/backblaze_b2.py +++ b/libcloud/storage/drivers/backblaze_b2.py @@ -317,7 +317,7 @@ def upload_object(self, file_path, container, object_name, extra=None, headers['Content-Type'] = content_type sha1 = hashlib.sha1() - sha1.update(data.encode()) + sha1.update(b(data)) headers['X-Bz-Content-Sha1'] = sha1.hexdigest() # Include optional meta-data (up to 10 items) diff --git a/libcloud/test/storage/test_backblaze_b2.py b/libcloud/test/storage/test_backblaze_b2.py index 6ce01e8dce..871f52b343 100644 --- a/libcloud/test/storage/test_backblaze_b2.py +++ b/libcloud/test/storage/test_backblaze_b2.py @@ -85,7 +85,7 @@ def test_download_object_as_stream(self): container = self.driver.list_containers()[0] obj = self.driver.list_container_objects(container=container)[0] result = self.driver.download_object_as_stream(obj=obj) - result = ''.join(list(result)) + result = ''.join([x.decode('utf-8') for x in list(result)]) self.assertEqual(result, 'ab') def test_upload_object(self): From 484d9746701778ff643ec8cc8a40d1937f673927 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Mon, 18 Jan 2016 11:35:45 +0100 Subject: [PATCH 10/13] Add tests for extension methods. --- .../fixtures/backblaze_b2/b2_hide_file.json | 7 ++ .../backblaze_b2/b2_list_file_versions.json | 69 +++++++++++++++++++ libcloud/test/storage/test_backblaze_b2.py | 41 +++++++++++ 3 files changed, 117 insertions(+) create mode 100644 libcloud/test/storage/fixtures/backblaze_b2/b2_hide_file.json create mode 100644 libcloud/test/storage/fixtures/backblaze_b2/b2_list_file_versions.json diff --git a/libcloud/test/storage/fixtures/backblaze_b2/b2_hide_file.json b/libcloud/test/storage/fixtures/backblaze_b2/b2_hide_file.json new file mode 100644 index 0000000000..c339410385 --- /dev/null +++ b/libcloud/test/storage/fixtures/backblaze_b2/b2_hide_file.json @@ -0,0 +1,7 @@ +{ + "action": "hide", + "fileId": "4_z481c37de2e1ab3bf5e150710_f1089c2d0af334258_d20160118_m103421_c001_v0001015_t0019", + "fileName": "2.txt", + "size": 0, + "uploadTimestamp": 1453113261000 +} diff --git a/libcloud/test/storage/fixtures/backblaze_b2/b2_list_file_versions.json b/libcloud/test/storage/fixtures/backblaze_b2/b2_list_file_versions.json new file mode 100644 index 0000000000..a5d8118ac4 --- /dev/null +++ b/libcloud/test/storage/fixtures/backblaze_b2/b2_list_file_versions.json @@ -0,0 +1,69 @@ +{ + "files": [ + { + "action": "hide", + "fileId": "4_z481c37de2e1ab3bf5e150710_f11356114bd70b633_d20151219_m171034_c001_v0001015_t0017", + "fileName": "1.txt", + "size": 0, + "uploadTimestamp": 1450545034000 + }, + { + "action": "upload", + "fileId": "4_z481c37de2e1ab3bf5e150710_f114de3a6d4984677_d20151219_m165328_c001_v0001011_t0013", + "fileName": "1.txt", + "size": 4, + "uploadTimestamp": 1450544008000 + }, + { + "action": "upload", + "fileId": "4_z481c37de2e1ab3bf5e150710_f112d803c648864bd_d20151219_m172606_c001_v0001013_t0006", + "fileName": "2.txt", + "size": 2, + "uploadTimestamp": 1450545966000 + }, + { + "action": "upload", + "fileId": "4_z481c37de2e1ab3bf5e150710_f112d1d656ee3d11a_d20160117_m215631_c001_v0001017_t0020", + "fileName": "test10.txt", + "size": 24, + "uploadTimestamp": 1453067791000 + }, + { + "action": "upload", + "fileId": "4_z481c37de2e1ab3bf5e150710_f108012807a71d788_d20160110_m155541_c001_v0001001_t0016", + "fileName": "test5.txt", + "size": 24, + "uploadTimestamp": 1452441341000 + }, + { + "action": "upload", + "fileId": "4_z481c37de2e1ab3bf5e150710_f118ba13b3b4343ae_d20151220_m135857_c001_v0001003_t0007", + "fileName": "test5.txt", + "size": 24, + "uploadTimestamp": 1450619937000 + }, + { + "action": "upload", + "fileId": "4_z481c37de2e1ab3bf5e150710_f1071961a62d3426d_d20160110_m155708_c001_v0001013_t0042", + "fileName": "test6.txt", + "size": 24, + "uploadTimestamp": 1452441428000 + }, + { + "action": "upload", + "fileId": "4_z481c37de2e1ab3bf5e150710_f106af7e634b95e05_d20160117_m180806_c001_v0001017_t0017", + "fileName": "test8.txt", + "size": 24, + "uploadTimestamp": 1453054086000 + }, + { + "action": "upload", + "fileId": "4_z481c37de2e1ab3bf5e150710_f114bdc6fc4c203d9_d20160117_m215616_c001_v0001017_t0006", + "fileName": "test9.txt", + "size": 24, + "uploadTimestamp": 1453067776000 + } + ], + "nextFileId": null, + "nextFileName": null +} diff --git a/libcloud/test/storage/test_backblaze_b2.py b/libcloud/test/storage/test_backblaze_b2.py index 871f52b343..6a6274627b 100644 --- a/libcloud/test/storage/test_backblaze_b2.py +++ b/libcloud/test/storage/test_backblaze_b2.py @@ -103,6 +103,33 @@ def test_delete_object(self): result = self.driver.delete_object(obj=obj) self.assertTrue(result) + def test_ex_hide_object(self): + container = self.driver.list_containers()[0] + container_id = container.extra['id'] + obj = self.driver.ex_hide_object(container_id=container_id, + object_name='2.txt') + self.assertEqual(obj.name, '2.txt') + + def test_ex_list_object_versions(self): + container = self.driver.list_containers()[0] + container_id = container.extra['id'] + objects = self.driver.ex_list_object_versions(container_id=container_id) + self.assertEqual(len(objects), 9) + + def test_ex_get_upload_data(self): + container = self.driver.list_containers()[0] + container_id = container.extra['id'] + data = self.driver.ex_get_upload_data(container_id=container_id) + self.assertEqual(data['authorizationToken'], 'nope') + self.assertEqual(data['bucketId'], '481c37de2e1ab3bf5e150710') + self.assertEqual(data['uploadUrl'], 'https://podxxx.backblaze.com/b2api/v1/b2_upload_file/abcd/defg') + + def test_ex_get_upload_url(self): + container = self.driver.list_containers()[0] + container_id = container.extra['id'] + url = self.driver.ex_get_upload_url(container_id=container_id) + self.assertEqual(url, 'https://podxxx.backblaze.com/b2api/v1/b2_upload_file/abcd/defg') + class BackblazeB2MockHttp(StorageMockHttp, MockHttpTestCase): fixtures = StorageFileFixtures('backblaze_b2') @@ -158,6 +185,20 @@ def _b2api_v1_b2_upload_file_abcd_defg(self, method, url, body, headers): raise AssertionError('Unsupported method') return (httplib.OK, body, {}, httplib.responses[httplib.OK]) + def _b2api_v1_b2_list_file_versions(self, method, url, body, headers): + if method == 'GET': + body = self.fixtures.load('b2_list_file_versions.json') + else: + raise AssertionError('Unsupported method') + return (httplib.OK, body, {}, httplib.responses[httplib.OK]) + + def _b2api_v1_b2_hide_file(self, method, url, body, headers): + if method == 'POST': + body = self.fixtures.load('b2_hide_file.json') + else: + raise AssertionError('Unsupported method') + return (httplib.OK, body, {}, httplib.responses[httplib.OK]) + class BackblazeB2MockRawResponse(MockRawResponse): def _file_test00001_2_txt(self, method, url, body, headers): From b13376bd1440194531330336ce389b4a4193142d Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Mon, 18 Jan 2016 11:49:44 +0100 Subject: [PATCH 11/13] Add some basic docs for Backblaze B2 driver. --- .../images/provider_logos/backblaze.png | Bin 0 -> 6864 bytes .../storage/backblaze_b2/instantiate.py | 8 ++++ docs/storage/drivers/backblaze_b2.rst | 40 ++++++++++++++++++ 3 files changed, 48 insertions(+) create mode 100644 docs/_static/images/provider_logos/backblaze.png create mode 100644 docs/examples/storage/backblaze_b2/instantiate.py create mode 100644 docs/storage/drivers/backblaze_b2.rst diff --git a/docs/_static/images/provider_logos/backblaze.png b/docs/_static/images/provider_logos/backblaze.png new file mode 100644 index 0000000000000000000000000000000000000000..5899a799bec1a98b90c85f8831267009f1b85e61 GIT binary patch literal 6864 zcma)hcRU;1-?rM-+FGSXLbcS47>OOBXse1UwO6blLhQYFa96EbRjm$tHA1Octr442 zwG}mj*fn16=f3aX^L#$f{l0Jhxbi*ce6Q<#&pF>e^0^ZG@PP(B9VZ40ScYJ7-@U7_c&_HnbaK_V~!YlOX{t0Et{%$=tcbQ{-PrisSAwOtlp#Ot3+;nQNQ--I%E^`eS z!8EKPFCEIHs+k}87m$laFsYsci^JLSKai|)2^jGwP#V($BmRLKeX6v;KOyw$S#%B; ztC{{A{4dJ?Q~rM{t3B&)+|R?~@vR&p?}-Nnx#N8e+d|SsOa&$7RxMFk<@tFq{1w%} z46#fm)ki5RE$()*TTHBKc$oI>v(wnXp8BWM!kxl2z4fc;fNIM_SAOdb?B|<=NHis; z=Om{AxEZ<*ZKTVXtUX>m4(JmVsVJl2*B)A6RgL+@yo68>W{gx&!%pw{~Ys0aSssYOJOpEo&jp5aKV$n>hww5rssdw!x=Pk;~f|$Jy6SVn- z3#TYE24IAsZa$D;GPeLL?CnrMcxm#aPcM-FCbr<&RB3e!RM=xlhSs@TVgto0Jeb6Z zhub3_=PMOLu>pNdkEvC%qPJ@(i?p2_%n!A6dA`prxE9XBS_IB>QU=1J%DtMqjv{8e zLz_(~h-N6`j1|VeL`M~tn{__zrdgo}ZAi!^Wp-rWuoS_5L_lik% zh{d2{;hCA5yEh&B=2P<1q)S31xS10$LKE0$DK+pxfBqxq+qaraAKJR|ljr0#M6H=s zV`}gFC+##ang^vx66@%D?u}kOJ{2u|z}(@EpL~z!s%e|FcvFOo3$28RZK@ZRzAk^t zAb;YiiQ+3!@nc!uG&FY^c|cUP73h0ijxZ=n^%bW*IyLC1P<*eI4^&9F$Zd0Msh|wJvkg@t@bv$=KP(V3Z=1Eeg^MhYIWW6GhsDR0N#`2_5!RhKlTTq|HS0x8*$Aa{SfMCZp&o>oMHIOw>$j4frvx_arb z^4%X)vuYsvl)SYg#qmf0VbpNr5DfT61P;hQ9@LIKlE$8)CnYA2ThuOdjY=?aRx)f6ACUw`_7muXad!?)w~-vWsS(Z3*O^ zYgZ9kj%~&31x2S;gAYW0qO-@)v}%%75RqE=jMJ!VE_}JPj<3Ui+S%x(!OQ?$aVZGO z4+pYyu4Kcg@WtAKqrpR|j4^!=Y7zb^l}b#xn-r*Xq4ZkZmY zjM1BmuxCwct~EAL<%Z%Kyc4DndTF;L1j8QLb~*@$MnSyFRWsEmtJ&gd+95r^wyq_G z#6L`eF)jNV2=?VocdYqw;^?hm7lQiKdLiG0Bgrs95C5C(8>b zLL~(W%xf^KdDT~_XRvJfaqOu??~80#@2j1s#S^2v%7=_=Pg|lAh@r*MANzqjFlv~T zcbVG4!w0C}t#h*}J!3lqVxPA<gg{hlzUz?3$}G|FSKx z*Ddy-y%4NfBp`y+X~>?1uHL5ok#kYLq)ju)PK3}P4hgO+TYPvppz}Tg!XCSQJ-1`s z`~5VJXQF;_qEAz}&}ji{94WVU&a=(}H!>?#y@Rj4(U)gy+TJf3EF~rWDtRJ(Er-Jv#x_K=usl!vG1IEsZyC5pr!Jb)Y z)V(zwl|bdr7eOz%MyB*=?@fkJH~KqO&XIENBli%GXS<6EEcgSzqOTE~U=A%kLA-;H zm9&lOR)7%=zKz+>p7U>Yjf+N zxeEJ*X!cejH&+KkY5H6){^jN2xoeTV2x;maWM_V{>pu2-5roK{dYWWPIldKrnaVFo zg{GX7n=JAbGOmGuSPWG-;qUnE?@gQ`es6g~ zxupOqEqQ83=CYGHraZ?`Vr_-lp1~)8yI^yaGL5f@J%?!%6j|vLQQx03e04YFyT->w zlD+oHe(M3E@a5f%Q|{ePK4gR*YSNqi2V_DZL_K(Lc>33TyRDIUsrfF4 zx#_T9iX%LNPDAG>ME{9bT`K!xn17&-IoK7}TREp$bAhPA0h(&YT0-DvIS4>*> zpU-Fq6Tcv@QBJW#zOdKNmPz+bntnd^x-uMH1zg%1rc zgJ~$;0>k_}(=D>)HocEbZj26i6Ka&1J%86b+NjY<6+1kjx}FZ>&|yEd*@K7-t2b-x zo=6W7*p%8w%xO3Gb>uUa=1es-+B0PsBN}>z5=x)@ROjSgjYKIe%NX>f2`l6c_G|+h z?e}ii)8PD4Ls!z_sg2qC+>j9NE2PZ!Z$s11ncCRHn`X$SY6fQqV!&lr;FLJ)qVg69 z>yFRYYFM^Ei)vx8tBl#ZoTzCmbwKu*SU7&Kv|py^`z%A!&IAD$=;A#KY8+N2Z`c!3 z_Q=-gmm_S}p&mF&Q&!@8_B%8o^u=GrlFDRoWOL}4!4qie8mC(>1#<%Db*F92c&z>7 zC&-|z%?@_nXuo4ZweC1gv5CwG*bAl=uX)Nu^cW%M5AoE>I2Fl0wVnQ`4}SL>2(94| z9lFP_wBoFuP~mBoG-eN1$XaDtlGw*g$0>iZ+*6!WsEGI!n)3!n)sySNxPu{k7P9wK z0_hnzh~`obrd=DNS!=jGCs!VBc5O?kNXQ-C8CuJqu(dOwl6f)Z*ORvfZX<>hz34Bh zBe!P33=O5y{onwx&vi{P^(E<%QB#0k&|vbSHp2O;(|C~(;eCcQ?>2OxY>bT!5VPtW zgXzlhe$YP#y)&7VsbZu(-+X=lDB)LbI^!YlezOjVo6^eC-a85Iz`^c!r?r65;!Pii zz)+KT=US+{MVuQiE-2>MZN1G`r)KV*%20S$f1bqpl4thjrU{q7b35rpUB|Rk5@2Ao z`(Rv`^kVNrduKPD^!30tvAq#`nO83+*-U+^Y`m^suHB~SwaPAZgJpi7kJAeiSeTj3 zNA0uFE~^#pUPv)d#MB6NBVUWJ$RT6Vwx<*;!m~$qaz}MrM8RBE>yHS5VzYP9J2aG` zX~n_})RDjQnXHQ5aOHHoEHz_TtNgUAq4@6Z_?{w?`OvPNi-47UJ*5nWP>?U{&6z{} z{UVF)jaL^_>oP!~6?fdGxZ6y79mV)~LL%fUrc1$`@YFekj3M+(BMXz0(jSV}ETw{Q zb2IDhEUkR$Pq6BDYZrigz3^-1;Qnwxb!FbA3QT9uL)VX2K!zD7O6kF;w;+4{PDbmA zaz`AVR;w-##4&AA`JSUOHqi@~#~vsC79Bffl-tbd{^21?OjF^&clWqvxo(lM*wf`C z1U%-P;ksd~vbn%CvXWfen_qlSdwFmrEWmVbZA!pW5j?cgN0-`#-4BTV-6~-HF~Oh1 zBekoD;Vo*!f->x$Y{wH2gW@Qj>s@S4ml>H_zwERrv+%gsTf6S{9Ii!DB^{<(zXbcFlV^RK|Ai5nSFQ1QT z@ia-mJjo;ahDg#$k1qt}o!4*CAz72Kt8D24 zzptijfQ~M+eoJTM>+WD}zmB6#b8*vJVL-h3&FkS*RkA^${CN}V{bpL_%UC;cKnT0) z{SHmE+0&2dGq@-p8#>~SR{VoN%Ur(z*;{Pq^-L_qHbc^89xT@fF)s#LzntA%4n?bp zNtJl7Em^7iNn!J{XKFaC@{CMN*|264iIr0|@>7LY$}y-R#lHR^Z!dBsUXsnlF8WIY z^9_cH{%KtzM#Zr*7?lIX;s*4Yp%1LvB=D!&q+T}$oim3R$ zDo$H};#zT$P6}?7n=L6mxnDnSE+N==ilnPj7)KcGX=p30{-HZYBJW=?O=FO6dMnh4 zo2m^Q>dq`f7r711eUrx{S;3 zXZvOYYGMkX9zkKv*-v4){>@pFH@?Y;I8keKN~#yMjGH$2gI&QTS_y@iQ)Y?7ch zz(~uI#LaKYH)kiLN65`n%D)v3jvd+^L?GY^yH(szFDdvuFYwU<>NMDFJdi!Q&%Bi~N;09VIV!kb71e|H4>YG^GEl7Pw zG`3cE#z8$D>Hm6fc|4Z6Nbap>lbt;KlnAe}KNqfeyP)V+@yaco*AbL=;x5som@!ja zw?yy?`SwhP-5iwsBI~>n>z|FN(7h=;*HC%q`}6Sf4w#A^+ml|d`xtuT&FBr4i`H#z za7I6oNnI>yjCnWKO8QkDHsO0iJ{P@<8N0ooTv4#nCa|Z?bIJ6De@MI(w&H~XNs2{} z7-h|#T<9ms`s&g)8|%lHC{sW2_e9@^pX3(9{zwt{+0`+6^5y-=X9o8g0h)b09Kyp8 z&@O{svz8PUkT~D%Og(dXCyK_3CYCE+oC?DcDn~Sg+;l(1hBnyq zq0Fn1`tuzrbD>}67U-(Gif;bMZ}I#6jleLEFj=hA?2Rt5377X^iBe zSo*c}1{!aR*Vu4a+qEX+)6b~VmQ^)912kcb zd0SCeI4B(R_uW4|)nB?sjPQra2*eyDx5=1&&q%bnu%f-94ZfLXOxCCarp$OxZf126 z2XVM_XG%!BzNF;g7~-XCOH?9W*U&KL6*=>oRdAoDz|UYFO3%9++Vj1`?sM7=&m!66 z*EP`|HRAUjk>41iKDU1fOh{{Y49lzHC4n*q#p+0sfu->fDxH#>GVF0RRR!cO>ZwYv z^}PG+7`@0NYJ|?}l>m{UmC@BHf*{*T_!^gp^O7UZz1=rF<8LqiW&Qq}LHj@b-~U*@ v|Kk7t$NK#<`u}&9|DVb+kB{=mD9ETjXCs0{`gG6#W71N8pjN7C8TNkwhZ=~F literal 0 HcmV?d00001 diff --git a/docs/examples/storage/backblaze_b2/instantiate.py b/docs/examples/storage/backblaze_b2/instantiate.py new file mode 100644 index 0000000000..2b0c9fc0ca --- /dev/null +++ b/docs/examples/storage/backblaze_b2/instantiate.py @@ -0,0 +1,8 @@ +from libcloud.storage.types import Provider +from libcloud.storage.providers import get_driver + +account_id = 'XXXXXX' +application_key = 'YYYYYY' + +cls = get_driver(Provider.BACKBLAZE_B2) +driver = cls(account_id, application_key) diff --git a/docs/storage/drivers/backblaze_b2.rst b/docs/storage/drivers/backblaze_b2.rst new file mode 100644 index 0000000000..af8d61b2d9 --- /dev/null +++ b/docs/storage/drivers/backblaze_b2.rst @@ -0,0 +1,40 @@ +Backblaze B2 Storage Driver Documentation +========================================= + +`Backblaze`_ is an online backup tool that allows Windows and Mac OS X users to +back up their data to an offsite data center. + +`Backblaze B2`_ is their cloud object storage offering similar to Amazon S3 and +other object storage services. + +.. figure:: /_static/images/provider_logos/backblaze.png + :align: center + :width: 300 + :target: https://www.backblaze.com/b2/cloud-storage.html + +Keep in mind that the service is currently in public beta and only users who +have signed up and received beta access can use it. To sign up for the beta +access, visit their website mentioned above. + +Instantiating a driver +---------------------- + +To instantiate the driver you need to pass your account id and application key +to the driver constructor as shown below. + +To access the account id, once we have admitted you into the beta you can login +to https://secure.backblaze.com/user_signin.htm, then click "buckets" and +"show account id and application key". + +.. literalinclude:: /examples/storage/backblaze_b2/instantiate.py + :language: python + +API Docs +-------- + +.. autoclass:: libcloud.storage.drivers.backblaze_b2.BackblazeB2StorageDriver + :members: + :inherited-members: + +.. _`Backblaze`: https://www.backblaze.com/ +.. _`Backblaze B2`: https://www.backblaze.com/b2/cloud-storage.html From d814d0613b5fdba704e9ba2aac848492d92305a4 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Mon, 18 Jan 2016 11:49:56 +0100 Subject: [PATCH 12/13] Remove trailing line breaks. --- docs/storage/drivers/auroraobjects.rst | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/storage/drivers/auroraobjects.rst b/docs/storage/drivers/auroraobjects.rst index d7dbeb2892..11eedbcda8 100644 --- a/docs/storage/drivers/auroraobjects.rst +++ b/docs/storage/drivers/auroraobjects.rst @@ -22,7 +22,6 @@ a LibcloudError. As a backend AuroraObjects uses `Ceph`_ for storage. - Instantiating a driver ---------------------- @@ -34,7 +33,6 @@ With these credentials you can instantiate the driver: .. literalinclude:: /examples/storage/auroraobjects/instantiate.py :language: python - Multipart uploads ----------------- @@ -58,7 +56,6 @@ Examples Please refer to the Amazon S3 storage driver documentation for examples. - API Docs -------- From 0d049e62d0bf015549a712fb61790552baad3091 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Mon, 18 Jan 2016 11:51:19 +0100 Subject: [PATCH 13/13] Re-generate supported providers tables. --- docs/storage/_supported_methods_cdn.rst | 2 ++ docs/storage/_supported_methods_main.rst | 2 ++ docs/storage/_supported_providers.rst | 2 ++ 3 files changed, 6 insertions(+) diff --git a/docs/storage/_supported_methods_cdn.rst b/docs/storage/_supported_methods_cdn.rst index d9b825bbc8..8b53926605 100644 --- a/docs/storage/_supported_methods_cdn.rst +++ b/docs/storage/_supported_methods_cdn.rst @@ -5,6 +5,7 @@ Provider enable container cdn enable object cdn get contain ============================= ==================== ================= ===================== ================== `PCextreme AuroraObjects`_ yes yes yes yes `Microsoft Azure (blobs)`_ no no no no +`Backblaze B2`_ no no no no `CloudFiles`_ yes no yes yes `CloudFiles (UK)`_ yes no yes yes `CloudFiles (US)`_ yes no yes yes @@ -24,6 +25,7 @@ Provider enable container cdn enable object cdn get contain .. _`PCextreme AuroraObjects`: https://www.pcextreme.com/aurora/objects .. _`Microsoft Azure (blobs)`: http://windows.azure.com/ +.. _`Backblaze B2`: https://www.backblaze.com/b2/ .. _`CloudFiles`: http://www.rackspace.com/ .. _`CloudFiles (UK)`: http://www.rackspace.com/ .. _`CloudFiles (US)`: http://www.rackspace.com/ diff --git a/docs/storage/_supported_methods_main.rst b/docs/storage/_supported_methods_main.rst index c202cbb56f..2da5f65802 100644 --- a/docs/storage/_supported_methods_main.rst +++ b/docs/storage/_supported_methods_main.rst @@ -5,6 +5,7 @@ Provider list containers list objects create container dele ============================= =============== ============ ================ ================ ============= ======================= =============== ========================= ============= `PCextreme AuroraObjects`_ yes yes yes yes yes yes yes yes yes `Microsoft Azure (blobs)`_ yes yes yes yes yes yes yes yes yes +`Backblaze B2`_ yes yes yes yes yes no yes yes yes `CloudFiles`_ yes yes yes yes yes yes yes yes yes `CloudFiles (UK)`_ yes yes yes yes yes yes yes yes yes `CloudFiles (US)`_ yes yes yes yes yes yes yes yes yes @@ -24,6 +25,7 @@ Provider list containers list objects create container dele .. _`PCextreme AuroraObjects`: https://www.pcextreme.com/aurora/objects .. _`Microsoft Azure (blobs)`: http://windows.azure.com/ +.. _`Backblaze B2`: https://www.backblaze.com/b2/ .. _`CloudFiles`: http://www.rackspace.com/ .. _`CloudFiles (UK)`: http://www.rackspace.com/ .. _`CloudFiles (US)`: http://www.rackspace.com/ diff --git a/docs/storage/_supported_providers.rst b/docs/storage/_supported_providers.rst index 0d8f29c0d3..4e4e35b525 100644 --- a/docs/storage/_supported_providers.rst +++ b/docs/storage/_supported_providers.rst @@ -5,6 +5,7 @@ Provider Documentation Pr ============================= =============================================== ================= ============================================== ==================================== `PCextreme AuroraObjects`_ :doc:`Click ` AURORAOBJECTS :mod:`libcloud.storage.drivers.auroraobjects` :class:`AuroraObjectsStorageDriver` `Microsoft Azure (blobs)`_ :doc:`Click ` AZURE_BLOBS :mod:`libcloud.storage.drivers.azure_blobs` :class:`AzureBlobsStorageDriver` +`Backblaze B2`_ :doc:`Click ` BACKBLAZE_B2 :mod:`libcloud.storage.drivers.backblaze_b2` :class:`BackblazeB2StorageDriver` `CloudFiles`_ CLOUDFILES :mod:`libcloud.storage.drivers.cloudfiles` :class:`CloudFilesStorageDriver` `CloudFiles (UK)`_ CLOUDFILES_UK :mod:`libcloud.storage.drivers.cloudfiles` :class:`CloudFilesUKStorageDriver` `CloudFiles (US)`_ CLOUDFILES_US :mod:`libcloud.storage.drivers.cloudfiles` :class:`CloudFilesUSStorageDriver` @@ -24,6 +25,7 @@ Provider Documentation Pr .. _`PCextreme AuroraObjects`: https://www.pcextreme.com/aurora/objects .. _`Microsoft Azure (blobs)`: http://windows.azure.com/ +.. _`Backblaze B2`: https://www.backblaze.com/b2/ .. _`CloudFiles`: http://www.rackspace.com/ .. _`CloudFiles (UK)`: http://www.rackspace.com/ .. _`CloudFiles (US)`: http://www.rackspace.com/