From d33705822cb3ee049ebc6920396c98603345c234 Mon Sep 17 00:00:00 2001 From: carycheng Date: Tue, 9 Oct 2018 16:46:29 -0700 Subject: [PATCH 01/54] chunked uploads endpoint --- boxsdk/object/__init__.py | 1 + boxsdk/object/collaboration_whitelist.py | 2 +- boxsdk/object/file.py | 35 ++++- boxsdk/object/folder.py | 34 ++++ boxsdk/object/upload_session.py | 146 ++++++++++++++++++ ...t_collaboration_whitelist_exempt_target.py | 2 +- test/unit/object/test_file.py | 22 +++ test/unit/object/test_folder.py | 23 +++ test/unit/object/test_upload_session.py | 116 ++++++++++++++ 9 files changed, 378 insertions(+), 3 deletions(-) create mode 100644 boxsdk/object/upload_session.py create mode 100644 test/unit/object/test_upload_session.py diff --git a/boxsdk/object/__init__.py b/boxsdk/object/__init__.py index 53d5d4e17..eca76a625 100644 --- a/boxsdk/object/__init__.py +++ b/boxsdk/object/__init__.py @@ -32,6 +32,7 @@ 'task', 'task_assignment', 'user', + 'upload_session', 'webhook', 'watermark', 'web_link', diff --git a/boxsdk/object/collaboration_whitelist.py b/boxsdk/object/collaboration_whitelist.py index 97713b04a..0b3245fc6 100644 --- a/boxsdk/object/collaboration_whitelist.py +++ b/boxsdk/object/collaboration_whitelist.py @@ -130,7 +130,7 @@ def add_exemption(self, user): url = self.get_url('collaboration_whitelist_exempt_targets') data = { 'user': { - 'id': user.object_id # pylint:disable=protected-access + 'id': user.object_id } } response = self._session.post(url, data=json.dumps(data)).json() diff --git a/boxsdk/object/file.py b/boxsdk/object/file.py index 72bfe5f42..15ca1af47 100644 --- a/boxsdk/object/file.py +++ b/boxsdk/object/file.py @@ -1,9 +1,10 @@ # coding: utf-8 -from __future__ import unicode_literals +from __future__ import unicode_literals, absolute_import import json +from boxsdk.config import API from .item import Item from ..util.api_call_decorator import api_call from ..pagination.marker_based_object_collection import MarkerBasedObjectCollection @@ -36,6 +37,38 @@ def preflight_check(self, size, name=None): file_id=self._object_id, ) + def create_upload_session(self, file_size, file_name=None): + """ + Create a new chunked upload session for uploading a new version of the file. + + :param file_size: + The size of the file that will be uploaded. + :type file_size: + `int` + :returns: + A :class:`ChunkedUploadSession` object. + :rtype: + :class:`ChunkedUploadSession` + """ + body_params = { + 'file_id': self.object_id, + 'file_size': file_size, + } + if file_name is not None: + body_params['file_name'] = file_name + response = self._session.post( + self.get_url('{0}s'.format('upload_session')).replace( + API.BASE_API_URL, + API.UPLOAD_URL, + ), + data=json.dumps(body_params), + ).json() + return self.translator.translate(response['type'])( + session=self.session, + object_id=response['id'], + response_object=response, + ) + def _get_accelerator_upload_url_for_update(self): """ Get Accelerator upload url for updating the file. diff --git a/boxsdk/object/folder.py b/boxsdk/object/folder.py index 777a5e57c..87ce9898f 100644 --- a/boxsdk/object/folder.py +++ b/boxsdk/object/folder.py @@ -5,6 +5,7 @@ import os from six import text_type +from boxsdk.config import API from boxsdk.object.group import Group from boxsdk.object.item import Item from boxsdk.object.user import User @@ -112,6 +113,39 @@ def preflight_check(self, size, name): parent_id=self._object_id, ) + def create_upload_session(self, file_size, file_name): + """ + Creates a new chunked upload session for upload a new file. + + :param file_size: + The size of the file that will be uploaded. + :type file_size: + `int` + :param file_name: + The name of the file that will be uploaded. + :type file_name: + `unicode` + :returns: + A :class:`ChunkedUploadSession` object. + :rtype: + :class:`ChunkedUploadSession` + """ + url = '{0}/files/upload_sessions'.format(API.UPLOAD_URL) + body_params = { + 'folder_id': self.object_id, + 'file_size': file_size, + 'file_name': file_name, + } + response = self._session.post( + url, + data=json.dumps(body_params), + ).json() + return self.translator.translate(response['type'])( + session=self.session, + object_id=response['id'], + response_object=response, + ) + def _get_accelerator_upload_url_fow_new_uploads(self): """ Get Accelerator upload url for uploading new files. diff --git a/boxsdk/object/upload_session.py b/boxsdk/object/upload_session.py new file mode 100644 index 000000000..c86bd5fde --- /dev/null +++ b/boxsdk/object/upload_session.py @@ -0,0 +1,146 @@ +# coding: utf-8 +from __future__ import unicode_literals, absolute_import + +import base64 +import hashlib +import json + +from .base_object import BaseObject +from ..config import API +from ..util.translator import Translator + +class UploadSession(BaseObject): + _item_type = 'upload_session' + _parent_item_type = 'file' + + def get_url(self, *args): + """ + Base class override. Endpoint is a little different - it's /files/upload_sessions. + + :rtype: + `unicode` + """ + return self._session.get_url( + '{0}s/{1}s'.format(self._parent_item_type, self._item_type), + self.object_id, + *args, + ).replace(API.BASE_API_URL, API.UPLOAD_URL) + + def get_parts(self): + """ + Get a list of parts uploaded so far. + + :returns: + Returns a `list` of parts uploaded so far. + :rtype: + `list` of `dict` + """ + response = self.session.get(self.get_url('parts')).json() + return response['entries'] + + def _calculate_part_sha1(self, content_stream): + """ + Calculate the SHA1 hash of the chunk stream for a given part of the file. + + :returns: + The unencoded SHA1 hash of the part. + :rtype: + `bytes` + """ + content_sha1 = hashlib.sha1() + stream_position = content_stream.tell() + hashed_length = 0 + while hashed_length < self.part_size: + chunk = content_stream.read(self.part_size - hashed_length) + if chunk is None: + continue + if len(chunk) == 0: + break + hashed_length += len(chunk) + content_sha1.update(chunk) + content_stream.seek(stream_position) + return content_sha1.digest() + + def upload_part(self, content_stream, offset, total_size, part_content_sha1=None): + """ + Upload a part of a file. + + :param content_stream: + File-like object containing the content of the part to be uploaded. + :type content_stream: + :class:`File` + :param offset: + Offset, in number of bytes, of the part compared to the beginning of the file. + :type offset: + `int` + :param total_size: + The size of the file that this part belongs to. + :type total_size: + `int` + :param part_content_sha1: + SHA-1 hash of the part's content. If not specified, this will be calculated. + :type part_content_sha1: + `unicode` + :returns: + The uploaded part. + :rtype: + `dict` + """ + if part_content_sha1 is None: + part_content_sha1 = self._calculate_part_sha1(content_stream) + + range_end = min(offset + self.part_size - 1, total_size - 1) + + return self._session.put( + self.get_url(), + headers={ + 'Content-Type': 'application/octet-stream', + 'Digest': 'SHA={0}'.format(base64.b64encode(part_content_sha1).decode('utf-8')), + 'Content-Range': 'bytes {0}-{1}/{2}'.format(offset, range_end, total_size), + }, + data=content_stream + ).json() + + def commit(self, parts, content_sha1): + """ + Commit a multiput upload. + + :param parts: + List of parts that were uploaded. + :type parts: + `Iterable` of `dict` + :param content_sha1: + SHA-1 has of the file contents that was uploaded. + :type content_sha1: + `unicode` + :returns: + A :class:`File` object. + :rtype: + :class:`File` + """ + response = self._session.post( + self.get_url('commit'), + headers={ + 'Content-Type': 'application/json', + 'Digest': 'SHA={0}'.format(base64.b64encode(content_sha1).decode('utf-8')), + }, + data=json.dumps({'parts': parts}), + ).json() + entry = response['entries'][0] + return self.translator.translate(entry['type'])( + session=self.session, + object_id=entry['id'], + response_object=entry, + ) + + def abort(self): + """ + Abort an upload session, cancelling the upload and removing any parts that have already been uploaded. + + :returns: + A boolean indication success of the upload abort. + :rtype: + `bool` + """ + response = self._session.delete(self.get_url()) + return response.ok diff --git a/test/unit/object/test_collaboration_whitelist_exempt_target.py b/test/unit/object/test_collaboration_whitelist_exempt_target.py index e0db97b16..51f06978f 100644 --- a/test/unit/object/test_collaboration_whitelist_exempt_target.py +++ b/test/unit/object/test_collaboration_whitelist_exempt_target.py @@ -24,6 +24,6 @@ def test_get(mock_box_session, test_collaboration_whitelist_exemption): def test_delete(mock_box_session, test_collaboration_whitelist_exemption): exemption_id = test_collaboration_whitelist_exemption.object_id - expected_url = mock_box_session.get_url('collaboration_whitelist_exempt_targets', exemption_id) + expected_url = '{0}/collaboration_whitelist_exempt_targets/{1}'.format(API.BASE_API_URL, exemption_id) test_collaboration_whitelist_exemption.delete() mock_box_session.delete.assert_called_once_with(expected_url, expect_json_response=False, headers=None, params={}) diff --git a/test/unit/object/test_file.py b/test/unit/object/test_file.py index 8c93f3f81..888dc937f 100644 --- a/test/unit/object/test_file.py +++ b/test/unit/object/test_file.py @@ -10,6 +10,7 @@ from boxsdk.object.comment import Comment from boxsdk.object.file import File from boxsdk.object.task import Task +from boxsdk.object.upload_session import UploadSession # pylint:disable=protected-access @@ -42,6 +43,27 @@ def test_delete_file(test_file, mock_box_session, etag, if_match_header): ) +def test_create_upload_session(test_file, mock_box_session): + expected_url = '{0}/files/{1}/upload_sessions'.format(API.UPLOAD_URL, test_file.object_id) + file_size = 197520 + expected_data = { + 'file_id': test_file.object_id, + 'file_size': file_size, + } + mock_box_session.post.return_value.json.return_value = { + 'id': 'F971964745A5CD0C001BBE4E58196BFD', + 'type': 'upload_session', + 'num_parts_processed': 0, + 'total_parts': 16, + 'part_size': 12345, + } + upload_session = test_file.create_upload_session(file_size) + mock_box_session.post.assert_called_once_with(expected_url, data=json.dumps(expected_data)) + assert isinstance(upload_session, UploadSession) + assert upload_session.part_size == 12345 + assert upload_session.total_parts == 16 + + def test_create_task(test_file, test_task, mock_box_session): # pylint:disable=redefined-outer-name expected_url = "{0}/tasks".format(API.BASE_API_URL) diff --git a/test/unit/object/test_folder.py b/test/unit/object/test_folder.py index 2b7249134..2592ebb30 100644 --- a/test/unit/object/test_folder.py +++ b/test/unit/object/test_folder.py @@ -14,6 +14,7 @@ from boxsdk.object.web_link import WebLink from boxsdk.object.collaboration import Collaboration, CollaborationRole from boxsdk.object.folder import Folder, FolderSyncState +from boxsdk.object.upload_session import UploadSession from boxsdk.session.box_response import BoxResponse @@ -188,6 +189,28 @@ def test_upload( assert 'entries' not in new_file +def test_create_upload_session(test_folder, mock_box_session): + expected_url = '{0}/files/upload_sessions'.format(API.UPLOAD_URL) + file_size = 197520 + file_name = 'test_file.pdf' + expected_data = { + 'folder_id': test_folder.object_id, + 'file_size': file_size, + 'file_name': file_name, + } + mock_box_session.post.return_value.json.return_value = { + 'id': 'F971964745A5CD0C001BBE4E58196BFD', + 'type': 'upload_session', + 'num_parts_processed': 0, + 'total_parts': 16, + 'part_size': 12345, + } + upload_session = test_folder.create_upload_session(file_size, file_name) + mock_box_session.post.assert_called_once_with(expected_url, data=json.dumps(expected_data)) + assert isinstance(upload_session, UploadSession) + assert upload_session.part_size == 12345 + assert upload_session.total_parts == 16 + def test_upload_stream_does_preflight_check_if_specified( mock_box_session, test_folder, diff --git a/test/unit/object/test_upload_session.py b/test/unit/object/test_upload_session.py new file mode 100644 index 000000000..889381398 --- /dev/null +++ b/test/unit/object/test_upload_session.py @@ -0,0 +1,116 @@ +# coding: utf-8 + +from __future__ import unicode_literals, absolute_import + +import base64 +import hashlib +from io import BytesIO +import json +import pytest + +from boxsdk.config import API +from boxsdk.object.file import File +from boxsdk.object.upload_session import UploadSession + + +@pytest.fixture() +def test_upload_session(mock_box_session): + upload_session_response_object = { + 'part_size': 8, + 'total_parts': 10, + } + return UploadSession(mock_box_session, '11493C07ED3EABB6E59874D3A1EF3581', upload_session_response_object) + + +def test_get_parts(test_upload_session, mock_box_session): + expected_url = '{0}/files/upload_sessions/{1}/parts'.format(API.UPLOAD_URL, test_upload_session.object_id) + + parts = mock_box_session.get.return_value.json.return_value = { + 'entries': [ + { + 'part': { + 'part_id': '8F0966B1', + 'offset': 0, + 'size': 8, + 'sha1': None, + }, + }, + ], + } + test_parts = test_upload_session.get_parts() + mock_box_session.get.assert_called_once_with(expected_url) + assert test_parts == parts['entries'] + + +def test_abort(test_upload_session, mock_box_session): + expected_url = '{0}/files/upload_sessions/{1}'.format(API.UPLOAD_URL, test_upload_session.object_id) + mock_box_session.delete.return_value.ok = True + result = test_upload_session.abort() + mock_box_session.delete.assert_called_once_with(expected_url) + assert result is True + + +def test_upload_part(test_upload_session, mock_box_session): + expected_url = '{0}/files/upload_sessions/{1}'.format(API.UPLOAD_URL, test_upload_session.object_id) + chunk = BytesIO(b'abcdefgh') + offset = 32 + total_size = 80 + expected_sha1 = 'QlrxKgdDUCsyLpOgFbz4aOMk1Wo=' + expected_headers = { + 'Content-Type': 'application/octet-stream', + 'Digest': 'SHA={}'.format(expected_sha1), + 'Content-Range': 'bytes 32-39/80', + } + mock_box_session.put.return_value.json.return_value = { + 'part': { + 'part_id': 'ABCDEF123', + 'offset': offset, + 'size': 8, + 'sha1': expected_sha1, + }, + } + part = test_upload_session.upload_part(chunk, offset, total_size) + + mock_box_session.put.assert_called_once_with(expected_url, data=chunk, headers=expected_headers) + assert part['part']['sha1'] == expected_sha1 + + +def test_commit(test_upload_session, mock_box_session): + expected_url = '{0}/files/upload_sessions/{1}/commit'.format(API.UPLOAD_URL, test_upload_session.object_id) + sha1 = hashlib.sha1() + sha1.update(b'fake_file_data') + file_id = '12345' + parts = [ + { + 'part_id': 'ABCDEF123', + 'offset': 0, + 'size': 8, + 'sha1': 'fake_sha1', + }, + { + 'part_id': 'ABCDEF456', + 'offset': 8, + 'size': 8, + 'sha1': 'fake_sha1', + }, + ] + expected_data = { + 'parts': parts, + } + expected_headers = { + 'Content-Type': 'application/json', + 'Digest': 'SHA={}'.format(base64.b64encode(sha1.digest()).decode('utf-8')), + } + + mock_box_session.post.return_value.json.return_value = { + 'entries': [ + { + 'type': 'file', + 'id': file_id, + }, + ], + } + created_file = test_upload_session.commit(parts, sha1.digest()) + mock_box_session.post.assert_called_once_with(expected_url, data=json.dumps(expected_data), headers=expected_headers) + assert isinstance(created_file, File) + assert created_file.id == file_id From 2b43cf98477ec39f8255623e5725dd370785f29e Mon Sep 17 00:00:00 2001 From: carycheng Date: Wed, 10 Oct 2018 17:36:05 -0700 Subject: [PATCH 02/54] added documentation for chunked upload endpoint --- boxsdk/client/client.py | 15 ++++++ boxsdk/object/file.py | 6 ++- boxsdk/object/upload_session.py | 2 +- docs/chunked_upload.md | 92 +++++++++++++++++++++++++++++++++ test/unit/object/test_file.py | 22 +++++--- test/unit/object/test_folder.py | 22 +++++--- 6 files changed, 143 insertions(+), 16 deletions(-) create mode 100644 docs/chunked_upload.md diff --git a/boxsdk/client/client.py b/boxsdk/client/client.py index 3a9dbb6a8..a37f123c8 100644 --- a/boxsdk/client/client.py +++ b/boxsdk/client/client.py @@ -101,6 +101,21 @@ def file(self, file_id): """ return self.translator.translate('file')(session=self._session, object_id=file_id) + def upload_session(self, session_id): + """ + Initialize a :class:`UploadSession` object, whose box id is session_id. + + :param session_id: + The box id of the :class:`UploadSession` object. + :type session_id: + `unicode` + :return: + A :class:`UploadSession` object with the given session id. + :rtype: + :class`UploadSession` + """ + return self.translator.get('upload_session')(session=self._session, object_id=session_id) + def comment(self, comment_id): """ Initialize a :class:`Comment` object, whose Box ID is comment_id. diff --git a/boxsdk/object/file.py b/boxsdk/object/file.py index 15ca1af47..560206f40 100644 --- a/boxsdk/object/file.py +++ b/boxsdk/object/file.py @@ -45,6 +45,10 @@ def create_upload_session(self, file_size, file_name=None): The size of the file that will be uploaded. :type file_size: `int` + :param file_name: + The name of the file version that will be uploaded. + :type file_name: + `unicode` or None :returns: A :class:`ChunkedUploadSession` object. :rtype: @@ -57,7 +61,7 @@ def create_upload_session(self, file_size, file_name=None): if file_name is not None: body_params['file_name'] = file_name response = self._session.post( - self.get_url('{0}s'.format('upload_session')).replace( + self.get_url('{0}'.format('upload_sessions')).replace( API.BASE_API_URL, API.UPLOAD_URL, ), diff --git a/boxsdk/object/upload_session.py b/boxsdk/object/upload_session.py index c86bd5fde..1d0129444 100644 --- a/boxsdk/object/upload_session.py +++ b/boxsdk/object/upload_session.py @@ -37,7 +37,7 @@ def get_parts(self): """ response = self.session.get(self.get_url('parts')).json() return response['entries'] - + def _calculate_part_sha1(self, content_stream): """ Calculate the SHA1 hash of the chunk stream for a given part of the file. diff --git a/docs/chunked_upload.md b/docs/chunked_upload.md new file mode 100644 index 000000000..d4b72b75b --- /dev/null +++ b/docs/chunked_upload.md @@ -0,0 +1,92 @@ +Chunked Upload +============== + +This API provides a way to reliably upload large files to Box by chunking them into a sequence of parts. When using this API instead of the single file upload API, a request failure means a client only needs to retry upload of a single part instead of the entire file. Parts can also be uploaded in parallel allowing for potential performance improvement. + +It is important to note that the Chunked Upload API is intended for large files with a minimum size of 20MB. + + + + + +- [Create Upload Session for File Version](#create-upload-session-for-file-version) +- [Get Upload Session](#get-upload-session) +- [Create Upload Session for File](#create-upload-session-for-file) +- [Upload Part](#upload-part) +- [Commit](#commit) +- [List Uploaded Parts](#list-uploaded-parts) +- [Abort](#abort) + + + +Create Upload Session for File Version +-------------------------------------- + +To create an upload session for uploading a large version, use `file.create_upload_session(file_size, file_name=None)` + +```python +file_size = 197520 +upload_session = client.file('11111').create_upload_session(file_size=file_size) +``` + +Get Upload Session +------------------ + +To get an upload session, use `upload_session.get()`. + +```python +upload_session = client.upload_session('11493C07ED3EABB6E59874D3A1EF3581').get() +``` + +Create Upload Session for File +------------------------------ + +To create an upload session for uploading a large file, use +`folder.create_upload_session(file_size, file_name)` + +```python +file_size = 197520 +file_name = 'test_file.pdf' +upload_session = client.folder('22222').create_upload_session(file_size=file_size, file_name=file_name) +``` + +Upload Part +----------- + +To upload a part of the file to this session, use `upload_session.upload_part(content_stream, offset, total_size, part_content_sha1=None)` + +```python +from io import BytesIO +chunk = BytesIO(b'abcdefgh') +offset = 32 +upload_part = client.upload_session('11493C07ED3EABB6E59874D3A1EF3581').upload_part(chunk, offset, total_size) +``` + +Commit +------ + +To commit the upload session to Box, use `upload_session.commit(parts, content_sha1)`. + +```python +import hashlib +parts = client.upload_session('11493C07ED3EABB6E59874D3A1EF3581').get_parts() +uploaded_file = client.upload_session('11493C07ED3EABB6E59874D3A1EF3581').commit(parts, sha1.digest()) +``` + +List Uploaded Parts +------------------- + +To return the list of parts uploaded so far, use `upload_session.get_parts()`. + +```python +parts = client.upload_session('11493C07ED3EABB6E59874D3A1EF3581').get_parts() +``` + +Abort +----- + +To abort a chunked upload, use `upload_session.abort()`. + +```python +client.upload_session('11493C07ED3EABB6E59874D3A1EF3581').abort() +``` diff --git a/test/unit/object/test_file.py b/test/unit/object/test_file.py index 888dc937f..3a11bea27 100644 --- a/test/unit/object/test_file.py +++ b/test/unit/object/test_file.py @@ -46,22 +46,30 @@ def test_delete_file(test_file, mock_box_session, etag, if_match_header): def test_create_upload_session(test_file, mock_box_session): expected_url = '{0}/files/{1}/upload_sessions'.format(API.UPLOAD_URL, test_file.object_id) file_size = 197520 + part_size = 12345 + total_parts = 16 + num_parts_processed = 0 + upload_session_type = 'upload_session' + upload_session_id = 'F971964745A5CD0C001BBE4E58196BFD' expected_data = { 'file_id': test_file.object_id, 'file_size': file_size, } mock_box_session.post.return_value.json.return_value = { - 'id': 'F971964745A5CD0C001BBE4E58196BFD', - 'type': 'upload_session', - 'num_parts_processed': 0, - 'total_parts': 16, - 'part_size': 12345, + 'id': upload_session_id, + 'type': upload_session_type, + 'num_parts_processed': num_parts_processed, + 'total_parts': total_parts, + 'part_size': part_size, } upload_session = test_file.create_upload_session(file_size) mock_box_session.post.assert_called_once_with(expected_url, data=json.dumps(expected_data)) assert isinstance(upload_session, UploadSession) - assert upload_session.part_size == 12345 - assert upload_session.total_parts == 16 + assert upload_session.part_size == part_size + assert upload_session.total_parts == total_parts + assert upload_session.num_parts_processed == num_parts_processed + assert upload_session.type == upload_session_type + assert upload_session.id == upload_session_id def test_create_task(test_file, test_task, mock_box_session): diff --git a/test/unit/object/test_folder.py b/test/unit/object/test_folder.py index 2592ebb30..6b6be236f 100644 --- a/test/unit/object/test_folder.py +++ b/test/unit/object/test_folder.py @@ -193,23 +193,31 @@ def test_create_upload_session(test_folder, mock_box_session): expected_url = '{0}/files/upload_sessions'.format(API.UPLOAD_URL) file_size = 197520 file_name = 'test_file.pdf' + upload_session_id = 'F971964745A5CD0C001BBE4E58196BFD' + upload_session_type = 'upload_session' + num_parts_processed = 0 + total_parts = 16 + part_size = 12345 expected_data = { 'folder_id': test_folder.object_id, 'file_size': file_size, 'file_name': file_name, } mock_box_session.post.return_value.json.return_value = { - 'id': 'F971964745A5CD0C001BBE4E58196BFD', - 'type': 'upload_session', - 'num_parts_processed': 0, - 'total_parts': 16, - 'part_size': 12345, + 'id': upload_session_id, + 'type': upload_session_type, + 'num_parts_processed': num_parts_processed, + 'total_parts': total_parts, + 'part_size': part_size, } upload_session = test_folder.create_upload_session(file_size, file_name) mock_box_session.post.assert_called_once_with(expected_url, data=json.dumps(expected_data)) assert isinstance(upload_session, UploadSession) - assert upload_session.part_size == 12345 - assert upload_session.total_parts == 16 + assert upload_session.part_size == part_size + assert upload_session.total_parts == total_parts + assert upload_session.num_parts_processed == num_parts_processed + assert upload_session.type == upload_session_type + assert upload_session.id == upload_session_id def test_upload_stream_does_preflight_check_if_specified( mock_box_session, From 0a9c9d33f5e15da3a6b17f73249352cf24122915 Mon Sep 17 00:00:00 2001 From: carycheng Date: Thu, 11 Oct 2018 13:24:34 -0700 Subject: [PATCH 03/54] removed collaboration whitelist changes --- boxsdk/object/collaboration_whitelist.py | 2 +- boxsdk/object/upload_session.py | 1 + test/unit/object/test_collaboration_whitelist_exempt_target.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/boxsdk/object/collaboration_whitelist.py b/boxsdk/object/collaboration_whitelist.py index 0b3245fc6..4124630a3 100644 --- a/boxsdk/object/collaboration_whitelist.py +++ b/boxsdk/object/collaboration_whitelist.py @@ -130,7 +130,7 @@ def add_exemption(self, user): url = self.get_url('collaboration_whitelist_exempt_targets') data = { 'user': { - 'id': user.object_id + 'id': user.object_id # pylint:disable=protected-access } } response = self._session.post(url, data=json.dumps(data)).json() diff --git a/boxsdk/object/upload_session.py b/boxsdk/object/upload_session.py index 1d0129444..649070dc1 100644 --- a/boxsdk/object/upload_session.py +++ b/boxsdk/object/upload_session.py @@ -9,6 +9,7 @@ from ..config import API from ..util.translator import Translator + class UploadSession(BaseObject): _item_type = 'upload_session' _parent_item_type = 'file' diff --git a/test/unit/object/test_collaboration_whitelist_exempt_target.py b/test/unit/object/test_collaboration_whitelist_exempt_target.py index 51f06978f..e0db97b16 100644 --- a/test/unit/object/test_collaboration_whitelist_exempt_target.py +++ b/test/unit/object/test_collaboration_whitelist_exempt_target.py @@ -24,6 +24,6 @@ def test_get(mock_box_session, test_collaboration_whitelist_exemption): def test_delete(mock_box_session, test_collaboration_whitelist_exemption): exemption_id = test_collaboration_whitelist_exemption.object_id - expected_url = '{0}/collaboration_whitelist_exempt_targets/{1}'.format(API.BASE_API_URL, exemption_id) + expected_url = mock_box_session.get_url('collaboration_whitelist_exempt_targets', exemption_id) test_collaboration_whitelist_exemption.delete() mock_box_session.delete.assert_called_once_with(expected_url, expect_json_response=False, headers=None, params={}) From 9e366e3359d503c7e6db38734d64d3c1d48708fd Mon Sep 17 00:00:00 2001 From: carycheng Date: Thu, 11 Oct 2018 14:46:53 -0700 Subject: [PATCH 04/54] changed the get_url function --- boxsdk/object/upload_session.py | 4 ++-- .../unit/object/test_collaboration_whitelist_exempt_target.py | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/boxsdk/object/upload_session.py b/boxsdk/object/upload_session.py index 649070dc1..2fb26859b 100644 --- a/boxsdk/object/upload_session.py +++ b/boxsdk/object/upload_session.py @@ -23,8 +23,8 @@ def get_url(self, *args): """ return self._session.get_url( '{0}s/{1}s'.format(self._parent_item_type, self._item_type), - self.object_id, - *args, + self._object_id, + *args ).replace(API.BASE_API_URL, API.UPLOAD_URL) def get_parts(self): diff --git a/test/unit/object/test_collaboration_whitelist_exempt_target.py b/test/unit/object/test_collaboration_whitelist_exempt_target.py index e0db97b16..1d7c9f8ee 100644 --- a/test/unit/object/test_collaboration_whitelist_exempt_target.py +++ b/test/unit/object/test_collaboration_whitelist_exempt_target.py @@ -1,4 +1,3 @@ - # coding: utf-8 from __future__ import unicode_literals, absolute_import From 2063aeba78f92203ff46376df861d14264e153bf Mon Sep 17 00:00:00 2001 From: carycheng Date: Thu, 11 Oct 2018 14:50:30 -0700 Subject: [PATCH 05/54] switched session --- boxsdk/object/upload_session.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/boxsdk/object/upload_session.py b/boxsdk/object/upload_session.py index 2fb26859b..34fe24d38 100644 --- a/boxsdk/object/upload_session.py +++ b/boxsdk/object/upload_session.py @@ -21,7 +21,7 @@ def get_url(self, *args): :rtype: `unicode` """ - return self._session.get_url( + return self.session.get_url( '{0}s/{1}s'.format(self._parent_item_type, self._item_type), self._object_id, *args From 3c1288fe391f5c72bed89399627caed969883191 Mon Sep 17 00:00:00 2001 From: carycheng Date: Thu, 11 Oct 2018 15:40:56 -0700 Subject: [PATCH 06/54] fixed for linter --- boxsdk/object/collaboration_whitelist.py | 2 +- boxsdk/object/upload_session.py | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/boxsdk/object/collaboration_whitelist.py b/boxsdk/object/collaboration_whitelist.py index 4124630a3..38d56056f 100644 --- a/boxsdk/object/collaboration_whitelist.py +++ b/boxsdk/object/collaboration_whitelist.py @@ -130,7 +130,7 @@ def add_exemption(self, user): url = self.get_url('collaboration_whitelist_exempt_targets') data = { 'user': { - 'id': user.object_id # pylint:disable=protected-access + 'id': user.object_id #pylint:disable=protected-access } } response = self._session.post(url, data=json.dumps(data)).json() diff --git a/boxsdk/object/upload_session.py b/boxsdk/object/upload_session.py index 34fe24d38..3305da7bf 100644 --- a/boxsdk/object/upload_session.py +++ b/boxsdk/object/upload_session.py @@ -7,7 +7,6 @@ from .base_object import BaseObject from ..config import API -from ..util.translator import Translator class UploadSession(BaseObject): @@ -51,11 +50,11 @@ def _calculate_part_sha1(self, content_stream): content_sha1 = hashlib.sha1() stream_position = content_stream.tell() hashed_length = 0 - while hashed_length < self.part_size: - chunk = content_stream.read(self.part_size - hashed_length) + while hashed_length < self.part_size: # pylint:disable=no-member + chunk = content_stream.read(self.part_size - hashed_length) # pylint:disable=no-member if chunk is None: continue - if len(chunk) == 0: + if not chunk: break hashed_length += len(chunk) content_sha1.update(chunk) @@ -90,7 +89,7 @@ def upload_part(self, content_stream, offset, total_size, part_content_sha1=None if part_content_sha1 is None: part_content_sha1 = self._calculate_part_sha1(content_stream) - range_end = min(offset + self.part_size - 1, total_size - 1) + range_end = min(offset + self.part_size - 1, total_size - 1) # pylint:disable=no-member return self._session.put( self.get_url(), From 0f5d349b650fb55ff0452fc11bd405ac7c90dd63 Mon Sep 17 00:00:00 2001 From: carycheng Date: Thu, 11 Oct 2018 15:51:18 -0700 Subject: [PATCH 07/54] fixed for linter --- test/unit/client/test_client.py | 4 +++- test/unit/object/test_file.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/test/unit/client/test_client.py b/test/unit/client/test_client.py index b8ce61986..4309e5340 100644 --- a/test/unit/client/test_client.py +++ b/test/unit/client/test_client.py @@ -25,6 +25,7 @@ from boxsdk.object.file import File from boxsdk.object.group import Group from boxsdk.object.user import User +from boxsdk.object.upload_session import UploadSession from boxsdk.object.trash import Trash from boxsdk.object.group_membership import GroupMembership from boxsdk.object.retention_policy import RetentionPolicy @@ -340,7 +341,8 @@ def device_pins_response(device_pin_id_1, device_pin_id_2): (Group, 'group'), (GroupMembership, 'group_membership'), (Enterprise, 'enterprise'), - (Webhook, 'webhook') + (Webhook, 'webhook'), + (UploadSession, 'upload_session'), ]) def test_factory_returns_the_correct_object(mock_client, test_class, factory_method_name): """ Tests the various id-only factory methods in the Client class """ diff --git a/test/unit/object/test_file.py b/test/unit/object/test_file.py index 3a11bea27..1b48b091e 100644 --- a/test/unit/object/test_file.py +++ b/test/unit/object/test_file.py @@ -51,9 +51,11 @@ def test_create_upload_session(test_file, mock_box_session): num_parts_processed = 0 upload_session_type = 'upload_session' upload_session_id = 'F971964745A5CD0C001BBE4E58196BFD' + file_name = 'test_file.pdf' expected_data = { 'file_id': test_file.object_id, 'file_size': file_size, + 'file_name': file_name } mock_box_session.post.return_value.json.return_value = { 'id': upload_session_id, @@ -62,7 +64,7 @@ def test_create_upload_session(test_file, mock_box_session): 'total_parts': total_parts, 'part_size': part_size, } - upload_session = test_file.create_upload_session(file_size) + upload_session = test_file.create_upload_session(file_size, file_name) mock_box_session.post.assert_called_once_with(expected_url, data=json.dumps(expected_data)) assert isinstance(upload_session, UploadSession) assert upload_session.part_size == part_size From f20d03f057ad20ced2652eadaf46b2a73ddf6e31 Mon Sep 17 00:00:00 2001 From: carycheng Date: Thu, 11 Oct 2018 15:53:43 -0700 Subject: [PATCH 08/54] Delete collaboration_whitelist.py --- boxsdk/object/collaboration_whitelist.py | 141 ----------------------- 1 file changed, 141 deletions(-) delete mode 100644 boxsdk/object/collaboration_whitelist.py diff --git a/boxsdk/object/collaboration_whitelist.py b/boxsdk/object/collaboration_whitelist.py deleted file mode 100644 index 38d56056f..000000000 --- a/boxsdk/object/collaboration_whitelist.py +++ /dev/null @@ -1,141 +0,0 @@ -# coding: utf-8 -from __future__ import unicode_literals, absolute_import - -import json - -from .base_endpoint import BaseEndpoint -from ..pagination.marker_based_object_collection import MarkerBasedObjectCollection -from ..util.api_call_decorator import api_call -from ..util.text_enum import TextEnum - - -class WhitelistDirection(TextEnum): - """ - Used to determine the direction of the whitelist. - """ - INBOUND = 'inbound' - OUTBOUNT = 'outbound' - BOTH = 'both' - - -class CollaborationWhitelist(BaseEndpoint): - """Represents the whitelist of email domains that users in an enterprise may collaborate with.""" - - @api_call - def get_entries(self, limit=None, marker=None, fields=None): - """ - Get the entries in the collaboration whitelist using limit-offset paging. - - :param limit: - The maximum number of entries to return per page. If not specified, then will use the server-side default. - :type limit: - `int` or None - :param marker: - The paging marker to start paging from. - :type marker: - `unicode` or None - :param fields: - List of fields to request. - :type fields: - `Iterable` of `unicode` - :returns: - An iterator of the entries in the whitelist. - :rtype: - :class:`BoxObjectCollection` - """ - return MarkerBasedObjectCollection( - session=self._session, - url=self.get_url('collaboration_whitelist_entries'), - limit=limit, - marker=marker, - fields=fields, - return_full_pages=False, - ) - - @api_call - def add_domain(self, domain, direction): - """ - Add a new domain to the collaboration whitelist. - - :param domain: - The email domain to add to the whitelist. - :type domain: - `unicode` - :param direction: - The direction in which collaboration should be allowed: 'inbound', 'outbound', or 'both'. - :type direction: - `unicode` - :returns: - The created whitelist entry for the domain. - :rtype: - :class:`CollaborationWhitelistEntry` - """ - url = self.get_url('collaboration_whitelist_entries') - data = { - 'domain': domain, - 'direction': direction - } - response = self._session.post(url, data=json.dumps(data)).json() - return self.translator.translate(response['type'])( - session=self._session, - object_id=response['id'], - response_object=response, - ) - - @api_call - def get_exemptions(self, limit=None, marker=None, fields=None): - """ - Get the list of exempted users who are not subject to the collaboration whitelist rules. - - :param limit: - The maximum number of entries to return per page. If not specified, then will use the server-side default. - :type limit: - `int` or None - :param marker: - The paging marker to start paging from. - :type marker: - `unicode` or None - :param fields: - List of fields to request. - :type fields: - `Iterable` of `unicode` - :returns: - An iterator of the exemptions to the whitelist. - :rtype: - :class:`BoxObjectCollection` - """ - return MarkerBasedObjectCollection( - session=self._session, - url=self.get_url('collaboration_whitelist_exempt_targets'), - limit=limit, - marker=marker, - fields=fields, - return_full_pages=False, - ) - - @api_call - def add_exemption(self, user): - """ - Exempt a user from the collaboration whitelist. - - :param user: - The user to exempt from the whitelist. - :type user: - :class:`User` - :returns: - The created whitelist exemption. - :rtype: - :class:`CollaborationWhitelistExemptTarget` - """ - url = self.get_url('collaboration_whitelist_exempt_targets') - data = { - 'user': { - 'id': user.object_id #pylint:disable=protected-access - } - } - response = self._session.post(url, data=json.dumps(data)).json() - return self.translator.translate(response['type'])( - session=self._session, - object_id=response['id'], - response_object=response, - ) From 1defb6fc2b1b830d64d05d49d08a4353c06a7ba3 Mon Sep 17 00:00:00 2001 From: carycheng Date: Thu, 11 Oct 2018 16:01:59 -0700 Subject: [PATCH 09/54] re-added collaboration --- boxsdk/object/collaboration_whitelist.py | 141 +++++++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 boxsdk/object/collaboration_whitelist.py diff --git a/boxsdk/object/collaboration_whitelist.py b/boxsdk/object/collaboration_whitelist.py new file mode 100644 index 000000000..97713b04a --- /dev/null +++ b/boxsdk/object/collaboration_whitelist.py @@ -0,0 +1,141 @@ +# coding: utf-8 +from __future__ import unicode_literals, absolute_import + +import json + +from .base_endpoint import BaseEndpoint +from ..pagination.marker_based_object_collection import MarkerBasedObjectCollection +from ..util.api_call_decorator import api_call +from ..util.text_enum import TextEnum + + +class WhitelistDirection(TextEnum): + """ + Used to determine the direction of the whitelist. + """ + INBOUND = 'inbound' + OUTBOUNT = 'outbound' + BOTH = 'both' + + +class CollaborationWhitelist(BaseEndpoint): + """Represents the whitelist of email domains that users in an enterprise may collaborate with.""" + + @api_call + def get_entries(self, limit=None, marker=None, fields=None): + """ + Get the entries in the collaboration whitelist using limit-offset paging. + + :param limit: + The maximum number of entries to return per page. If not specified, then will use the server-side default. + :type limit: + `int` or None + :param marker: + The paging marker to start paging from. + :type marker: + `unicode` or None + :param fields: + List of fields to request. + :type fields: + `Iterable` of `unicode` + :returns: + An iterator of the entries in the whitelist. + :rtype: + :class:`BoxObjectCollection` + """ + return MarkerBasedObjectCollection( + session=self._session, + url=self.get_url('collaboration_whitelist_entries'), + limit=limit, + marker=marker, + fields=fields, + return_full_pages=False, + ) + + @api_call + def add_domain(self, domain, direction): + """ + Add a new domain to the collaboration whitelist. + + :param domain: + The email domain to add to the whitelist. + :type domain: + `unicode` + :param direction: + The direction in which collaboration should be allowed: 'inbound', 'outbound', or 'both'. + :type direction: + `unicode` + :returns: + The created whitelist entry for the domain. + :rtype: + :class:`CollaborationWhitelistEntry` + """ + url = self.get_url('collaboration_whitelist_entries') + data = { + 'domain': domain, + 'direction': direction + } + response = self._session.post(url, data=json.dumps(data)).json() + return self.translator.translate(response['type'])( + session=self._session, + object_id=response['id'], + response_object=response, + ) + + @api_call + def get_exemptions(self, limit=None, marker=None, fields=None): + """ + Get the list of exempted users who are not subject to the collaboration whitelist rules. + + :param limit: + The maximum number of entries to return per page. If not specified, then will use the server-side default. + :type limit: + `int` or None + :param marker: + The paging marker to start paging from. + :type marker: + `unicode` or None + :param fields: + List of fields to request. + :type fields: + `Iterable` of `unicode` + :returns: + An iterator of the exemptions to the whitelist. + :rtype: + :class:`BoxObjectCollection` + """ + return MarkerBasedObjectCollection( + session=self._session, + url=self.get_url('collaboration_whitelist_exempt_targets'), + limit=limit, + marker=marker, + fields=fields, + return_full_pages=False, + ) + + @api_call + def add_exemption(self, user): + """ + Exempt a user from the collaboration whitelist. + + :param user: + The user to exempt from the whitelist. + :type user: + :class:`User` + :returns: + The created whitelist exemption. + :rtype: + :class:`CollaborationWhitelistExemptTarget` + """ + url = self.get_url('collaboration_whitelist_exempt_targets') + data = { + 'user': { + 'id': user.object_id # pylint:disable=protected-access + } + } + response = self._session.post(url, data=json.dumps(data)).json() + return self.translator.translate(response['type'])( + session=self._session, + object_id=response['id'], + response_object=response, + ) From 8b1799082ee94606693c80f6126016729e9b4a0c Mon Sep 17 00:00:00 2001 From: carycheng Date: Thu, 11 Oct 2018 16:03:55 -0700 Subject: [PATCH 10/54] removed collaboration whitelist from this pr --- test/unit/object/test_collaboration_whitelist_exempt_target.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/unit/object/test_collaboration_whitelist_exempt_target.py b/test/unit/object/test_collaboration_whitelist_exempt_target.py index 1d7c9f8ee..e0db97b16 100644 --- a/test/unit/object/test_collaboration_whitelist_exempt_target.py +++ b/test/unit/object/test_collaboration_whitelist_exempt_target.py @@ -1,3 +1,4 @@ + # coding: utf-8 from __future__ import unicode_literals, absolute_import From 714bdcff4f2620bf9882296a7f19a5ead6ede026 Mon Sep 17 00:00:00 2001 From: carycheng Date: Thu, 11 Oct 2018 16:40:03 -0700 Subject: [PATCH 11/54] fixed for linter --- test/unit/object/test_folder.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/unit/object/test_folder.py b/test/unit/object/test_folder.py index 6b6be236f..19b9a764e 100644 --- a/test/unit/object/test_folder.py +++ b/test/unit/object/test_folder.py @@ -219,6 +219,7 @@ def test_create_upload_session(test_folder, mock_box_session): assert upload_session.type == upload_session_type assert upload_session.id == upload_session_id + def test_upload_stream_does_preflight_check_if_specified( mock_box_session, test_folder, From 03acd661bfb1c362868fcae63e3acaf19eda0935 Mon Sep 17 00:00:00 2001 From: carycheng Date: Tue, 16 Oct 2018 16:02:47 -0700 Subject: [PATCH 12/54] added paging for chunked upload parts --- boxsdk/object/file.py | 13 +- boxsdk/object/folder.py | 9 +- boxsdk/object/upload_session.py | 57 +++++++-- boxsdk/pagination/box_object_collection.py | 4 +- ...rt_limit_offset_based_object_collection.py | 10 ++ boxsdk/pagination/chunked_upload_part_page.py | 12 ++ docs/{chunked_upload.md => file.md} | 69 +++++----- test/unit/object/test_upload_session.py | 119 +++++++++++++++--- 8 files changed, 212 insertions(+), 81 deletions(-) create mode 100644 boxsdk/pagination/chunked_upload_part_limit_offset_based_object_collection.py create mode 100644 boxsdk/pagination/chunked_upload_part_page.py rename docs/{chunked_upload.md => file.md} (57%) diff --git a/boxsdk/object/file.py b/boxsdk/object/file.py index 560206f40..f8e4aca95 100644 --- a/boxsdk/object/file.py +++ b/boxsdk/object/file.py @@ -42,11 +42,11 @@ def create_upload_session(self, file_size, file_name=None): Create a new chunked upload session for uploading a new version of the file. :param file_size: - The size of the file that will be uploaded. + The size of the file in bytes that will be uploaded. :type file_size: `int` :param file_name: - The name of the file version that will be uploaded. + The new name of the file version that will be uploaded. :type file_name: `unicode` or None :returns: @@ -60,13 +60,8 @@ def create_upload_session(self, file_size, file_name=None): } if file_name is not None: body_params['file_name'] = file_name - response = self._session.post( - self.get_url('{0}'.format('upload_sessions')).replace( - API.BASE_API_URL, - API.UPLOAD_URL, - ), - data=json.dumps(body_params), - ).json() + url = self.get_url('{0}'.format('upload_sessions')).replace(API.BASE_API_URL, API.UPLOAD_URL) + response = self._session.post(url, data=json.dumps(body_params)).json() return self.translator.translate(response['type'])( session=self.session, object_id=response['id'], diff --git a/boxsdk/object/folder.py b/boxsdk/object/folder.py index 87ce9898f..2b6ae6a98 100644 --- a/boxsdk/object/folder.py +++ b/boxsdk/object/folder.py @@ -118,11 +118,11 @@ def create_upload_session(self, file_size, file_name): Creates a new chunked upload session for upload a new file. :param file_size: - The size of the file that will be uploaded. + The size of the file in bytes that will be uploaded. :type file_size: `int` :param file_name: - The name of the file that will be uploaded. + The new name of the file that will be uploaded. :type file_name: `unicode` :returns: @@ -136,10 +136,7 @@ def create_upload_session(self, file_size, file_name): 'file_size': file_size, 'file_name': file_name, } - response = self._session.post( - url, - data=json.dumps(body_params), - ).json() + response = self._session.post(url, data=json.dumps(body_params)).json() return self.translator.translate(response['type'])( session=self.session, object_id=response['id'], diff --git a/boxsdk/object/upload_session.py b/boxsdk/object/upload_session.py index 3305da7bf..59825a4a7 100644 --- a/boxsdk/object/upload_session.py +++ b/boxsdk/object/upload_session.py @@ -7,6 +7,7 @@ from .base_object import BaseObject from ..config import API +from ..pagination.chunked_upload_part_limit_offset_based_object_collection import ChunkedUploadPartLimitOffsetBasedObjectCollection class UploadSession(BaseObject): @@ -26,22 +27,44 @@ def get_url(self, *args): *args ).replace(API.BASE_API_URL, API.UPLOAD_URL) - def get_parts(self): + def get_parts(self, limit=None, offset=None, fields=None): """ Get a list of parts uploaded so far. + :param limit: + The maximum number of items to return per page. If not specified, then will use the server-side default. + :type limit: + `int` or None + :param offset: + The index at which to start returning items. + :type offset: + `int` or None + :param fields: + Fields to include on the returned items. + :type fields: + `Iterable` of `unicode` :returns: Returns a `list` of parts uploaded so far. :rtype: `list` of `dict` """ - response = self.session.get(self.get_url('parts')).json() - return response['entries'] + return ChunkedUploadPartLimitOffsetBasedObjectCollection( + session=self.session, + url=self.get_url('parts'), + limit=limit, + fields=fields, + offset=offset, + return_full_pages=False, + ) def _calculate_part_sha1(self, content_stream): """ Calculate the SHA1 hash of the chunk stream for a given part of the file. + :param content_stream: + File-like object containing the content of the part to be uploaded. + :type content_stream: + :class:`File` :returns: The unencoded SHA1 hash of the part. :rtype: @@ -99,32 +122,46 @@ def upload_part(self, content_stream, offset, total_size, part_content_sha1=None 'Content-Range': 'bytes {0}-{1}/{2}'.format(offset, range_end, total_size), }, data=content_stream - ).json() + ) - def commit(self, parts, content_sha1): + def commit(self, content_sha1, parts=None, file_attributes=None): """ Commit a multiput upload. - :param parts: - List of parts that were uploaded. - :type parts: - `Iterable` of `dict` :param content_sha1: SHA-1 has of the file contents that was uploaded. :type content_sha1: `unicode` + :param parts: + List of parts that were uploaded. + :type parts: + `Iterable` of `dict` or None + :param file_attributes: + An array of attributes to set on the created file. + :type file_attributes: + `List` of `unicode` :returns: A :class:`File` object. :rtype: :class:`File` """ + body = {} + partsList = [] + if file_attributes is not None: + body['attributes'] = file_attributes + if parts is None: + parts = self.get_parts() + [partsList.append(part) for part in parts] + body['parts'] = partsList + else: + body['parts'] = parts response = self._session.post( self.get_url('commit'), headers={ 'Content-Type': 'application/json', 'Digest': 'SHA={0}'.format(base64.b64encode(content_sha1).decode('utf-8')), }, - data=json.dumps({'parts': parts}), + data=json.dumps(body), ).json() entry = response['entries'][0] return self.translator.translate(entry['type'])( diff --git a/boxsdk/pagination/box_object_collection.py b/boxsdk/pagination/box_object_collection.py index c04001911..4d1dcba40 100644 --- a/boxsdk/pagination/box_object_collection.py +++ b/boxsdk/pagination/box_object_collection.py @@ -27,6 +27,8 @@ class BoxObjectCollection(collections.Iterator, object): will be used to retrieve the next page of Box objects. This pointer can be used when requesting new BoxObjectCollection instances that start off from a particular page, instead of from the very beginning. """ + _page_constructor = Page + def __init__( self, session, @@ -101,7 +103,7 @@ def _items_generator(self): self._update_pointer_to_next_page(response_object) self._has_retrieved_all_items = not self._has_more_pages(response_object) - page = Page(self._session, response_object) + page = self._page_constructor(self._session, response_object) if self._return_full_pages: yield page diff --git a/boxsdk/pagination/chunked_upload_part_limit_offset_based_object_collection.py b/boxsdk/pagination/chunked_upload_part_limit_offset_based_object_collection.py new file mode 100644 index 000000000..d432ff996 --- /dev/null +++ b/boxsdk/pagination/chunked_upload_part_limit_offset_based_object_collection.py @@ -0,0 +1,10 @@ +# coding: utf-8 + +from __future__ import unicode_literals, absolute_import + +from .chunked_upload_part_page import ChunkedUploadPartPage +from .limit_offset_based_object_collection import LimitOffsetBasedObjectCollection + + +class ChunkedUploadPartLimitOffsetBasedObjectCollection(LimitOffsetBasedObjectCollection): + _page_constructor = ChunkedUploadPartPage diff --git a/boxsdk/pagination/chunked_upload_part_page.py b/boxsdk/pagination/chunked_upload_part_page.py new file mode 100644 index 000000000..0b682e767 --- /dev/null +++ b/boxsdk/pagination/chunked_upload_part_page.py @@ -0,0 +1,12 @@ +# coding: utf-8 + +from __future__ import unicode_literals, absolute_import + +from .page import Page + + +class ChunkedUploadPartPage(Page): + def __getitem__(self, key): + item_json = self._response_object[self._item_entries_key_name][key] + + return item_json diff --git a/docs/chunked_upload.md b/docs/file.md similarity index 57% rename from docs/chunked_upload.md rename to docs/file.md index d4b72b75b..68be5aa8c 100644 --- a/docs/chunked_upload.md +++ b/docs/file.md @@ -1,45 +1,42 @@ Chunked Upload -============== +-------------- -This API provides a way to reliably upload large files to Box by chunking them into a sequence of parts. When using this API instead of the single file upload API, a request failure means a client only needs to retry upload of a single part instead of the entire file. Parts can also be uploaded in parallel allowing for potential performance improvement. - -It is important to note that the Chunked Upload API is intended for large files with a minimum size of 20MB. +For large files or in cases where the network connection is less reliable, +you may want to upload the file in parts. This allows a single part to fail +without aborting the entire upload, and failed parts can then be retried. -- [Create Upload Session for File Version](#create-upload-session-for-file-version) -- [Get Upload Session](#get-upload-session) -- [Create Upload Session for File](#create-upload-session-for-file) -- [Upload Part](#upload-part) -- [Commit](#commit) -- [List Uploaded Parts](#list-uploaded-parts) -- [Abort](#abort) +- [Manual Process](#manual-process) + - [Create Upload Session for File Version](#create-upload-session-for-file-version) + - [Create Upload Session for File](#create-upload-session-for-file) + - [Upload Part](#upload-part) + - [Commit Upload Session](#commit-upload-session) + - [Abort Upload Session](#abort-upload-session) + - [List Upload Parts](#list-upload-parts) -Create Upload Session for File Version --------------------------------------- +### Manual Process -To create an upload session for uploading a large version, use `file.create_upload_session(file_size, file_name=None)` +For more complicated upload scenarios, such as those being coordinated across +multiple processes or when an unrecoverable error occurs with the automatic +uploader, the endpoints for chunked upload operations are also exposed directly. -```python -file_size = 197520 -upload_session = client.file('11111').create_upload_session(file_size=file_size) -``` +The individual endpoint methods are detailed below: -Get Upload Session ------------------- +#### Create Upload Session for File Version -To get an upload session, use `upload_session.get()`. +To create an upload session for uploading a large version, use `file.create_upload_session(file_size, file_name=None)` ```python -upload_session = client.upload_session('11493C07ED3EABB6E59874D3A1EF3581').get() +file_size = 197520 +upload_session = client.file('11111').create_upload_session(file_size=file_size) ``` -Create Upload Session for File ------------------------------- +#### Create Upload Session for File To create an upload session for uploading a large file, use `folder.create_upload_session(file_size, file_name)` @@ -50,8 +47,7 @@ file_name = 'test_file.pdf' upload_session = client.folder('22222').create_upload_session(file_size=file_size, file_name=file_name) ``` -Upload Part ------------ +#### Upload Part To upload a part of the file to this session, use `upload_session.upload_part(content_stream, offset, total_size, part_content_sha1=None)` @@ -62,10 +58,9 @@ offset = 32 upload_part = client.upload_session('11493C07ED3EABB6E59874D3A1EF3581').upload_part(chunk, offset, total_size) ``` -Commit ------- +#### Commit Upload Session -To commit the upload session to Box, use `upload_session.commit(parts, content_sha1)`. +To commit the upload session to Box, use `upload_session.commit(content_sha1, parts=None, file_attributes=None)`. ```python import hashlib @@ -73,20 +68,18 @@ parts = client.upload_session('11493C07ED3EABB6E59874D3A1EF3581').get_parts() uploaded_file = client.upload_session('11493C07ED3EABB6E59874D3A1EF3581').commit(parts, sha1.digest()) ``` -List Uploaded Parts -------------------- +#### Abort Upload Session -To return the list of parts uploaded so far, use `upload_session.get_parts()`. +To abort a chunked upload, use `upload_session.abort()`. ```python -parts = client.upload_session('11493C07ED3EABB6E59874D3A1EF3581').get_parts() +client.upload_session('11493C07ED3EABB6E59874D3A1EF3581').abort() ``` -Abort ------ +#### List Upload Parts -To abort a chunked upload, use `upload_session.abort()`. +To return the list of parts uploaded so far, use `upload_session.get_parts(limit=None, offset=None, fields=None)`. ```python -client.upload_session('11493C07ED3EABB6E59874D3A1EF3581').abort() -``` +parts = client.upload_session('11493C07ED3EABB6E59874D3A1EF3581').get_parts() +``` \ No newline at end of file diff --git a/test/unit/object/test_upload_session.py b/test/unit/object/test_upload_session.py index 889381398..fd3f04558 100644 --- a/test/unit/object/test_upload_session.py +++ b/test/unit/object/test_upload_session.py @@ -11,6 +11,7 @@ from boxsdk.config import API from boxsdk.object.file import File from boxsdk.object.upload_session import UploadSession +from boxsdk.object.base_object import BaseObject @pytest.fixture() @@ -24,22 +25,25 @@ def test_upload_session(mock_box_session): def test_get_parts(test_upload_session, mock_box_session): expected_url = '{0}/files/upload_sessions/{1}/parts'.format(API.UPLOAD_URL, test_upload_session.object_id) - - parts = mock_box_session.get.return_value.json.return_value = { - 'entries': [ - { - 'part': { - 'part_id': '8F0966B1', - 'offset': 0, - 'size': 8, - 'sha1': None, - }, - }, - ], + mock_entry = { + 'part_id': '8F0966B1', + 'offset': 0, + 'size': 8, + 'sha1': None, + } + mock_box_session.get.return_value.json.return_value = { + 'entries': [mock_entry], + 'offset': 0, + 'total_count': 1, + 'limit': 1000, } test_parts = test_upload_session.get_parts() - mock_box_session.get.assert_called_once_with(expected_url) - assert test_parts == parts['entries'] + part = test_parts.next() + mock_box_session.get.assert_called_once_with(expected_url, params={'offset': None}) + assert isinstance(part, dict) + assert part['part_id'] == mock_entry['part_id'] + assert part['size'] == mock_entry['size'] + assert part['offset'] == mock_entry['offset'] def test_abort(test_upload_session, mock_box_session): @@ -61,7 +65,7 @@ def test_upload_part(test_upload_session, mock_box_session): 'Digest': 'SHA={}'.format(expected_sha1), 'Content-Range': 'bytes 32-39/80', } - mock_box_session.put.return_value.json.return_value = { + mock_box_session.put.return_value = { 'part': { 'part_id': 'ABCDEF123', 'offset': offset, @@ -80,6 +84,8 @@ def test_commit(test_upload_session, mock_box_session): sha1 = hashlib.sha1() sha1.update(b'fake_file_data') file_id = '12345' + file_type = 'file' + file_attributes = ['content_modified_at'] parts = [ { 'part_id': 'ABCDEF123', @@ -95,6 +101,7 @@ def test_commit(test_upload_session, mock_box_session): }, ] expected_data = { + 'attributes': file_attributes, 'parts': parts, } expected_headers = { @@ -105,12 +112,90 @@ def test_commit(test_upload_session, mock_box_session): mock_box_session.post.return_value.json.return_value = { 'entries': [ { - 'type': 'file', + 'type': file_type, 'id': file_id, + 'content_modified_at': '2017-11-02T15:04:38-07:00', }, ], } - created_file = test_upload_session.commit(parts, sha1.digest()) + created_file = test_upload_session.commit(content_sha1=sha1.digest(), parts=parts, file_attributes=file_attributes) mock_box_session.post.assert_called_once_with(expected_url, data=json.dumps(expected_data), headers=expected_headers) assert isinstance(created_file, File) assert created_file.id == file_id + assert created_file.type == file_type + assert created_file.content_modified_at == '2017-11-02T15:04:38-07:00' + + +def test_commit_with_missing_params(test_upload_session, mock_box_session): + expected_get_url = '{0}/files/upload_sessions/{1}/parts'.format(API.UPLOAD_URL, test_upload_session.object_id) + expected_url = '{0}/files/upload_sessions/{1}/commit'.format(API.UPLOAD_URL, test_upload_session.object_id) + sha1 = hashlib.sha1() + sha1.update(b'fake_file_data') + file_id = '12345' + file_type = 'file' + parts = [ + { + 'part_id': '8F0966B1', + 'offset': 0, + 'size': 8, + 'sha1': None, + }, + ] + expected_data = { + 'parts': parts, + } + expected_headers = { + 'Content-Type': 'application/json', + 'Digest': 'SHA={}'.format(base64.b64encode(sha1.digest()).decode('utf-8')), + } + mock_entry = { + 'part_id': '8F0966B1', + 'offset': 0, + 'size': 8, + 'sha1': None, + } + mock_box_session.get.return_value.json.return_value = { + 'entries': [mock_entry], + 'offset': 0, + 'total_count': 1, + 'limit': 1000, + } + mock_box_session.post.return_value.json.return_value = { + 'entries': [ + { + 'type': file_type, + 'id': file_id, + }, + ], + } + created_file = test_upload_session.commit(content_sha1=sha1.digest()) + mock_box_session.get.assert_called_once_with(expected_get_url, params={'offset': None}) + mock_box_session.post.assert_called_once_with(expected_url, data=json.dumps(expected_data), headers=expected_headers) + assert isinstance(created_file, File) + assert created_file.id == file_id + assert created_file.type == file_type + + +def test_upload_part(test_upload_session, mock_box_session): + expected_url = '{0}/files/upload_sessions/{1}'.format(API.UPLOAD_URL, test_upload_session.object_id) + chunk = BytesIO(b'abcdefgh') + offset = 32 + total_size = 80 + expected_sha1 = 'QlrxKgdDUCsyLpOgFbz4aOMk1Wo=' + expected_headers = { + 'Content-Type': 'application/octet-stream', + 'Digest': 'SHA={}'.format(expected_sha1), + 'Content-Range': 'bytes 32-39/80', + } + mock_box_session.put.return_value = { + 'part': { + 'part_id': 'ABCDEF123', + 'offset': offset, + 'size': 8, + 'sha1': expected_sha1, + }, + } + part = test_upload_session.upload_part(chunk, offset, total_size) + + mock_box_session.put.assert_called_once_with(expected_url, data=chunk, headers=expected_headers) + assert part['part']['sha1'] == expected_sha1 From 6e9fe4ddce720957bcb5d1845d402271e887009a Mon Sep 17 00:00:00 2001 From: carycheng Date: Tue, 16 Oct 2018 16:21:23 -0700 Subject: [PATCH 13/54] converted to for loop --- boxsdk/object/upload_session.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/boxsdk/object/upload_session.py b/boxsdk/object/upload_session.py index 59825a4a7..ebaf083f9 100644 --- a/boxsdk/object/upload_session.py +++ b/boxsdk/object/upload_session.py @@ -146,13 +146,14 @@ def commit(self, content_sha1, parts=None, file_attributes=None): :class:`File` """ body = {} - partsList = [] + parts_list = [] if file_attributes is not None: body['attributes'] = file_attributes if parts is None: parts = self.get_parts() - [partsList.append(part) for part in parts] - body['parts'] = partsList + for part in parts: + parts_list.append(part) + body['parts'] = parts_list else: body['parts'] = parts response = self._session.post( From c21b0f325166dd46d4ef6dd41076518e24ed5837 Mon Sep 17 00:00:00 2001 From: carycheng Date: Tue, 16 Oct 2018 16:31:25 -0700 Subject: [PATCH 14/54] remove unused import --- test/unit/object/test_upload_session.py | 26 ------------------------- 1 file changed, 26 deletions(-) diff --git a/test/unit/object/test_upload_session.py b/test/unit/object/test_upload_session.py index fd3f04558..15c83bd6b 100644 --- a/test/unit/object/test_upload_session.py +++ b/test/unit/object/test_upload_session.py @@ -11,7 +11,6 @@ from boxsdk.config import API from boxsdk.object.file import File from boxsdk.object.upload_session import UploadSession -from boxsdk.object.base_object import BaseObject @pytest.fixture() @@ -174,28 +173,3 @@ def test_commit_with_missing_params(test_upload_session, mock_box_session): assert isinstance(created_file, File) assert created_file.id == file_id assert created_file.type == file_type - - -def test_upload_part(test_upload_session, mock_box_session): - expected_url = '{0}/files/upload_sessions/{1}'.format(API.UPLOAD_URL, test_upload_session.object_id) - chunk = BytesIO(b'abcdefgh') - offset = 32 - total_size = 80 - expected_sha1 = 'QlrxKgdDUCsyLpOgFbz4aOMk1Wo=' - expected_headers = { - 'Content-Type': 'application/octet-stream', - 'Digest': 'SHA={}'.format(expected_sha1), - 'Content-Range': 'bytes 32-39/80', - } - mock_box_session.put.return_value = { - 'part': { - 'part_id': 'ABCDEF123', - 'offset': offset, - 'size': 8, - 'sha1': expected_sha1, - }, - } - part = test_upload_session.upload_part(chunk, offset, total_size) - - mock_box_session.put.assert_called_once_with(expected_url, data=chunk, headers=expected_headers) - assert part['part']['sha1'] == expected_sha1 From 837dbc2a90155a2867b55081947fb647b003d29a Mon Sep 17 00:00:00 2001 From: carycheng Date: Thu, 18 Oct 2018 11:22:43 -0700 Subject: [PATCH 15/54] removed unused function --- boxsdk/object/upload_session.py | 61 ++++++++----------------- test/unit/object/test_upload_session.py | 3 +- 2 files changed, 22 insertions(+), 42 deletions(-) diff --git a/boxsdk/object/upload_session.py b/boxsdk/object/upload_session.py index ebaf083f9..65dfc8ead 100644 --- a/boxsdk/object/upload_session.py +++ b/boxsdk/object/upload_session.py @@ -57,41 +57,14 @@ def get_parts(self, limit=None, offset=None, fields=None): return_full_pages=False, ) - def _calculate_part_sha1(self, content_stream): - """ - Calculate the SHA1 hash of the chunk stream for a given part of the file. - - :param content_stream: - File-like object containing the content of the part to be uploaded. - :type content_stream: - :class:`File` - :returns: - The unencoded SHA1 hash of the part. - :rtype: - `bytes` - """ - content_sha1 = hashlib.sha1() - stream_position = content_stream.tell() - hashed_length = 0 - while hashed_length < self.part_size: # pylint:disable=no-member - chunk = content_stream.read(self.part_size - hashed_length) # pylint:disable=no-member - if chunk is None: - continue - if not chunk: - break - hashed_length += len(chunk) - content_sha1.update(chunk) - content_stream.seek(stream_position) - return content_sha1.digest() - - def upload_part(self, content_stream, offset, total_size, part_content_sha1=None): + def upload_part(self, part_bytes, offset, total_size, part_content_sha1=None): """ Upload a part of a file. - :param content_stream: - File-like object containing the content of the part to be uploaded. - :type content_stream: - :class:`File` + :param part_bytes: + Part bytes + :type part_bytes: + `bytes` :param offset: Offset, in number of bytes, of the part compared to the beginning of the file. :type offset: @@ -109,8 +82,11 @@ def upload_part(self, content_stream, offset, total_size, part_content_sha1=None :rtype: `dict` """ + if part_content_sha1 is None: - part_content_sha1 = self._calculate_part_sha1(content_stream) + sha1 = hashlib.sha1() + sha1.update(part_bytes) + part_content_sha1 = sha1.digest() range_end = min(offset + self.part_size - 1, total_size - 1) # pylint:disable=no-member @@ -121,10 +97,10 @@ def upload_part(self, content_stream, offset, total_size, part_content_sha1=None 'Digest': 'SHA={0}'.format(base64.b64encode(part_content_sha1).decode('utf-8')), 'Content-Range': 'bytes {0}-{1}/{2}'.format(offset, range_end, total_size), }, - data=content_stream + data=part_bytes ) - def commit(self, content_sha1, parts=None, file_attributes=None): + def commit(self, content_sha1, parts=None, file_attributes=None, etag=None): """ Commit a multiput upload. @@ -137,9 +113,9 @@ def commit(self, content_sha1, parts=None, file_attributes=None): :type parts: `Iterable` of `dict` or None :param file_attributes: - An array of attributes to set on the created file. + An `dict` of attributes to set on file upload. :type file_attributes: - `List` of `unicode` + `dict` :returns: A :class:`File` object. :rtype: @@ -156,12 +132,15 @@ def commit(self, content_sha1, parts=None, file_attributes=None): body['parts'] = parts_list else: body['parts'] = parts + headers = { + 'Content-Type': 'application/json', + 'Digest': 'SHA={0}'.format(base64.b64encode(content_sha1).decode('utf-8')), + } + if etag is not None: + headers['If-Match'] = etag response = self._session.post( self.get_url('commit'), - headers={ - 'Content-Type': 'application/json', - 'Digest': 'SHA={0}'.format(base64.b64encode(content_sha1).decode('utf-8')), - }, + headers=headers, data=json.dumps(body), ).json() entry = response['entries'][0] diff --git a/test/unit/object/test_upload_session.py b/test/unit/object/test_upload_session.py index 15c83bd6b..0e612372f 100644 --- a/test/unit/object/test_upload_session.py +++ b/test/unit/object/test_upload_session.py @@ -55,7 +55,8 @@ def test_abort(test_upload_session, mock_box_session): def test_upload_part(test_upload_session, mock_box_session): expected_url = '{0}/files/upload_sessions/{1}'.format(API.UPLOAD_URL, test_upload_session.object_id) - chunk = BytesIO(b'abcdefgh') + part_bytes = BytesIO(b'abcdefgh') + chunk = part_bytes.read(20) offset = 32 total_size = 80 expected_sha1 = 'QlrxKgdDUCsyLpOgFbz4aOMk1Wo=' From 40334b510ff97d9a08a030a8457c892cc5f25312 Mon Sep 17 00:00:00 2001 From: carycheng Date: Thu, 18 Oct 2018 12:38:48 -0700 Subject: [PATCH 16/54] added tests for optional params in commit --- boxsdk/object/upload_session.py | 4 ++++ test/unit/object/test_upload_session.py | 10 ++++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/boxsdk/object/upload_session.py b/boxsdk/object/upload_session.py index 65dfc8ead..a32394550 100644 --- a/boxsdk/object/upload_session.py +++ b/boxsdk/object/upload_session.py @@ -116,6 +116,10 @@ def commit(self, content_sha1, parts=None, file_attributes=None, etag=None): An `dict` of attributes to set on file upload. :type file_attributes: `dict` + :param etag: + etag lets you ensure that your app only alters files/folders on Box if you have the current version. + :type etag: + `unicode` :returns: A :class:`File` object. :rtype: diff --git a/test/unit/object/test_upload_session.py b/test/unit/object/test_upload_session.py index 0e612372f..5baf5f605 100644 --- a/test/unit/object/test_upload_session.py +++ b/test/unit/object/test_upload_session.py @@ -85,7 +85,8 @@ def test_commit(test_upload_session, mock_box_session): sha1.update(b'fake_file_data') file_id = '12345' file_type = 'file' - file_attributes = ['content_modified_at'] + file_etag = '7' + file_attributes = {'description': 'This is a test description.'} parts = [ { 'part_id': 'ABCDEF123', @@ -107,6 +108,7 @@ def test_commit(test_upload_session, mock_box_session): expected_headers = { 'Content-Type': 'application/json', 'Digest': 'SHA={}'.format(base64.b64encode(sha1.digest()).decode('utf-8')), + 'If-Match': '7', } mock_box_session.post.return_value.json.return_value = { @@ -114,16 +116,16 @@ def test_commit(test_upload_session, mock_box_session): { 'type': file_type, 'id': file_id, - 'content_modified_at': '2017-11-02T15:04:38-07:00', + 'description': 'This is a test description.', }, ], } - created_file = test_upload_session.commit(content_sha1=sha1.digest(), parts=parts, file_attributes=file_attributes) + created_file = test_upload_session.commit(content_sha1=sha1.digest(), parts=parts, file_attributes=file_attributes, etag=file_etag) mock_box_session.post.assert_called_once_with(expected_url, data=json.dumps(expected_data), headers=expected_headers) assert isinstance(created_file, File) assert created_file.id == file_id assert created_file.type == file_type - assert created_file.content_modified_at == '2017-11-02T15:04:38-07:00' + assert created_file.description == 'This is a test description.' def test_commit_with_missing_params(test_upload_session, mock_box_session): From a81804a2a126b765728c315758e3d6e631640306 Mon Sep 17 00:00:00 2001 From: carycheng Date: Thu, 18 Oct 2018 14:57:58 -0700 Subject: [PATCH 17/54] fixed docs --- docs/file.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/file.md b/docs/file.md index 68be5aa8c..4a2ecbb22 100644 --- a/docs/file.md +++ b/docs/file.md @@ -49,18 +49,21 @@ upload_session = client.folder('22222').create_upload_session(file_size=file_siz #### Upload Part -To upload a part of the file to this session, use `upload_session.upload_part(content_stream, offset, total_size, part_content_sha1=None)` +To upload a part of the file to this session, use `upload_session.upload_part(part_bytes, offset, total_size, part_content_sha1=None)` ```python from io import BytesIO -chunk = BytesIO(b'abcdefgh') offset = 32 -upload_part = client.upload_session('11493C07ED3EABB6E59874D3A1EF3581').upload_part(chunk, offset, total_size) +part_bytes = BytesIO(b'abcdefgh') +upload_session = client.upload_session('11493C07ED3EABB6E59874D3A1EF3581') +chunk = part_bytes.read(upload_session.part_size) +offset = 32 +upload_part = upload_session.upload_part(chunk, offset, total_size) ``` #### Commit Upload Session -To commit the upload session to Box, use `upload_session.commit(content_sha1, parts=None, file_attributes=None)`. +To commit the upload session to Box, use `upload_session.commit(content_sha1, parts=None, file_attributes=None, etag=None)`. ```python import hashlib From a118b5d11c667b8f54c50cc7be8fd46a287bca9e Mon Sep 17 00:00:00 2001 From: Matt Willer Date: Fri, 19 Oct 2018 10:55:42 -0700 Subject: [PATCH 18/54] Update boxsdk/object/file.py Co-Authored-By: carycheng --- boxsdk/object/file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/boxsdk/object/file.py b/boxsdk/object/file.py index f8e4aca95..eab35a18c 100644 --- a/boxsdk/object/file.py +++ b/boxsdk/object/file.py @@ -60,7 +60,7 @@ def create_upload_session(self, file_size, file_name=None): } if file_name is not None: body_params['file_name'] = file_name - url = self.get_url('{0}'.format('upload_sessions')).replace(API.BASE_API_URL, API.UPLOAD_URL) + url = self.get_url('upload_sessions').replace(API.BASE_API_URL, API.UPLOAD_URL) response = self._session.post(url, data=json.dumps(body_params)).json() return self.translator.translate(response['type'])( session=self.session, From 6e69607edd92e8fb89e7c79dcd5b5c3eb7907802 Mon Sep 17 00:00:00 2001 From: Matt Willer Date: Fri, 19 Oct 2018 10:55:49 -0700 Subject: [PATCH 19/54] Update boxsdk/object/file.py Co-Authored-By: carycheng --- boxsdk/object/file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/boxsdk/object/file.py b/boxsdk/object/file.py index eab35a18c..d563929d2 100644 --- a/boxsdk/object/file.py +++ b/boxsdk/object/file.py @@ -50,7 +50,7 @@ def create_upload_session(self, file_size, file_name=None): :type file_name: `unicode` or None :returns: - A :class:`ChunkedUploadSession` object. + A :class:`UploadSession` object. :rtype: :class:`ChunkedUploadSession` """ From 537cd7f4bc488cdba048b7b44f245f7b88fcdfe3 Mon Sep 17 00:00:00 2001 From: Matt Willer Date: Fri, 19 Oct 2018 10:55:53 -0700 Subject: [PATCH 20/54] Update boxsdk/object/file.py Co-Authored-By: carycheng --- boxsdk/object/file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/boxsdk/object/file.py b/boxsdk/object/file.py index d563929d2..f08de62e7 100644 --- a/boxsdk/object/file.py +++ b/boxsdk/object/file.py @@ -52,7 +52,7 @@ def create_upload_session(self, file_size, file_name=None): :returns: A :class:`UploadSession` object. :rtype: - :class:`ChunkedUploadSession` + :class:`UploadSession` """ body_params = { 'file_id': self.object_id, From 8430bf5979a9cc36086c488fc5f95d31ac00c3f8 Mon Sep 17 00:00:00 2001 From: Matt Willer Date: Fri, 19 Oct 2018 10:55:57 -0700 Subject: [PATCH 21/54] Update boxsdk/object/folder.py Co-Authored-By: carycheng --- boxsdk/object/folder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/boxsdk/object/folder.py b/boxsdk/object/folder.py index 2b6ae6a98..36c7027b8 100644 --- a/boxsdk/object/folder.py +++ b/boxsdk/object/folder.py @@ -126,7 +126,7 @@ def create_upload_session(self, file_size, file_name): :type file_name: `unicode` :returns: - A :class:`ChunkedUploadSession` object. + A :class:`UploadSession` object. :rtype: :class:`ChunkedUploadSession` """ From 85c2156850b6f704fd88a8e4ee411798504669a0 Mon Sep 17 00:00:00 2001 From: Matt Willer Date: Fri, 19 Oct 2018 10:56:01 -0700 Subject: [PATCH 22/54] Update boxsdk/object/folder.py Co-Authored-By: carycheng --- boxsdk/object/folder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/boxsdk/object/folder.py b/boxsdk/object/folder.py index 36c7027b8..ce3b26982 100644 --- a/boxsdk/object/folder.py +++ b/boxsdk/object/folder.py @@ -128,7 +128,7 @@ def create_upload_session(self, file_size, file_name): :returns: A :class:`UploadSession` object. :rtype: - :class:`ChunkedUploadSession` + :class:`UploadSession` """ url = '{0}/files/upload_sessions'.format(API.UPLOAD_URL) body_params = { From 6b9f788e3be200835d9b69d9357dd3ebb1ae67fb Mon Sep 17 00:00:00 2001 From: Matt Willer Date: Fri, 19 Oct 2018 11:05:34 -0700 Subject: [PATCH 23/54] Update boxsdk/object/upload_session.py Co-Authored-By: carycheng --- boxsdk/object/upload_session.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/boxsdk/object/upload_session.py b/boxsdk/object/upload_session.py index a32394550..10a810a62 100644 --- a/boxsdk/object/upload_session.py +++ b/boxsdk/object/upload_session.py @@ -78,7 +78,7 @@ def upload_part(self, part_bytes, offset, total_size, part_content_sha1=None): :type part_content_sha1: `unicode` :returns: - The uploaded part. + The uploaded part record. :rtype: `dict` """ From cbad90195e9cd05c502dc87df3702d7279c9d4a6 Mon Sep 17 00:00:00 2001 From: Matt Willer Date: Fri, 19 Oct 2018 11:05:49 -0700 Subject: [PATCH 24/54] Update boxsdk/object/upload_session.py Co-Authored-By: carycheng --- boxsdk/object/upload_session.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/boxsdk/object/upload_session.py b/boxsdk/object/upload_session.py index 10a810a62..ae7bed1e0 100644 --- a/boxsdk/object/upload_session.py +++ b/boxsdk/object/upload_session.py @@ -105,7 +105,7 @@ def commit(self, content_sha1, parts=None, file_attributes=None, etag=None): Commit a multiput upload. :param content_sha1: - SHA-1 has of the file contents that was uploaded. + SHA-1 hash of the file contents that was uploaded. :type content_sha1: `unicode` :param parts: From f04c04724161d209f6a46d3684c78100cb5de35f Mon Sep 17 00:00:00 2001 From: Matt Willer Date: Fri, 19 Oct 2018 11:05:54 -0700 Subject: [PATCH 25/54] Update boxsdk/object/upload_session.py Co-Authored-By: carycheng --- boxsdk/object/upload_session.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/boxsdk/object/upload_session.py b/boxsdk/object/upload_session.py index ae7bed1e0..3ae1edcc6 100644 --- a/boxsdk/object/upload_session.py +++ b/boxsdk/object/upload_session.py @@ -113,7 +113,7 @@ def commit(self, content_sha1, parts=None, file_attributes=None, etag=None): :type parts: `Iterable` of `dict` or None :param file_attributes: - An `dict` of attributes to set on file upload. + A `dict` of attributes to set on the uploaded file. :type file_attributes: `dict` :param etag: From 7810d804cdd0279e239cad4b130943d638fadc5a Mon Sep 17 00:00:00 2001 From: Matt Willer Date: Fri, 19 Oct 2018 11:06:31 -0700 Subject: [PATCH 26/54] Update boxsdk/object/upload_session.py Co-Authored-By: carycheng --- boxsdk/object/upload_session.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/boxsdk/object/upload_session.py b/boxsdk/object/upload_session.py index 3ae1edcc6..087fb97af 100644 --- a/boxsdk/object/upload_session.py +++ b/boxsdk/object/upload_session.py @@ -119,7 +119,7 @@ def commit(self, content_sha1, parts=None, file_attributes=None, etag=None): :param etag: etag lets you ensure that your app only alters files/folders on Box if you have the current version. :type etag: - `unicode` + `unicode` or None :returns: A :class:`File` object. :rtype: From 19c3203ab807b99edd60f14dc779d7556bd9d08b Mon Sep 17 00:00:00 2001 From: Matt Willer Date: Fri, 19 Oct 2018 11:07:47 -0700 Subject: [PATCH 27/54] Update boxsdk/object/upload_session.py Co-Authored-By: carycheng --- boxsdk/object/upload_session.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/boxsdk/object/upload_session.py b/boxsdk/object/upload_session.py index 087fb97af..fa5d3e6f3 100644 --- a/boxsdk/object/upload_session.py +++ b/boxsdk/object/upload_session.py @@ -149,7 +149,7 @@ def commit(self, content_sha1, parts=None, file_attributes=None, etag=None): ).json() entry = response['entries'][0] return self.translator.translate(entry['type'])( - session=self.session, + session=self._session, object_id=entry['id'], response_object=entry, ) From 68f8c8656e7d871ba98d6115ea8916de898fbb0f Mon Sep 17 00:00:00 2001 From: Matt Willer Date: Fri, 19 Oct 2018 11:07:52 -0700 Subject: [PATCH 28/54] Update boxsdk/object/file.py Co-Authored-By: carycheng --- boxsdk/object/file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/boxsdk/object/file.py b/boxsdk/object/file.py index f08de62e7..7045b5434 100644 --- a/boxsdk/object/file.py +++ b/boxsdk/object/file.py @@ -63,7 +63,7 @@ def create_upload_session(self, file_size, file_name=None): url = self.get_url('upload_sessions').replace(API.BASE_API_URL, API.UPLOAD_URL) response = self._session.post(url, data=json.dumps(body_params)).json() return self.translator.translate(response['type'])( - session=self.session, + session=self._session, object_id=response['id'], response_object=response, ) From f1b342081a4c94e0033f5f123c736cab6ef769ae Mon Sep 17 00:00:00 2001 From: Matt Willer Date: Fri, 19 Oct 2018 11:07:56 -0700 Subject: [PATCH 29/54] Update boxsdk/object/folder.py Co-Authored-By: carycheng --- boxsdk/object/folder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/boxsdk/object/folder.py b/boxsdk/object/folder.py index ce3b26982..a4e7a36f5 100644 --- a/boxsdk/object/folder.py +++ b/boxsdk/object/folder.py @@ -138,7 +138,7 @@ def create_upload_session(self, file_size, file_name): } response = self._session.post(url, data=json.dumps(body_params)).json() return self.translator.translate(response['type'])( - session=self.session, + session=self._session, object_id=response['id'], response_object=response, ) From c5ed08dcca6974a45514d9f420f79c046e46790f Mon Sep 17 00:00:00 2001 From: Matt Willer Date: Fri, 19 Oct 2018 11:08:00 -0700 Subject: [PATCH 30/54] Update boxsdk/object/upload_session.py Co-Authored-By: carycheng --- boxsdk/object/upload_session.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/boxsdk/object/upload_session.py b/boxsdk/object/upload_session.py index fa5d3e6f3..f9de6a427 100644 --- a/boxsdk/object/upload_session.py +++ b/boxsdk/object/upload_session.py @@ -21,7 +21,7 @@ def get_url(self, *args): :rtype: `unicode` """ - return self.session.get_url( + return self._session.get_url( '{0}s/{1}s'.format(self._parent_item_type, self._item_type), self._object_id, *args From 14bb1ea52d9b9768eb23d40ea24a469e5b00b039 Mon Sep 17 00:00:00 2001 From: Matt Willer Date: Fri, 19 Oct 2018 11:08:04 -0700 Subject: [PATCH 31/54] Update boxsdk/object/upload_session.py Co-Authored-By: carycheng --- boxsdk/object/upload_session.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/boxsdk/object/upload_session.py b/boxsdk/object/upload_session.py index f9de6a427..c06155161 100644 --- a/boxsdk/object/upload_session.py +++ b/boxsdk/object/upload_session.py @@ -49,7 +49,7 @@ def get_parts(self, limit=None, offset=None, fields=None): `list` of `dict` """ return ChunkedUploadPartLimitOffsetBasedObjectCollection( - session=self.session, + session=self._session, url=self.get_url('parts'), limit=limit, fields=fields, From 07b7daa712fe02118126b3b9c5a54a64078fcdcb Mon Sep 17 00:00:00 2001 From: Matt Willer Date: Fri, 19 Oct 2018 11:27:14 -0700 Subject: [PATCH 32/54] Update docs/file.md Co-Authored-By: carycheng --- docs/file.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/file.md b/docs/file.md index 4a2ecbb22..03b124bcf 100644 --- a/docs/file.md +++ b/docs/file.md @@ -33,7 +33,7 @@ To create an upload session for uploading a large version, use `file.create_uplo ```python file_size = 197520 -upload_session = client.file('11111').create_upload_session(file_size=file_size) +upload_session = client.file('11111').create_upload_session(file_size) ``` #### Create Upload Session for File @@ -85,4 +85,4 @@ To return the list of parts uploaded so far, use `upload_session.get_parts(limit ```python parts = client.upload_session('11493C07ED3EABB6E59874D3A1EF3581').get_parts() -``` \ No newline at end of file +``` From 347f959653bb38fc1e853399cac90d564db0cec8 Mon Sep 17 00:00:00 2001 From: Matt Willer Date: Fri, 19 Oct 2018 11:27:19 -0700 Subject: [PATCH 33/54] Update docs/file.md Co-Authored-By: carycheng --- docs/file.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/file.md b/docs/file.md index 03b124bcf..604b85857 100644 --- a/docs/file.md +++ b/docs/file.md @@ -44,7 +44,7 @@ To create an upload session for uploading a large file, use ```python file_size = 197520 file_name = 'test_file.pdf' -upload_session = client.folder('22222').create_upload_session(file_size=file_size, file_name=file_name) +upload_session = client.folder('22222').create_upload_session(file_size, file_name) ``` #### Upload Part From ab28462070bb23f7bd4223c099673de1bd22f81a Mon Sep 17 00:00:00 2001 From: Jeff Meadows Date: Fri, 19 Oct 2018 15:23:54 -0700 Subject: [PATCH 34/54] Update boxsdk/object/folder.py Co-Authored-By: carycheng --- boxsdk/object/folder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/boxsdk/object/folder.py b/boxsdk/object/folder.py index a4e7a36f5..ec449dde6 100644 --- a/boxsdk/object/folder.py +++ b/boxsdk/object/folder.py @@ -130,7 +130,7 @@ def create_upload_session(self, file_size, file_name): :rtype: :class:`UploadSession` """ - url = '{0}/files/upload_sessions'.format(API.UPLOAD_URL) + url = '{0}/files/upload_sessions'.format(self._session.api_config.UPLOAD_URL)) body_params = { 'folder_id': self.object_id, 'file_size': file_size, From d6af5648008afffb2b085b0914df43cbd736e2e6 Mon Sep 17 00:00:00 2001 From: Jeff Meadows Date: Fri, 19 Oct 2018 15:24:01 -0700 Subject: [PATCH 35/54] Update boxsdk/object/upload_session.py Co-Authored-By: carycheng --- boxsdk/object/upload_session.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/boxsdk/object/upload_session.py b/boxsdk/object/upload_session.py index c06155161..7a4427cb8 100644 --- a/boxsdk/object/upload_session.py +++ b/boxsdk/object/upload_session.py @@ -25,7 +25,7 @@ def get_url(self, *args): '{0}s/{1}s'.format(self._parent_item_type, self._item_type), self._object_id, *args - ).replace(API.BASE_API_URL, API.UPLOAD_URL) + ).replace(self.session.api_config.BASE_API_URL, self.session.api_config.UPLOAD_URL) def get_parts(self, limit=None, offset=None, fields=None): """ From 3e8b73d4b7dfd239a905dda2c28b703a901ce745 Mon Sep 17 00:00:00 2001 From: Jeff Meadows Date: Fri, 19 Oct 2018 15:24:05 -0700 Subject: [PATCH 36/54] Update boxsdk/object/upload_session.py Co-Authored-By: carycheng --- boxsdk/object/upload_session.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/boxsdk/object/upload_session.py b/boxsdk/object/upload_session.py index 7a4427cb8..d86c612e4 100644 --- a/boxsdk/object/upload_session.py +++ b/boxsdk/object/upload_session.py @@ -97,7 +97,7 @@ def upload_part(self, part_bytes, offset, total_size, part_content_sha1=None): 'Digest': 'SHA={0}'.format(base64.b64encode(part_content_sha1).decode('utf-8')), 'Content-Range': 'bytes {0}-{1}/{2}'.format(offset, range_end, total_size), }, - data=part_bytes + data=part_bytes, ) def commit(self, content_sha1, parts=None, file_attributes=None, etag=None): From b23e32c980fb414d7fe561159687ae4227c0012c Mon Sep 17 00:00:00 2001 From: Jeff Meadows Date: Fri, 19 Oct 2018 15:33:04 -0700 Subject: [PATCH 37/54] Update boxsdk/object/upload_session.py Co-Authored-By: carycheng --- boxsdk/object/upload_session.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/boxsdk/object/upload_session.py b/boxsdk/object/upload_session.py index d86c612e4..1d86b4e96 100644 --- a/boxsdk/object/upload_session.py +++ b/boxsdk/object/upload_session.py @@ -163,5 +163,5 @@ def abort(self): :rtype: `bool` """ - response = self._session.delete(self.get_url()) + return self.delete() return response.ok From c881f3b5ce7f62a253909a44ece70256a654d470 Mon Sep 17 00:00:00 2001 From: carycheng Date: Fri, 19 Oct 2018 16:41:28 -0700 Subject: [PATCH 38/54] updates to chunked uploader --- boxsdk/object/upload_session.py | 43 +++++++++---------- ...rt_limit_offset_based_object_collection.py | 10 ----- ...unked_upload_part_page.py => dict_page.py} | 2 +- .../limit_offset_based_dict_collection.py | 10 +++++ docs/file.md | 16 +++---- test/unit/object/test_upload_session.py | 3 +- 6 files changed, 40 insertions(+), 44 deletions(-) delete mode 100644 boxsdk/pagination/chunked_upload_part_limit_offset_based_object_collection.py rename boxsdk/pagination/{chunked_upload_part_page.py => dict_page.py} (86%) create mode 100644 boxsdk/pagination/limit_offset_based_dict_collection.py diff --git a/boxsdk/object/upload_session.py b/boxsdk/object/upload_session.py index a32394550..54b02012d 100644 --- a/boxsdk/object/upload_session.py +++ b/boxsdk/object/upload_session.py @@ -7,7 +7,7 @@ from .base_object import BaseObject from ..config import API -from ..pagination.chunked_upload_part_limit_offset_based_object_collection import ChunkedUploadPartLimitOffsetBasedObjectCollection +from ..pagination.limit_offset_based_dict_collection import LimitOffsetBasedDictCollection class UploadSession(BaseObject): @@ -27,7 +27,7 @@ def get_url(self, *args): *args ).replace(API.BASE_API_URL, API.UPLOAD_URL) - def get_parts(self, limit=None, offset=None, fields=None): + def get_parts(self, limit=None, offset=None): """ Get a list of parts uploaded so far. @@ -48,12 +48,12 @@ def get_parts(self, limit=None, offset=None, fields=None): :rtype: `list` of `dict` """ - return ChunkedUploadPartLimitOffsetBasedObjectCollection( + return LimitOffsetBasedDictCollection( session=self.session, url=self.get_url('parts'), limit=limit, - fields=fields, offset=offset, + fields=None, return_full_pages=False, ) @@ -66,7 +66,8 @@ def upload_part(self, part_bytes, offset, total_size, part_content_sha1=None): :type part_bytes: `bytes` :param offset: - Offset, in number of bytes, of the part compared to the beginning of the file. + Offset, in number of bytes, of the part compared to the beginning of the file. This number should be a + multiple of the part size. :type offset: `int` :param total_size: @@ -76,7 +77,7 @@ def upload_part(self, part_bytes, offset, total_size, part_content_sha1=None): :param part_content_sha1: SHA-1 hash of the part's content. If not specified, this will be calculated. :type part_content_sha1: - `unicode` + `unicode` or None :returns: The uploaded part. :rtype: @@ -89,16 +90,17 @@ def upload_part(self, part_bytes, offset, total_size, part_content_sha1=None): part_content_sha1 = sha1.digest() range_end = min(offset + self.part_size - 1, total_size - 1) # pylint:disable=no-member - - return self._session.put( + headers = { + 'Content-Type': 'application/octet-stream', + 'Digest': 'SHA={0}'.format(base64.b64encode(part_content_sha1).decode('utf-8')), + 'Content-Range': 'bytes {0}-{1}/{2}'.format(offset, range_end, total_size), + } + response = self._session.put( self.get_url(), - headers={ - 'Content-Type': 'application/octet-stream', - 'Digest': 'SHA={0}'.format(base64.b64encode(part_content_sha1).decode('utf-8')), - 'Content-Range': 'bytes {0}-{1}/{2}'.format(offset, range_end, total_size), - }, - data=part_bytes + headers=headers, + data=part_bytes, ) + return response['part'] def commit(self, content_sha1, parts=None, file_attributes=None, etag=None): """ @@ -117,11 +119,11 @@ def commit(self, content_sha1, parts=None, file_attributes=None, etag=None): :type file_attributes: `dict` :param etag: - etag lets you ensure that your app only alters files/folders on Box if you have the current version. + If specified, instruct the Box API to delete the folder only if the current version's etag matches. :type etag: `unicode` :returns: - A :class:`File` object. + The newly-uploaded file object. :rtype: :class:`File` """ @@ -129,13 +131,10 @@ def commit(self, content_sha1, parts=None, file_attributes=None, etag=None): parts_list = [] if file_attributes is not None: body['attributes'] = file_attributes - if parts is None: - parts = self.get_parts() - for part in parts: - parts_list.append(part) - body['parts'] = parts_list - else: + if parts is not None: body['parts'] = parts + else: + body['parts'] = [parts_list.append(part) for part in self.get_parts()] headers = { 'Content-Type': 'application/json', 'Digest': 'SHA={0}'.format(base64.b64encode(content_sha1).decode('utf-8')), diff --git a/boxsdk/pagination/chunked_upload_part_limit_offset_based_object_collection.py b/boxsdk/pagination/chunked_upload_part_limit_offset_based_object_collection.py deleted file mode 100644 index d432ff996..000000000 --- a/boxsdk/pagination/chunked_upload_part_limit_offset_based_object_collection.py +++ /dev/null @@ -1,10 +0,0 @@ -# coding: utf-8 - -from __future__ import unicode_literals, absolute_import - -from .chunked_upload_part_page import ChunkedUploadPartPage -from .limit_offset_based_object_collection import LimitOffsetBasedObjectCollection - - -class ChunkedUploadPartLimitOffsetBasedObjectCollection(LimitOffsetBasedObjectCollection): - _page_constructor = ChunkedUploadPartPage diff --git a/boxsdk/pagination/chunked_upload_part_page.py b/boxsdk/pagination/dict_page.py similarity index 86% rename from boxsdk/pagination/chunked_upload_part_page.py rename to boxsdk/pagination/dict_page.py index 0b682e767..d2603e4a5 100644 --- a/boxsdk/pagination/chunked_upload_part_page.py +++ b/boxsdk/pagination/dict_page.py @@ -5,7 +5,7 @@ from .page import Page -class ChunkedUploadPartPage(Page): +class DictPage(Page): def __getitem__(self, key): item_json = self._response_object[self._item_entries_key_name][key] diff --git a/boxsdk/pagination/limit_offset_based_dict_collection.py b/boxsdk/pagination/limit_offset_based_dict_collection.py new file mode 100644 index 000000000..487667e57 --- /dev/null +++ b/boxsdk/pagination/limit_offset_based_dict_collection.py @@ -0,0 +1,10 @@ +# coding: utf-8 + +from __future__ import unicode_literals, absolute_import + +from .dict_page import DictPage +from .limit_offset_based_object_collection import LimitOffsetBasedObjectCollection + + +class LimitOffsetBasedDictCollection(LimitOffsetBasedObjectCollection): + _page_constructor = DictPage diff --git a/docs/file.md b/docs/file.md index 4a2ecbb22..e0a202b9b 100644 --- a/docs/file.md +++ b/docs/file.md @@ -21,9 +21,7 @@ without aborting the entire upload, and failed parts can then be retried. ### Manual Process -For more complicated upload scenarios, such as those being coordinated across -multiple processes or when an unrecoverable error occurs with the automatic -uploader, the endpoints for chunked upload operations are also exposed directly. + The individual endpoint methods are detailed below: @@ -32,7 +30,7 @@ The individual endpoint methods are detailed below: To create an upload session for uploading a large version, use `file.create_upload_session(file_size, file_name=None)` ```python -file_size = 197520 +file_size = 26000000 upload_session = client.file('11111').create_upload_session(file_size=file_size) ``` @@ -42,7 +40,7 @@ To create an upload session for uploading a large file, use `folder.create_upload_session(file_size, file_name)` ```python -file_size = 197520 +file_size = 26000000 file_name = 'test_file.pdf' upload_session = client.folder('22222').create_upload_session(file_size=file_size, file_name=file_name) ``` @@ -52,12 +50,10 @@ upload_session = client.folder('22222').create_upload_session(file_size=file_siz To upload a part of the file to this session, use `upload_session.upload_part(part_bytes, offset, total_size, part_content_sha1=None)` ```python -from io import BytesIO -offset = 32 -part_bytes = BytesIO(b'abcdefgh') +offset = 25165824 +part_bytes = b'abcdefgh' upload_session = client.upload_session('11493C07ED3EABB6E59874D3A1EF3581') chunk = part_bytes.read(upload_session.part_size) -offset = 32 upload_part = upload_session.upload_part(chunk, offset, total_size) ``` @@ -81,7 +77,7 @@ client.upload_session('11493C07ED3EABB6E59874D3A1EF3581').abort() #### List Upload Parts -To return the list of parts uploaded so far, use `upload_session.get_parts(limit=None, offset=None, fields=None)`. +To return the list of parts uploaded so far, use `upload_session.get_parts(limit=None, offset=None)`. ```python parts = client.upload_session('11493C07ED3EABB6E59874D3A1EF3581').get_parts() diff --git a/test/unit/object/test_upload_session.py b/test/unit/object/test_upload_session.py index 5baf5f605..d0a3dc042 100644 --- a/test/unit/object/test_upload_session.py +++ b/test/unit/object/test_upload_session.py @@ -76,7 +76,8 @@ def test_upload_part(test_upload_session, mock_box_session): part = test_upload_session.upload_part(chunk, offset, total_size) mock_box_session.put.assert_called_once_with(expected_url, data=chunk, headers=expected_headers) - assert part['part']['sha1'] == expected_sha1 + assert isinstance(part, dict) + assert part['sha1'] == expected_sha1 def test_commit(test_upload_session, mock_box_session): From 601db2a5b74cb978fec7f03b5419b974aca46a7c Mon Sep 17 00:00:00 2001 From: carycheng Date: Thu, 25 Oct 2018 23:15:44 -0700 Subject: [PATCH 39/54] almost working example of chunked uploader --- boxsdk/object/upload_session.py | 2 +- chunked_uploader.py | 42 +++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 chunked_uploader.py diff --git a/boxsdk/object/upload_session.py b/boxsdk/object/upload_session.py index 54b02012d..f29f4b1a2 100644 --- a/boxsdk/object/upload_session.py +++ b/boxsdk/object/upload_session.py @@ -100,7 +100,7 @@ def upload_part(self, part_bytes, offset, total_size, part_content_sha1=None): headers=headers, data=part_bytes, ) - return response['part'] + return response.json()['part'] def commit(self, content_sha1, parts=None, file_attributes=None, etag=None): """ diff --git a/chunked_uploader.py b/chunked_uploader.py new file mode 100644 index 000000000..c4e27a775 --- /dev/null +++ b/chunked_uploader.py @@ -0,0 +1,42 @@ +import hashlib +import os + +from boxsdk import OAuth2 +from boxsdk import DevelopmentClient + +oauth = OAuth2(client_id='g4cz3htv6mp6sdlv9ft84izffbxicsj5', client_secret='4Qa7WxGwsp7m4aLmEdm9doOpKFs0jXS1', access_token='hvRp0vjDuVZatk7OwScaBU1ioOUBCQZ1') +client = DevelopmentClient(oauth) +copied_length = 0 +test_file_path = '/Users/ccheng/Desktop/CLICCOSXLinux.mp4' +total_size = os.stat(test_file_path).st_size +sha1 = hashlib.sha1() +content_stream = open(test_file_path, 'rb') +file_to_upload = client.file('330568970271').get() +upload_session = file_to_upload.create_upload_session(total_size) +chunk = None +while_loop_counter = 0 + +for part_num in range(upload_session.total_parts): + + copied_length = 0 + while copied_length < upload_session.part_size: + print('INNER WHILEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA') + + chunk = content_stream.read(upload_session.part_size - copied_length) + if chunk is None: + continue + if len(chunk) == 0: + break + copied_length += len(chunk) + if copied_length < upload_session.part_size and copied_length > 0: + break + print('Copied Length: ' + str(copied_length)) + print('CHUNK SIZE: ' + str(len(chunk))) + while_loop_counter += 1 + + uploaded_part = upload_session.upload_part(chunk, part_num*upload_session.part_size, total_size) + print('PART: ' + str(uploaded_part)) + print('LOOP_COUNTER: ' + str(while_loop_counter)) + updated_sha1 = sha1.update(chunk) +content_sha1 = sha1.digest() +uploaded_file = upload_session.commit(content_sha1) From e51ad8def23d22111b6dbf3bdd5a024673d52c33 Mon Sep 17 00:00:00 2001 From: carycheng Date: Fri, 26 Oct 2018 13:19:58 -0700 Subject: [PATCH 40/54] working chunked uploader example in docs --- boxsdk/object/folder.py | 2 +- boxsdk/object/upload_session.py | 4 ++-- chunked_uploader.py | 42 --------------------------------- docs/file.md | 38 +++++++++++++++++++++++++++++ 4 files changed, 41 insertions(+), 45 deletions(-) delete mode 100644 chunked_uploader.py diff --git a/boxsdk/object/folder.py b/boxsdk/object/folder.py index ec449dde6..c95e4996d 100644 --- a/boxsdk/object/folder.py +++ b/boxsdk/object/folder.py @@ -130,7 +130,7 @@ def create_upload_session(self, file_size, file_name): :rtype: :class:`UploadSession` """ - url = '{0}/files/upload_sessions'.format(self._session.api_config.UPLOAD_URL)) + url = '{0}/files/upload_sessions'.format(self._session.api_config.UPLOAD_URL) body_params = { 'folder_id': self.object_id, 'file_size': file_size, diff --git a/boxsdk/object/upload_session.py b/boxsdk/object/upload_session.py index 5b9b775aa..30f9bc820 100644 --- a/boxsdk/object/upload_session.py +++ b/boxsdk/object/upload_session.py @@ -57,7 +57,7 @@ def get_parts(self, limit=None, offset=None): return_full_pages=False, ) - def upload_part(self, part_bytes, offset, total_size, part_content_sha1=None): + def upload_part_bytes(self, part_bytes, offset, total_size, part_content_sha1=None): """ Upload a part of a file. @@ -109,7 +109,7 @@ def commit(self, content_sha1, parts=None, file_attributes=None, etag=None): :param content_sha1: SHA-1 hash of the file contents that was uploaded. :type content_sha1: - `unicode` + `bytes` :param parts: List of parts that were uploaded. :type parts: diff --git a/chunked_uploader.py b/chunked_uploader.py deleted file mode 100644 index c4e27a775..000000000 --- a/chunked_uploader.py +++ /dev/null @@ -1,42 +0,0 @@ -import hashlib -import os - -from boxsdk import OAuth2 -from boxsdk import DevelopmentClient - -oauth = OAuth2(client_id='g4cz3htv6mp6sdlv9ft84izffbxicsj5', client_secret='4Qa7WxGwsp7m4aLmEdm9doOpKFs0jXS1', access_token='hvRp0vjDuVZatk7OwScaBU1ioOUBCQZ1') -client = DevelopmentClient(oauth) -copied_length = 0 -test_file_path = '/Users/ccheng/Desktop/CLICCOSXLinux.mp4' -total_size = os.stat(test_file_path).st_size -sha1 = hashlib.sha1() -content_stream = open(test_file_path, 'rb') -file_to_upload = client.file('330568970271').get() -upload_session = file_to_upload.create_upload_session(total_size) -chunk = None -while_loop_counter = 0 - -for part_num in range(upload_session.total_parts): - - copied_length = 0 - while copied_length < upload_session.part_size: - print('INNER WHILEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA') - - chunk = content_stream.read(upload_session.part_size - copied_length) - if chunk is None: - continue - if len(chunk) == 0: - break - copied_length += len(chunk) - if copied_length < upload_session.part_size and copied_length > 0: - break - print('Copied Length: ' + str(copied_length)) - print('CHUNK SIZE: ' + str(len(chunk))) - while_loop_counter += 1 - - uploaded_part = upload_session.upload_part(chunk, part_num*upload_session.part_size, total_size) - print('PART: ' + str(uploaded_part)) - print('LOOP_COUNTER: ' + str(while_loop_counter)) - updated_sha1 = sha1.update(chunk) -content_sha1 = sha1.digest() -uploaded_file = upload_session.commit(content_sha1) diff --git a/docs/file.md b/docs/file.md index 158eeb7d5..5ca34ad46 100644 --- a/docs/file.md +++ b/docs/file.md @@ -21,7 +21,45 @@ without aborting the entire upload, and failed parts can then be retried. ### Manual Process +For more complicated upload scenarios, such as those being coordinated across multiple processes or when an unrecoverable error occurs with the automatic uploader, the endpoints for chunked upload operations are also exposed directly. +For example, this is roughly how a chunked upload is done manually: + +```python +import hashlib +import os + + +test_file_path = '/path/to/large_file.mp4' +total_size = os.stat(test_file_path).st_size +sha1 = hashlib.sha1() +content_stream = open(test_file_path, 'rb') +upload_session = client.folder(folder_id='11111').create_upload_session(file_size=total_size, file_name='test_file_name.mp4') +part_array = [] + +for part_num in range(upload_session.total_parts): + + copied_length = 0 + chunk = b'' + while copied_length < upload_session.part_size: + bytes_read = content_stream.read(upload_session.part_size - copied_length) + if bytes_read is None: + # stream returns none when no bytes are ready currently but there are + # potentially more bytes in the stream to be read. + continue + if len(bytes_read) == 0: + # stream is exhausted. + break + chunk += bytes_read + copied_length += len(bytes_read) + + uploaded_part = upload_session.upload_part_bytes(chunk, part_num*upload_session.part_size, total_size) + part_array.append(uploaded_part) + updated_sha1 = sha1.update(chunk) +content_sha1 = sha1.digest() +uploaded_file = upload_session.commit(content_sha1=content_sha1, parts=part_array) +print('File ID: {0} and File Name: {1}', uploaded_file.id, uploaded_file.name) +``` The individual endpoint methods are detailed below: From 80125b1dbb66deb74d5bfb748bb718d6220659cb Mon Sep 17 00:00:00 2001 From: carycheng Date: Fri, 26 Oct 2018 13:22:12 -0700 Subject: [PATCH 41/54] added assertion for sessions --- test/unit/object/test_file.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/unit/object/test_file.py b/test/unit/object/test_file.py index 1b48b091e..1a15b8c22 100644 --- a/test/unit/object/test_file.py +++ b/test/unit/object/test_file.py @@ -67,6 +67,7 @@ def test_create_upload_session(test_file, mock_box_session): upload_session = test_file.create_upload_session(file_size, file_name) mock_box_session.post.assert_called_once_with(expected_url, data=json.dumps(expected_data)) assert isinstance(upload_session, UploadSession) + assert upload_session._session == mock_box_session assert upload_session.part_size == part_size assert upload_session.total_parts == total_parts assert upload_session.num_parts_processed == num_parts_processed From e11611e71542276afcb88dccb1c4cd865e2e7bbb Mon Sep 17 00:00:00 2001 From: carycheng Date: Fri, 26 Oct 2018 14:13:01 -0700 Subject: [PATCH 42/54] fixed translator --- boxsdk/object/file.py | 3 +-- boxsdk/object/folder.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/boxsdk/object/file.py b/boxsdk/object/file.py index 08cd16971..ec1e85c78 100644 --- a/boxsdk/object/file.py +++ b/boxsdk/object/file.py @@ -63,9 +63,8 @@ def create_upload_session(self, file_size, file_name=None): body_params['file_name'] = file_name url = self.get_url('upload_sessions').replace(API.BASE_API_URL, API.UPLOAD_URL) response = self._session.post(url, data=json.dumps(body_params)).json() - return self.translator.translate(response['type'])( + return self.translator.translate( session=self._session, - object_id=response['id'], response_object=response, ) diff --git a/boxsdk/object/folder.py b/boxsdk/object/folder.py index c0b9116e6..03f6d0e67 100644 --- a/boxsdk/object/folder.py +++ b/boxsdk/object/folder.py @@ -138,9 +138,8 @@ def create_upload_session(self, file_size, file_name): 'file_name': file_name, } response = self._session.post(url, data=json.dumps(body_params)).json() - return self.translator.translate(response['type'])( + return self.translator.translate( session=self._session, - object_id=response['id'], response_object=response, ) From 8041657c8c5c8ffbf73c6402fd46e85dfa473152 Mon Sep 17 00:00:00 2001 From: carycheng Date: Fri, 26 Oct 2018 14:47:47 -0700 Subject: [PATCH 43/54] fixed tests assertion errors for upload_part_bytes and translator swap --- boxsdk/object/upload_session.py | 3 +-- docs/file.md | 2 +- test/unit/object/test_upload_session.py | 10 ++++++---- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/boxsdk/object/upload_session.py b/boxsdk/object/upload_session.py index 30f9bc820..1280e5063 100644 --- a/boxsdk/object/upload_session.py +++ b/boxsdk/object/upload_session.py @@ -147,9 +147,8 @@ def commit(self, content_sha1, parts=None, file_attributes=None, etag=None): data=json.dumps(body), ).json() entry = response['entries'][0] - return self.translator.translate(entry['type'])( + return self.translator.translate( session=self._session, - object_id=entry['id'], response_object=entry, ) diff --git a/docs/file.md b/docs/file.md index 5ca34ad46..fb28d01f8 100644 --- a/docs/file.md +++ b/docs/file.md @@ -58,7 +58,7 @@ for part_num in range(upload_session.total_parts): updated_sha1 = sha1.update(chunk) content_sha1 = sha1.digest() uploaded_file = upload_session.commit(content_sha1=content_sha1, parts=part_array) -print('File ID: {0} and File Name: {1}', uploaded_file.id, uploaded_file.name) +print('File ID: {0} and File Name: {1}'.format(uploaded_file.id, uploaded_file.name)) ``` The individual endpoint methods are detailed below: diff --git a/test/unit/object/test_upload_session.py b/test/unit/object/test_upload_session.py index d0a3dc042..30bc500cc 100644 --- a/test/unit/object/test_upload_session.py +++ b/test/unit/object/test_upload_session.py @@ -49,11 +49,11 @@ def test_abort(test_upload_session, mock_box_session): expected_url = '{0}/files/upload_sessions/{1}'.format(API.UPLOAD_URL, test_upload_session.object_id) mock_box_session.delete.return_value.ok = True result = test_upload_session.abort() - mock_box_session.delete.assert_called_once_with(expected_url) + mock_box_session.delete.assert_called_once_with(expected_url, expect_json_response=False, headers=None, params={}) assert result is True -def test_upload_part(test_upload_session, mock_box_session): +def test_upload_part_bytes(test_upload_session, mock_box_session): expected_url = '{0}/files/upload_sessions/{1}'.format(API.UPLOAD_URL, test_upload_session.object_id) part_bytes = BytesIO(b'abcdefgh') chunk = part_bytes.read(20) @@ -65,7 +65,7 @@ def test_upload_part(test_upload_session, mock_box_session): 'Digest': 'SHA={}'.format(expected_sha1), 'Content-Range': 'bytes 32-39/80', } - mock_box_session.put.return_value = { + mock_box_session.put.return_value.json.return_value = { 'part': { 'part_id': 'ABCDEF123', 'offset': offset, @@ -73,11 +73,13 @@ def test_upload_part(test_upload_session, mock_box_session): 'sha1': expected_sha1, }, } - part = test_upload_session.upload_part(chunk, offset, total_size) + part = test_upload_session.upload_part_bytes(chunk, offset, total_size) mock_box_session.put.assert_called_once_with(expected_url, data=chunk, headers=expected_headers) assert isinstance(part, dict) assert part['sha1'] == expected_sha1 + assert part['size'] == 8 + assert part['part_id'] == 'ABCDED123' def test_commit(test_upload_session, mock_box_session): From b5039211dc4436a17c9a383c8e446773c5a9c2e5 Mon Sep 17 00:00:00 2001 From: carycheng Date: Fri, 26 Oct 2018 16:04:42 -0700 Subject: [PATCH 44/54] changed from list comprehension to for loop --- boxsdk/object/upload_session.py | 4 +- test/unit/object/test_upload_session.py | 50 ++++++++++++++++++++++++- 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/boxsdk/object/upload_session.py b/boxsdk/object/upload_session.py index 1280e5063..f4e76ed4f 100644 --- a/boxsdk/object/upload_session.py +++ b/boxsdk/object/upload_session.py @@ -134,7 +134,9 @@ def commit(self, content_sha1, parts=None, file_attributes=None, etag=None): if parts is not None: body['parts'] = parts else: - body['parts'] = [parts_list.append(part) for part in self.get_parts()] + for part in self.get_parts(): + parts_list.append(part) + body['parts'] = parts_list headers = { 'Content-Type': 'application/json', 'Digest': 'SHA={0}'.format(base64.b64encode(content_sha1).decode('utf-8')), diff --git a/test/unit/object/test_upload_session.py b/test/unit/object/test_upload_session.py index 30bc500cc..dcb92152b 100644 --- a/test/unit/object/test_upload_session.py +++ b/test/unit/object/test_upload_session.py @@ -79,7 +79,7 @@ def test_upload_part_bytes(test_upload_session, mock_box_session): assert isinstance(part, dict) assert part['sha1'] == expected_sha1 assert part['size'] == 8 - assert part['part_id'] == 'ABCDED123' + assert part['part_id'] == 'ABCDEF123' def test_commit(test_upload_session, mock_box_session): @@ -131,6 +131,54 @@ def test_commit(test_upload_session, mock_box_session): assert created_file.description == 'This is a test description.' +def test_commit(test_upload_session, mock_box_session): + expected_url = '{0}/files/upload_sessions/{1}/commit'.format(API.UPLOAD_URL, test_upload_session.object_id) + sha1 = hashlib.sha1() + sha1.update(b'fake_file_data') + file_id = '12345' + file_type = 'file' + file_etag = '7' + file_attributes = {'description': 'This is a test description.'} + parts = [ + { + 'part_id': 'ABCDEF123', + 'offset': 0, + 'size': 8, + 'sha1': 'fake_sha1', + }, + { + 'part_id': 'ABCDEF456', + 'offset': 8, + 'size': 8, + 'sha1': 'fake_sha1', + }, + ] + expected_data = { + 'attributes': file_attributes, + 'parts': parts, + } + expected_headers = { + 'Content-Type': 'application/json', + 'Digest': 'SHA={}'.format(base64.b64encode(sha1.digest()).decode('utf-8')), + 'If-Match': '7', + } + mock_box_session.post.return_value.json.return_value = { + 'entries': [ + { + 'type': file_type, + 'id': file_id, + 'description': 'This is a test description.', + }, + ], + } + created_file = test_upload_session.commit(content_sha1=sha1.digest(), parts=parts, file_attributes=file_attributes, etag=file_etag) + mock_box_session.post.assert_called_once_with(expected_url, data=json.dumps(expected_data), headers=expected_headers) + assert isinstance(created_file, File) + assert created_file.id == file_id + assert created_file.type == file_type + assert created_file.description == 'This is a test description.' + + def test_commit_with_missing_params(test_upload_session, mock_box_session): expected_get_url = '{0}/files/upload_sessions/{1}/parts'.format(API.UPLOAD_URL, test_upload_session.object_id) expected_url = '{0}/files/upload_sessions/{1}/commit'.format(API.UPLOAD_URL, test_upload_session.object_id) From 4140d0e31f5bf7a9d9e7a1ea4856eeaf1a4dc5db Mon Sep 17 00:00:00 2001 From: carycheng Date: Fri, 26 Oct 2018 16:25:32 -0700 Subject: [PATCH 45/54] removed unused imports --- boxsdk/object/upload_session.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/boxsdk/object/upload_session.py b/boxsdk/object/upload_session.py index f4e76ed4f..4982c74fa 100644 --- a/boxsdk/object/upload_session.py +++ b/boxsdk/object/upload_session.py @@ -6,7 +6,6 @@ import json from .base_object import BaseObject -from ..config import API from ..pagination.limit_offset_based_dict_collection import LimitOffsetBasedDictCollection @@ -164,4 +163,3 @@ def abort(self): `bool` """ return self.delete() - return response.ok From bca4b0383be756cae684e1a8fde54d5a0ac57f39 Mon Sep 17 00:00:00 2001 From: carycheng Date: Fri, 26 Oct 2018 16:31:13 -0700 Subject: [PATCH 46/54] removed unused import --- boxsdk/object/folder.py | 1 - 1 file changed, 1 deletion(-) diff --git a/boxsdk/object/folder.py b/boxsdk/object/folder.py index 03f6d0e67..b04bb6125 100644 --- a/boxsdk/object/folder.py +++ b/boxsdk/object/folder.py @@ -5,7 +5,6 @@ import os from six import text_type -from boxsdk.config import API from boxsdk.object.group import Group from boxsdk.object.item import Item from boxsdk.object.user import User From 42f3c2a22022b0789b9f59c75931125f25e5cdad Mon Sep 17 00:00:00 2001 From: carycheng Date: Fri, 26 Oct 2018 16:37:00 -0700 Subject: [PATCH 47/54] removed double test --- test/unit/object/test_upload_session.py | 49 ------------------------- 1 file changed, 49 deletions(-) diff --git a/test/unit/object/test_upload_session.py b/test/unit/object/test_upload_session.py index dcb92152b..a9a5c6b68 100644 --- a/test/unit/object/test_upload_session.py +++ b/test/unit/object/test_upload_session.py @@ -82,55 +82,6 @@ def test_upload_part_bytes(test_upload_session, mock_box_session): assert part['part_id'] == 'ABCDEF123' -def test_commit(test_upload_session, mock_box_session): - expected_url = '{0}/files/upload_sessions/{1}/commit'.format(API.UPLOAD_URL, test_upload_session.object_id) - sha1 = hashlib.sha1() - sha1.update(b'fake_file_data') - file_id = '12345' - file_type = 'file' - file_etag = '7' - file_attributes = {'description': 'This is a test description.'} - parts = [ - { - 'part_id': 'ABCDEF123', - 'offset': 0, - 'size': 8, - 'sha1': 'fake_sha1', - }, - { - 'part_id': 'ABCDEF456', - 'offset': 8, - 'size': 8, - 'sha1': 'fake_sha1', - }, - ] - expected_data = { - 'attributes': file_attributes, - 'parts': parts, - } - expected_headers = { - 'Content-Type': 'application/json', - 'Digest': 'SHA={}'.format(base64.b64encode(sha1.digest()).decode('utf-8')), - 'If-Match': '7', - } - - mock_box_session.post.return_value.json.return_value = { - 'entries': [ - { - 'type': file_type, - 'id': file_id, - 'description': 'This is a test description.', - }, - ], - } - created_file = test_upload_session.commit(content_sha1=sha1.digest(), parts=parts, file_attributes=file_attributes, etag=file_etag) - mock_box_session.post.assert_called_once_with(expected_url, data=json.dumps(expected_data), headers=expected_headers) - assert isinstance(created_file, File) - assert created_file.id == file_id - assert created_file.type == file_type - assert created_file.description == 'This is a test description.' - - def test_commit(test_upload_session, mock_box_session): expected_url = '{0}/files/upload_sessions/{1}/commit'.format(API.UPLOAD_URL, test_upload_session.object_id) sha1 = hashlib.sha1() From 1edfd8b31e3415aa6641a26b1827072b017cde3e Mon Sep 17 00:00:00 2001 From: Matt Willer Date: Sun, 28 Oct 2018 12:48:52 -0700 Subject: [PATCH 48/54] Update boxsdk/object/folder.py Co-Authored-By: carycheng --- boxsdk/object/folder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/boxsdk/object/folder.py b/boxsdk/object/folder.py index b04bb6125..396ccc7ff 100644 --- a/boxsdk/object/folder.py +++ b/boxsdk/object/folder.py @@ -122,7 +122,7 @@ def create_upload_session(self, file_size, file_name): :type file_size: `int` :param file_name: - The new name of the file that will be uploaded. + The name of the file that will be uploaded. :type file_name: `unicode` :returns: From 444d4448d15cabbb8aca1f717bc9edf935ff677c Mon Sep 17 00:00:00 2001 From: carycheng Date: Sun, 28 Oct 2018 13:04:32 -0700 Subject: [PATCH 49/54] added docstring changes --- boxsdk/object/upload_session.py | 10 ++++------ docs/web_link.md | 14 ++++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/boxsdk/object/upload_session.py b/boxsdk/object/upload_session.py index 4982c74fa..ab8fe0de0 100644 --- a/boxsdk/object/upload_session.py +++ b/boxsdk/object/upload_session.py @@ -43,9 +43,9 @@ def get_parts(self, limit=None, offset=None): :type fields: `Iterable` of `unicode` :returns: - Returns a `list` of parts uploaded so far. + Returns a :class:`BoxObjectCollection` object containing the uploaded parts. :rtype: - `list` of `dict` + :class:`BoxObjectCollection` """ return LimitOffsetBasedDictCollection( session=self.session, @@ -76,7 +76,7 @@ def upload_part_bytes(self, part_bytes, offset, total_size, part_content_sha1=No :param part_content_sha1: SHA-1 hash of the part's content. If not specified, this will be calculated. :type part_content_sha1: - `unicode` or None + `bytes` or None :returns: The uploaded part record. :rtype: @@ -133,9 +133,7 @@ def commit(self, content_sha1, parts=None, file_attributes=None, etag=None): if parts is not None: body['parts'] = parts else: - for part in self.get_parts(): - parts_list.append(part) - body['parts'] = parts_list + body['parts'] = [part for part in self.get_parts()] headers = { 'Content-Type': 'application/json', 'Digest': 'SHA={0}'.format(base64.b64encode(content_sha1).decode('utf-8')), diff --git a/docs/web_link.md b/docs/web_link.md index eac7ac9ed..f629ebb64 100644 --- a/docs/web_link.md +++ b/docs/web_link.md @@ -19,19 +19,21 @@ similarly to file objects. Create Web Link --------------- -Calling `folder.create_web_link(target_url, name=None, description=None)` will let you create a new web link with a specified name and description. +To create a web link object, first call `[client.folder(folder_id)]`[folder] to construct the appropriate ['Folder'][folder_class] object, and then calling [`folder.create_web_link(target_url, name=None, description=None)`][create] will let you create a new web link with a specified name and description. This method return an updated [`WebLink`][web_link_class] object populated with data from the API, leaving the original unmodified. ```python -folder_id = '1234' -target_url = 'https://example.com' -link_name = 'Example Link' -link_description = 'This is the description' -web_link = client.folder(folder_id).create_web_link(target_url, link_name, link_description) +web_link = client.folder('12345').create_web_link('https://example.com', 'Example Link', 'This is the description') ``` +[folder]: https://box-python-sdk.readthedocs.io/en/latest/boxsdk.client.html#boxsdk.client.client.Client.Folder +[folder_class]: https://box-python-sdk.readthedocs.io/en/latest/boxsdk.object.html#boxsdk.object.folder.Folder +[create]: https://box-python-sdk.readthedocs.io/en/latest/boxsdk.object.html#boxsdk.object.folder.Folder.create_web_link +[web_link_class]: https://box-python-sdk.readthedocs.io/en/latest/boxsdk.object.html#boxsdk.object.web_link.WebLink + Get Web Link ------------ +To get a web link object, first call `[client.web_link(web_link_id)]`[web_link] to construct the appropriate [`WebLink`][web_link_class] object, and then calling [] Calling `web_link.get()` can be used to retrieve information regarding a specific weblink. ```python From 8fc4394b66e9069b7fb24f768aa1cc1fab84e457 Mon Sep 17 00:00:00 2001 From: carycheng Date: Sun, 28 Oct 2018 13:08:51 -0700 Subject: [PATCH 50/54] reverted web link docs --- docs/web_link.md | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/docs/web_link.md b/docs/web_link.md index f629ebb64..fd02b2778 100644 --- a/docs/web_link.md +++ b/docs/web_link.md @@ -19,21 +19,19 @@ similarly to file objects. Create Web Link --------------- -To create a web link object, first call `[client.folder(folder_id)]`[folder] to construct the appropriate ['Folder'][folder_class] object, and then calling [`folder.create_web_link(target_url, name=None, description=None)`][create] will let you create a new web link with a specified name and description. This method return an updated [`WebLink`][web_link_class] object populated with data from the API, leaving the original unmodified. +Calling `folder.create_web_link(target_url, name=None, description=None)` will let you create a new web link with a specified name and description. ```python -web_link = client.folder('12345').create_web_link('https://example.com', 'Example Link', 'This is the description') +folder_id = '1234' +target_url = 'https://example.com' +link_name = 'Example Link' +link_description = 'This is the description' +web_link = client.folder(folder_id).create_web_link(target_url, link_name, link_description) ``` -[folder]: https://box-python-sdk.readthedocs.io/en/latest/boxsdk.client.html#boxsdk.client.client.Client.Folder -[folder_class]: https://box-python-sdk.readthedocs.io/en/latest/boxsdk.object.html#boxsdk.object.folder.Folder -[create]: https://box-python-sdk.readthedocs.io/en/latest/boxsdk.object.html#boxsdk.object.folder.Folder.create_web_link -[web_link_class]: https://box-python-sdk.readthedocs.io/en/latest/boxsdk.object.html#boxsdk.object.web_link.WebLink - Get Web Link ------------ -To get a web link object, first call `[client.web_link(web_link_id)]`[web_link] to construct the appropriate [`WebLink`][web_link_class] object, and then calling [] Calling `web_link.get()` can be used to retrieve information regarding a specific weblink. ```python @@ -59,4 +57,4 @@ Calling `web_link.delete()` can be used to delete a specified weblink. ```python web_link_id = '1234' client.web_link(web_link_id).delete() -``` +``` \ No newline at end of file From 0596f6562aa6de7911260776b535bfdfa308c6a7 Mon Sep 17 00:00:00 2001 From: carycheng Date: Sun, 28 Oct 2018 13:10:17 -0700 Subject: [PATCH 51/54] reverted web link docs from chunked uploader pr --- docs/web_link.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/web_link.md b/docs/web_link.md index fd02b2778..eac7ac9ed 100644 --- a/docs/web_link.md +++ b/docs/web_link.md @@ -57,4 +57,4 @@ Calling `web_link.delete()` can be used to delete a specified weblink. ```python web_link_id = '1234' client.web_link(web_link_id).delete() -``` \ No newline at end of file +``` From 9be9f00f1e86804d6df718561a88d53a3a24f854 Mon Sep 17 00:00:00 2001 From: carycheng Date: Sun, 28 Oct 2018 16:00:50 -0700 Subject: [PATCH 52/54] removed unused variable --- boxsdk/object/upload_session.py | 1 - 1 file changed, 1 deletion(-) diff --git a/boxsdk/object/upload_session.py b/boxsdk/object/upload_session.py index ab8fe0de0..3dc19daba 100644 --- a/boxsdk/object/upload_session.py +++ b/boxsdk/object/upload_session.py @@ -127,7 +127,6 @@ def commit(self, content_sha1, parts=None, file_attributes=None, etag=None): :class:`File` """ body = {} - parts_list = [] if file_attributes is not None: body['attributes'] = file_attributes if parts is not None: From 9af64d7faeb5326d0e9bde09ffa1dc96ab268706 Mon Sep 17 00:00:00 2001 From: carycheng Date: Mon, 29 Oct 2018 11:28:00 -0700 Subject: [PATCH 53/54] swapped from global config to local config --- boxsdk/object/file.py | 2 +- boxsdk/object/folder.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/boxsdk/object/file.py b/boxsdk/object/file.py index ec1e85c78..2b24d02d7 100644 --- a/boxsdk/object/file.py +++ b/boxsdk/object/file.py @@ -61,7 +61,7 @@ def create_upload_session(self, file_size, file_name=None): } if file_name is not None: body_params['file_name'] = file_name - url = self.get_url('upload_sessions').replace(API.BASE_API_URL, API.UPLOAD_URL) + url = self.get_url('upload_sessions').replace(self.session.api_config.BASE_API_URL, self.session.api_config.UPLOAD_URL) response = self._session.post(url, data=json.dumps(body_params)).json() return self.translator.translate( session=self._session, diff --git a/boxsdk/object/folder.py b/boxsdk/object/folder.py index 396ccc7ff..3a4b687ad 100644 --- a/boxsdk/object/folder.py +++ b/boxsdk/object/folder.py @@ -130,7 +130,7 @@ def create_upload_session(self, file_size, file_name): :rtype: :class:`UploadSession` """ - url = '{0}/files/upload_sessions'.format(self._session.api_config.UPLOAD_URL) + url = '{0}/files/upload_sessions'.format(self.session.api_config.UPLOAD_URL) body_params = { 'folder_id': self.object_id, 'file_size': file_size, From 26db804b3ff4f3205c9d335857765cc124768728 Mon Sep 17 00:00:00 2001 From: carycheng Date: Mon, 29 Oct 2018 12:26:59 -0700 Subject: [PATCH 54/54] removed unused import --- boxsdk/object/file.py | 1 - 1 file changed, 1 deletion(-) diff --git a/boxsdk/object/file.py b/boxsdk/object/file.py index 2b24d02d7..876131015 100644 --- a/boxsdk/object/file.py +++ b/boxsdk/object/file.py @@ -4,7 +4,6 @@ import json -from boxsdk.config import API from .item import Item from ..util.api_call_decorator import api_call from ..pagination.marker_based_object_collection import MarkerBasedObjectCollection