diff --git a/CHANGELOG b/CHANGELOG index 85beb6591..b049273e7 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -2,6 +2,10 @@ ChangeLog ********* +0.36.1 (2017-12-11) +=================== +- Fix: Update OneDrive metadata to report the correct materialized name. + 0.36.0 (2017-12-05) =================== - Feature: WaterButler now supports two new read-only providers, GitLab and Microsoft OneDrive! diff --git a/tests/core/streams/test_zip.py b/tests/core/streams/test_zip.py index ea6194f15..a1a77e71f 100644 --- a/tests/core/streams/test_zip.py +++ b/tests/core/streams/test_zip.py @@ -1,15 +1,14 @@ -import pytest - import io import os -import tempfile import zipfile -from tests.utils import temp_files +import pytest from waterbutler.core import streams from waterbutler.core.utils import AsyncIterator +from tests.utils import temp_files + class TestZipStreamReader: @@ -127,3 +126,41 @@ async def test_multiple_large_files(self, temp_files): for file in files: assert zip.open(file['filename']).read() == file['contents'] + + @pytest.mark.asyncio + async def test_zip_files(self, temp_files): + + files = [] + for filename in ['file1.ext', 'zip.zip', 'file2.ext']: + path = temp_files.add_file(filename) + contents = os.urandom(2 ** 5) + with open(path, 'wb') as f: + f.write(contents) + files.append({ + 'filename': filename, + 'path': path, + 'contents': contents, + 'handle': open(path, 'rb') + }) + + stream = streams.ZipStreamReader( + AsyncIterator( + (file['filename'], streams.FileStreamReader(file['handle'])) + for file in files + ) + ) + data = await stream.read() + for file in files: + file['handle'].close() + zip = zipfile.ZipFile(io.BytesIO(data)) + + # Verify CRCs: `.testzip()` returns `None` if there are no bad files in the zipfile + assert zip.testzip() is None + + for file in files: + assert zip.open(file['filename']).read() == file['contents'] + compression_type = zip.open(file['filename'])._compress_type + if file['filename'].endswith('.zip'): + assert compression_type == zipfile.ZIP_STORED + else: + assert compression_type != zipfile.ZIP_STORED diff --git a/tests/providers/onedrive/test_provider.py b/tests/providers/onedrive/test_provider.py index d7607ba41..9ad99b869 100644 --- a/tests/providers/onedrive/test_provider.py +++ b/tests/providers/onedrive/test_provider.py @@ -96,7 +96,6 @@ async def test_validate_v1_path_file(self, root_provider, root_provider_fixtures file_metadata = root_provider_fixtures['file_metadata'] item_url = root_provider._build_item_url(file_id) - print('item url: {}'.format(item_url)) aiohttpretty.register_json_uri('GET', item_url, body=file_metadata, status=200) file_path = '/{}'.format(file_id) @@ -124,7 +123,6 @@ async def test_validate_v1_path_folder(self, root_provider, root_provider_fixtur folder_metadata = root_provider_fixtures['folder_metadata'] item_url = root_provider._build_item_url(folder_id) - print('item url: {}'.format(item_url)) aiohttpretty.register_json_uri('GET', item_url, body=folder_metadata, status=200) folder_path = '/{}/'.format(folder_id) @@ -168,7 +166,6 @@ async def test_validate_v1_path_folder(self, subfolder_provider, subfolder_provi folder_metadata = subfolder_provider_fixtures['folder_metadata'] item_url = subfolder_provider._build_item_url(folder_id) - print('item url: {}'.format(item_url)) aiohttpretty.register_json_uri('GET', item_url, body=folder_metadata, status=200) folder_path = '/{}/'.format(folder_id) @@ -198,7 +195,6 @@ async def test_validate_v1_path_file_is_child(self, subfolder_provider, file_metadata = subfolder_provider_fixtures['file_metadata'] item_url = subfolder_provider._build_item_url(file_id) - print('item url: {}'.format(item_url)) aiohttpretty.register_json_uri('GET', item_url, body=file_metadata, status=200) file_path = '/{}'.format(file_id) @@ -228,11 +224,9 @@ async def test_validate_v1_path_file_is_grandchild(self, subfolder_provider, subfile_metadata = subfolder_provider_fixtures['subfile_metadata'] item_url = subfolder_provider._build_item_url(subfile_id) - print('item url: {}'.format(item_url)) aiohttpretty.register_json_uri('GET', item_url, body=subfile_metadata, status=200) root_url = subfolder_provider._build_item_url(subfolder_provider_fixtures['root_id']) - print('root url: {}'.format(root_url)) aiohttpretty.register_json_uri('GET', root_url, body=subfolder_provider_fixtures['root_metadata'], status=200) @@ -359,10 +353,12 @@ async def test_metadata_root(self, subfolder_provider, subfolder_provider_fixtur folder_metadata = result[0] assert folder_metadata.kind == 'folder' assert folder_metadata.name == 'crushers' + assert folder_metadata.materialized_path == '/crushers/' file_metadata = result[1] assert file_metadata.kind == 'file' assert file_metadata.name == 'bicuspid.txt' + assert file_metadata.materialized_path == '/bicuspid.txt' @pytest.mark.aiohttpretty @pytest.mark.asyncio @@ -382,6 +378,7 @@ async def test_metadata_folder(self, subfolder_provider, subfolder_provider_fixt file_metadata = result[0] assert file_metadata.kind == 'file' assert file_metadata.name == 'molars.txt' + assert file_metadata.materialized_path == '/crushers/molars.txt' @pytest.mark.aiohttpretty @pytest.mark.asyncio @@ -398,6 +395,7 @@ async def test_metadata_file(self, subfolder_provider, subfolder_provider_fixtur result = await subfolder_provider.metadata(path) assert result.kind == 'file' assert result.name == 'bicuspid.txt' + assert result.materialized_path == '/bicuspid.txt' class TestRevisions: diff --git a/waterbutler/__init__.py b/waterbutler/__init__.py index 0bd3bb0bd..bfc7521f1 100644 --- a/waterbutler/__init__.py +++ b/waterbutler/__init__.py @@ -1,2 +1,2 @@ -__version__ = '0.36.0' +__version__ = '0.36.1' __import__('pkg_resources').declare_namespace(__name__) diff --git a/waterbutler/core/streams/zip.py b/waterbutler/core/streams/zip.py index 8db013b6d..4ea1c1c9f 100644 --- a/waterbutler/core/streams/zip.py +++ b/waterbutler/core/streams/zip.py @@ -1,13 +1,11 @@ -import asyncio -import binascii -import struct +import zlib import time +import struct +import asyncio import zipfile -import zlib +import binascii -from waterbutler.core.streams.base import BaseStream -from waterbutler.core.streams.base import MultiStream -from waterbutler.core.streams.base import StringStream +from waterbutler.core.streams.base import BaseStream, MultiStream, StringStream # for some reason python3.5 has this as (1 << 31) - 1, which is 0x7fffffff ZIP64_LIMIT = 0xffffffff - 1 @@ -118,27 +116,35 @@ async def _read(self, n=-1, *args, **kwargs): class ZipLocalFile(MultiStream): - """A local file entry in a zip archive. Constructs the local file header, file data stream, - and data descriptor. + """A local file entry in a zip archive. Constructs the local file header, + file data stream, and data descriptor. - Note: This class is tightly coupled to ZipStreamReader and should not be used separately. + Note: This class is tightly coupled to ZipStreamReader and should not be + used separately. """ def __init__(self, file_tuple): + filename, stream = file_tuple # Build a ZipInfo instance to use for the file's header and footer self.zinfo = zipfile.ZipInfo( filename=filename, date_time=time.localtime(time.time())[:6], ) - # If the file is a directory, set the directory flag, turn off compression - if self.zinfo.filename[-1] == '/': - self.zinfo.external_attr = 0o40775 << 16 # drwxrwxr-x - self.zinfo.external_attr |= 0x10 # Directory flag + + # If the file is a `.zip`, set permission and turn off compression + if self.zinfo.filename.endswith('.zip'): + self.zinfo.external_attr = 0o600 << 16 # -rw------- self.zinfo.compress_type = zipfile.ZIP_STORED self.compressor = None + # If the file is a directory, set the directory flag and turn off compression + elif self.zinfo.filename[-1] == '/': + self.zinfo.external_attr = 0o40775 << 16 # drwxrwxr-x + self.zinfo.external_attr |= 0x10 # Directory flag + self.zinfo.compress_type = zipfile.ZIP_STORED + self.compressor = None + # For other types, set permission and define a compressor else: - self.zinfo.external_attr = 0o600 << 16 # rw------- - # define a compressor + self.zinfo.external_attr = 0o600 << 16 # -rw------- self.zinfo.compress_type = zipfile.ZIP_DEFLATED self.compressor = zlib.compressobj( zlib.Z_DEFAULT_COMPRESSION, @@ -148,6 +154,7 @@ def __init__(self, file_tuple): self.zinfo.header_offset = 0 self.zinfo.flag_bits |= 0x08 + # Initial CRC: value will be updated as file is streamed self.zinfo.CRC = 0 diff --git a/waterbutler/providers/onedrive/provider.py b/waterbutler/providers/onedrive/provider.py index 43ec5a750..6cdfd2234 100644 --- a/waterbutler/providers/onedrive/provider.py +++ b/waterbutler/providers/onedrive/provider.py @@ -268,7 +268,7 @@ async def metadata(self, path: OneDrivePath, **kwargs): # type: ignore code=HTTPStatus.NOT_FOUND, ) - return self._construct_metadata(data) + return self._construct_metadata(data, path) async def revisions(self, # type: ignore path: OneDrivePath, @@ -393,21 +393,24 @@ def _build_drive_url(self, *segments, **query) -> str: def _build_item_url(self, *segments, **query) -> str: return provider.build_url(settings.BASE_DRIVE_URL, 'items', *segments, **query) - def _construct_metadata(self, data: dict): - """Take a file/folder metadata response from OneDrive and return a `OneDriveFileMetadata` - object if the repsonse represents a file or a list of `OneDriveFileMetadata` and - `OneDriveFolderMetadata` objects if the response represents a folder. """ + def _construct_metadata(self, data: dict, path): + """Take a file/folder metadata response from OneDrive and a path object representing the + queried path and return a `OneDriveFileMetadata` object if the repsonse represents a file + or a list of `OneDriveFileMetadata` and `OneDriveFolderMetadata` objects if the response + represents a folder. """ if 'folder' in data.keys(): ret = [] if 'children' in data.keys(): for item in data['children']: - if 'folder' in item.keys(): - ret.append(OneDriveFolderMetadata(item, self.folder)) # type: ignore + is_folder = 'folder' in item.keys() + child_path = path.child(item['name'], _id=item['id'], folder=is_folder) + if is_folder: + ret.append(OneDriveFolderMetadata(item, child_path)) # type: ignore else: - ret.append(OneDriveFileMetadata(item, self.folder)) # type: ignore + ret.append(OneDriveFileMetadata(item, child_path)) # type: ignore return ret - return OneDriveFileMetadata(data, self.folder) + return OneDriveFileMetadata(data, path) async def _revisions_json(self, path: OneDrivePath, **kwargs) -> dict: """Fetch a list of revisions for the file at ``path``. diff --git a/waterbutler/server/api/v1/provider/movecopy.py b/waterbutler/server/api/v1/provider/movecopy.py index b68e3f1c7..4d868e276 100644 --- a/waterbutler/server/api/v1/provider/movecopy.py +++ b/waterbutler/server/api/v1/provider/movecopy.py @@ -107,11 +107,16 @@ async def move_or_copy(self): if path is None: raise exceptions.InvalidParameters('"path" field is required for moves or copies') if not path.endswith('/'): - raise exceptions.InvalidParameters('"path" field requires a trailing slash to ' - 'indicate it is a folder') + raise exceptions.InvalidParameters( + '"path" field requires a trailing slash to indicate it is a folder' + ) # TODO optimize for same provider and resource + # for copy action, `auth_action` is the same as `provider_action` + if auth_action == 'copy' and self.path.is_root and not self.json.get('rename'): + raise exceptions.InvalidParameters('"rename" field is required for copying root') + # Note: attached to self so that _send_hook has access to these self.dest_resource = self.json.get('resource', self.resource) self.dest_auth = await auth_handler.get( @@ -154,8 +159,8 @@ async def move_or_copy(self): self.dest_meta = metadata if created: - self.set_status(HTTPStatus.CREATED) + self.set_status(int(HTTPStatus.CREATED)) else: - self.set_status(HTTPStatus.OK) + self.set_status(int(HTTPStatus.OK)) self.write({'data': metadata.json_api_serialized(self.dest_resource)})