Merge branch 'develop' into feature/metadata-logging

CHANGELOG

@@ -2,6 +2,10 @@
 ChangeLog
 *********
 
+0.36.1 (2017-12-11)
+===================
+- Fix: Update OneDrive metadata to report the correct materialized name.
+
 0.36.0 (2017-12-05)
 ===================
 - Feature: WaterButler now supports two new read-only providers, GitLab and Microsoft OneDrive!

tests/core/streams/test_zip.py

@@ -1,15 +1,14 @@
-import pytest
-
 import io
 import os
-import tempfile
 import zipfile
 
-from tests.utils import temp_files
+import pytest
 
 from waterbutler.core import streams
 from waterbutler.core.utils import AsyncIterator
 
+from tests.utils import temp_files
+
 
 class TestZipStreamReader:
 
@@ -127,3 +126,41 @@ async def test_multiple_large_files(self, temp_files):
 
         for file in files:
             assert zip.open(file['filename']).read() == file['contents']
+
+    @pytest.mark.asyncio
+    async def test_zip_files(self, temp_files):
+
+        files = []
+        for filename in ['file1.ext', 'zip.zip', 'file2.ext']:
+            path = temp_files.add_file(filename)
+            contents = os.urandom(2 ** 5)
+            with open(path, 'wb') as f:
+                f.write(contents)
+            files.append({
+                'filename': filename,
+                'path': path,
+                'contents': contents,
+                'handle': open(path, 'rb')
+            })
+
+        stream = streams.ZipStreamReader(
+            AsyncIterator(
+                (file['filename'], streams.FileStreamReader(file['handle']))
+                for file in files
+            )
+        )
+        data = await stream.read()
+        for file in files:
+            file['handle'].close()
+        zip = zipfile.ZipFile(io.BytesIO(data))
+
+        # Verify CRCs: `.testzip()` returns `None` if there are no bad files in the zipfile
+        assert zip.testzip() is None
+
+        for file in files:
+            assert zip.open(file['filename']).read() == file['contents']
+            compression_type = zip.open(file['filename'])._compress_type
+            if file['filename'].endswith('.zip'):
+                assert compression_type == zipfile.ZIP_STORED
+            else:
+                assert compression_type != zipfile.ZIP_STORED

tests/providers/onedrive/test_provider.py

@@ -96,7 +96,6 @@ async def test_validate_v1_path_file(self, root_provider, root_provider_fixtures
         file_metadata = root_provider_fixtures['file_metadata']
 
         item_url = root_provider._build_item_url(file_id)
-        print('item url: {}'.format(item_url))
         aiohttpretty.register_json_uri('GET', item_url, body=file_metadata, status=200)
 
         file_path = '/{}'.format(file_id)
@@ -124,7 +123,6 @@ async def test_validate_v1_path_folder(self, root_provider, root_provider_fixtur
         folder_metadata = root_provider_fixtures['folder_metadata']
 
         item_url = root_provider._build_item_url(folder_id)
-        print('item url: {}'.format(item_url))
         aiohttpretty.register_json_uri('GET', item_url, body=folder_metadata, status=200)
 
         folder_path = '/{}/'.format(folder_id)
@@ -168,7 +166,6 @@ async def test_validate_v1_path_folder(self, subfolder_provider, subfolder_provi
         folder_metadata = subfolder_provider_fixtures['folder_metadata']
 
         item_url = subfolder_provider._build_item_url(folder_id)
-        print('item url: {}'.format(item_url))
         aiohttpretty.register_json_uri('GET', item_url, body=folder_metadata, status=200)
 
         folder_path = '/{}/'.format(folder_id)
@@ -198,7 +195,6 @@ async def test_validate_v1_path_file_is_child(self, subfolder_provider,
         file_metadata = subfolder_provider_fixtures['file_metadata']
 
         item_url = subfolder_provider._build_item_url(file_id)
-        print('item url: {}'.format(item_url))
         aiohttpretty.register_json_uri('GET', item_url, body=file_metadata, status=200)
 
         file_path = '/{}'.format(file_id)
@@ -228,11 +224,9 @@ async def test_validate_v1_path_file_is_grandchild(self, subfolder_provider,
         subfile_metadata = subfolder_provider_fixtures['subfile_metadata']
 
         item_url = subfolder_provider._build_item_url(subfile_id)
-        print('item url: {}'.format(item_url))
         aiohttpretty.register_json_uri('GET', item_url, body=subfile_metadata, status=200)
 
         root_url = subfolder_provider._build_item_url(subfolder_provider_fixtures['root_id'])
-        print('root url: {}'.format(root_url))
         aiohttpretty.register_json_uri('GET', root_url,
                                        body=subfolder_provider_fixtures['root_metadata'],
                                        status=200)
@@ -359,10 +353,12 @@ async def test_metadata_root(self, subfolder_provider, subfolder_provider_fixtur
         folder_metadata = result[0]
         assert folder_metadata.kind == 'folder'
         assert folder_metadata.name == 'crushers'
+        assert folder_metadata.materialized_path == '/crushers/'
 
         file_metadata = result[1]
         assert file_metadata.kind == 'file'
         assert file_metadata.name == 'bicuspid.txt'
+        assert file_metadata.materialized_path == '/bicuspid.txt'
 
     @pytest.mark.aiohttpretty
     @pytest.mark.asyncio
@@ -382,6 +378,7 @@ async def test_metadata_folder(self, subfolder_provider, subfolder_provider_fixt
         file_metadata = result[0]
         assert file_metadata.kind == 'file'
         assert file_metadata.name == 'molars.txt'
+        assert file_metadata.materialized_path == '/crushers/molars.txt'
 
     @pytest.mark.aiohttpretty
     @pytest.mark.asyncio
@@ -398,6 +395,7 @@ async def test_metadata_file(self, subfolder_provider, subfolder_provider_fixtur
         result = await subfolder_provider.metadata(path)
         assert result.kind == 'file'
         assert result.name == 'bicuspid.txt'
+        assert result.materialized_path == '/bicuspid.txt'
 
 
 class TestRevisions:

waterbutler/__init__.py

@@ -1,2 +1,2 @@
-__version__ = '0.36.0'
+__version__ = '0.36.1'
 __import__('pkg_resources').declare_namespace(__name__)

waterbutler/core/streams/zip.py

@@ -1,13 +1,11 @@
-import asyncio
-import binascii
-import struct
+import zlib
 import time
+import struct
+import asyncio
 import zipfile
-import zlib
+import binascii
 
-from waterbutler.core.streams.base import BaseStream
-from waterbutler.core.streams.base import MultiStream
-from waterbutler.core.streams.base import StringStream
+from waterbutler.core.streams.base import BaseStream, MultiStream, StringStream
 
 # for some reason python3.5 has this as (1 << 31) - 1, which is 0x7fffffff
 ZIP64_LIMIT = 0xffffffff - 1
@@ -118,27 +116,35 @@ async def _read(self, n=-1, *args, **kwargs):
 
 
 class ZipLocalFile(MultiStream):
-    """A local file entry in a zip archive.  Constructs the local file header, file data stream,
-    and data descriptor.
+    """A local file entry in a zip archive. Constructs the local file header,
+    file data stream, and data descriptor.
 
-    Note: This class is tightly coupled to ZipStreamReader and should not be used separately.
+    Note: This class is tightly coupled to ZipStreamReader and should not be
+    used separately.
     """
     def __init__(self, file_tuple):
+
         filename, stream = file_tuple
         # Build a ZipInfo instance to use for the file's header and footer
         self.zinfo = zipfile.ZipInfo(
             filename=filename,
             date_time=time.localtime(time.time())[:6],
         )
-        # If the file is a directory, set the directory flag, turn off compression
-        if self.zinfo.filename[-1] == '/':
-            self.zinfo.external_attr = 0o40775 << 16   # drwxrwxr-x
-            self.zinfo.external_attr |= 0x10           # Directory flag
+
+        # If the file is a `.zip`, set permission and turn off compression
+        if self.zinfo.filename.endswith('.zip'):
+            self.zinfo.external_attr = 0o600 << 16      # -rw-------
             self.zinfo.compress_type = zipfile.ZIP_STORED
             self.compressor = None
+        # If the file is a directory, set the directory flag and turn off compression
+        elif self.zinfo.filename[-1] == '/':
+            self.zinfo.external_attr = 0o40775 << 16    # drwxrwxr-x
+            self.zinfo.external_attr |= 0x10            # Directory flag
+            self.zinfo.compress_type = zipfile.ZIP_STORED
+            self.compressor = None
+        # For other types, set permission and define a compressor
         else:
-            self.zinfo.external_attr = 0o600 << 16  # rw-------
-            # define a compressor
+            self.zinfo.external_attr = 0o600 << 16      # -rw-------
             self.zinfo.compress_type = zipfile.ZIP_DEFLATED
             self.compressor = zlib.compressobj(
                 zlib.Z_DEFAULT_COMPRESSION,
@@ -148,6 +154,7 @@ def __init__(self, file_tuple):
 
         self.zinfo.header_offset = 0
         self.zinfo.flag_bits |= 0x08
+
         # Initial CRC: value will be updated as file is streamed
         self.zinfo.CRC = 0
 

waterbutler/providers/onedrive/provider.py

@@ -268,7 +268,7 @@ async def metadata(self, path: OneDrivePath, **kwargs):  # type: ignore
                 code=HTTPStatus.NOT_FOUND,
             )
 
-        return self._construct_metadata(data)
+        return self._construct_metadata(data, path)
 
     async def revisions(self,  # type: ignore
                         path: OneDrivePath,
@@ -393,21 +393,24 @@ def _build_drive_url(self, *segments, **query) -> str:
     def _build_item_url(self, *segments, **query) -> str:
         return provider.build_url(settings.BASE_DRIVE_URL, 'items', *segments, **query)
 
-    def _construct_metadata(self, data: dict):
-        """Take a file/folder metadata response from OneDrive and return a `OneDriveFileMetadata`
-        object if the repsonse represents a file or a list of `OneDriveFileMetadata` and
-        `OneDriveFolderMetadata` objects if the response represents a folder. """
+    def _construct_metadata(self, data: dict, path):
+        """Take a file/folder metadata response from OneDrive and a path object representing the
+        queried path and return a `OneDriveFileMetadata` object if the repsonse represents a file
+        or a list of `OneDriveFileMetadata` and `OneDriveFolderMetadata` objects if the response
+        represents a folder. """
         if 'folder' in data.keys():
             ret = []
             if 'children' in data.keys():
                 for item in data['children']:
-                    if 'folder' in item.keys():
-                        ret.append(OneDriveFolderMetadata(item, self.folder))  # type: ignore
+                    is_folder = 'folder' in item.keys()
+                    child_path = path.child(item['name'], _id=item['id'], folder=is_folder)
+                    if is_folder:
+                        ret.append(OneDriveFolderMetadata(item, child_path))  # type: ignore
                     else:
-                        ret.append(OneDriveFileMetadata(item, self.folder))  # type: ignore
+                        ret.append(OneDriveFileMetadata(item, child_path))  # type: ignore
             return ret
 
-        return OneDriveFileMetadata(data, self.folder)
+        return OneDriveFileMetadata(data, path)
 
     async def _revisions_json(self, path: OneDrivePath, **kwargs) -> dict:
         """Fetch a list of revisions for the file at ``path``.

waterbutler/server/api/v1/provider/movecopy.py

@@ -107,11 +107,16 @@ async def move_or_copy(self):
             if path is None:
                 raise exceptions.InvalidParameters('"path" field is required for moves or copies')
             if not path.endswith('/'):
-                raise exceptions.InvalidParameters('"path" field requires a trailing slash to '
-                                                   'indicate it is a folder')
+                raise exceptions.InvalidParameters(
+                    '"path" field requires a trailing slash to indicate it is a folder'
+                )
 
             # TODO optimize for same provider and resource
 
+            # for copy action, `auth_action` is the same as `provider_action`
+            if auth_action == 'copy' and self.path.is_root and not self.json.get('rename'):
+                raise exceptions.InvalidParameters('"rename" field is required for copying root')
+
             # Note: attached to self so that _send_hook has access to these
             self.dest_resource = self.json.get('resource', self.resource)
             self.dest_auth = await auth_handler.get(
@@ -154,8 +159,8 @@ async def move_or_copy(self):
         self.dest_meta = metadata
 
         if created:
-            self.set_status(HTTPStatus.CREATED)
+            self.set_status(int(HTTPStatus.CREATED))
         else:
-            self.set_status(HTTPStatus.OK)
+            self.set_status(int(HTTPStatus.OK))
 
         self.write({'data': metadata.json_api_serialized(self.dest_resource)})