Merge pull request #56 from RangelReale/windows2

Windows compatibility for LOCAL driver

setup.py

@@ -80,7 +80,7 @@ def read(*names, **kwargs):
         'local': [
             'filelock>=3.0.0',  # Public Domain
             'itsdangerous>=1.1.0',  # BSD License
-            'xattr>=0.9.6',  # MIT
+            'xattr>=0.9.6; sys_platform != "win32"',  # MIT
         ],
         'microsoft': [
             'azure>=4.0.0',  # MIT

src/cloudstorage/drivers/local.py

@@ -9,10 +9,12 @@
 from contextlib import contextmanager
 from datetime import datetime, timedelta, timezone
 from typing import Dict, Iterable, List
+import simplejson as json
 
 import filelock
 import itsdangerous
-import xattr
+if os.name != 'nt':
+    import xattr
 from inflection import underscore
 
 from cloudstorage import Blob, Container, Driver, messages
@@ -68,7 +70,8 @@ def lock_local_file(path: str) -> filelock.FileLock:
     if lock.is_locked:
         lock.release()
 
-    os.remove(lock.lock_file)
+    if os.path.exists(lock.lock_file):
+        os.remove(lock.lock_file)
 
 
 class LocalDriver(Driver):
@@ -114,6 +117,7 @@ def __init__(self, key: str, secret: str = None, salt: str = None,
 
         self.base_path = key
         self.salt = salt
+        self.is_windows = os.name == 'nt'
 
         try:
             if not os.path.exists(key):
@@ -165,6 +169,29 @@ def _make_serializer(self) -> itsdangerous.URLSafeTimedSerializer:
                 'digest_method': 'SHA1'
             })
 
+    def _make_xattr(self, filename: str):
+        """
+        Make a xattr-like object depending on the current platform.
+        :param filename:
+        :return:
+        """
+        if self.is_windows:
+            return XattrWindows(filename)
+        return xattr.xattr(filename)
+
+    def _check_path_accessible(self, path: str) -> bool:
+        """
+        Check if the path is accessible. In windows custom files are used to simulate file attributes,
+        these must not be accessed.
+        :param filename:
+        :return:
+        """
+        if self.is_windows:
+            p = pathlib.Path(path)
+            if p.name.startswith('.') and p.name.endswith('.xattr'):
+                return False
+        return True
+
     def _get_folders(self) -> Iterable[str]:
         """Iterate over first level folders found in base path.
 
@@ -173,6 +200,8 @@ def _get_folders(self) -> Iterable[str]:
         """
         for container_name in os.listdir(self.base_path):
             full_path = os.path.join(self.base_path, container_name)
+            if not self._check_path_accessible(full_path):
+                continue
             if not os.path.isdir(full_path):
                 continue
 
@@ -194,6 +223,8 @@ def _get_folder_path(self, container: Container,
         :raises NotFoundError: If the container doesn't exist.
         """
         full_path = os.path.join(self.base_path, container.name)
+        if validate and not self._check_path_accessible(full_path):
+            raise NotFoundError(messages.CONTAINER_NOT_FOUND % container.name)
         if validate and not os.path.isdir(full_path):
             raise NotFoundError(messages.CONTAINER_NOT_FOUND % container.name)
 
@@ -221,7 +252,7 @@ def _set_file_attributes(self, filename: str, attributes: Dict) -> None:
         :raises CloudStorageError: If the local file system does not support
           extended filesystem attributes.
         """
-        xattrs = xattr.xattr(filename)
+        xattrs = self._make_xattr(filename)
 
         for key, value in attributes.items():
             if not value:
@@ -288,6 +319,8 @@ def _make_container(self, folder_name: str) -> Container:
         :raises FileNotFoundError: If container does not exist.
         """
         full_path = os.path.join(self.base_path, folder_name)
+        if not self._check_path_accessible(full_path):
+            raise NotFoundError(messages.CONTAINER_NOT_FOUND % folder_name)
 
         try:
             stat = os.stat(full_path)
@@ -312,6 +345,9 @@ def _make_blob(self, container: Container, object_name: str) -> Blob:
         :rtype: :class:`.Blob`
         """
         full_path = os.path.join(self.base_path, container.name, object_name)
+        if not self._check_path_accessible(full_path):
+            raise NotFoundError(messages.BLOB_NOT_FOUND % (object_name,
+                                                           container.name))
 
         object_path = pathlib.Path(full_path)
 
@@ -327,7 +363,7 @@ def _make_blob(self, container: Container, object_name: str) -> Blob:
         cache_control = None
 
         try:
-            attributes = xattr.xattr(full_path)
+            attributes = self._make_xattr(full_path)
 
             for attr_key, attr_value in attributes.items():
                 value_str = None
@@ -385,6 +421,8 @@ def create_container(self, container_name: str, acl: str = None,
             logger.info(messages.OPTION_NOT_SUPPORTED, 'meta_data')
 
         full_path = os.path.join(self.base_path, container_name)
+        if not self._check_path_accessible(full_path):
+            raise CloudStorageError(messages.CONTAINER_NAME_INVALID)
 
         self._make_path(full_path, ignore_existing=True)
         try:
@@ -483,6 +521,8 @@ def get_blobs(self, container: Container) -> Iterable[Blob]:
 
             for name in files:
                 full_path = os.path.join(folder, name)
+                if not self._check_path_accessible(full_path):
+                    continue
                 object_name = pathlib.Path(full_path).name
                 yield self._make_blob(container, object_name)
 
@@ -518,6 +558,10 @@ def delete_blob(self, blob: Blob) -> None:
             except OSError as err:
                 logger.exception(err)
 
+        if self.is_windows:
+            xattr = XattrWindows(path)
+            xattr.remove_attributes()
+
     def blob_cdn_url(self, blob: Blob) -> str:
         return os.path.join(self.base_path, blob.container.name, blob.name)
 
@@ -609,3 +653,56 @@ def validate_signature(self, signature):
     _PUT_OBJECT_KEYS = {
         'metadata': 'meta_data',
     }
+
+
+class XattrWindows:
+    """
+    Simulate xattr on windows.
+    A file named ".<filename>.xattr" will be created on the same directory as the source file.
+    """
+    def __init__(self, filename) -> None:
+        self.filename = filename
+        p = pathlib.Path(filename)
+        self.xattr_filename = os.path.join(p.parent, '.{}.xattr'.format(p.name))
+
+    def __setitem__(self, key, value) -> None:
+        """
+        Write an attribute to the json file.
+        """
+        data = self._load()
+        data[key] = value
+        with open(self.xattr_filename, 'w') as outfile:
+            json.dump(data, outfile)
+
+    def items(self):
+        """
+        Return a list of the attributes.
+        :return:
+        """
+        # xattr returns items as bytes, must convert all str first
+        items = self._load()
+        ret = {}
+        for itemname, itemvalue in items.items():
+            if isinstance(itemvalue, str):
+                ret[itemname] = itemvalue.encode('utf-8')
+            else:
+                ret[itemname] = itemvalue
+        return  ret.items()
+
+    def _load(self) -> Dict:
+        """
+        Load json file if it exists
+        :return:
+        """
+        if os.path.exists(self.xattr_filename):
+            with open(self.xattr_filename) as json_file:
+                return json.load(json_file)
+        return {}
+
+    def remove_attributes(self):
+        if os.path.exists(self.xattr_filename):
+            with lock_local_file(self.xattr_filename):
+                try:
+                    os.unlink(self.xattr_filename)
+                except OSError as err:
+                    logger.exception(err)

tests/test_drivers_local.py

@@ -37,6 +37,8 @@ def storage():
 
 
 def test_driver_validate_credentials():
+    if os.name == 'nt':
+        pytest.skip("skipping Windows incompatible test")
     driver = LocalDriver(key=LOCAL_KEY)
     assert driver.validate_credentials() is None
 
@@ -148,6 +150,26 @@ def test_blob_upload_path(container, text_filename):
     assert blob.checksum == TEXT_MD5_CHECKSUM
 
 
+def test_blob_windows_xattr(container, text_filename):
+    if os.name != 'nt':
+        pytest.skip("skipping Windows-only test")
+    blob = container.upload_blob(text_filename, meta_data={'test': 'testvalue'})
+    try:
+        internal_blob = container.get_blob('.{}.xattr'.format(TEXT_FILENAME))
+        pytest.fail('should not be possible to get internal xattr file')
+    except NotFoundError:
+        pass
+
+
+def test_blob_windows_xattr_list(container, text_filename):
+    if os.name != 'nt':
+        pytest.skip("skipping Windows-only test")
+    blob = container.upload_blob(text_filename, meta_data={'test': 'testvalue'})
+    for blobitem in container:
+        if blobitem.name.startswith('.') and blobitem.name.endswith('.xattr'):
+            pytest.fail('should not be possible to get internal xattr file')
+
+
 def test_blob_upload_stream(container, binary_stream):
     blob = container.upload_blob(filename=binary_stream,
                                  blob_name=BINARY_STREAM_FILENAME,