Merge pull request #2738 from activeloopai/fy_thread_safety

Don't use cached relpath in parallel code
activeloopai · Jan 13, 2024 · fc047ab · fc047ab
2 parents cab837a + 20b3345
commit fc047ab
Show file tree

Hide file tree

Showing 4 changed files with 4 additions and 4 deletions.
diff --git a/deeplake/core/storage/azure.py b/deeplake/core/storage/azure.py
@@ -180,7 +180,7 @@ def _all_keys(self, prefix: str = ""):
         self._check_update_creds()
         prefix = posixpath.join(self.root_folder, prefix)
         return {
-            relpath(blob.name, self.root_folder)
+            posixpath.relpath(blob.name, self.root_folder)
             for blob in self.container_client.list_blobs(
                 name_starts_with=prefix, include=["metadata"]
             )

diff --git a/deeplake/core/storage/gcs.py b/deeplake/core/storage/gcs.py
@@ -325,7 +325,7 @@ def _get_path_from_key(self, key):
 
     def _all_keys(self):
         self._blob_objects = self.client_bucket.list_blobs(prefix=self.path)
-        return {relpath(obj.name, self.path) for obj in self._blob_objects}
+        return {posixpath.relpath(obj.name, self.path) for obj in self._blob_objects}
 
     def _set_hub_creds_info(
         self,

diff --git a/deeplake/core/storage/local.py b/deeplake/core/storage/local.py
@@ -175,7 +175,7 @@ def _all_keys(self, refresh: bool = False) -> Set[str]:
             for root, dirs, files in os.walk(full_path):
                 for file in files:
                     key_set.add(
-                        relpath(
+                        posixpath.relpath(
                             posixpath.join(pathlib.Path(root).as_posix(), file),
                             pathlib.Path(full_path).as_posix(),
                         )

diff --git a/deeplake/util/transform.py b/deeplake/util/transform.py
@@ -348,7 +348,7 @@ def store_data_slice_with_pbar(pg_callback, transform_input: Tuple) -> Dict:
     if isinstance(data_slice, deeplake.Dataset):
         data_slice = add_cache_to_dataset_slice(data_slice, tensors)
 
-    rel_tensors = [relpath(tensor, group_index) for tensor in visible_tensors]
+    rel_tensors = [posixpath.relpath(tensor, group_index) for tensor in visible_tensors]
 
     transform_dataset = TransformDataset(
         rel_tensors,