From 6bd93f3d202ba5419b44f5b5da1fad6602012c87 Mon Sep 17 00:00:00 2001 From: Levon Ghukasyan Date: Wed, 10 May 2023 09:37:40 +0400 Subject: [PATCH 1/5] corrected tensor calculation logic --- deeplake/integrations/pytorch/shuffle_buffer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deeplake/integrations/pytorch/shuffle_buffer.py b/deeplake/integrations/pytorch/shuffle_buffer.py index 9793a45a1a..9de1810670 100644 --- a/deeplake/integrations/pytorch/shuffle_buffer.py +++ b/deeplake/integrations/pytorch/shuffle_buffer.py @@ -118,6 +118,8 @@ def _num_torch_tensors(self, sample): return 0 if isinstance(sample, TorchTensor): return 1 + elif isinstance(sample, bytes): + return 0 elif isinstance(sample, dict): return sum(self._num_torch_tensors(tensor) for tensor in sample.values()) elif isinstance(sample, Sequence): From 7e02e05343c5851f0f982ddbb227e93a49a452ec Mon Sep 17 00:00:00 2001 From: Levon Ghukasyan Date: Wed, 10 May 2023 11:22:19 +0400 Subject: [PATCH 2/5] conversion --- deeplake/tests/common.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/deeplake/tests/common.py b/deeplake/tests/common.py index eaaa5f6397..fac807d1d2 100644 --- a/deeplake/tests/common.py +++ b/deeplake/tests/common.py @@ -142,9 +142,7 @@ def __exit__(self, *args, **kwargs): def convert_data_according_to_torch_version(batch): - import torch - - if torch.__version__ < "2.0.0": + if isinstance(batch, List): return batch[0] else: return batch From 244d99dccb7a69a2b78032e2b93f44948ffe541e Mon Sep 17 00:00:00 2001 From: Levon Ghukasyan Date: Wed, 10 May 2023 14:04:53 +0400 Subject: [PATCH 3/5] test fix --- deeplake/enterprise/test_pytorch.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/deeplake/enterprise/test_pytorch.py b/deeplake/enterprise/test_pytorch.py index a803d1e1c0..af59626e1e 100644 --- a/deeplake/enterprise/test_pytorch.py +++ b/deeplake/enterprise/test_pytorch.py @@ -6,7 +6,11 @@ from deeplake.util.remove_cache import get_base_storage from deeplake.core.index.index import IndexEntry -from deeplake.tests.common import requires_torch, requires_libdeeplake +from deeplake.tests.common import ( + requires_torch, + requires_libdeeplake, + convert_data_according_to_torch_version, +) from deeplake.core.dataset import Dataset from deeplake.constants import KB @@ -63,7 +67,6 @@ def index_transform(sample): @pytest.mark.parametrize( "ds", ["hub_cloud_ds", "hub_cloud_gcs_ds"], - indirect=True, ) def test_pytorch_small(ds): with ds: @@ -458,7 +461,7 @@ def test_pytorch_decode(hub_cloud_ds, compressed_image_paths, compression): ptds = hub_cloud_ds.dataloader().pytorch(decode_method={"image": "tobytes"}) for i, batch in enumerate(ptds): - image = batch["image"] + image = convert_data_according_to_torch_version(batch["image"]) assert isinstance(image, bytes) if i < 5 and not compression: np.testing.assert_array_equal( From 88be4b2275d81d0458d86e7dc162f2ae2d2f8dae Mon Sep 17 00:00:00 2001 From: Levon Ghukasyan Date: Thu, 11 May 2023 10:20:43 +0400 Subject: [PATCH 4/5] add back --- deeplake/enterprise/test_pytorch.py | 1 + 1 file changed, 1 insertion(+) diff --git a/deeplake/enterprise/test_pytorch.py b/deeplake/enterprise/test_pytorch.py index af59626e1e..f22757d4ae 100644 --- a/deeplake/enterprise/test_pytorch.py +++ b/deeplake/enterprise/test_pytorch.py @@ -67,6 +67,7 @@ def index_transform(sample): @pytest.mark.parametrize( "ds", ["hub_cloud_ds", "hub_cloud_gcs_ds"], + indirect=True, ) def test_pytorch_small(ds): with ds: From baa69dc76a524c3f83cf5cb0cb7ceefd5bedfdbd Mon Sep 17 00:00:00 2001 From: Levon Ghukasyan Date: Thu, 11 May 2023 12:48:06 +0400 Subject: [PATCH 5/5] add str --- deeplake/integrations/pytorch/shuffle_buffer.py | 2 ++ deeplake/requirements/tests.txt | 2 ++ 2 files changed, 4 insertions(+) diff --git a/deeplake/integrations/pytorch/shuffle_buffer.py b/deeplake/integrations/pytorch/shuffle_buffer.py index 9de1810670..08f13f7e8b 100644 --- a/deeplake/integrations/pytorch/shuffle_buffer.py +++ b/deeplake/integrations/pytorch/shuffle_buffer.py @@ -120,6 +120,8 @@ def _num_torch_tensors(self, sample): return 1 elif isinstance(sample, bytes): return 0 + elif isinstance(sample, str): + return 0 elif isinstance(sample, dict): return sum(self._num_torch_tensors(tensor) for tensor in sample.values()) elif isinstance(sample, Sequence): diff --git a/deeplake/requirements/tests.txt b/deeplake/requirements/tests.txt index 1ccbf435d9..c4de825ac1 100644 --- a/deeplake/requirements/tests.txt +++ b/deeplake/requirements/tests.txt @@ -16,3 +16,5 @@ boto3-stubs[essential] lz4 rich wandb + +pandas; python_version >= '3.11' and sys_platform == 'win32'