From 9715983913889860cc70c2b0f07872d03e9f3fbc Mon Sep 17 00:00:00 2001 From: FayazRahman Date: Mon, 15 Jan 2024 10:43:48 +0530 Subject: [PATCH 1/2] init --- deeplake/core/transform/transform_dataset.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/deeplake/core/transform/transform_dataset.py b/deeplake/core/transform/transform_dataset.py index 0484591e35..085866887d 100644 --- a/deeplake/core/transform/transform_dataset.py +++ b/deeplake/core/transform/transform_dataset.py @@ -237,7 +237,10 @@ def flush(self, clear_on_fail=True): name = self._get_engine_name(name) updated_tensors[name] = 0 chunk_engine = all_chunk_engines[name] - callback = chunk_engine._transform_callback + if chunk_engine.tensor_meta.links: + callback = chunk_engine._transform_callback + else: + callback = None meta = chunk_engine.tensor_meta if meta.length == 0 and meta.dtype is None: From 6b5d4d757bd3bc3bf5752a27caf618ff803733a7 Mon Sep 17 00:00:00 2001 From: FayazRahman Date: Fri, 19 Jan 2024 09:46:19 +0530 Subject: [PATCH 2/2] fix numpy chunk compression inconsistency --- deeplake/core/chunk/chunk_compressed_chunk.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/deeplake/core/chunk/chunk_compressed_chunk.py b/deeplake/core/chunk/chunk_compressed_chunk.py index 1ef30f6412..8853c3b3c8 100644 --- a/deeplake/core/chunk/chunk_compressed_chunk.py +++ b/deeplake/core/chunk/chunk_compressed_chunk.py @@ -151,6 +151,9 @@ def extend_if_has_space_byte_compression_numpy( update_tensor_meta: bool = True, ): sample = incoming_samples[0] + shape = sample.shape + if not shape: + shape = (1,) chunk_dtype = self.dtype sample_dtype = sample.dtype if chunk_dtype == sample_dtype: @@ -220,7 +223,7 @@ def extend_if_has_space_byte_compression_numpy( if num_samples: self.register_in_meta_and_headers( sample_nbytes, - sample.shape, + shape, update_tensor_meta=update_tensor_meta, num_samples=num_samples, )