Merge pull request #2343 from activeloopai/fy_fix_tensor_type

Fix dtype bug

deeplake/api/tests/test_api.py

@@ -2459,6 +2459,22 @@ def test_sequence_numpy_bug(memory_ds):
         assert ds.abc.numpy(aslist=True) == [[1, 2], [1, 2, 3], [1, 2, 3, 4]]
 
 
+def test_tensor_dtype_bug(local_path):
+    from nibabel.testing import data_path
+
+    with deeplake.empty(local_path, overwrite=True) as ds:
+        ds.create_tensor("abc", htype="link[nifti]", sample_compression="nii.gz")
+        ds.abc.append(deeplake.link(f"{data_path}/standard.nii.gz"))
+
+    assert ds.abc[0].numpy().shape == (4, 5, 7)
+    assert ds.abc.dtype == np.dtype("<U1")
+
+    ds2 = ds.copy(f"{local_path}_2", overwrite=True)
+
+    assert ds2.abc[0].numpy().shape == (4, 5, 7)
+    assert ds2.abc.dtype == np.dtype("<U1")
+
+
 def test_iterate_with_groups(memory_ds):
     with memory_ds as ds:
         ds.create_tensor("x/y/z")

deeplake/core/dataset/dataset.py

@@ -1128,6 +1128,7 @@ def create_tensor_like(
         del meta["version"]
         del meta["name"]
         del meta["links"]
+        meta["dtype"] = np.dtype(meta["typestr"]) if meta["typestr"] else meta["dtype"]
 
         destination_tensor = self._create_tensor(
             name,

deeplake/core/fast_forwarding.py

@@ -91,6 +91,8 @@ def ffw_tensor_meta(tensor_meta, version):
         tensor_meta.is_link = False
     if not hasattr(tensor_meta, "is_sequence"):
         tensor_meta.is_sequence = False
+    if not hasattr(tensor_meta, "typestr"):
+        tensor_meta.typestr = None
     required_meta_keys = tensor_meta._required_meta_keys
     tensor_meta._required_meta_keys = tuple(
         set(
@@ -101,6 +103,7 @@ def ffw_tensor_meta(tensor_meta, version):
                 "links",
                 "is_link",
                 "is_sequence",
+                "typestr",
             )
         )
     )

deeplake/core/meta/tensor_meta.py

@@ -33,6 +33,7 @@ class TensorMeta(Meta):
     name: Optional[str] = None
     htype: str
     dtype: str
+    typestr: str
     min_shape: List[int]
     max_shape: List[int]
     length: int
@@ -98,6 +99,7 @@ def set_dtype(self, dtype: np.dtype):
             )
 
         self.dtype = dtype.name
+        self.typestr = dtype.str
         self.is_dirty = True
 
     def set_dtype_str(self, dtype_name: str):

deeplake/core/tensor.py

@@ -506,7 +506,7 @@ def dtype(self) -> Optional[np.dtype]:
         if self.base_htype in ("json", "list"):
             return np.dtype(str)
         if self.meta.dtype:
-            return np.dtype(self.meta.dtype)
+            return np.dtype(self.meta.typestr or self.meta.dtype)
         return None
 
     @property

deeplake/htype.py

@@ -133,6 +133,7 @@ class htype:
     "sample_compression": None,
     "chunk_compression": None,
     "dtype": None,
+    "typestr": None,
     "max_chunk_size": None,
     "tiling_threshold": None,
     "is_sequence": False,

deeplake/util/transform.py

@@ -39,6 +39,8 @@
 import posixpath
 import time
 
+import numpy as np
+
 try:
     import pandas as pd  # type: ignore
 except ImportError:
@@ -400,7 +402,9 @@ def create_worker_chunk_engines(
                 tiling_threshold = storage_chunk_engine.tiling_threshold
                 new_tensor_meta = TensorMeta(
                     htype=existing_meta.htype,
-                    dtype=existing_meta.dtype,
+                    dtype=np.dtype(existing_meta.typestr)
+                    if existing_meta.typestr
+                    else existing_meta.dtype,
                     sample_compression=existing_meta.sample_compression,
                     chunk_compression=existing_meta.chunk_compression,
                     max_chunk_size=chunk_size,