[AL-1579] Auto htype

hub/api/tests/test_api.py

@@ -811,3 +811,17 @@ def test_empty_extend(memory_ds):
         ds.create_tensor("y")
         ds.y.extend(np.zeros((len(ds), 3)))
     assert len(ds) == 0
+
+
+def test_auto_htype(memory_ds):
+    ds = memory_ds
+    with ds:
+        ds.create_tensor("x")
+        ds.create_tensor("y")
+        ds.create_tensor("z")
+        ds.x.append("hello")
+        ds.y.append({"a": [1, 2]})
+        ds.z.append([1, 2, 3])
+    assert ds.x.htype == "text"
+    assert ds.y.htype == "json"
+    assert ds.z.htype == "generic"

hub/core/chunk_engine.py

@@ -18,7 +18,7 @@
 from hub.core.meta.encode.chunk_id import ChunkIdEncoder
 from hub.core.meta.tensor_meta import TensorMeta
 from hub.core.storage.lru_cache import LRUCache
-from hub.util.casting import get_dtype
+from hub.util.casting import get_dtype, get_htype
 from hub.util.chunk_engine import (
     check_samples_type,
     make_sequence,
@@ -353,6 +353,8 @@ def _convert_to_list(self, samples):
 
     def _sanitize_samples(self, samples):
         check_samples_type(samples)
+        if self.tensor_meta.htype is None:
+            self.tensor_meta.set_htype(get_htype(samples))
         if self.tensor_meta.dtype is None:
             self.tensor_meta.set_dtype(get_dtype(samples))
         if self._convert_to_list(samples):
@@ -369,7 +371,6 @@ def extend(self, samples):
         current_chunk = self.last_chunk() or self._create_new_chunk()
         updated_chunks = {current_chunk}
         enc = self.chunk_id_encoder
-
         while len(samples) > 0:
             num_samples_added = current_chunk.extend_if_has_space(samples)
             if num_samples_added == 0:

hub/core/dataset/dataset.py

@@ -231,7 +231,7 @@ def __getitem__(
     def create_tensor(
         self,
         name: str,
-        htype: str = DEFAULT_HTYPE,
+        htype: str = UNSPECIFIED,
         dtype: Union[str, np.dtype] = UNSPECIFIED,
         sample_compression: str = UNSPECIFIED,
         chunk_compression: str = UNSPECIFIED,
@@ -289,8 +289,8 @@ def create_tensor(
 
         # Seperate meta and info
 
-        htype_config = HTYPE_CONFIGURATIONS[htype].copy()
-        info_keys = htype_config.pop("_info", [])
+        htype_config = HTYPE_CONFIGURATIONS.get(htype, {})
+        info_keys = htype_config.copy().pop("_info", [])
         info_kwargs = {}
         meta_kwargs = {}
         for k, v in kwargs.items():

hub/core/meta/tensor_meta.py

@@ -26,6 +26,7 @@
 )
 from hub.htype import (
     HTYPE_CONFIGURATIONS,
+    DEFAULT_HTYPE,
 )
 from hub.htype import HTYPE_CONFIGURATIONS, REQUIRE_USER_SPECIFICATION, UNSPECIFIED
 from hub.core.meta.meta import Meta
@@ -57,22 +58,13 @@ def __init__(
             **kwargs: Any key that the provided `htype` has can be overridden via **kwargs. For more information, check out `hub.htype`.
         """
 
-        if htype != UNSPECIFIED:
-            _validate_htype_exists(htype)
-            _validate_htype_overwrites(htype, kwargs)
-            _replace_unspecified_values(htype, kwargs)
-            _validate_required_htype_overwrites(htype, kwargs)
-            _format_values(htype, kwargs)
-
-            required_meta = _required_meta_from_htype(htype)
-            required_meta.update(kwargs)
+        super().__init__()
 
-            self._required_meta_keys = tuple(required_meta.keys())
-            self.__dict__.update(required_meta)
+        if htype != UNSPECIFIED:
+            self.set_htype(htype, **kwargs)
         else:
-            self._required_meta_keys = tuple()
-
-        super().__init__()
+            self.set_htype(DEFAULT_HTYPE, **kwargs)
+            self.htype = None  # type: ignore
 
     def set_dtype(self, dtype: np.dtype):
         """Should only be called once."""
@@ -88,6 +80,38 @@ def set_dtype(self, dtype: np.dtype):
 
         self.dtype = dtype.name
 
+    def set_htype(self, htype: str, **kwargs):
+        """Should only be called once."""
+        ffw_tensor_meta(self)
+
+        if getattr(self, "htype", None) is not None:
+            raise ValueError(
+                f"Tensor meta already has a htype ({self.htype}). Incoming: {htype}."
+            )
+
+        if getattr(self, "length", 0) > 0:
+            raise ValueError("Htype was None, but length was > 0.")
+
+        if not kwargs:
+            kwargs = HTYPE_CONFIGURATIONS[htype]
+
+        _validate_htype_exists(htype)
+        _validate_htype_overwrites(htype, kwargs)
+        _replace_unspecified_values(htype, kwargs)
+        _validate_required_htype_overwrites(htype, kwargs)
+        _format_values(htype, kwargs)
+
+        required_meta = _required_meta_from_htype(htype)
+        required_meta.update(kwargs)
+
+        self._required_meta_keys = tuple(required_meta.keys())
+
+        for k in self._required_meta_keys:
+            if getattr(self, k, None):
+                required_meta.pop(k, None)
+
+        self.__dict__.update(required_meta)
+
     def update_shape_interval(self, shape: Tuple[int, ...]):
         ffw_tensor_meta(self)
 
@@ -219,7 +243,6 @@ def _replace_unspecified_values(htype: str, htype_overwrite: dict):
 
 def _validate_required_htype_overwrites(htype: str, htype_overwrite: dict):
     """Raises errors if `htype_overwrite` has invalid values."""
-
     sample_compression = htype_overwrite["sample_compression"]
     sample_compression = COMPRESSION_ALIASES.get(sample_compression, sample_compression)
     if sample_compression not in hub.compressions:

hub/util/casting.py

@@ -37,6 +37,21 @@ def get_dtype(val: Union[np.ndarray, Sequence, Sample]) -> np.dtype:
         raise TypeError(f"Cannot infer numpy dtype for {val}")
 
 
+def get_htype(val: Union[np.ndarray, Sequence, Sample]) -> str:
+    if isinstance(val, np.ndarray):
+        return "generic"
+    types = set((map(type, val)))
+    if dict in types:
+        return "json"
+    if types == set((str,)):
+        return "text"
+    if np.object in [  # type: ignore
+        np.array(x).dtype if not isinstance(x, np.ndarray) else x.dtype for x in val
+    ]:
+        return "json"
+    return "generic"
+
+
 def intelligent_cast(
     sample: Any, dtype: Union[np.dtype, str], htype: str
 ) -> np.ndarray: