[AL-1619][AL-1586] Removes cachable, makes Info more dict like

hub/api/info.py

@@ -1,78 +1,40 @@
-from hub.util.version_control import auto_checkout
-from hub.core.storage.lru_cache import LRUCache
-from typing import Any, Dict, Optional, Union, Sequence
-from hub.core.storage.cachable import CachableCallback, use_callback
+from hub.util.exceptions import InfoError
+from hub.core.storage.hub_memory_object import HubMemoryObject
+from typing import Any, Dict, Optional
 
 
-class Info(CachableCallback):
+class Info(HubMemoryObject):
     def __init__(self):
-        """Contains **optional** key/values that datasets/tensors use for human-readability.
-        See the `Meta` class for required key/values for datasets/tensors.
-
-        Note:
-            Since `Info` is rarely written to and mostly by the user, every modifier will call `cache[key] = self`.
-            Must call `initialize_callback_location` before using any methods.
-        """
         self._info = {}
+        self._dataset = None
         super().__init__()
 
+    def prepare_for_write(self):
+        if self._dataset is not None:
+            storage = self._dataset.storage
+            storage.check_readonly()
+            if not self._dataset.version_state["commit_node"].is_head_node:
+                raise InfoError("Cannot modify info from a non-head commit.")
+            self.is_dirty = True
+
+    def end_write(self):
+        if self._dataset is not None:
+            storage = self._dataset.storage
+            storage.maybe_flush()
+
     @property
     def nbytes(self):
-        # TODO: optimize this
         return len(self.tobytes())
 
-    @use_callback(check_only=True)
     def __len__(self):
         return len(self._info)
 
-    @use_callback(check_only=True)
     def __getstate__(self) -> Dict[str, Any]:
         return self._info
 
     def __setstate__(self, state: Dict[str, Any]):
         self._info = state
 
-    @use_callback()
-    def update(self, *args, **kwargs):
-        """Store optional dataset/tensor information. Will be accessible after loading your data from a new script!
-        Inputs must be supported by JSON.
-
-
-        Note:
-            This method has the same functionality as `dict().update(...)` Reference: https://www.geeksforgeeks.org/python-dictionary-update-method/.
-            A full list of supported value types can be found here: https://docs.python.org/3/library/json.html#json.JSONEncoder.
-
-        Examples:
-            Normal update usage:
-                >>> ds.info
-                {}
-                >>> ds.info.update(key=0)
-                >>> ds.info
-                {"key": 0}
-                >>> ds.info.update({"key1": 5, "key2": [1, 2, "test"]})
-                >>> ds.info
-                {"key": 0, "key1": 5, "key2": [1, 2, "test"]}
-
-            Alternate update usage:
-                >>> ds.info
-                {}
-                >>> ds.info.update(list=[1, 2, "apple"])
-                >>> ds.info
-                {"list": [1, 2, "apple"]}
-                >>> l = ds.info.list
-                >>> l
-                [1, 2, "apple"]
-                >>> l.append(5)
-                >>> l
-                [1, 2, "apple", 5]
-                >>> ds.info.update()  # required to be persistent!
-
-        """
-        self._cache.check_readonly()
-        if self._dataset is not None:
-            auto_checkout(self._dataset)
-        self._info.update(*args, **kwargs)
-
     def __getattribute__(self, name: str) -> Any:
         """Allows access to info values using the `.` syntax. Example: `info.description`."""
 
@@ -82,48 +44,33 @@ def __getattribute__(self, name: str) -> Any:
             return self.__getitem__(name)
         return super().__getattribute__(name)
 
+    # implement all the methods of dictionary
     def __getitem__(self, key: str):
         return self._info[key]
 
-    def get(self, key: str, default: Optional[Any] = None):
-        return self._info.get(key, default)
-
     def __str__(self):
         return self._info.__str__()
 
     def __repr__(self):
         return self._info.__repr__()
 
-    @use_callback()
-    def delete(self, key: Optional[Union[Sequence[str], str]] = None):
-        """Deletes a key or list of keys. If no key(s) is passed, all keys are deleted."""
-        self._cache.check_readonly()
-        if self._dataset is not None:
-            auto_checkout(self._dataset)
-        if key is None:
-            self._info.clear()
-        elif isinstance(key, str):
-            del self._info[key]
-        elif isinstance(key, Sequence):
-            for k in key:
-                del self._info[k]
-        else:
-            raise KeyError(key)
-
-    @use_callback()
-    def __setitem__(self, key: str, value):
-        self._cache.check_readonly()
-        if self._dataset is not None:
-            auto_checkout(self._dataset)
+    def __setitem__(self, key, value):
+        self.prepare_for_write()
         self._info[key] = value
+        self.end_write()
 
-    def __setattr__(self, key: str, value):
-        if key in {"_key", "_cache", "_info", "_dataset"}:
-            object.__setattr__(self, key, value)
-        else:
-            self[key] = value
+    def __delitem__(self, key):
+        self.prepare_for_write()
+        del self._info[key]
+        self.end_write()
 
-    def __getattr__(self, key: str):
+    def __contains__(self, key):
+        return key in self._info
+
+    def __iter__(self):
+        return iter(self._info)
+
+    def __getattr__(self, key):
         try:
             return object.__getattribute__(self, key)
         except AttributeError:
@@ -132,6 +79,60 @@ def __getattr__(self, key: str):
                 return self._info
             return self[key]
 
+    def __setattr__(self, key: str, value):
+        if key in {"_info", "_dataset", "is_dirty"}:
+            object.__setattr__(self, key, value)
+        else:
+            self.prepare_for_write()
+            self[key] = value
+            self.end_write()
+
+    def get(self, key, default=None):
+        return self._info.get(key, default)
+
+    def setdefault(self, key, default=None):
+        self.prepare_for_write()
+        ret = self._info.setdefault(key, default)
+        self.end_write()
+        return ret
+
+    def clear(self):
+        self.prepare_for_write()
+        self._info.clear()
+        self.end_write()
+
+    def pop(self, key, default=None):
+        self.prepare_for_write()
+        popped = self._info.pop(key, default)
+        self.end_write()
+        return popped
+
+    def popitem(self):
+        self.prepare_for_write()
+        popped = self._info.popitem()
+        self.end_write()
+        return popped
+
+    def update(self, *args, **kwargs):
+        self.prepare_for_write()
+        self._info.update(*args, **kwargs)
+        self.end_write()
+
+    def keys(self):
+        return self._info.keys()
+
+    def values(self):
+        return self._info.values()
+
+    def items(self):
+        return self._info.items()
+
+    def replace_with(self, d):
+        self.prepare_for_write()
+        self._info.clear()
+        self._info.update(d)
+        self.end_write()
+
     # the below methods are used by cloudpickle dumps
     def __origin__(self):
         return None
@@ -155,11 +156,8 @@ def __args__(self):
         return None
 
 
-def load_info(info_key: str, cache: LRUCache, dataset):
-    if info_key in cache:
-        info = cache.get_cachable(info_key, Info, callback_arg=dataset)
-    else:
-        info = Info()
-        info.initialize_callback_location(info_key, cache, dataset)
-
+def load_info(key, dataset):
+    storage = dataset.storage
+    info = storage.get_hub_object(key, Info) if key in storage else Info()
+    info._dataset = dataset
     return info

hub/api/tests/test_api.py

@@ -595,6 +595,9 @@ def test_like(local_path):
 
     src_ds.d.info.update(key=1)
 
+    assert src_ds.info.key == 0
+    assert src_ds.d.info.key == 1
+
     dest_ds = hub.like(dest_path, src_ds)
 
     assert tuple(dest_ds.tensors.keys()) == ("a", "b", "c", "d")

hub/api/tests/test_info.py

@@ -20,6 +20,7 @@ def test_dataset(local_ds_generator):
         test_list.extend(["user made change without `update`"])
 
     ds.info.update({"1_-+": 5})
+    assert len(ds.info) == 7
 
     ds = local_ds_generator()
 
@@ -41,13 +42,14 @@ def test_dataset(local_ds_generator):
     assert len(ds.info) == 7
     assert ds.info.test == [99]
 
-    ds.info.delete("test")
+    ds.info.pop("test")
     assert len(ds.info) == 6
 
-    ds.info.delete(["1_-+", "xyz"])
+    ds.info.pop("1_-+")
+    ds.info.pop("xyz")
     assert len(ds.info) == 4
 
-    ds.info.delete()
+    ds.info.clear()
     assert len(ds.info) == 0
 
 
@@ -89,13 +91,14 @@ def test_tensor(local_ds_generator):
 
     assert t1.info.key == 99
 
-    t2.info.delete("key")
+    t2.info.pop("key")
     assert len(t2.info) == 3
 
-    t2.info.delete(["key2", "key3"])
+    t2.info.pop("key2")
+    t2.info.pop("key3")
     assert len(t2.info) == 1
 
-    t2.info.delete()
+    t2.info.clear()
     assert len(t2.info) == 0
 
 
@@ -147,3 +150,44 @@ def test_class_label(local_ds_generator):
         ds.labels.info.class_names == ds.labels.info["class_names"] == ["c", "b", "a"]
     )
     assert ds.labels2.info.class_names == ds.labels2.info["class_names"] == []
+
+
+def test_info_new_methods(local_ds_generator):
+    ds = local_ds_generator()
+
+    ds.info[0] = "hello"
+    ds.info[1] = "world"
+    assert len(ds.info) == 2
+    assert set(ds.info.keys()) == {0, 1}
+    assert 0 in ds.info
+    assert 1 in ds.info
+
+    assert ds.info[0] == "hello"
+    assert ds.info[1] == "world"
+
+    del ds.info[0]
+    assert len(ds.info) == 1
+    assert 1 in ds.info
+    assert ds.info[1] == "world"
+
+    for it in ds.info:
+        assert it == 1
+
+    ds.info.setdefault(0, "yo")
+    assert len(ds.info) == 2
+    assert 0 in ds.info
+    assert 1 in ds.info
+    assert ds.info[0] == "yo"
+    assert ds.info[1] == "world"
+
+    ds.info.popitem()
+    assert len(ds.info) == 1
+    assert 1 in ds.info
+    assert ds.info[1] == "world"
+
+    for k, v in ds.info.items():
+        assert k == 1
+        assert v == "world"
+
+    for v in ds.info.values():
+        assert v == "world"

hub/core/chunk/base_chunk.py

@@ -20,7 +20,7 @@
     serialize_text,
     serialize_tensor,
 )
-from hub.core.storage.cachable import Cachable
+from hub.core.storage.hub_memory_object import HubMemoryObject
 from hub.core.tiling.sample_tiles import SampleTiles
 from hub.util.exceptions import TensorInvalidSampleShapeError
 
@@ -40,7 +40,7 @@
 SerializedOutput = Tuple[bytes, Tuple]
 
 
-class BaseChunk(Cachable):
+class BaseChunk(HubMemoryObject):
     def __init__(
         self,
         min_chunk_size: int,
@@ -51,6 +51,7 @@ def __init__(
         encoded_byte_positions: Optional[np.ndarray] = None,
         data: Optional[memoryview] = None,
     ):
+        super().__init__()
         self._data_bytes: Union[bytearray, bytes, memoryview] = data or bytearray()
         self.version = hub.__version__
         self.min_chunk_size = min_chunk_size
@@ -140,6 +141,7 @@ def frombuffer(cls, buffer: bytes, chunk_args: list, copy=True):  # type: ignore
         version, shapes, byte_positions, data = deserialize_chunk(buffer, copy=copy)
         chunk = cls(*chunk_args, shapes, byte_positions, data=data)  # type: ignore
         chunk.version = version
+        chunk.is_dirty = False
         return chunk
 
     @abstractmethod
@@ -163,6 +165,7 @@ def _make_data_bytearray(self):
     def prepare_for_write(self):
         ffw_chunk(self)
         self._make_data_bytearray()
+        self.is_dirty = True
 
     def register_sample_to_headers(
         self, incoming_num_bytes: Optional[int], sample_shape: Tuple[int]
@@ -259,7 +262,7 @@ def register_in_meta_and_headers(self, sample_nbytes: Optional[int], shape):
         """Registers a new sample in meta and headers"""
         self.register_sample_to_headers(sample_nbytes, shape)
         if self._update_tensor_meta_length:
-            self.tensor_meta.length += 1
+            self.tensor_meta.update_length(1)
         self.tensor_meta.update_shape_interval(shape)
 
     def update_in_meta_and_headers(
@@ -308,7 +311,7 @@ def write_tile(self, sample: SampleTiles):
         if sample.is_first_write:
             self.tensor_meta.update_shape_interval(sample.sample_shape)
             if self._update_tensor_meta_length:
-                self.tensor_meta.length += 1
+                self.tensor_meta.update_length(1)
 
     def _pop_sample(self):
         self.prepare_for_write()