Avoid VDB Index copy with 'like' API

deeplake/core/dataset/dataset.py

@@ -1168,6 +1168,8 @@ def create_tensor_like(
         del meta["version"]
         del meta["name"]
         del meta["links"]
+        if "vdb_indexes" in meta:
+            del meta["vdb_indexes"]
         meta["dtype"] = np.dtype(meta["typestr"]) if meta["typestr"] else meta["dtype"]
 
         destination_tensor = self._create_tensor(

deeplake/core/index_maintenance.py

@@ -134,7 +134,9 @@
         if not below_threshold:
             return INDEX_OP_TYPE.CREATE_INDEX
     else:
-        if (
+        if changed_data_len == 0:
+            return INDEX_OP_TYPE.NOOP
+        elif (
             not index_regeneration
             and check_index_params(self)
             and check_incr_threshold(len(self.dataset), changed_data_len)
@@ -157,7 +159,7 @@
         if not below_threshold:
             return INDEX_OP_TYPE.CREATE_INDEX
 
-    if not check_vdb_indexes(self):
+    if not check_vdb_indexes(self) or changed_data_len == 0:
         return INDEX_OP_TYPE.NOOP
 
     if index_delete:

deeplake/core/tensor.py

@@ -350,9 +350,10 @@ def extend(
         Raises:
             TensorDtypeMismatchError: Dtype for array must be equal to or castable to this tensor's dtype.
         """
+        old_dataset_length = self.num_samples
         self._extend(samples, progressbar=progressbar, ignore_errors=ignore_errors)
         if index_maintenance.validate_embedding_tensor(self):
-            row_ids = list(range(self.num_samples, self.num_samples + len(samples)))
+            row_ids = list(range(old_dataset_length, old_dataset_length + len(samples)))
             index_maintenance.index_operation_dataset(  # TODO: this might pick the wrong tensor when we support
                 self.dataset,  #       index for multiple tensors in the future
                 dml_type=_INDEX_OPERATION_MAPPING["ADD"],

deeplake/core/vectorstore/test_deeplake_vectorstore.py

@@ -1789,6 +1789,60 @@ def test_vdb_index_incr_maint_tensor_append(local_path, capsys, hub_cloud_dev_to
 
     vector_store.delete_by_path(local_path, token=ds.token)
 
+@requires_libdeeplake
+def test_vdb_index_like(local_path, capsys, hub_cloud_dev_token):
+    number_of_data = 1000
+    texts, embeddings, ids, metadatas, _ = utils.create_data(
+        number_of_data=number_of_data, embedding_dim=EMBEDDING_DIM
+    )
+
+    # initialize vector store object with vdb index threshold as 200.
+    vector_store = DeepLakeVectorStore(
+        path=local_path,
+        overwrite=True,
+        verbose=True,
+        exec_option="compute_engine",
+        index_params={"threshold": 200, "distance_metric": "L2"},
+        token=hub_cloud_dev_token,
+    )
+
+    vector_store.add(embedding=embeddings, text=texts, id=ids, metadata=metadatas)
+
+    assert len(vector_store) == number_of_data
+    assert set(vector_store.dataset.tensors) == set(
+        [
+            "embedding",
+            "id",
+            "metadata",
+            "text",
+        ]
+    )
+    assert set(vector_store.tensors()) == set(
+        [
+            "embedding",
+            "id",
+            "metadata",
+            "text",
+        ]
+    )
+
+    # Check if the index is recreated properly.
+    ds = vector_store.dataset
+    es = ds.embedding.get_vdb_indexes()
+    assert len(es) == 1
+    assert es[0]["id"] == "hnsw_1"
+    assert es[0]["distance"] == "l2_norm"
+    assert es[0]["type"] == "hnsw"
+
+    ds = deeplake.load(path=local_path, read_only=True)
+
+    ds2 = deeplake.like('mem://dummy', ds, overwrite=True)
+
+    for tensor in ds2.tensors:
+        ds2[tensor].extend(ds[tensor].data()['value'])
+
+    vector_store.delete_by_path(local_path, token=ds.token)
+
 
 def assert_vectorstore_structure(vector_store, number_of_data):
     assert len(vector_store) == number_of_data