[AL-2322] Improved dataset directory validation

.gitignore

@@ -132,6 +132,7 @@ wandb/
 *Python-3.7*
 *mem:/*
 hub_pytest/
+deeplake/test-dataset
 
 # Translations
 *.mo
@@ -199,6 +200,7 @@ logs/
 .vscode/
 .creds/
 .idea/
+*.iml
 .nvimrc
 .vimrc
 waymo/
@@ -213,5 +215,8 @@ benchmarks/hub_data
 benchmarks/torch_data
 .benchmarks/
 
+deeplake/test-dataset
+
 # API docs
 api_docs/
+

deeplake/util/keys.py

@@ -183,9 +183,16 @@ def get_sequence_encoder_key(key: str, commit_id: str) -> str:
 
 
 def dataset_exists(storage) -> bool:
+    """
+    Returns true if a dataset exists at the given location.
+    NOTE: This does not verify if it is a VALID dataset, only that it exists and is likely a deeplake directory.
+    To verify the content, use :func:`dataset_valid`
+    """
     try:
-        storage[get_dataset_meta_key(FIRST_COMMIT_ID)]
-        return True
+        return (
+            get_dataset_meta_key(FIRST_COMMIT_ID) in storage
+            or get_version_control_info_key() in storage
+        )
     except S3GetAccessError as err:
         raise AuthorizationException("The dataset storage cannot be accessed") from err
     except (KeyError, S3GetError) as err:

deeplake/util/tests/test_keys.py

@@ -0,0 +1,21 @@
+import deeplake
+from deeplake.util.keys import dataset_exists
+
+
+def test_dataset_exists():
+    ds = deeplake.dataset("mem://x")
+    assert dataset_exists(ds.storage)
+
+    # Single files missing is fine
+    del ds.storage["version_control_info.json"]
+    assert dataset_exists(ds.storage)
+
+    ds = deeplake.dataset("mem://x")
+    del ds.storage["dataset_meta.json"]
+    assert dataset_exists(ds.storage)
+
+    # Enough files are missing and it's no longer valid
+    ds = deeplake.dataset("mem://x")
+    del ds.storage["dataset_meta.json"]
+    del ds.storage["version_control_info.json"]
+    assert not dataset_exists(ds.storage)