[AL-2322] Improved dataset directory validation

.gitignore

@@ -199,6 +199,7 @@ logs/
 .vscode/
 .creds/
 .idea/
+*.iml
 .nvimrc
 .vimrc
 waymo/
@@ -213,5 +214,8 @@ benchmarks/hub_data
 benchmarks/torch_data
 .benchmarks/
 
+deeplake/test-dataset
+
 # API docs
 api_docs/
+

deeplake/core/dataset/dataset.py

@@ -107,6 +107,7 @@
 )
 from deeplake.util.keys import (
     dataset_exists,
+    dataset_validate,
     get_dataset_info_key,
     get_dataset_meta_key,
     tensor_exists,
@@ -1760,6 +1761,7 @@ def diff(
     def _populate_meta(self, verbose=True):
         """Populates the meta information for the dataset."""
         if dataset_exists(self.storage):
+            dataset_validate(self.storage)
             load_meta(self)
 
         elif not self.storage.empty():

deeplake/util/keys.py

@@ -29,6 +29,7 @@
     S3GetError,
     S3GetAccessError,
     AuthorizationException,
+    DatasetCorruptError,
 )
 
 
@@ -183,15 +184,36 @@ def get_sequence_encoder_key(key: str, commit_id: str) -> str:
 
 
 def dataset_exists(storage) -> bool:
+    """
+    Returns true if a dataset exists at the given location.
+    NOTE: This does not verify if it is a VALID dataset, only that it exists and is likely a deeplake directory.
+    To verify the content, use :func:`dataset_valid`
+    """
     try:
-        storage[get_dataset_meta_key(FIRST_COMMIT_ID)]
-        return True
+        return (
+                get_dataset_meta_key(FIRST_COMMIT_ID) in storage
+                or get_version_control_info_key() in storage
+        )
     except S3GetAccessError as err:
         raise AuthorizationException("The dataset storage cannot be accessed") from err
-    except (KeyError, S3GetError) as err:
+    except S3GetError:
         return False
 
 
+def dataset_validate(storage) -> None:
+    """
+    Checks the structure of the dataset and throws a descriptive DatasetCorruptError for any problems.
+    """
+    try:
+        for file in [get_dataset_meta_key(FIRST_COMMIT_ID), get_version_control_info_key()]:
+            if file not in storage:
+                raise DatasetCorruptError(f"Invalid dataset: missing file {file}")
+    except S3GetAccessError as err:
+        raise AuthorizationException("The dataset storage cannot be accessed") from err
+    except S3GetError:
+        raise DatasetCorruptError("Error reading from S3")
+
+
 def tensor_exists(key: str, storage, commit_id: str) -> bool:
     try:
         storage[get_tensor_meta_key(key, commit_id)]