Add support for spatial dataset

retrieverdash/dashboard_script/dashboard_script.py

@@ -1,13 +1,17 @@
 import json
 import os
 from multiprocessing import Pool
+from shutil import rmtree
+from tempfile import mkdtemp
 
 from filelock import FileLock
 from retriever import datasets
+from retriever import download
 
 from .status_dashboard_tools import get_dataset_md5
 from .status_dashboard_tools import diff_generator
 from .status_dashboard_tools import create_dirs
+from .status_dashboard_tools import dataset_type
 
 file_location = os.path.dirname(os.path.realpath(__file__))
 
@@ -25,10 +29,23 @@ def check_dataset(dataset):
             with open("dataset_details.json", 'w') as json_file:
                 dataset_detail = dict()
                 json.dump(dataset_detail, json_file)
-        md5 = get_dataset_md5(dataset)
-        if dataset.name not in dataset_detail \
-                or md5 != dataset_detail[dataset.name]['md5']:
-            diff = diff_generator(dataset)
+
+        if dataset_type(dataset) == 'spatial':
+            workdir = None
+            try:
+                workdir = mkdtemp(dir=file_location)
+                os.chdir(workdir)
+                download(dataset)
+            except Exception:
+                raise
+            finally:
+                if workdir:
+                    rmtree(workdir)
+        else:
+            md5 = get_dataset_md5(dataset)
+            if dataset.name not in dataset_detail \
+                    or md5 != dataset_detail[dataset.name]['md5']:
+                diff = diff_generator(dataset)
         status = True
     except Exception as e:
         reason = str(e)

retrieverdash/dashboard_script/status_dashboard_tools.py

@@ -146,3 +146,27 @@ def create_json(path="dataset_details.json"):
             data[dataset.name] = {"md5": get_dataset_md5(dataset)}
         with open(path, 'w') as json_file:
             json.dump(data, json_file, sort_keys=True, indent=4)
+
+
+def dataset_type(dataset):
+    """
+    Parameters
+    ----------
+    dataset : dataset script object
+
+    Returns
+    -------
+    str : The type of dataset.
+
+    Example
+    -------
+    >>> for dataset in datasets():
+    ...     if dataset.name=='aquatic-animal-excretion':
+    ...         print(dataset_type(dataset))
+    ...
+    tabular
+    """
+    for _, table_obj in dataset.tables.items():
+        if table_obj.dataset_type in ["RasterDataset", "VectorDataset"]:
+            return "spatial"
+    return "tabular"