Added an option for recursive downloads

data_loader_plugin/base.py

@@ -1,34 +1,46 @@
+#
+# File: https://github.com/cloud-helpers/python-plugin-data-loader/blob/main/data_loader_plugin/base.py
+# Authors: Stanislav Khrapov, Denis Arnaud
+#
+
 import logging
 from abc import ABC, abstractmethod
 from pathlib import Path
 from typing import Tuple
 
-
 class DataLoaderBase(ABC):
-    def __init__(self, local_path: Path, external_url: str) -> None:
+    def __init__(self, local_path: Path, external_url: str,
+                 recursive: bool=False) -> None:
         self._local_path = local_path
         self._external_url = external_url
+        self._recursive = recursive
         self._logger = logging.getLogger(self.__class__.__name__)
 
     def load(self) -> Tuple[bool, str]:
+        is_success = False
+
         try:
+            message = "Will try to create the destination directory on the " \
+                f"local file-system: {self._local_path}..."
+            self._logger.info(message)
+
             self._local_path.parent.mkdir(exist_ok=True)
-            self._logger.info(
-                f"Will try loading data from {self._external_url} to {self._local_path}."
-            )
+            f"Will try loading data from {self._external_url} to {self._local_path}..."
+            self._logger.info(message)
+
             self._load()
-            message = (
-                f"OK. Loaded data from {self._external_url} to {self._local_path}."
-            )
+            message = "Done. The data has been downloaded from " \
+                f"{self._external_url} to {self._local_path}."
             self._logger.info(message)
-            return True, message
+            is_success = True
+
         except Exception as error:
-            message = (
-                f"Failed to load data from {self._external_url} to {self._local_path} "
-                f"with {error.__class__}({error})."
-            )
+            message = f"Failed to download data from {self._external_url} " \
+                f"to {self._local_path} with {error.__class__} ({error})."
             self._logger.info(message)
-            return False, message
+            is_success = False
+
+        return is_success, message
 
     @abstractmethod
     def _load(self) -> None:

setup.py

@@ -8,7 +8,7 @@
 
 setuptools.setup(
     name='data-loader-plugin',
-    version='0.0.1.post1',
+    version='0.0.2',
     license='MIT',
     description='Python plugin/extra to load data files from an external source (such as AWS S3) to a local directory',
     long_description_content_type='text/markdown',