add recursive upstream in previous build download

duck_jenkins/_app.py

@@ -79,6 +79,11 @@ def pull_upstream(
                     artifact=artifact
                 )
                 if not recursive or not files[0]:
+                    logging.info(
+                        "Upstream recursive pull exit: [recursive=%s], [file_exist=%s]",
+                        recursive,
+                        files[0]
+                    )
                     break
                 json_file = files[0]
             else:
@@ -91,6 +96,7 @@ def pull_previous(
             build_number: int,
             overwrite: bool,
             artifact: bool,
+            upstream: bool = False,
             trial=5,
             size=0,
     ):
@@ -101,6 +107,7 @@ def pull_previous(
         :param build_number: build number from the job to start pulling backward
         :param overwrite: True to re-download and replace the existing file
         :param artifact: True to include artefact metadata download
+        :param upstream: True to include upstream build download
         :param trial: When a build is unavailable(404), we don't know it is skipped or deleted
         :param size: Total previous build info to be downloaded
         :return:
@@ -124,6 +131,14 @@ def pull_previous(
                 logging.info('Build exist with remaining trial: %s', trial)
                 if trial == 0:
                     break
+            elif upstream:
+                self.pull_upstream(
+                    project_name=project_name,
+                    build_number=previous_build,
+                    overwrite=overwrite,
+                    artifact=artifact,
+                    recursive=True
+                )
             previous_build -= 1
             counter += 1
             if size < counter:
@@ -224,7 +239,7 @@ def _pull(
         :param data_directory: File system directory to store the downloaded data
         :param artifact: True to include artefact metadata download
         :param overwrite: True to re-download and replace the existing file
-        :return:
+        :return pulled json file, pulled artifact file, is build exist prior pull
         """
         json_file = get_json_file(
             data_directory,
@@ -265,14 +280,14 @@ def pull(
             build_number: int,
             artifact: bool = False,
             overwrite: bool = False,
-    ):
+    ) -> Tuple[str, str, bool]:
         """
         Download build information and artefacts metadata as json file and csv file
         :param project_name: Jenkins's Job name
         :param build_number: build number from the job
         :param artifact: True to include artefact metadata download
         :param overwrite: True to re-download and replace the existing file
-        :return:
+        :return pulled json file, pulled artifact file, is build exist prior pull
         """
         json_file = get_json_file(self.data_directory, self.domain_name, project_name, build_number)
         logging.info('Overwrite: %s', overwrite)
@@ -300,7 +315,7 @@ def __init__(self, cursor: DuckDBPyConnection, jenkins_data_directory: str = '.'
         """
 
         :param cursor: DuckDB connection's cursor
-        :param jenkins_data_directory: directory where all the extracted jenkins' data located.
+        :param jenkins_data_directory: directory where all the extracted jenkins data located.
         """
         self.cursor = cursor
         self.data_directory = jenkins_data_directory
@@ -318,7 +333,7 @@ def insert_build(
         into a DuckDB
         :param job_dir: Job directory contains json files and csv files
         :param jenkins_domain_name: identify which jenkins server to import
-        :param data_dir: JenkinsData extracted root directory which contains all jenkins' server domain names
+        :param data_dir: JenkinsData extracted root directory which contains all jenkins server domain names
         :param cursor: DuckDB connection's cursor
         :param overwrite: True to re-insert, False to skip when a record is existed
         :return:

setup.py

@@ -5,7 +5,7 @@
 
 setuptools.setup(
     name="duck-jenkins",
-    version="0.0.19",
+    version="0.0.20",
     install_requires=[
         "duckdb",
         "pandas",

tests/test_jenkins.py

@@ -196,3 +196,20 @@ def test_pull_previous_build_overwrite(jenkins_data):
     )
     _dir = f"{jenkins_data.data_directory}/{jenkins_data.domain_name}/{FEATURE_BRANCH_PROJECT}"
     assert len(glob.glob(f"{_dir}/*")) == 3
+
+
+@responses.activate
+@pytest.mark.parametrize("size", [0, 2])
+@pytest.mark.parametrize("overwrite", [False, True])
+def test_pull_previous_build_with_upstream_recursive(jenkins_data, size, overwrite):
+    init_responses(jenkins_data.domain_name)
+    jenkins_data.pull_previous(
+        project_name=FEATURE_BRANCH_PROJECT,
+        build_number=3,
+        artifact=False,
+        overwrite=True,
+        upstream=True,
+        size=size
+    )
+    _dir = f"{jenkins_data.data_directory}/{jenkins_data.domain_name}/{FEATURE_BRANCH_PROJECT}"
+    assert len(glob.glob(f"{_dir}/*")) == 2