Adjust DAGs for handling pathlib objects in XCOMs

sul-dlss · May 15, 2024 · bc59b08 · bc59b08
1 parent 2e34333
commit bc59b08
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 4 deletions.
diff --git a/libsys_airflow/dags/vendor/data_fetcher.py b/libsys_airflow/dags/vendor/data_fetcher.py
@@ -61,14 +61,17 @@
     def setup():
         context = get_current_context()
         params = context["params"]
-        params["download_path"] = download_path(
+        dag_run_download_path = download_path(
             params["vendor_uuid"], params["vendor_interface_uuid"]
         )
         params["environment"] = os.getenv('HONEYBADGER_ENVIRONMENT', 'development')
 
-        logger.info(f"Params are {params}")
+        dag_run_download_path.mkdir(exist_ok=True)
+
+        # XCOM cannot serialize pathlib Path object
+        params["download_path"] = str(dag_run_download_path)
 
-        os.makedirs(params["download_path"], exist_ok=True)
+        logger.info(f"Params are {params}")
 
         return params
 

diff --git a/libsys_airflow/dags/vendor/default_data_processor.py b/libsys_airflow/dags/vendor/default_data_processor.py
@@ -125,7 +125,9 @@ def setup():
             params["archive_regex"] = processing_options.get("archive_regex")
 
         logger.info(f"Params are {params}")
-        assert os.path.exists(os.path.join(params["download_path"], params["filename"]))
+        assert (params["download_path"] / params["filename"]).exists()
+        # XCOM cannot serialize pathlib Paths
+        params["download_path"] = str(params["download_path"])
 
         return params