set source_objects to match all CSV files

GoogleCloudPlatform · Jun 24, 2021 · 3c83177 · 3c83177
1 parent f309da6
commit 3c83177
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/datasets/bls/cpsaat18/cpsaat18_dag.py b/datasets/bls/cpsaat18/cpsaat18_dag.py
@@ -36,7 +36,7 @@
     load_csv_to_bq = gcs_to_bq.GoogleCloudStorageToBigQueryOperator(
         task_id="load_csv_to_bq",
         bucket="{{ var.json.bls.source_bucket }}",
-        source_objects=["cpsaat18/2020.csv"],
+        source_objects=["cpsaat18/*.csv"],
         source_format="CSV",
         destination_project_dataset_table="bls.cpsaat18",
         skip_leading_rows=1,

diff --git a/datasets/bls/cpsaat18/pipeline.yaml b/datasets/bls/cpsaat18/pipeline.yaml
@@ -36,7 +36,7 @@ dag:
       args:
         task_id: "load_csv_to_bq"
         bucket: "{{ var.json.bls.source_bucket }}"
-        source_objects: ["cpsaat18/2020.csv"]
+        source_objects: ["cpsaat18/*.csv"]
         source_format: "CSV"
         destination_project_dataset_table: "bls.cpsaat18"
         skip_leading_rows: 1