fix: Resolved reference to destination bucket causing failure in prod…

…uction. (#507)
GoogleCloudPlatform · Oct 13, 2022 · 69128bc · 69128bc
1 parent 8d02866
commit 69128bc
Show file tree

Hide file tree

Showing 6 changed files with 6 additions and 6 deletions.
diff --git a/datasets/covid19_italy/pipelines/data_by_province/data_by_province_dag.py b/datasets/covid19_italy/pipelines/data_by_province/data_by_province_dag.py
@@ -84,7 +84,7 @@
     # Task to copy bq uploadable data file to bucket in EU
     copy_data_file_EU = bash.BashOperator(
         task_id="copy_data_file_EU",
-        bash_command="gsutil cp gs://{{ var.value.composer_bucket }}/data/covid19_italy/data_by_province/data_output.csv gs://public-datasets-dev-covid19-italy-eu/province/",
+        bash_command="gsutil cp gs://{{ var.value.composer_bucket }}/data/covid19_italy/data_by_province/data_output.csv {{ var.json.covid19_italy.destination_gcs_province }}",
     )
 
     # Task to load CSV data to a BigQuery table

diff --git a/datasets/covid19_italy/pipelines/data_by_province/pipeline.yaml b/datasets/covid19_italy/pipelines/data_by_province/pipeline.yaml
@@ -115,7 +115,7 @@ dag:
       description: "Task to copy bq uploadable data file to bucket in EU"
       args:
         task_id: "copy_data_file_EU"
-        bash_command: "gsutil cp gs://{{ var.value.composer_bucket }}/data/covid19_italy/data_by_province/data_output.csv gs://public-datasets-dev-covid19-italy-eu/province/"
+        bash_command: "gsutil cp gs://{{ var.value.composer_bucket }}/data/covid19_italy/data_by_province/data_output.csv {{ var.json.covid19_italy.destination_gcs_province }}"
 
     - operator: "GoogleCloudStorageToBigQueryOperator"
       description: "Task to load CSV data to a BigQuery table"

diff --git a/datasets/covid19_italy/pipelines/data_by_region/data_by_region_dag.py b/datasets/covid19_italy/pipelines/data_by_region/data_by_region_dag.py
@@ -115,7 +115,7 @@
     # Task to copy bq uploadable data file to bucket in EU
     copy_data_file_EU = bash.BashOperator(
         task_id="copy_data_file_EU",
-        bash_command="gsutil cp gs://{{ var.value.composer_bucket }}/data/covid19_italy/data_by_region/data_output.csv gs://public-datasets-dev-covid19-italy-eu/region/",
+        bash_command="gsutil cp gs://{{ var.value.composer_bucket }}/data/covid19_italy/data_by_region/data_output.csv {{ var.json.covid19_italy.destination_gcs_region }}",
     )
 
     # Task to load CSV data to a BigQuery table

diff --git a/datasets/covid19_italy/pipelines/data_by_region/pipeline.yaml b/datasets/covid19_italy/pipelines/data_by_region/pipeline.yaml
@@ -135,7 +135,7 @@ dag:
       description: "Task to copy bq uploadable data file to bucket in EU"
       args:
         task_id: "copy_data_file_EU"
-        bash_command: "gsutil cp gs://{{ var.value.composer_bucket }}/data/covid19_italy/data_by_region/data_output.csv gs://public-datasets-dev-covid19-italy-eu/region/"
+        bash_command: "gsutil cp gs://{{ var.value.composer_bucket }}/data/covid19_italy/data_by_region/data_output.csv {{ var.json.covid19_italy.destination_gcs_region }}"
 
     - operator: "GoogleCloudStorageToBigQueryOperator"
       description: "Task to load CSV data to a BigQuery table"

diff --git a/datasets/covid19_italy/pipelines/national_trends/national_trends_dag.py b/datasets/covid19_italy/pipelines/national_trends/national_trends_dag.py
@@ -110,7 +110,7 @@
     # Task to copy bq uploadable data file to bucket in EU
     copy_data_file_EU = bash.BashOperator(
         task_id="copy_data_file_EU",
-        bash_command="gsutil cp gs://{{ var.value.composer_bucket }}/data/covid19_italy/national_trends/data_output.csv gs://public-datasets-dev-covid19-italy-eu/national/",
+        bash_command="gsutil cp gs://{{ var.value.composer_bucket }}/data/covid19_italy/national_trends/data_output.csv {{ var.json.covid19_italy.destination_gcs_national_trends }}",
     )
 
     # Task to load CSV data to a BigQuery table

diff --git a/datasets/covid19_italy/pipelines/national_trends/pipeline.yaml b/datasets/covid19_italy/pipelines/national_trends/pipeline.yaml
@@ -121,7 +121,7 @@ dag:
       description: "Task to copy bq uploadable data file to bucket in EU"
       args:
         task_id: "copy_data_file_EU"
-        bash_command: "gsutil cp gs://{{ var.value.composer_bucket }}/data/covid19_italy/national_trends/data_output.csv gs://public-datasets-dev-covid19-italy-eu/national/"
+        bash_command: "gsutil cp gs://{{ var.value.composer_bucket }}/data/covid19_italy/national_trends/data_output.csv {{ var.json.covid19_italy.destination_gcs_national_trends }}"
 
     - operator: "GoogleCloudStorageToBigQueryOperator"
       description: "Task to load CSV data to a BigQuery table"