diff --git a/datasets/covid19_italy/pipelines/data_by_province/data_by_province_dag.py b/datasets/covid19_italy/pipelines/data_by_province/data_by_province_dag.py index f21aefdb3..8354365a1 100644 --- a/datasets/covid19_italy/pipelines/data_by_province/data_by_province_dag.py +++ b/datasets/covid19_italy/pipelines/data_by_province/data_by_province_dag.py @@ -84,7 +84,7 @@ # Task to copy bq uploadable data file to bucket in EU copy_data_file_EU = bash.BashOperator( task_id="copy_data_file_EU", - bash_command="gsutil cp gs://{{ var.value.composer_bucket }}/data/covid19_italy/data_by_province/data_output.csv gs://public-datasets-dev-covid19-italy-eu/province/", + bash_command="gsutil cp gs://{{ var.value.composer_bucket }}/data/covid19_italy/data_by_province/data_output.csv {{ var.json.covid19_italy.destination_gcs_province }}", ) # Task to load CSV data to a BigQuery table diff --git a/datasets/covid19_italy/pipelines/data_by_province/pipeline.yaml b/datasets/covid19_italy/pipelines/data_by_province/pipeline.yaml index 9113647c0..579ed8443 100644 --- a/datasets/covid19_italy/pipelines/data_by_province/pipeline.yaml +++ b/datasets/covid19_italy/pipelines/data_by_province/pipeline.yaml @@ -115,7 +115,7 @@ dag: description: "Task to copy bq uploadable data file to bucket in EU" args: task_id: "copy_data_file_EU" - bash_command: "gsutil cp gs://{{ var.value.composer_bucket }}/data/covid19_italy/data_by_province/data_output.csv gs://public-datasets-dev-covid19-italy-eu/province/" + bash_command: "gsutil cp gs://{{ var.value.composer_bucket }}/data/covid19_italy/data_by_province/data_output.csv {{ var.json.covid19_italy.destination_gcs_province }}" - operator: "GoogleCloudStorageToBigQueryOperator" description: "Task to load CSV data to a BigQuery table" diff --git a/datasets/covid19_italy/pipelines/data_by_region/data_by_region_dag.py b/datasets/covid19_italy/pipelines/data_by_region/data_by_region_dag.py index 886ea17c8..53852cff7 100644 --- a/datasets/covid19_italy/pipelines/data_by_region/data_by_region_dag.py +++ b/datasets/covid19_italy/pipelines/data_by_region/data_by_region_dag.py @@ -115,7 +115,7 @@ # Task to copy bq uploadable data file to bucket in EU copy_data_file_EU = bash.BashOperator( task_id="copy_data_file_EU", - bash_command="gsutil cp gs://{{ var.value.composer_bucket }}/data/covid19_italy/data_by_region/data_output.csv gs://public-datasets-dev-covid19-italy-eu/region/", + bash_command="gsutil cp gs://{{ var.value.composer_bucket }}/data/covid19_italy/data_by_region/data_output.csv {{ var.json.covid19_italy.destination_gcs_region }}", ) # Task to load CSV data to a BigQuery table diff --git a/datasets/covid19_italy/pipelines/data_by_region/pipeline.yaml b/datasets/covid19_italy/pipelines/data_by_region/pipeline.yaml index 545c1e5ac..5f930c8f8 100644 --- a/datasets/covid19_italy/pipelines/data_by_region/pipeline.yaml +++ b/datasets/covid19_italy/pipelines/data_by_region/pipeline.yaml @@ -135,7 +135,7 @@ dag: description: "Task to copy bq uploadable data file to bucket in EU" args: task_id: "copy_data_file_EU" - bash_command: "gsutil cp gs://{{ var.value.composer_bucket }}/data/covid19_italy/data_by_region/data_output.csv gs://public-datasets-dev-covid19-italy-eu/region/" + bash_command: "gsutil cp gs://{{ var.value.composer_bucket }}/data/covid19_italy/data_by_region/data_output.csv {{ var.json.covid19_italy.destination_gcs_region }}" - operator: "GoogleCloudStorageToBigQueryOperator" description: "Task to load CSV data to a BigQuery table" diff --git a/datasets/covid19_italy/pipelines/national_trends/national_trends_dag.py b/datasets/covid19_italy/pipelines/national_trends/national_trends_dag.py index 6f5748465..92757b753 100644 --- a/datasets/covid19_italy/pipelines/national_trends/national_trends_dag.py +++ b/datasets/covid19_italy/pipelines/national_trends/national_trends_dag.py @@ -110,7 +110,7 @@ # Task to copy bq uploadable data file to bucket in EU copy_data_file_EU = bash.BashOperator( task_id="copy_data_file_EU", - bash_command="gsutil cp gs://{{ var.value.composer_bucket }}/data/covid19_italy/national_trends/data_output.csv gs://public-datasets-dev-covid19-italy-eu/national/", + bash_command="gsutil cp gs://{{ var.value.composer_bucket }}/data/covid19_italy/national_trends/data_output.csv {{ var.json.covid19_italy.destination_gcs_national_trends }}", ) # Task to load CSV data to a BigQuery table diff --git a/datasets/covid19_italy/pipelines/national_trends/pipeline.yaml b/datasets/covid19_italy/pipelines/national_trends/pipeline.yaml index 855911b94..c4ec72c91 100644 --- a/datasets/covid19_italy/pipelines/national_trends/pipeline.yaml +++ b/datasets/covid19_italy/pipelines/national_trends/pipeline.yaml @@ -121,7 +121,7 @@ dag: description: "Task to copy bq uploadable data file to bucket in EU" args: task_id: "copy_data_file_EU" - bash_command: "gsutil cp gs://{{ var.value.composer_bucket }}/data/covid19_italy/national_trends/data_output.csv gs://public-datasets-dev-covid19-italy-eu/national/" + bash_command: "gsutil cp gs://{{ var.value.composer_bucket }}/data/covid19_italy/national_trends/data_output.csv {{ var.json.covid19_italy.destination_gcs_national_trends }}" - operator: "GoogleCloudStorageToBigQueryOperator" description: "Task to load CSV data to a BigQuery table"