diff --git a/pipeline/dags/task_factories.py b/pipeline/dags/task_factories.py index d384aac0f..11c94173f 100644 --- a/pipeline/dags/task_factories.py +++ b/pipeline/dags/task_factories.py @@ -90,10 +90,10 @@ def fn(filename: str): def gcloud_upload_dir(layer_id: str, layer_variable: str, directory: str): return BashOperator( task_id='gcloud_upload', - bash_command='gcloud auth activate-service-account --key-file $KEY_FILE && gsutil -m rsync -d -r $UPLOAD_DIR $BUCKET', + bash_command='gcloud auth activate-service-account --key-file $KEY_FILE && gsutil -q -m cp -r $UPLOAD_DIR/* $BUCKET', env={ "UPLOAD_DIR": directory, - "BUCKET": 'gs://{{ dag_run.conf["output_bucket"] }}/{{ dag_run.conf["layer_version"] }}/' + f'{layer_id}.{layer_variable}', + "BUCKET": 'gs://{{ dag_run.conf["output_bucket"] }}/{{ dag_run.conf["layer_version"] }}/' + f'{layer_id}.{layer_variable}/', "KEY_FILE": '/opt/airflow/plugins/service-account.json', "CLOUDSDK_PYTHON": '/usr/local/bin/python' }