feat(pipeline): use gsutil cp instead of rsync

ubilabs · Mar 14, 2024 · ee4e576 · ee4e576
1 parent bf97d69
commit ee4e576
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/pipeline/dags/task_factories.py b/pipeline/dags/task_factories.py
@@ -90,10 +90,10 @@ def fn(filename: str):
 def gcloud_upload_dir(layer_id: str, layer_variable: str, directory: str):
     return BashOperator(
         task_id='gcloud_upload',
-        bash_command='gcloud auth activate-service-account --key-file $KEY_FILE && gsutil -m rsync -d -r $UPLOAD_DIR $BUCKET',
+        bash_command='gcloud auth activate-service-account --key-file $KEY_FILE && gsutil -q -m cp -r $UPLOAD_DIR/* $BUCKET',
         env={
             "UPLOAD_DIR": directory,
-            "BUCKET": 'gs://{{ dag_run.conf["output_bucket"] }}/{{ dag_run.conf["layer_version"] }}/' + f'{layer_id}.{layer_variable}',
+            "BUCKET": 'gs://{{ dag_run.conf["output_bucket"] }}/{{ dag_run.conf["layer_version"] }}/' + f'{layer_id}.{layer_variable}/',
             "KEY_FILE": '/opt/airflow/plugins/service-account.json',
             "CLOUDSDK_PYTHON": '/usr/local/bin/python'
         }