make gsutil rsync recursive (#297)

* make gsutil rsync recursive * reformatted with black * add threaded flag for gsutil rsync
drivendataorg · Mar 16, 2023 · 4d04432 · 4d04432
1 parent 851c922
commit 4d04432
Show file tree

Hide file tree

Showing 3 changed files with 2 additions and 4 deletions.
diff --git a/ccds/monkey_patch.py b/ccds/monkey_patch.py
@@ -97,7 +97,6 @@ def prompt_for_config(context, no_input=False):
 
     # Second pass; handle the dictionaries.
     for key, raw in context["cookiecutter"].items():
-
         try:
             if isinstance(raw, dict):
                 # We are dealing with a dict variable

diff --git a/tests/test_creation.py b/tests/test_creation.py
@@ -131,7 +131,6 @@ def verify_makefile_commands(root, config):
     elif config["environment_manager"] == "virtualenv":
         harness_path = test_path / "virtualenv_harness.sh"
     elif config["environment_manager"] == "pipenv":
-
         harness_path = test_path / "pipenv_harness.sh"
     elif config["environment_manager"] == "none":
         return True

diff --git a/{{ cookiecutter.repo_name }}/Makefile b/{{ cookiecutter.repo_name }}/Makefile
@@ -53,7 +53,7 @@ sync_data_down:
 	az storage blob download-batch -s {{ cookiecutter.dataset_storage.azure.container }}/data/ \
 		-d data/
 	{% elif cookiecutter.dataset_storage.gcs -%}
-	gsutil rsync gs://{{ cookiecutter.dataset_storage.gcs.bucket }}/data/ data/
+	gsutil -m rsync -r gs://{{ cookiecutter.dataset_storage.gcs.bucket }}/data/ data/
 	{% endif %}
 
 ## Upload Data to storage system
@@ -65,7 +65,7 @@ sync_data_up:
 	az storage blob upload-batch -d {{ cookiecutter.dataset_storage.azure.container }}/data/ \
 		-s data/
 	{% elif cookiecutter.dataset_storage.gcs -%}
-	gsutil rsync data/ gs://{{ cookiecutter.dataset_storage.gcs.bucket }}/data/
+	gsutil -m rsync -r data/ gs://{{ cookiecutter.dataset_storage.gcs.bucket }}/data/
 	{% endif %}
 {% endif %}