diff --git a/scripts/runManifestPull.sh b/scripts/runManifestPull.sh old mode 100644 new mode 100755 index 6f43143..b0c4bf4 --- a/scripts/runManifestPull.sh +++ b/scripts/runManifestPull.sh @@ -17,9 +17,13 @@ export MY_VENV=~/pyVenvForThree export PYTHONPATH=.:${MY_VENV}/lib +TABLE='your-project-id.your-dataset.your-manifest-table' # BQ table with your manifest +TARG_DIR='/path-to-your-home-dir/destination' # Has to be on a filesystem with enough sapce +PAYING='your-project-id' # Needed for requester pays though it is free to crossload to a cloud VM + cd ~ pushd ${MY_VENV} > /dev/null source bin/activate popd > /dev/null -python3 ~/IDC-Examples/scripts/pullManifestToVM.py +python3 ~/IDC-Examples/scripts/pullManifestToVM.py --table ${TABLE} --destination ${TARG_DIR} --paying ${PAYING} deactivate diff --git a/scripts/setupVM.sh b/scripts/setupVM.sh index 09d15d4..700ac59 100755 --- a/scripts/setupVM.sh +++ b/scripts/setupVM.sh @@ -59,5 +59,7 @@ if [ ! -d "${HOME}/pyVenvForThree" ]; then python3 -m pip install google-api-python-client python3 -m pip install google-cloud-storage python3 -m pip install google-cloud-bigquery + python3 -m pip install hurry.filesize + python3 -m pip install tqdm deactivate fi