Skip to content

Commit

Permalink
Accept ExtData path as input argument in S3 download script
Browse files Browse the repository at this point in the history
so it is easy to change root data path (for downloading to other systems)
  • Loading branch information
JiaweiZhuang committed Dec 16, 2018
1 parent 74dfdad commit 859fc7b
Show file tree
Hide file tree
Showing 8 changed files with 63 additions and 13 deletions.
7 changes: 6 additions & 1 deletion scripts/download_data/from_S3/CHEM_INPUTS.sh
Original file line number Diff line number Diff line change
@@ -1,6 +1,11 @@
#!/bin/bash

DATA_ROOT=~/ExtData
if [ "$1" ]; then
DATA_ROOT=$1
else
echo 'Must specify path to ExtData/ directory'
exit 1
fi

time aws s3 cp --request-payer=requester --recursive \
s3://gcgrid/GEOS_NATIVE $DATA_ROOT/CHEM_INPUTS \
Expand Down
7 changes: 6 additions & 1 deletion scripts/download_data/from_S3/GCHP-specific.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,12 @@
# Pull GCHP input data sample from S3
# Assume that data for GC-classic are already there (basic HEMCO data + metfield)

DATA_ROOT=~/ExtData
if [ "$1" ]; then
DATA_ROOT=$1
else
echo 'Must specify path to ExtData/ directory'
exit 1
fi

aws s3 cp --request-payer=requester --recursive \
s3://gcgrid/GEOSCHEM_RESTARTS/ $DATA_ROOT/GEOSCHEM_RESTARTS \
Expand Down
7 changes: 6 additions & 1 deletion scripts/download_data/from_S3/HEMCO.sh
Original file line number Diff line number Diff line change
@@ -1,6 +1,11 @@
#!/bin/bash

DATA_ROOT=~/ExtData
if [ "$1" ]; then
DATA_ROOT=$1
else
echo 'Must specify path to ExtData/ directory'
exit 1
fi

# exclude huge data directories that are not in use by default
# for CEDS, only get the most recent year
Expand Down
20 changes: 15 additions & 5 deletions scripts/download_data/from_S3/all_input_GC-classic.sh
Original file line number Diff line number Diff line change
@@ -1,7 +1,17 @@
#/bin/bash

./restart_file.sh
./metfields.sh
./CHEM_INPUTS.sh
./HEMCO.sh
./fix_GMI.sh
# Usage example:
# $ ./all_input_GC-classic.sh ~/ExtData

if [ "$1" ]; then
DATA_ROOT=$1
else
echo 'Must specify path to ExtData/ directory'
exit 1
fi

./restart_file.sh $DATA_ROOT
./metfields.sh $DATA_ROOT
./CHEM_INPUTS.sh $DATA_ROOT
./HEMCO.sh $DATA_ROOT
./fix_GMI.sh $DATA_ROOT
14 changes: 12 additions & 2 deletions scripts/download_data/from_S3/all_input_GCHP.sh
Original file line number Diff line number Diff line change
@@ -1,4 +1,14 @@
#/bin/bash

./all_input_GC-classic.sh
./GCHP-specific.sh
# Usage example:
# $ ./all_input_GCHP.sh ~/ExtData

if [ "$1" ]; then
DATA_ROOT=$1
else
echo 'Must specify path to ExtData/ directory'
exit 1
fi

./all_input_GC-classic.sh $DATA_ROOT
./GCHP-specific.sh $DATA_ROOT
7 changes: 6 additions & 1 deletion scripts/download_data/from_S3/fix_GMI.sh
Original file line number Diff line number Diff line change
@@ -1,7 +1,12 @@
#!/bin/bash
# HEMCO/GMI has some softlinks that are ignored by S3

DATA_ROOT=~/ExtData
if [ "$1" ]; then
DATA_ROOT=$1
else
echo 'Must specify path to ExtData/ directory'
exit 1
fi

cd $DATA_ROOT/HEMCO/GMI/v2015-02

Expand Down
7 changes: 6 additions & 1 deletion scripts/download_data/from_S3/metfields.sh
Original file line number Diff line number Diff line change
@@ -1,6 +1,11 @@
#!/bin/bash

DATA_ROOT=~/ExtData
if [ "$1" ]; then
DATA_ROOT=$1
else
echo 'Must specify path to ExtData/ directory'
exit 1
fi

# GEOSFP 4x5 CN field
aws s3 cp --request-payer=requester --recursive \
Expand Down
7 changes: 6 additions & 1 deletion scripts/download_data/from_S3/restart_file.sh
Original file line number Diff line number Diff line change
@@ -1,6 +1,11 @@
#!/bin/bash

DATA_ROOT=~/ExtData
if [ "$1" ]; then
DATA_ROOT=$1
else
echo 'Must specify path to ExtData/ directory'
exit 1
fi

aws s3 cp --request-payer=requester --recursive \
s3://gcgrid/GEOSCHEM_RESTARTS/ $DATA_ROOT/GEOSCHEM_RESTARTS \
Expand Down

0 comments on commit 859fc7b

Please sign in to comment.