Skip to content

Commit

Permalink
Resource updates to support WCOSS2 (#1070)
Browse files Browse the repository at this point in the history 
* Add WCOSS2 BASE_CPLIC to config.coupled_ic
* Remove errant ) from WCOSS2 hosts file for COMINsyn
* Updates to config.fv3 from WCOSS2 testing
* Add "is_exclusive" setting to config.resources
* Add WCOSS2 to machine npe_node_max check in config.resources
* Update resources in config.resources
* Update build_ufs.sh to set S2SW as default app on WCOSS2
* Set hpssarch to NO by default on WCOSS2

Refs #419
  • Loading branch information
@KateFriedman-NOAA
KateFriedman-NOAA committed Oct 24, 2022
1 parent d01de06 commit 5c03697
Show file tree
Hide file tree
Showing 10 changed files with 218 additions and 109 deletions.
2 changes: 1 addition & 1 deletion ecf/scripts/enkfgdas/analysis/recenter/jenkfgdas_sfc.ecf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
#PBS -q %QUEUE%
#PBS -A %PROJ%-%PROJENVIR%
#PBS -l walltime=00:06:00
#PBS -l select=1:mpiprocs=80:ompthreads=1:ncpus=80:mem=60GB
#PBS -l select=1:mpiprocs=80:ompthreads=1:ncpus=80:mem=80GB
#PBS -l place=vscatter
#PBS -l debug=true

Expand Down
3 changes: 2 additions & 1 deletion ecf/scripts/gfs/atmos/post_processing/grib2_wafs/jgfs_atmos_wafs_blending_0p25.ecf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
#PBS -q %QUEUE%
#PBS -A %PROJ%-%PROJENVIR%
#PBS -l walltime=00:30:00
#PBS -l select=1:mpiprocs=1:ompthreads=1:ncpus=1:mem=1GB
#PBS -l select=1:mpiprocs=1:ompthreads=1:ncpus=1:mem=15GB
#PBS -l place=vscatter
#PBS -l debug=true

Expand Down Expand Up @@ -35,6 +35,7 @@ module list
#############################################################
export cyc=%CYC%
export cycle=t%CYC%z
export ICAO2023=no

############################################################
# CALL executable job script here
Expand Down
5 changes: 4 additions & 1 deletion ecf/scripts/gfs/atmos/post_processing/grib2_wafs/jgfs_atmos_wafs_grib2.ecf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
#PBS -q %QUEUE%
#PBS -A %PROJ%-%PROJENVIR%
#PBS -l walltime=00:30:00
#PBS -l select=1:mpiprocs=1:ompthreads=1:ncpus=1:mem=5GB
#PBS -l select=1:mpiprocs=18:ompthreads=1:ncpus=18:mem=80GB
#PBS -l place=vscatter
#PBS -l debug=true

Expand All @@ -28,6 +28,7 @@ module load cray-pals/${cray_pals_ver}
module load libjpeg/${libjpeg_ver}
module load grib_util/${grib_util_ver}
module load wgrib2/${wgrib2_ver}
module load cfp/${cfp_ver}

module list

Expand All @@ -36,6 +37,8 @@ module list
#############################################################
export cyc=%CYC%
export cycle=t%CYC%z
export USE_CFP=YES
export ICAO2023=no

############################################################
# CALL executable job script here
Expand Down
5 changes: 4 additions & 1 deletion ecf/scripts/gfs/atmos/post_processing/grib2_wafs/jgfs_atmos_wafs_grib2_0p25.ecf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
#PBS -q %QUEUE%
#PBS -A %PROJ%-%PROJENVIR%
#PBS -l walltime=00:30:00
#PBS -l select=1:mpiprocs=1:ompthreads=1:ncpus=1:mem=1GB
#PBS -l select=1:mpiprocs=11:ompthreads=1:ncpus=11:mem=80GB
#PBS -l place=vscatter
#PBS -l debug=true

Expand All @@ -28,6 +28,7 @@ module load cray-pals/${cray_pals_ver}
module load libjpeg/${libjpeg_ver}
module load grib_util/${grib_util_ver}
module load wgrib2/${wgrib2_ver}
module load cfp/${cfp_ver}

module list

Expand All @@ -36,6 +37,8 @@ module list
#############################################################
export cyc=%CYC%
export cycle=t%CYC%z
export USE_CFP=YES
export ICAO2023=no

############################################################
# CALL executable job script here
Expand Down
1 change: 1 addition & 0 deletions ecf/scripts/gfs/atmos/post_processing/jgfs_atmos_wafs_gcip.ecf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ module list
export cyc=%CYC%
export cycle=t%CYC%z
export USE_CFP=YES
export ICAO2023=no

############################################################
# CALL executable job script here
Expand Down
4 changes: 3 additions & 1 deletion parm/config/config.coupled_ic
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,9 @@ echo "BEGIN: config.coupled_ic"
# Get task specific resources
source ${EXPDIR}/config.resources coupled_ic

if [[ "${machine}" == "HERA" ]]; then
if [[ "${machine}" == "WCOSS2" ]]; then
export BASE_CPLIC="/lfs/h2/emc/global/noscrub/emc.global/IC/COUPLED"
elif [[ "${machine}" == "HERA" ]]; then
export BASE_CPLIC="/scratch1/NCEPDEV/climate/role.ufscpara/IC"
elif [[ "${machine}" == "ORION" ]]; then
export BASE_CPLIC="/work/noaa/global/wkolczyn/noscrub/global-workflow/IC"
Expand Down
64 changes: 38 additions & 26 deletions parm/config/config.fv3
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,28 +20,30 @@ case_in=$1
echo "BEGIN: config.fv3"


if [[ "$machine" = "JET" ]]; then
if [[ "$PARTITION_BATCH" = "xjet" ]]; then
if [[ "${machine}" = "WCOSS2" ]]; then
export npe_node_max=128
elif [[ "${machine}" = "JET" ]]; then
if [[ "${PARTITION_BATCH}" = "xjet" ]]; then
export npe_node_max=24
elif [[ "$PARTITION_BATCH" = "vjet" || "$PARTITION_BATCH" = "sjet" ]]; then
elif [[ "${PARTITION_BATCH}" = "vjet" || "${PARTITION_BATCH}" = "sjet" ]]; then
export npe_node_max=16
elif [[ "$PARTITION_BATCH" = "kjet" ]]; then
elif [[ "${PARTITION_BATCH}" = "kjet" ]]; then
export npe_node_max=40
fi
elif [[ "$machine" = "HERA" ]]; then
elif [[ "${machine}" = "HERA" ]]; then
export npe_node_max=40
elif [[ "$machine" = "S4" ]]; then
if [[ "$PARTITION_BATCH" = "s4" ]]; then
elif [[ "${machine}" = "S4" ]]; then
if [[ "${PARTITION_BATCH}" = "s4" ]]; then
export npe_node_max=32
elif [[ "$PARTITION_BATCH" = "ivy" ]]; then
elif [[ "${PARTITION_BATCH}" = "ivy" ]]; then
export npe_node_max=20
fi
elif [[ "$machine" = "ORION" ]]; then
elif [[ "${machine}" = "ORION" ]]; then
export npe_node_max=40
fi

# (Standard) Model resolution dependent variables
case $case_in in
case ${case_in} in
"C48")
export DELTIM=450
export layout_x=3
Expand All @@ -52,9 +54,11 @@ case $case_in in
export nth_fv3_gfs=2
export cdmbgwd="0.071,2.1,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling
export WRITE_GROUP=1
export WRTTASK_PER_GROUP=$npe_node_max
export WRTTASK_PER_GROUP=64
if [[ "${WRTTASK_PER_GROUP}" -gt "${npe_node_max}" ]]; then export WRTTASK_PER_GROUP=${npe_node_max} ; fi
export WRITE_GROUP_GFS=1
export WRTTASK_PER_GROUP_GFS=$npe_node_max
export WRTTASK_PER_GROUP_GFS=64
if [[ "${WRTTASK_PER_GROUP_GFS}" -gt "${npe_node_max}" ]]; then export WRTTASK_PER_GROUP_GFS=${npe_node_max} ; fi
export WRTIOBUF="1M"
;;
"C96")
Expand All @@ -67,9 +71,11 @@ case $case_in in
export nth_fv3_gfs=1
export cdmbgwd="0.14,1.8,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling
export WRITE_GROUP=1
export WRTTASK_PER_GROUP=$npe_node_max
export WRTTASK_PER_GROUP=64
if [[ "${WRTTASK_PER_GROUP}" -gt "${npe_node_max}" ]]; then export WRTTASK_PER_GROUP=${npe_node_max} ; fi
export WRITE_GROUP_GFS=1
export WRTTASK_PER_GROUP_GFS=$npe_node_max
export WRTTASK_PER_GROUP_GFS=64
if [[ "${WRTTASK_PER_GROUP_GFS}" -gt "${npe_node_max}" ]]; then export WRTTASK_PER_GROUP_GFS=${npe_node_max} ; fi
export WRTIOBUF="4M"
export n_split=6
;;
Expand All @@ -79,28 +85,32 @@ case $case_in in
export layout_y=6
export layout_x_gfs=4
export layout_y_gfs=6
export nth_fv3=2
export nth_fv3=1
export nth_fv3_gfs=2
export cdmbgwd="0.23,1.5,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling
export WRITE_GROUP=1
export WRTTASK_PER_GROUP=$npe_node_max
export WRTTASK_PER_GROUP=64
if [[ "${WRTTASK_PER_GROUP}" -gt "${npe_node_max}" ]]; then export WRTTASK_PER_GROUP=${npe_node_max} ; fi
export WRITE_GROUP_GFS=2
export WRTTASK_PER_GROUP_GFS=$npe_node_max
export WRTTASK_PER_GROUP_GFS=64
if [[ "${WRTTASK_PER_GROUP_GFS}" -gt "${npe_node_max}" ]]; then export WRTTASK_PER_GROUP_GFS=${npe_node_max} ; fi
export WRTIOBUF="8M"
;;
"C384")
export DELTIM=${DELTIM:-300}
export DELTIM=${DELTIM:-200}
export layout_x=6
export layout_y=8
export layout_x_gfs=${layout_x_gfs:-8}
export layout_y_gfs=${layout_y_gfs:-12}
export nth_fv3=2
export nth_fv3=1
export nth_fv3_gfs=${nth_fv3_gfs:-2}
export cdmbgwd="1.1,0.72,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling
export WRITE_GROUP=1
export WRTTASK_PER_GROUP=$npe_node_max
export WRITE_GROUP=2
export WRTTASK_PER_GROUP=64
if [[ "${WRTTASK_PER_GROUP}" -gt "${npe_node_max}" ]]; then export WRTTASK_PER_GROUP=${npe_node_max} ; fi
export WRITE_GROUP_GFS=${WRITE_GROUP_GFS:-2}
export WRTTASK_PER_GROUP_GFS=${WRTTASK_PER_GROUP_GFS:-$npe_node_max}
export WRTTASK_PER_GROUP_GFS=${WRTTASK_PER_GROUP_GFS:-64}
if [[ "${WRTTASK_PER_GROUP_GFS}" -gt "${npe_node_max}" ]]; then export WRTTASK_PER_GROUP_GFS=${npe_node_max} ; fi
export WRTIOBUF=${WRTIOBUF:-"16M"}
;;
"C768")
Expand All @@ -113,9 +123,11 @@ case $case_in in
export nth_fv3_gfs=4
export cdmbgwd="4.0,0.15,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling
export WRITE_GROUP=2
export WRTTASK_PER_GROUP=$(echo "2*${npe_node_max}" |bc)
export WRTTASK_PER_GROUP=64
if [[ "${WRTTASK_PER_GROUP}" -gt "${npe_node_max}" ]]; then export WRTTASK_PER_GROUP=${npe_node_max} ; fi
export WRITE_GROUP_GFS=4
export WRTTASK_PER_GROUP_GFS=$(echo "2*${npe_node_max}" |bc)
export WRTTASK_PER_GROUP_GFS=64
if [[ "${WRTTASK_PER_GROUP_GFS}" -gt "${npe_node_max}" ]]; then export WRTTASK_PER_GROUP_GFS=${npe_node_max} ; fi
export WRTIOBUF="32M"
;;
"C1152")
Expand Down Expand Up @@ -164,11 +176,11 @@ export kchunk3d=1

# Determine whether to use parallel NetCDF based on resolution
case ${case_in} in
"C48" | "C96" | "C192")
"C48" | "C96" | "C192" | "C384")
export OUTPUT_FILETYPE_ATM="netcdf"
export OUTPUT_FILETYPE_SFC="netcdf"
;;
"C384" | "C768" | "C1152" | "C3072")
"C768" | "C1152" | "C3072")
export OUTPUT_FILETYPE_ATM="netcdf_parallel"
export OUTPUT_FILETYPE_SFC="netcdf_parallel"
;;
Expand Down
Loading

0 comments on commit 5c03697

Please sign in to comment.