From 06666e56fcf1a57bd4b22fcb64b7e0cee873202f Mon Sep 17 00:00:00 2001 From: David Huber Date: Wed, 15 Sep 2021 13:25:40 +0000 Subject: [PATCH 01/63] Initial port to Jet. (#357) --- driver/product/run_postsnd.sh.jet | 53 ++++++++++---------- env/JET.env | 82 ++++++++++++++++++++----------- jobs/JGFS_ATMOS_CYCLONE_TRACKER | 3 ++ modulefiles/module-setup.csh.inc | 12 +++++ modulefiles/module_base.jet | 33 +++++++++++-- parm/config/config.fcst | 3 ++ parm/config/config.prepbufr | 3 +- parm/config/config.resources | 40 +++++++++++++++ sorc/build_all.sh | 6 --- sorc/build_ww3prepost.sh | 1 + sorc/machine-setup.sh | 20 ++++---- ush/rocoto/setup_expt.py | 21 ++++++++ ush/rocoto/setup_expt_fcstonly.py | 2 +- 13 files changed, 203 insertions(+), 76 deletions(-) diff --git a/driver/product/run_postsnd.sh.jet b/driver/product/run_postsnd.sh.jet index 23038d6d2b..cffa85119a 100755 --- a/driver/product/run_postsnd.sh.jet +++ b/driver/product/run_postsnd.sh.jet @@ -1,39 +1,38 @@ -#!/bin/sh -#PBS -o postsnd.out.01 -#PBS -e postsnd.out.01 -#PBS -N postsnd -#PBS -A hfv3gfs -#PBS -l partition=xjet:sjet:vjet:tjet -#PBS -l nodes=6:ppn=2 -#PBS -l walltime=02:30:00 +#!/bin/ksh -l +#SBATCH -A fv3-cam # Use a valid account +#SBATCH --time=2:30:00 # For big directories you may want 8:00:00 hrs +#SBATCH -N 40 # Probably 2 to 4 depending on amount of data +#SBATCH --partition=xjet +#SBATCH -n 24 +#SBATCH -o %x.o%j +#SBATCH -J postsnd ############################################ # Loading module ############################################ . $MODULESHOME/init/ksh -module load newdefaults intel/15.0.3.187 impi/5.1.3.181 szip hdf5 netcdf4/4.2.1.1 +module load intel/18.0.5.274 +impi/2018.4.274 -module use /lfs3/projects/hfv3gfs/nwprod/lib/modulefiles +module use /lfs4/HFIP/hfv3gfs/nwprod/NCEPLIBS/modulefiles + +# Loding nceplibs modules +module load sigio/2.1.1 module load bacio/v2.0.2 module load sp/v2.0.2 -module load ip/v2.0.0 -module load w3nco/v2.0.6 module load w3emc/v2.2.0 -module load nemsio/v2.2.2 - -module use /lfs3/projects/hwrf-vd/soft/modulefiles - -module use /lfs3/projects/hwrf-vd/soft/modulefiles +module load w3nco/v2.0.6 +module load bufr/v10.2.5 +module load nemsio/v2.2.4 +module load gempak/7.4.2 module load prod_util -module load grib_util -module load hpss -# mpiserial on Jet includes a cfp wrapper -module load mpiserial - -module load gempak/7.4.2 +module load netcdf_parallel/4.7.4 +module load hdf5_parallel/1.10.6 +module load intelpython/3.6.5 +#TODO Check if anaconda2 is required ######################################## # Runs GFS BUFR SOUNDINGS ######################################## @@ -48,8 +47,9 @@ export MP_STDOUTMODE=ordered #export machine="WCOSS_C" #machine="WCOSS_DELL_P3" machine="JET" -export npe_postsnd=12 +export npe_postsnd=24 export npe_postsndcfp=10 +export OUTPUT_FILE=${OUTPUT_FILE:-netcdf} if [ $machine == "WCOSS_C" ]; then ##For WCOSS-Cray################## export FHMAX_HF_GFS=120 @@ -71,7 +71,7 @@ export APRUN_POSTSNDCFP="$launcher $npe_postsndcfp" else ##For Theia, Jet################ export FHMAX_HF_GFS=0 -export FHOUT_HF_GFS=0 +export FHOUT_HF_GFS=3 export launcher="mpirun -np" export APRUN_POSTSND="$launcher $npe_postsnd" export APRUN_POSTSNDCFP="$launcher $npe_postsndcfp" @@ -97,6 +97,7 @@ export util_ver=${util_ver:-v1.0.0} # obtain unique process id (pid) and make temp directories # export pid=$$ +#TODO Change location export DATA_IN=${DATA_IN:-/mnt/lfs3/projects/hfv3gfs/$USER/scrub} export DATA=$DATA_IN/postsnd.${pid} mkdir -p $DATA @@ -135,6 +136,7 @@ mkdir -p $pcom # Set up the UTILITIES ################################### +#TODO Change location export HOMEbufrsnd=/mnt/lfs3/projects/hfv3gfs/Guang.Ping.Lou/fv3gfs #export HOMEbufrsnd=/scratch4/NCEPDEV/meso/save/Guang.Ping.Lou/fv3gfs #export HOMEbufrsnd=/gpfs/hps3/emc/meso/noscrub/Guang.Ping.Lou/fv3gfs @@ -149,6 +151,7 @@ export HOMEbufrsnd=/mnt/lfs3/projects/hfv3gfs/Guang.Ping.Lou/fv3gfs ##export COMIN=/gpfs/hps/nco/ops/com/gfs/para/${RUN}.${PDY} ##export COMIN=/gpfs/hps/nco/ops/com/gfs/prod/${RUN}.${PDY} ##export COMIN=/scratch4/NCEPDEV/meso/save/Guang.Ping.Lou/fv3data/${RUN}.${PDY}/$cyc +#TODO Change location export COMIN=/mnt/lfs3/projects/hfv3gfs/glopara/comrot/fv3q2fy19retro5/${RUN}.${PDY}/$cyc export COMOUT=$DATA_IN/com2/${NET}/${envir}/${RUN}.${PDY} diff --git a/env/JET.env b/env/JET.env index aa47a46530..20d6d2186d 100755 --- a/env/JET.env +++ b/env/JET.env @@ -20,7 +20,7 @@ elif [[ "$PARTITION_BATCH" = "vjet" ]]; then fi export launcher="srun -l --export=ALL" -# Configure STACK +# Configure MPI environment export OMP_STACKSIZE=2048000 export NTHSTACK=1024000000 @@ -35,35 +35,44 @@ if [ $step = "prep" -o $step = "prepbufr" ]; then nth_max=$(($npe_node_max / $npe_node_prep)) export POE="NO" - export BACK=${BACK:-"NO"} + export BACK="NO" export sys_tp="JET" elif [ $step = "waveinit" -o $step = "waveprep" -o $step = "wavepostsbs" -o $step = "wavepostbndpnt" -o $step = "wavepostbndpntbll" -o $step = "wavepostpnt" ]; then - +#TODO Test multi-prog option + export mpmd="--multi-prog" + export CFP_MP="YES" if [ $step = "waveprep" ]; then export MP_PULSE=0 ; fi export wavempexec=${launcher} export wave_mpmd=${mpmd} elif [ $step = "anal" ]; then + export MKL_NUM_THREADS=4 + export MKL_CBWR=AUTO + + export CFP_MP=${CFP_MP:-"YES"} + export USE_CFP=${USE_CFP:-"YES"} + export APRUNCFP="$launcher -n \$ncmd --multi-prog" + nth_max=$(($npe_node_max / $npe_node_anal)) - export NTHREADS_GSI=${nth_gsi:-$nth_max} + export NTHREADS_GSI=${nth_anal:-$nth_max} [[ $NTHREADS_GSI -gt $nth_max ]] && export NTHREADS_GSI=$nth_max - export APRUN_GSI="$launcher ${npe_gsi:-${npe_anal:-$PBS_NP}}" + export APRUN_GSI="$launcher" export NTHREADS_CALCINC=${nth_calcinc:-1} [[ $NTHREADS_CALCINC -gt $nth_max ]] && export NTHREADS_CALCINC=$nth_max - export APRUN_CALCINC="$launcher \$ncmd" + export APRUN_CALCINC="$launcher" export NTHREADS_CYCLE=${nth_cycle:-12} [[ $NTHREADS_CYCLE -gt $npe_node_max ]] && export NTHREADS_CYCLE=$npe_node_max npe_cycle=${ntiles:-6} - export APRUN_CYCLE="$launcher $npe_cycle" + export APRUN_CYCLE="$launcher -n $npe_cycle" export NTHREADS_GAUSFCANL=1 npe_gausfcanl=${npe_gausfcanl:-1} - export APRUN_GAUSFCANL="$launcher $npe_gausfcanl" + export APRUN_GAUSFCANL="$launcher -n $npe_gausfcanl" export NTHREADS_CHGRES=${nth_echgres:-1} [[ $NTHREADS_CHGRES -gt $npe_node_max ]] && export NTHREADS_CHGRES=$npe_node_max @@ -75,27 +84,44 @@ elif [ $step = "gldas" ]; then export NTHREADS_GLDAS=${nth_gldas:-$nth_max} [[ $NTHREADS_GLDAS -gt $nth_max ]] && export NTHREADS_GLDAS=$nth_max - export APRUN_GLDAS="$launcher $npe_gldas" + export APRUN_GLDAS="$launcher -n $npe_gldas" export NTHREADS_GAUSSIAN=${nth_gaussian:-1} [[ $NTHREADS_GAUSSIAN -gt $nth_max ]] && export NTHREADS_GAUSSIAN=$nth_max - export APRUN_GAUSSIAN="$launcher $npe_gaussian" + export APRUN_GAUSSIAN="$launcher -n $npe_gaussian" + +# Must run data processing with exactly the number of tasks as time +# periods being processed. + + npe_gldas_data_proc=$(($gldas_spinup_hours + 12)) + export APRUN_GLDAS_DATA_PROC="$launcher -n $npe_gldas_data_proc --multi-prog" elif [ $step = "eobs" ]; then + export MKL_NUM_THREADS=4 + export MKL_CBWR=AUTO + nth_max=$(($npe_node_max / $npe_node_eobs)) - export NTHREADS_GSI=${nth_gsi:-$nth_max} + export NTHREADS_GSI=${nth_eobs:-$nth_max} [[ $NTHREADS_GSI -gt $nth_max ]] && export NTHREADS_GSI=$nth_max - export APRUN_GSI="$launcher ${npe_gsi:-${npe_eobs:-$PBS_NP}}" + export APRUN_GSI="$launcher" + + export CFP_MP=${CFP_MP:-"YES"} + export USE_CFP=${USE_CFP:-"YES"} + export APRUNCFP="$launcher -n \$ncmd --multi-prog" elif [ $step = "eupd" ]; then nth_max=$(($npe_node_max / $npe_node_eupd)) - export NTHREADS_ENKF=${nth_enkf:-$nth_max} + export NTHREADS_ENKF=${nth_eupd:-$nth_max} [[ $NTHREADS_ENKF -gt $nth_max ]] && export NTHREADS_ENKF=$nth_max - export APRUN_ENKF="$launcher ${npe_enkf:-${npe_eupd:-$PBS_NP}}" + export APRUN_ENKF="$launcher" + + export CFP_MP=${CFP_MP:-"YES"} + export USE_CFP=${USE_CFP:-"YES"} + export APRUNCFP="$launcher -n \$ncmd --multi-prog" elif [ $step = "fcst" ]; then @@ -108,20 +134,20 @@ elif [ $step = "fcst" ]; then export NTHREADS_REGRID_NEMSIO=${nth_regrid_nemsio:-1} [[ $NTHREADS_REGRID_NEMSIO -gt $nth_max ]] && export NTHREADS_REGRID_NEMSIO=$nth_max - export APRUN_REGRID_NEMSIO="$launcher $LEVS" + export APRUN_REGRID_NEMSIO="$launcher" export NTHREADS_REMAP=${nth_remap:-2} [[ $NTHREADS_REMAP -gt $nth_max ]] && export NTHREADS_REMAP=$nth_max - export APRUN_REMAP="$launcher ${npe_remap:-${npe_fcst:-$PBS_NP}}" + export APRUN_REMAP="$launcher" elif [ $step = "efcs" ]; then nth_max=$(($npe_node_max / $npe_node_efcs)) - export NTHREADS_FV3=${nth_fv3:-$nth_max} + export NTHREADS_FV3=${nth_efcs:-$nth_max} [[ $NTHREADS_FV3 -gt $nth_max ]] && export NTHREADS_FV3=$nth_max export cores_per_node=$npe_node_max - export APRUN_FV3="$launcher ${npe_fv3:-${npe_efcs:-$PBS_NP}}" + export APRUN_FV3="$launcher" export NTHREADS_REGRID_NEMSIO=${nth_regrid_nemsio:-1} [[ $NTHREADS_REGRID_NEMSIO -gt $nth_max ]] && export NTHREADS_REGRID_NEMSIO=$nth_max @@ -137,7 +163,7 @@ elif [ $step = "post" ]; then export NTHREADS_DWN=${nth_dwn:-1} [[ $NTHREADS_DWN -gt $nth_max ]] && export NTHREADS_DWN=$nth_max - export APRUN_NP="$launcher" + export APRUN_DWN="$launcher" elif [ $step = "ecen" ]; then @@ -145,7 +171,7 @@ elif [ $step = "ecen" ]; then export NTHREADS_ECEN=${nth_ecen:-$nth_max} [[ $NTHREADS_ECEN -gt $nth_max ]] && export NTHREADS_ECEN=$nth_max - export APRUN_ECEN="$launcher ${npe_ecen:-$PBS_NP}" + export APRUN_ECEN="$launcher" export NTHREADS_CHGRES=${nth_chgres:-12} [[ $NTHREADS_CHGRES -gt $npe_node_max ]] && export NTHREADS_CHGRES=$npe_node_max @@ -153,7 +179,7 @@ elif [ $step = "ecen" ]; then export NTHREADS_CALCINC=${nth_calcinc:-1} [[ $NTHREADS_CALCINC -gt $nth_max ]] && export NTHREADS_CALCINC=$nth_max - export APRUN_CALCINC="$launcher ${npe_ecen:-$PBS_NP}" + export APRUN_CALCINC="$launcher" elif [ $step = "esfc" ]; then @@ -161,12 +187,11 @@ elif [ $step = "esfc" ]; then export NTHREADS_ESFC=${nth_esfc:-$nth_max} [[ $NTHREADS_ESFC -gt $nth_max ]] && export NTHREADS_ESFC=$nth_max - export APRUN_ESFC="$launcher ${npe_esfc:-$PBS_NP}" + export APRUN_ESFC="$launcher -n $npe_esfc" - export NTHREADS_CYCLE=${nth_cycle:-12} + export NTHREADS_CYCLE=${nth_cycle:-14} [[ $NTHREADS_CYCLE -gt $npe_node_max ]] && export NTHREADS_CYCLE=$npe_node_max - export APRUN_CYCLE="$launcher $npe_esfc" - + export APRUN_CYCLE="$launcher -n $npe_esfc" elif [ $step = "epos" ]; then @@ -174,7 +199,7 @@ elif [ $step = "epos" ]; then export NTHREADS_EPOS=${nth_epos:-$nth_max} [[ $NTHREADS_EPOS -gt $nth_max ]] && export NTHREADS_EPOS=$nth_max - export APRUN_EPOS="$launcher ${npe_epos:-$PBS_NP}" + export APRUN_EPOS="$launcher" elif [ $step = "init" ]; then @@ -186,11 +211,11 @@ elif [ $step = "postsnd" ]; then export NTHREADS_POSTSND=${nth_postsnd:-1} [[ $NTHREADS_POSTSND -gt $nth_max ]] && export NTHREADS_POSTSND=$nth_max - export APRUN_POSTSND="$launcher $npe_postsnd" + export APRUN_POSTSND="$launcher" export NTHREADS_POSTSNDCFP=${nth_postsndcfp:-1} [[ $NTHREADS_POSTSNDCFP -gt $nth_max ]] && export NTHREADS_POSTSNDCFP=$nth_max - export APRUN_POSTSNDCFP="$launcher $npe_postsndcfp" + export APRUN_POSTSNDCFP="$launcher" elif [ $step = "awips" ]; then @@ -199,5 +224,4 @@ elif [ $step = "awips" ]; then elif [ $step = "gempak" ]; then echo "WARNING: $step is not enabled on $machine!" - fi diff --git a/jobs/JGFS_ATMOS_CYCLONE_TRACKER b/jobs/JGFS_ATMOS_CYCLONE_TRACKER index 7626ddc6bd..c4596c0b2c 100755 --- a/jobs/JGFS_ATMOS_CYCLONE_TRACKER +++ b/jobs/JGFS_ATMOS_CYCLONE_TRACKER @@ -148,6 +148,9 @@ elif [ $machine = HERA ]; then # For HERA elif [ $machine = ORION ]; then # For ORION machine=orion ${USHens_tracker}/extrkr_gfs.sh ${loopnum} ${cmodel} ${CDATE} ${pert} ${DATA} +elif [ $machine = JET ]; then # For JET + machine=hera + ${USHens_tracker}/extrkr_gfs.sh ${loopnum} ${cmodel} ${CDATE} ${pert} ${DATA} fi export err=$?; err_chk diff --git a/modulefiles/module-setup.csh.inc b/modulefiles/module-setup.csh.inc index 3a0bcdd772..aafe5c7fc5 100644 --- a/modulefiles/module-setup.csh.inc +++ b/modulefiles/module-setup.csh.inc @@ -19,6 +19,18 @@ else if ( { test -d /work } ) then source /apps/lmod/init/$__ms_shell endif module purge +else if ( { test -d /scratch1 } ) then + # We are on NOAA Hera + if ( ! { module help >& /dev/null } ) then + source /apps/lmod/lmod/init/$__ms_shell + endif + module purge +else if ( { test -d /jetmon } ) then + # We are on NOAA Jet + if ( ! { module help >& /dev/null } ) then + source /apps/lmod/lmod/init/$__ms_shell + endif + module purge else if ( { test -d /gpfs/hps -a -e /etc/SuSE-release } ) then # We are on NOAA Luna or Surge if ( ! { module help >& /dev/null } ) then diff --git a/modulefiles/module_base.jet b/modulefiles/module_base.jet index 1aa2475a8f..6e6fd66a96 100644 --- a/modulefiles/module_base.jet +++ b/modulefiles/module_base.jet @@ -13,13 +13,38 @@ module load hpc-impi/2018.4.274 module load wgrib2/2.0.8 module load hpss module load nco/4.9.1 -module load cdo/1.9.5 module load gempak/7.4.2 module load prod_util/1.2.2 module load grib_util/1.2.2 module load g2tmpl/1.10.0 module load crtm/2.3.0 -module load esmf/8_2_0_beta_snapshot_14 -module load fms/2021.03 +setenv CRTM_FIX /lfs4/HFIP/hfv3gfs/glopara/crtm_v2.3.0 + +module load jasper/2.0.25 +module load zlib/1.2.11 +module load png/1.6.35 + +module load hdf5/1.10.6 module load netcdf/4.7.4 -module load rocoto +module load pio/2.5.2 +module load esmf/8_1_1 +module load fms/2020.04.03 + +module load bacio/2.4.1 +module load g2/3.4.1 +module load ip/3.3.3 +module load nemsio/2.5.2 +module load sp/2.3.3 +module load w3emc/2.7.3 +module load w3nco/2.4.1 +module load upp/10.0.6 + +setenv WGRIB2 wgrib2 + +# python +module use /contrib/anaconda/modulefiles +module load anaconda/5.3.1 + +# waveprep +module load cdo/1.9.5 + diff --git a/parm/config/config.fcst b/parm/config/config.fcst index 7c07c4f658..769613ab03 100755 --- a/parm/config/config.fcst +++ b/parm/config/config.fcst @@ -226,6 +226,9 @@ if [[ "$OUTPUT_FILE" == "netcdf" ]]; then export OUTPUT_FILETYPES=" 'netcdf_parallel' 'netcdf' " fi fi + if [[ "$machine" == "JET" ]]; then + export OUTPUT_FILETYPES=" 'netcdf_parallel' 'netcdf' " + fi fi diff --git a/parm/config/config.prepbufr b/parm/config/config.prepbufr index 904d946774..a48f8b76f7 100755 --- a/parm/config/config.prepbufr +++ b/parm/config/config.prepbufr @@ -10,10 +10,9 @@ echo "BEGIN: config.prepbufr" # Set variables +export GESROOT=${GESROOT:-"/dev/null"} if [ $machine = "HERA" ]; then export GESROOT=/scratch1/NCEPDEV/rstprod # set by module prod_envir on WCOSS_C -elif [ $machine = "ORION" ]; then - export GESROOT=/dev/null fi echo "END: config.prepbufr" diff --git a/parm/config/config.resources b/parm/config/config.resources index 71c96acba1..a552a5b482 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -131,13 +131,37 @@ elif [ $step = "anal" ]; then export npe_anal=1000 export nth_anal=5 export npe_anal_gfs=1000 + #For the analysis jobs, the number of tasks and cores used must be equal. + #On the Jet partitions (except kjet), this is accomplished by adjusting the task + #and thread count based on cores/node. + if [[ "$machine" = "Jet" && $PARTITION_BATCH != "kjet" ]]; then + export nth_anal=4 + if [[ "$PARTITION_BATCH" = "xjet" ]]; then + export npe_anal=1002 + export npe_anal_gfs=1002 + fi + fi if [ $CASE = "C384" ]; then export npe_anal=400 export npe_anal_gfs=400 + if [[ "$machine" = "Jet" ]]; then + if [[ "$PARTITION_BATCH" = "xjet" ]]; then + export nth_anal=4 + export npe_anal=402 + export npe_anal_gfs=402 + elif [[ "$PARTITION_BATCH" = "vjet" || "$PARTITION_BATCH" = "sjet" ]]; then + export nth_anal=4 + fi + fi fi if [ $CASE = "C192" -o $CASE = "C96" -o $CASE = "C48" ]; then export npe_anal=84 export npe_anal_gfs=84 + if [[ "$machine" = "Jet" ]]; then + if [[ "$PARTITION_BATCH" != "kjet" ]]; then + export nth_anal=4 + fi + fi fi if [[ "$machine" = "WCOSS_DELL_P3" ]]; then export nth_anal=4 @@ -413,6 +437,22 @@ elif [ $step = "eobs" -o $step = "eomg" ]; then export nth_eobs=2 if [[ "$machine" = "WCOSS_DELL_P3" ]]; then export nth_eobs=7; fi export npe_node_eobs=$(echo "$npe_node_max / $nth_eobs" | bc) + #The number of tasks and number of CPUs used must be the same for eobs + #For xjet, adjust the number of tasks + if [[ "$machine" = "JET" ]]; then + if [[ "$PARTITION_BATCH" = "xjet" ]]; then + if [[ "$CASE" = "C768" ]]; then + export npe_eobs=204 + elif [[ "$CASE" = "C384" ]]; then + export npe_eobs=108 + elif [[ "$CASE" = "C192" ]]; then + export npe_eobs=48 + elif [[ "$CASE" = "C96" || "$CASE" = "C48" ]]; then + export npe_eobs=24 + fi + fi + export npe_node_eobs=$(echo "$npe_node_max / $nth_eobs" | bc) + fi if [[ "$machine" == "WCOSS_C" ]]; then export memory_eobs="3072M"; fi elif [ $step = "ediag" ]; then diff --git a/sorc/build_all.sh b/sorc/build_all.sh index 6c8bfabcd0..875f261e52 100755 --- a/sorc/build_all.sh +++ b/sorc/build_all.sh @@ -58,12 +58,6 @@ source ./machine-setup.sh > /dev/null 2>&1 #------------------------------------ . ./partial_build.sh $@ -if [ $target = jet ]; then - Build_gldas=false - Build_gfs_util=false - Build_ww3_prepost=false -fi - #------------------------------------ # Exception Handling Init #------------------------------------ diff --git a/sorc/build_ww3prepost.sh b/sorc/build_ww3prepost.sh index 09ce879e64..d31f627b55 100755 --- a/sorc/build_ww3prepost.sh +++ b/sorc/build_ww3prepost.sh @@ -17,6 +17,7 @@ set -x if [ $target = hera ]; then target=hera.intel ; fi if [ $target = orion ]; then target=orion.intel ; fi if [ $target = stampede ]; then target=stampede.intel ; fi +if [ $target = jet ]; then target=jet.intel ; fi cd ufs_model.fd/WW3 export WW3_DIR=$( pwd -P )/model diff --git a/sorc/machine-setup.sh b/sorc/machine-setup.sh index 59e9ad8947..67bd3eefb8 100644 --- a/sorc/machine-setup.sh +++ b/sorc/machine-setup.sh @@ -16,6 +16,8 @@ else __ms_shell=sh fi +set -x + target="" USERNAME=`echo $LOGNAME | awk '{ print tolower($0)'}` ##--------------------------------------------------------------------------- @@ -172,7 +174,7 @@ export FCOMP=ftn # END GWV ADD ##--------------------------------------------------------------------------- -elif [[ -d /lfs3 ]] ; then +elif [[ -d /jetmon ]] ; then # We are on NOAA Jet if ( ! eval module help > /dev/null 2>&1 ) ; then echo load the module command 1>&2 @@ -180,14 +182,14 @@ elif [[ -d /lfs3 ]] ; then fi target=jet module purge -module load intel/15.0.3.187 -module load impi -#export NCEPLIBS=/mnt/lfs3/projects/hfv3gfs/gwv/ljtjet/lib - export NCEPLIBS=/mnt/lfs3/projects/hfv3gfs/gwv/ljtjet/lib -export NCEPLIBS=/mnt/lfs3/projects/hfv3gfs/gwv/NCEPLIBS.15X - module use $NCEPLIBS/modulefiles -export WRFPATH=$NCEPLIBS/wrf.shared.new/v1.1.1/src -export myFC=mpiifort + module use /lfs4/HFIP/hfv3gfs/nwprod/hpc-stack/libs/modulefiles/stack + module load hpc/1.1.0 + module load hpc-intel/18.0.5.274 + module load hpc-impi/2018.4.274 + module load cmake/3.20.1 + + export myFC=mpiifort + export FCOMP=mpiifort else echo WARNING: UNKNOWN PLATFORM 1>&2 diff --git a/ush/rocoto/setup_expt.py b/ush/rocoto/setup_expt.py index 323ab960d0..af302f2d54 100755 --- a/ush/rocoto/setup_expt.py +++ b/ush/rocoto/setup_expt.py @@ -239,6 +239,27 @@ def edit_baseconfig(): chgrp_rstprod = 'YES' chgrp_cmd = 'chgrp rstprod' hpssarch = 'YES' + elif machine == 'JET': + base_git = '/lfs4/HFIP/hfv3gfs/glopara/git' + base_svn = '/lfs4/HFIP/hfv3gfs/glopara/svn' + dmpdir = '/lfs4/HFIP/hfv3gfs/glopara/dump' + nwprod = '/lfs4/HFIP/hfv3gfs/glopara/nwpara' + comroot = '/lfs4/HFIP/hfv3gfs/glopara/com' + homedir = '/lfs4/HFIP/hfv3gfs$USER' + stmp = '/lfs4/HFIP/hfv3gfs/$USER/stmp' + ptmp = '/lfs4/HFIP/hfv3gfs/$USER/ptmp' + noscrub = '$HOMEDIR' + account = 'hfv3gfs' + queue = 'batch' + queue_service = 'service' + # Set partition_batch; can be xjet or kjet (xjet is the default) + if partition is not None: + partition_batch = partition + else: + partition_batch = 'xjet' + chgrp_rstprod = 'YES' + chgrp_cmd = 'chgrp rstprod' + hpssarch = 'YES' elif machine == 'ORION': base_git = '/work/noaa/global/glopara/git' base_svn = '/work/noaa/global/glopara/svn' diff --git a/ush/rocoto/setup_expt_fcstonly.py b/ush/rocoto/setup_expt_fcstonly.py index 539bd4a9b8..0ef9624b5b 100755 --- a/ush/rocoto/setup_expt_fcstonly.py +++ b/ush/rocoto/setup_expt_fcstonly.py @@ -229,7 +229,7 @@ def edit_baseconfig(): hpssarch = 'YES' elif machine == 'JET': base_git = '/lfs4/HFIP/hfv3gfs/glopara/git' - base_svn = '/dev/null/global/save/glopara/svn/' + base_svn = '/lfs4/HFIP/hfv3gfs/glopara/svn/' dmpdir = '/lfs4/HFIP/hfv3gfs/glopara/dump' nwprod = '/lfs4/HFIP/hfv3gfs/glopara/nwpara' comroot = '/lfs4/HFIP/hfv3gfs/glopara/com' From b08f11a1688fe31b4f294ea11a3b47e6bdd63b34 Mon Sep 17 00:00:00 2001 From: David Huber Date: Fri, 15 Oct 2021 17:39:44 +0000 Subject: [PATCH 02/63] Update resources for Jet. #357 This also runs the C48 ensemble forecasts with 2 threads on all systems to allow enough memory for the jobs. --- parm/config/config.fv3 | 4 ++-- parm/config/config.resources | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/parm/config/config.fv3 b/parm/config/config.fv3 index 7a84bcac2f..7a80148024 100755 --- a/parm/config/config.fv3 +++ b/parm/config/config.fv3 @@ -47,8 +47,8 @@ case $case_in in export layout_y=2 export layout_x_gfs=3 export layout_y_gfs=2 - export nth_fv3=1 - export nth_fv3_gfs=1 + export nth_fv3=2 + export nth_fv3_gfs=2 export cdmbgwd="0.071,2.1,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export WRITE_GROUP=1 export WRTTASK_PER_GROUP=$npe_node_max diff --git a/parm/config/config.resources b/parm/config/config.resources index a552a5b482..7e028e512d 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -472,9 +472,9 @@ elif [ $step = "eupd" ]; then if [[ "$machine" = "WCOSS_DELL_P3" ]]; then export nth_eupd=7 fi - if [[ "$machine" = "HERA" ]]; then + if [ "$machine" = "HERA" -o "$machine" = "JET" ]; then export npe_eupd=150 - export nth_eupd=40 + export nth_eupd=$npe_node_max fi elif [ $CASE = "C384" ]; then export npe_eupd=270 @@ -482,16 +482,16 @@ elif [ $step = "eupd" ]; then if [[ "$machine" = "WCOSS_DELL_P3" ]]; then export nth_eupd=9 fi - if [[ "$machine" = "HERA" ]]; then + if [ "$machine" = "HERA" -o "$machine" = "JET" ]; then export npe_eupd=100 - export nth_eupd=40 + export nth_eupd=$npe_node_max fi elif [ $CASE = "C192" -o $CASE = "C96" -o $CASE = "C48" ]; then export npe_eupd=42 export nth_eupd=2 - if [[ "$machine" = "HERA" ]]; then + if [ "$machine" = "HERA" -o "$machine" = "JET" ]; then export npe_eupd=40 - export nth_eupd=40 + export nth_eupd=$npe_node_max fi fi export npe_node_eupd=$(echo "$npe_node_max / $nth_eupd" | bc) From 1fac19889f388be20c0ceb131c7253a6be87a3af Mon Sep 17 00:00:00 2001 From: David Huber Date: Fri, 15 Oct 2021 17:41:43 +0000 Subject: [PATCH 03/63] Added a WW3 modulefile for Jet. #357 --- modulefiles/modulefile.ww3.jet | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 modulefiles/modulefile.ww3.jet diff --git a/modulefiles/modulefile.ww3.jet b/modulefiles/modulefile.ww3.jet new file mode 100644 index 0000000000..266f9d4722 --- /dev/null +++ b/modulefiles/modulefile.ww3.jet @@ -0,0 +1,31 @@ +#%Module###################################################################### +# module for ww3 before base uses hpc-stack +module load cmake/3.16.1 + +module use /lfs4/HFIP/hfv3gfs/nwprod/hpc-stack/libs/modulefiles/stack + +module load hpc/1.1.0 +module load hpc-intel/18.0.5.274 +module load hpc-impi/2018.4.274 + +module load jasper/2.0.25 +module load zlib/1.2.11 +module load png/1.6.35 + +module load hdf5/1.10.6 +module load netcdf/4.7.4 +module load pio/2.5.2 +module load esmf/8_1_1 +module load fms/2020.04.03 + +module load bacio/2.4.1 +module load crtm/2.3.0 +module load g2/3.4.1 +module load g2tmpl/1.9.1 +module load ip/3.3.3 +module load nemsio/2.5.2 +module load sp/2.3.3 +module load w3emc/2.7.3 +module load w3nco/2.4.1 +module load wgrib2/2.0.8 +module load upp/10.0.6 From a87115828e72164dae4c6e61eefec8042f77cd5e Mon Sep 17 00:00:00 2001 From: David Huber Date: Fri, 15 Oct 2021 18:10:27 +0000 Subject: [PATCH 04/63] Updated checkout.sh for the Jet port. #357 --- sorc/checkout.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/sorc/checkout.sh b/sorc/checkout.sh index 57503a03f5..59be740f28 100755 --- a/sorc/checkout.sh +++ b/sorc/checkout.sh @@ -41,9 +41,9 @@ fi echo gsi checkout ... if [[ ! -d gsi.fd ]] ; then rm -f ${topdir}/checkout-gsi.log - git clone --recursive https://github.com/NOAA-EMC/GSI.git gsi.fd >> ${logdir}/checkout-gsi.log 2>&1 + git clone --recursive https://github.com/DavidHuber-NOAA/GSI.git gsi.fd >> ${topdir}/checkout-gsi.log 2>&1 cd gsi.fd - git checkout 9c1fc15d42573b398037319bbf8d5143ad126fb6 + git checkout port_9c1fc15_2_jet git submodule update cd ${topdir} else @@ -53,9 +53,9 @@ fi echo gldas checkout ... if [[ ! -d gldas.fd ]] ; then rm -f ${topdir}/checkout-gldas.log - git clone https://github.com/NOAA-EMC/GLDAS.git gldas.fd >> ${logdir}/checkout-gldas.fd.log 2>&1 + git clone https://github.com/DavidHuber-NOAA/GLDAS.git gldas.fd >> ${topdir}/checkout-gldas.fd.log 2>&1 cd gldas.fd - git checkout gldas_gfsv16_release.v1.15.0 + git checkout port_v1.15.0_2_jet cd ${topdir} else echo 'Skip. Directory gldas.fd already exists.' @@ -75,9 +75,9 @@ fi echo UPP checkout ... if [[ ! -d gfs_post.fd ]] ; then rm -f ${topdir}/checkout-gfs_post.log - git clone https://github.com/NOAA-EMC/UPP.git gfs_post.fd >> ${logdir}/checkout-gfs_post.log 2>&1 + git clone https://github.com/DavidHuber-NOAA/UPP.git gfs_post.fd >> ${topdir}/checkout-gfs_post.log 2>&1 cd gfs_post.fd - git checkout upp_v10.0.8 + git checkout upp_v10.0.8_port_2_jet git submodule update --init CMakeModules ################################################################################ # checkout_gtg From 03fbeb11dcf4e8d5cf97738739da049575f5883a Mon Sep 17 00:00:00 2001 From: David Huber Date: Mon, 18 Oct 2021 13:09:46 +0000 Subject: [PATCH 05/63] Removed debug "set -x" from checkout.sh --- sorc/checkout.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/sorc/checkout.sh b/sorc/checkout.sh index 59be740f28..355b5ba247 100755 --- a/sorc/checkout.sh +++ b/sorc/checkout.sh @@ -1,6 +1,5 @@ #!/bin/sh #set -xue -set -x while getopts "om:" option; do case $option in From 5f92e1ba57f0adf4e81e60f8bee4e95cbde53782 Mon Sep 17 00:00:00 2001 From: David Huber Date: Mon, 18 Oct 2021 13:40:52 +0000 Subject: [PATCH 06/63] Updated Externals.cfg for Jet port. --- Externals.cfg | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Externals.cfg b/Externals.cfg index 7cc545cd8d..6359409368 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -8,22 +8,22 @@ protocol = git required = True [GSI] -hash = 9c1fc15d42573b398037319bbf8d5143ad126fb6 +tag = port_9c1fc15_2_jet local_path = sorc/gsi.fd -repo_url = https://github.com/NOAA-EMC/GSI.git +repo_url = https://github.com/DavidHuber-NOAA/GSI.git protocol = git required = True [GLDAS] -tag = gldas_gfsv16_release.v1.15.0 +tag = port_v1.15.0_2_jet local_path = sorc/gldas.fd -repo_url = https://github.com/NOAA-EMC/GLDAS.git +repo_url = https://github.com/DavidHuber-NOAA/GLDAS.git protocol = git required = True [UPP] #No externals setting = .gitmodules will be invoked for CMakeModules and comupp/src/lib/crtm2 submodules -hash = ff42e0227d6100285d4179a2572b700fd5a959cb +tag = port_ff42e022_2_jet local_path = sorc/gfs_post.fd repo_url = https://github.com/NOAA-EMC/UPP.git protocol = git From 0072da5d60c73e0ee9065ff37a669ddc51fa9b49 Mon Sep 17 00:00:00 2001 From: David Huber Date: Mon, 18 Oct 2021 13:41:18 +0000 Subject: [PATCH 07/63] Cleaned up Jet port. (#357) --- env/JET.env | 4 +++- modulefiles/module_base.jet | 4 ++-- parm/config/config.resources | 16 ++++++++-------- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/env/JET.env b/env/JET.env index 20d6d2186d..fd48784153 100755 --- a/env/JET.env +++ b/env/JET.env @@ -15,8 +15,10 @@ step=$1 if [[ "$PARTITION_BATCH" = "xjet" ]]; then export npe_node_max=24 -elif [[ "$PARTITION_BATCH" = "vjet" ]]; then +elif [[ "$PARTITION_BATCH" = "vjet" || "$PARTITION_BATCH" = "sjet" ]]; then export npe_node_max=16 +elif [[ "$PARTITION_BATCH" = "kjet" ]]; then + export npe_node_max=40 fi export launcher="srun -l --export=ALL" diff --git a/modulefiles/module_base.jet b/modulefiles/module_base.jet index 6e6fd66a96..a0f62b2b39 100644 --- a/modulefiles/module_base.jet +++ b/modulefiles/module_base.jet @@ -16,7 +16,7 @@ module load nco/4.9.1 module load gempak/7.4.2 module load prod_util/1.2.2 module load grib_util/1.2.2 -module load g2tmpl/1.10.0 +module load g2tmpl/1.10.0 module load crtm/2.3.0 setenv CRTM_FIX /lfs4/HFIP/hfv3gfs/glopara/crtm_v2.3.0 @@ -46,5 +46,5 @@ module use /contrib/anaconda/modulefiles module load anaconda/5.3.1 # waveprep -module load cdo/1.9.5 +module load cdo/1.9.5 diff --git a/parm/config/config.resources b/parm/config/config.resources index 7e028e512d..74696ceb05 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -152,7 +152,7 @@ elif [ $step = "anal" ]; then elif [[ "$PARTITION_BATCH" = "vjet" || "$PARTITION_BATCH" = "sjet" ]]; then export nth_anal=4 fi - fi + fi fi if [ $CASE = "C192" -o $CASE = "C96" -o $CASE = "C48" ]; then export npe_anal=84 @@ -161,7 +161,7 @@ elif [ $step = "anal" ]; then if [[ "$PARTITION_BATCH" != "kjet" ]]; then export nth_anal=4 fi - fi + fi fi if [[ "$machine" = "WCOSS_DELL_P3" ]]; then export nth_anal=4 @@ -190,12 +190,12 @@ elif [ $step = "analdiag" ]; then elif [ $step = "gldas" ]; then export wtime_gldas="00:10:00" - export npe_gldas=96 - export nth_gldas=1 + export npe_gldas=96 + export nth_gldas=1 export npe_node_gldas=$npe_node_max - export npe_gaussian=96 - export nth_gaussian=1 - export npe_node_gaussian=24 + export npe_gaussian=96 + export nth_gaussian=1 + export npe_node_gaussian=24 if [[ "$machine" = "WCOSS_DELL_P3" ]]; then export npe_gldas=112 ; fi if [[ "$machine" == "WCOSS_C" ]]; then export memory_gldas="3072M"; fi @@ -366,7 +366,7 @@ elif [ $step = "vrfy" ]; then fi elif [ $step = "metp" ]; then - + export nth_metp=1 export wtime_metp="03:00:00" export npe_metp=4 From c03c96d93450e62dc59dad3894ee1888ce188d91 Mon Sep 17 00:00:00 2001 From: David Huber Date: Thu, 2 Dec 2021 14:33:48 +0000 Subject: [PATCH 08/63] Update module load order for Jet (#357 #479) --- modulefiles/module_base.jet | 46 ++++++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/modulefiles/module_base.jet b/modulefiles/module_base.jet index a0f62b2b39..394453c7f6 100644 --- a/modulefiles/module_base.jet +++ b/modulefiles/module_base.jet @@ -10,41 +10,55 @@ module load hpc/1.1.0 # Load intel compiler and mpi module load hpc-intel/18.0.5.274 module load hpc-impi/2018.4.274 -module load wgrib2/2.0.8 + +# Load hpss module load hpss -module load nco/4.9.1 -module load gempak/7.4.2 -module load prod_util/1.2.2 -module load grib_util/1.2.2 -module load g2tmpl/1.10.0 -module load crtm/2.3.0 -setenv CRTM_FIX /lfs4/HFIP/hfv3gfs/glopara/crtm_v2.3.0 +# Standard 3rd party libraries module load jasper/2.0.25 module load zlib/1.2.11 module load png/1.6.35 +# Gempak +module load gempak/7.4.2 + +# HDF5 and netCDF module load hdf5/1.10.6 module load netcdf/4.7.4 + +# NCEP Libs that have no prerequisite libraries +module load prod_util/1.2.2 +module load grib_util/1.2.2 +module load crtm/2.3.0 +setenv CRTM_FIX /lfs4/HFIP/hfv3gfs/glopara/crtm_v2.3.0 +module load g2tmpl/1.10.0 module load pio/2.5.2 module load esmf/8_1_1 module load fms/2020.04.03 +module load g2/3.4.3 +module load ip/3.3.3 +module load sp/2.3.3 + +# NCO and WGRIB2 need netCDF +module load nco/4.9.1 +module load wgrib2/2.0.8 +setenv WGRIB2 wgrib2 +# NEMSIO needs BACIO and W3NCO module load bacio/2.4.1 -module load g2/3.4.1 -module load ip/3.3.3 +module load w3nco/2.4.1 module load nemsio/2.5.2 -module load sp/2.3.3 + +# W3EMC needs SIGIO and NEMSIO +module load sigio/2.3.2 module load w3emc/2.7.3 -module load w3nco/2.4.1 -module load upp/10.0.6 -setenv WGRIB2 wgrib2 +# UPP needs G2, G2Tmpl, IP, SP, SIGIO, BACIO, W3NCO, NEMSIO, W3EMC, and CRTM +module load upp/10.0.6 # python module use /contrib/anaconda/modulefiles module load anaconda/5.3.1 -# waveprep +# Load CDO for waveprep module load cdo/1.9.5 - From e122d83ca378f6424c66c25fff10bb66d53fbd69 Mon Sep 17 00:00:00 2001 From: David Huber Date: Thu, 2 Dec 2021 14:43:40 +0000 Subject: [PATCH 09/63] Added Fit2Obs and TC_Tracker locations for Jet (#357) Currently, these are in my personal area. --- parm/config/config.vrfy | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/parm/config/config.vrfy b/parm/config/config.vrfy index fb307abb7a..fe4c2815cf 100755 --- a/parm/config/config.vrfy +++ b/parm/config/config.vrfy @@ -35,7 +35,7 @@ export RUNMOS="NO" # whether to run entire MOS package if [ $VRFYFITS = "YES" ]; then export fit_ver="newm.1.3" - export fitdir="$BASE_GIT/verif/global/Fit2Obs/${fit_ver}/batrun" + export fitdir="/lfs1/NESDIS/nesdis-rdo2/David.Huber/Fit2Obs/newm.1.3/batrun" export PRVT=$HOMEgfs/fix/fix_gsi/prepobs_errtable.global export HYBLEVS=$HOMEgfs/fix/fix_am/global_hyblev.l${LEVS}.txt export CUE2RUN=$QUEUE @@ -56,6 +56,8 @@ if [ $VRFYFITS = "YES" ]; then export PREPQFITSH="$fitdir/subfits_hera_slurm" elif [ $machine = "ORION" ]; then export PREPQFITSH="$fitdir/subfits_orion_netcdf" + elif [ $machine = "JET" ]; then + export PREPQFITSH="$fitdir/subfits_jet_netcdf" else echo "Fit2Obs NOT supported on this machine" fi @@ -143,7 +145,7 @@ fi #------------------------------------------------- export ens_tracker_ver=v1.1.15.4 -export HOMEens_tracker=$BASE_GIT/TC_tracker/TC_tracker.${ens_tracker_ver} +export HOMEens_tracker=/lfs1/NESDIS/nesdis-rdo2/David.Huber/TC_Tracker/TC_tracker.v1.1.15.4 if [ "$VRFYTRAK" = "YES" ]; then From 44999551751b12314bdfe06fbbcdc5e1aa1cd7aa Mon Sep 17 00:00:00 2001 From: David Huber Date: Fri, 3 Dec 2021 19:35:37 +0000 Subject: [PATCH 10/63] Corrected resources for analysis jobs on jet. (#357) --- parm/config/config.resources | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/parm/config/config.resources b/parm/config/config.resources index 74696ceb05..e2964688bd 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -132,9 +132,9 @@ elif [ $step = "anal" ]; then export nth_anal=5 export npe_anal_gfs=1000 #For the analysis jobs, the number of tasks and cores used must be equal. - #On the Jet partitions (except kjet), this is accomplished by adjusting the task + #On the Jet partitions, this is accomplished by adjusting the task #and thread count based on cores/node. - if [[ "$machine" = "Jet" && $PARTITION_BATCH != "kjet" ]]; then + if [[ "$machine" = "JET" && $PARTITION_BATCH != "kjet" ]]; then export nth_anal=4 if [[ "$PARTITION_BATCH" = "xjet" ]]; then export npe_anal=1002 @@ -144,7 +144,7 @@ elif [ $step = "anal" ]; then if [ $CASE = "C384" ]; then export npe_anal=400 export npe_anal_gfs=400 - if [[ "$machine" = "Jet" ]]; then + if [[ "$machine" = "JET" ]]; then if [[ "$PARTITION_BATCH" = "xjet" ]]; then export nth_anal=4 export npe_anal=402 @@ -157,9 +157,12 @@ elif [ $step = "anal" ]; then if [ $CASE = "C192" -o $CASE = "C96" -o $CASE = "C48" ]; then export npe_anal=84 export npe_anal_gfs=84 - if [[ "$machine" = "Jet" ]]; then + if [[ "$machine" = "JET" ]]; then if [[ "$PARTITION_BATCH" != "kjet" ]]; then export nth_anal=4 + else + export npe_anal=88 + export npe_anal_gfs=88 fi fi fi From 2d318353452c9a48dc3a4e9cc83616728e60fd9d Mon Sep 17 00:00:00 2001 From: David Huber Date: Tue, 14 Dec 2021 13:09:53 +0000 Subject: [PATCH 11/63] New location for SYNDAT data on Jet. #357 --- parm/config/config.vrfy | 1 + 1 file changed, 1 insertion(+) diff --git a/parm/config/config.vrfy b/parm/config/config.vrfy index fe4c2815cf..cbd27a6932 100755 --- a/parm/config/config.vrfy +++ b/parm/config/config.vrfy @@ -58,6 +58,7 @@ if [ $VRFYFITS = "YES" ]; then export PREPQFITSH="$fitdir/subfits_orion_netcdf" elif [ $machine = "JET" ]; then export PREPQFITSH="$fitdir/subfits_jet_netcdf" + export COMINsyn="/lfs4/HFIP/hwrf-data/hwrf-input/SYNDAT-PLUS" else echo "Fit2Obs NOT supported on this machine" fi From 4d442e3f5550296f504790cba6b711c1ad3d62fa Mon Sep 17 00:00:00 2001 From: David Huber Date: Tue, 11 Jan 2022 21:22:22 +0000 Subject: [PATCH 12/63] Updated the Jet port with the coupled changes. (#357) --- env/JET.env | 7 +++++-- modulefiles/module_base.jet | 8 ++++---- modulefiles/modulefile.ww3.jet | 18 +++--------------- modulefiles/workflow_utils.jet | 2 ++ ush/load_fv3gfs_modules.sh | 2 +- ush/rocoto/setup_expt.py | 2 +- ush/rocoto/setup_expt_fcstonly.py | 6 +++++- 7 files changed, 21 insertions(+), 24 deletions(-) diff --git a/env/JET.env b/env/JET.env index fd48784153..6e568e6b91 100755 --- a/env/JET.env +++ b/env/JET.env @@ -132,7 +132,10 @@ elif [ $step = "fcst" ]; then export NTHREADS_FV3=${nth_fv3:-$nth_max} [[ $NTHREADS_FV3 -gt $nth_max ]] && export NTHREADS_FV3=$nth_max export cores_per_node=$npe_node_max - export APRUN_FV3="$launcher" + if [[ $CDUMP == "gfs" ]]; then + npe_fcst=$npe_fcst_gfs + fi + export APRUN_FV3="$launcher -n $npe_fcst" export NTHREADS_REGRID_NEMSIO=${nth_regrid_nemsio:-1} [[ $NTHREADS_REGRID_NEMSIO -gt $nth_max ]] && export NTHREADS_REGRID_NEMSIO=$nth_max @@ -149,7 +152,7 @@ elif [ $step = "efcs" ]; then export NTHREADS_FV3=${nth_efcs:-$nth_max} [[ $NTHREADS_FV3 -gt $nth_max ]] && export NTHREADS_FV3=$nth_max export cores_per_node=$npe_node_max - export APRUN_FV3="$launcher" + export APRUN_FV3="$launcher -n $npe_efcs" export NTHREADS_REGRID_NEMSIO=${nth_regrid_nemsio:-1} [[ $NTHREADS_REGRID_NEMSIO -gt $nth_max ]] && export NTHREADS_REGRID_NEMSIO=$nth_max diff --git a/modulefiles/module_base.jet b/modulefiles/module_base.jet index 394453c7f6..581dddf7c1 100644 --- a/modulefiles/module_base.jet +++ b/modulefiles/module_base.jet @@ -33,9 +33,9 @@ module load crtm/2.3.0 setenv CRTM_FIX /lfs4/HFIP/hfv3gfs/glopara/crtm_v2.3.0 module load g2tmpl/1.10.0 module load pio/2.5.2 -module load esmf/8_1_1 -module load fms/2020.04.03 -module load g2/3.4.3 +module load esmf/8_2_0_beta_snapshot_14 +module load fms/2021.03 +module load g2/3.4.1 module load ip/3.3.3 module load sp/2.3.3 @@ -54,7 +54,7 @@ module load sigio/2.3.2 module load w3emc/2.7.3 # UPP needs G2, G2Tmpl, IP, SP, SIGIO, BACIO, W3NCO, NEMSIO, W3EMC, and CRTM -module load upp/10.0.6 +module load upp/10.0.8 # python module use /contrib/anaconda/modulefiles diff --git a/modulefiles/modulefile.ww3.jet b/modulefiles/modulefile.ww3.jet index 266f9d4722..f658d5f994 100644 --- a/modulefiles/modulefile.ww3.jet +++ b/modulefiles/modulefile.ww3.jet @@ -12,20 +12,8 @@ module load jasper/2.0.25 module load zlib/1.2.11 module load png/1.6.35 -module load hdf5/1.10.6 -module load netcdf/4.7.4 -module load pio/2.5.2 -module load esmf/8_1_1 -module load fms/2020.04.03 - module load bacio/2.4.1 -module load crtm/2.3.0 module load g2/3.4.1 -module load g2tmpl/1.9.1 -module load ip/3.3.3 -module load nemsio/2.5.2 -module load sp/2.3.3 -module load w3emc/2.7.3 -module load w3nco/2.4.1 -module load wgrib2/2.0.8 -module load upp/10.0.6 + +module load hdf5/1.10.6 +module load netcdf/4.7.4 diff --git a/modulefiles/workflow_utils.jet b/modulefiles/workflow_utils.jet index 6b1bed3440..7770fd9e7d 100644 --- a/modulefiles/workflow_utils.jet +++ b/modulefiles/workflow_utils.jet @@ -27,6 +27,8 @@ module load ip/3.3.3 module load nemsio/2.5.2 module load nemsiogfs/2.5.3 module load ncio/1.0.0 +module load landsfcutil/2.4.1 +module load wgrib2/2.0.8 module load sigio/2.3.2 module load g2/3.4.1 module load bufr/11.4.0 diff --git a/ush/load_fv3gfs_modules.sh b/ush/load_fv3gfs_modules.sh index 2f1a301db0..a13a6ee3ab 100755 --- a/ush/load_fv3gfs_modules.sh +++ b/ush/load_fv3gfs_modules.sh @@ -14,7 +14,7 @@ source "$HOMEgfs/modulefiles/module-setup.sh.inc" # Load our modules: module use "$HOMEgfs/modulefiles" -if [[ -d /lfs3 ]] ; then +if [[ -d /lfs4 ]] ; then # We are on NOAA Jet module load module_base.jet elif [[ -d /scratch1 ]] ; then diff --git a/ush/rocoto/setup_expt.py b/ush/rocoto/setup_expt.py index af302f2d54..591b337283 100755 --- a/ush/rocoto/setup_expt.py +++ b/ush/rocoto/setup_expt.py @@ -245,7 +245,7 @@ def edit_baseconfig(): dmpdir = '/lfs4/HFIP/hfv3gfs/glopara/dump' nwprod = '/lfs4/HFIP/hfv3gfs/glopara/nwpara' comroot = '/lfs4/HFIP/hfv3gfs/glopara/com' - homedir = '/lfs4/HFIP/hfv3gfs$USER' + homedir = '/lfs4/HFIP/hfv3gfs/$USER' stmp = '/lfs4/HFIP/hfv3gfs/$USER/stmp' ptmp = '/lfs4/HFIP/hfv3gfs/$USER/ptmp' noscrub = '$HOMEDIR' diff --git a/ush/rocoto/setup_expt_fcstonly.py b/ush/rocoto/setup_expt_fcstonly.py index 0ef9624b5b..0d0b0fae2c 100755 --- a/ush/rocoto/setup_expt_fcstonly.py +++ b/ush/rocoto/setup_expt_fcstonly.py @@ -223,7 +223,11 @@ def edit_baseconfig(): account = 'hfv3gfs' queue = 'batch' queue_service = 'service' - partition_batch = 'xjet' + # Set partition_batch; can be xjet or kjet (xjet is the default) + if partition is not None: + partition_batch = partition + else: + partition_batch = 'xjet' chgrp_rstprod = 'YES' chgrp_cmd = 'chgrp rstprod' hpssarch = 'YES' From da41a15c48d8b23f67016197d9baf075e472c42f Mon Sep 17 00:00:00 2001 From: David Huber Date: Wed, 19 Oct 2022 15:27:06 +0000 Subject: [PATCH 13/63] Updated Jet module files #357 --- modulefiles/module_base.jet.lua | 28 +++++++++++++++++++++++++--- modulefiles/modulefile.ww3.jet | 19 ------------------- modulefiles/modulefile.ww3.jet.lua | 23 +++++++++++++++++++++++ 3 files changed, 48 insertions(+), 22 deletions(-) delete mode 100644 modulefiles/modulefile.ww3.jet create mode 100644 modulefiles/modulefile.ww3.jet.lua diff --git a/modulefiles/module_base.jet.lua b/modulefiles/module_base.jet.lua index 2a972dbd1a..fdb4d974d5 100644 --- a/modulefiles/module_base.jet.lua +++ b/modulefiles/module_base.jet.lua @@ -7,27 +7,49 @@ prepend_path("MODULEPATH", "/lfs4/HFIP/hfv3gfs/nwprod/hpc-stack/libs/modulefiles load(pathJoin("hpc", "1.1.0")) load(pathJoin("hpc-intel", "18.0.5.274")) load(pathJoin("hpc-impi", "2018.4.274")) +load(pathJoin("cmake", "3.20.1")) -load(hpss) +load(pathJoin("hpss", "hpss")) load(pathJoin("nco", "4.9.1")) load(pathJoin("gempak", "7.4.2")) +load(pathJoin("ncl", "6.6.2")) load(pathJoin("prod_util", "1.2.2")) load(pathJoin("grib_util", "1.2.2")) load(pathJoin("crtm", "2.3.0")) +setenv("CRTM_FIX","/lfs4/HFIP/hfv3gfs/nwprod/NCEPLIBS/fix/crtm_v2.3.0") + +load(pathJoin("jasper", "2.0.25")) +load(pathJoin("zlib", "1.2.11")) +load(pathJoin("png", "1.6.35")) load(pathJoin("hdf5", "1.10.6")) load(pathJoin("netcdf", "4.7.4")) -load(pathJoin("esmf", "8_2_0_beta_snapshot_14")) +load(pathJoin("pio", "2.5.7")) +load(pathJoin("esmf", "8.3.0b09")) load(pathJoin("fms", "2021.03")) +load(pathJoin("bacio", "2.4.1")) +load(pathJoin("g2", "3.4.2")) load(pathJoin("g2tmpl", "1.10.0")) +load(pathJoin("ip", "3.3.3")) +load(pathJoin("nemsio", "2.5.2")) +load(pathJoin("sp", "2.3.3")) +load(pathJoin("w3emc", "2.7.3")) +load(pathJoin("w3nco", "2.4.1")) +load(pathJoin("ncdiag", "1.0.0")) load(pathJoin("wgrib2", "2.0.8")) +setenv("WGRIB2","wgrib2") load(pathJoin("cdo", "1.9.5")) -load(rocoto) +load(pathJoin("R", "4.0.2")) + +--TODO need to port/build prepobs +--prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/prepobs/v1.0.1/modulefiles")) +--load(pathJoin("prepobs", "1.0.1")) + whatis("Description: GFS run environment") diff --git a/modulefiles/modulefile.ww3.jet b/modulefiles/modulefile.ww3.jet deleted file mode 100644 index f658d5f994..0000000000 --- a/modulefiles/modulefile.ww3.jet +++ /dev/null @@ -1,19 +0,0 @@ -#%Module###################################################################### -# module for ww3 before base uses hpc-stack -module load cmake/3.16.1 - -module use /lfs4/HFIP/hfv3gfs/nwprod/hpc-stack/libs/modulefiles/stack - -module load hpc/1.1.0 -module load hpc-intel/18.0.5.274 -module load hpc-impi/2018.4.274 - -module load jasper/2.0.25 -module load zlib/1.2.11 -module load png/1.6.35 - -module load bacio/2.4.1 -module load g2/3.4.1 - -module load hdf5/1.10.6 -module load netcdf/4.7.4 diff --git a/modulefiles/modulefile.ww3.jet.lua b/modulefiles/modulefile.ww3.jet.lua new file mode 100644 index 0000000000..5664302b47 --- /dev/null +++ b/modulefiles/modulefile.ww3.jet.lua @@ -0,0 +1,23 @@ +help([[ +Build environment for WW3 on Hera +]]) + +prepend_path("MODULEPATH", "/lfs4/HFIP/hfv3gfs/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack") + +load(pathJoin("hpc", "1.1.0")) +load(pathJoin("hpc-intel", "2022.1.2")) +load(pathJoin("hpc-impi", "2022.1.2")) + +load(pathJoin("cmake", "3.20.1")) + +load(pathJoin("jasper", "2.0.25")) +load(pathJoin("zlib", "1.2.11")) +load(pathJoin("libpng", "1.6.37")) + +load(pathJoin("bacio", "2.4.1")) +load(pathJoin("g2", "3.4.5")) + +load(pathJoin("hdf5", "1.10.6")) +load(pathJoin("netcdf", "4.7.4")) + +load(pathJoin("w3emc", "2.9.2")) From 3205f8c73b99566a649b1414dbb6d88e7cb0cff3 Mon Sep 17 00:00:00 2001 From: David Huber Date: Wed, 19 Oct 2022 15:31:19 +0000 Subject: [PATCH 14/63] Fixed Jet WW3 module file. #357 --- modulefiles/modulefile.ww3.jet.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modulefiles/modulefile.ww3.jet.lua b/modulefiles/modulefile.ww3.jet.lua index 5664302b47..d35c0d3b04 100644 --- a/modulefiles/modulefile.ww3.jet.lua +++ b/modulefiles/modulefile.ww3.jet.lua @@ -1,8 +1,8 @@ help([[ -Build environment for WW3 on Hera +Build environment for WW3 on Jet ]]) -prepend_path("MODULEPATH", "/lfs4/HFIP/hfv3gfs/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack") +prepend_path("MODULEPATH", "/lfs4/HFIP/hfv3gfs/nwprod/hpc-stack/libs/modulefiles/stack") load(pathJoin("hpc", "1.1.0")) load(pathJoin("hpc-intel", "2022.1.2")) From f4cef9438fe20546f969bde0eb168e4328fe8bda Mon Sep 17 00:00:00 2001 From: David Huber Date: Mon, 23 Jan 2023 13:25:06 +0000 Subject: [PATCH 15/63] Updated Jet module file. #357 --- modulefiles/module_base.jet.lua | 14 ++++++++------ modulefiles/modulefile.ww3.jet.lua | 23 ----------------------- 2 files changed, 8 insertions(+), 29 deletions(-) delete mode 100644 modulefiles/modulefile.ww3.jet.lua diff --git a/modulefiles/module_base.jet.lua b/modulefiles/module_base.jet.lua index fdb4d974d5..305f8c5900 100644 --- a/modulefiles/module_base.jet.lua +++ b/modulefiles/module_base.jet.lua @@ -2,14 +2,14 @@ help([[ Load environment to run GFS on Jet ]]) -prepend_path("MODULEPATH", "/lfs4/HFIP/hfv3gfs/nwprod/hpc-stack/libs/modulefiles/stack") +prepend_path("MODULEPATH", "/lfs4/HFIP/hfv3gfs/role.epic/hpc-stack/libs/intel-18.0.5.274/modulefiles/stack") -load(pathJoin("hpc", "1.1.0")) +load(pathJoin("hpc", "1.2.0")) load(pathJoin("hpc-intel", "18.0.5.274")) load(pathJoin("hpc-impi", "2018.4.274")) load(pathJoin("cmake", "3.20.1")) -load(pathJoin("hpss", "hpss")) +load(pathJoin("hpss")) load(pathJoin("nco", "4.9.1")) load(pathJoin("gempak", "7.4.2")) load(pathJoin("ncl", "6.6.2")) @@ -22,7 +22,7 @@ setenv("CRTM_FIX","/lfs4/HFIP/hfv3gfs/nwprod/NCEPLIBS/fix/crtm_v2.3.0") load(pathJoin("jasper", "2.0.25")) load(pathJoin("zlib", "1.2.11")) -load(pathJoin("png", "1.6.35")) +load(pathJoin("libpng", "1.6.35")) load(pathJoin("hdf5", "1.10.6")) load(pathJoin("netcdf", "4.7.4")) @@ -48,8 +48,10 @@ load(pathJoin("cdo", "1.9.5")) load(pathJoin("R", "4.0.2")) --TODO need to port/build prepobs ---prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/prepobs/v1.0.1/modulefiles")) ---load(pathJoin("prepobs", "1.0.1")) +prepend_path("MODULEPATH", pathJoin("/lfs1/NESDIS/nesdis-rdo2/David.Huber/prepobs/v1.0.1/modulefiles")) +load(pathJoin("prepobs", "1.0.1")) +prepend_path("MODULEPATH", "/contrib/anaconda/modulefiles") +load(pathJoin("anaconda", "5.3.1")) whatis("Description: GFS run environment") diff --git a/modulefiles/modulefile.ww3.jet.lua b/modulefiles/modulefile.ww3.jet.lua deleted file mode 100644 index d35c0d3b04..0000000000 --- a/modulefiles/modulefile.ww3.jet.lua +++ /dev/null @@ -1,23 +0,0 @@ -help([[ -Build environment for WW3 on Jet -]]) - -prepend_path("MODULEPATH", "/lfs4/HFIP/hfv3gfs/nwprod/hpc-stack/libs/modulefiles/stack") - -load(pathJoin("hpc", "1.1.0")) -load(pathJoin("hpc-intel", "2022.1.2")) -load(pathJoin("hpc-impi", "2022.1.2")) - -load(pathJoin("cmake", "3.20.1")) - -load(pathJoin("jasper", "2.0.25")) -load(pathJoin("zlib", "1.2.11")) -load(pathJoin("libpng", "1.6.37")) - -load(pathJoin("bacio", "2.4.1")) -load(pathJoin("g2", "3.4.5")) - -load(pathJoin("hdf5", "1.10.6")) -load(pathJoin("netcdf", "4.7.4")) - -load(pathJoin("w3emc", "2.9.2")) From 0c14584ff4446734962500b834991d4f6dce78ab Mon Sep 17 00:00:00 2001 From: David Huber Date: Mon, 23 Jan 2023 13:27:26 +0000 Subject: [PATCH 16/63] Add host file for Jet. #357 --- workflow/hosts/jet.yaml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 workflow/hosts/jet.yaml diff --git a/workflow/hosts/jet.yaml b/workflow/hosts/jet.yaml new file mode 100644 index 0000000000..1cb60b1080 --- /dev/null +++ b/workflow/hosts/jet.yaml @@ -0,0 +1,23 @@ +BASE_GIT: '/lfs4/HFIP/hfv3gfs/glopara/git' +DMPDIR: '/lfs4/HFIP/hfv3gfs/glopara/dump' +PACKAGEROOT: '/lfs4/HFIP/hfv3gfs/glopara/nwpara' +COMROOT: '/lfs4/HFIP/hfv3gfs/glopara/com' +COMINsyn: '${COMROOT}/gfs/prod/syndat' +HOMEDIR: '/lfs4/HFIP/hfv3gfs/${USER}' +STMP: '/lfs4/HFIP/hfv3gfs/${USER}/stmp' +PTMP: '/lfs4/HFIP/hfv3gfs/${USER}/ptmp' +NOSCRUB: $HOMEDIR +ACCOUNT: hfv3gfs +SCHEDULER: slurm +QUEUE: batch +QUEUE_SERVICE: batch +PARTITION_BATCH: xjet +PARTITION_SERVICE: service +CHGRP_RSTPROD: 'YES' +CHGRP_CMD: 'chgrp rstprod' +HPSSARCH: 'YES' +LOCALARCH: 'NO' +ATARDIR: '/NCEPDEV/${HPSS_PROJECT}/1year/${USER}/${machine}/scratch/${PSLOT}' +MAKE_NSSTBUFR: 'NO' +MAKE_ACFTBUFR: 'NO' +SUPPORTED_RESOLUTIONS: ['C768', 'C384', 'C192', 'C96', 'C48'] From 89320b2448b77e25510e579e170eb2375de90cfa Mon Sep 17 00:00:00 2001 From: David Huber Date: Fri, 3 Feb 2023 20:56:18 +0000 Subject: [PATCH 17/63] Temporarilly disable verif-global, GLDAS, and fit2obs on Jet. #357 --- parm/config/config.base.emc.dyn | 6 +++++- parm/config/config.metp | 12 +++++++++--- parm/config/config.resources | 4 ++++ parm/config/config.vrfy | 6 +++++- 4 files changed, 23 insertions(+), 5 deletions(-) diff --git a/parm/config/config.base.emc.dyn b/parm/config/config.base.emc.dyn index 48fc04e69e..6b9343d05f 100644 --- a/parm/config/config.base.emc.dyn +++ b/parm/config/config.base.emc.dyn @@ -289,7 +289,11 @@ export IAU_DELTHRS_ENKF=6 export lobsdiag_forenkf=".true." # run GLDAS to spin up land ICs -export DO_GLDAS="YES" +if [[ $machine == "JET" ]]; then + export DO_GLDAS="NO" +else + export DO_GLDAS="YES" +fi export gldas_cyc=00 # Exception handling that when DO_GLDAS is set, the FHOUT must be 1 diff --git a/parm/config/config.metp b/parm/config/config.metp index 4be7151ffa..2afeb9da3e 100644 --- a/parm/config/config.metp +++ b/parm/config/config.metp @@ -8,9 +8,15 @@ echo "BEGIN: config.metp" # Get task specific resources . $EXPDIR/config.resources metp -export RUN_GRID2GRID_STEP1="YES" # Run grid-to-grid verification using METplus -export RUN_GRID2OBS_STEP1="YES" # Run grid-to-obs verification using METplus -export RUN_PRECIP_STEP1="YES" # Run precip verification using METplus +if [[ $machine == "JET" ]]; then + export RUN_GRID2GRID_STEP1="NO" + export RUN_GRID2OBS_STEP1="NO" + export RUN_PRECIP_STEP1="NO" +else + export RUN_GRID2GRID_STEP1="YES" # Run grid-to-grid verification using METplus + export RUN_GRID2OBS_STEP1="YES" # Run grid-to-obs verification using METplus + export RUN_PRECIP_STEP1="YES" # Run precip verification using METplus +fi #---------------------------------------------------------- diff --git a/parm/config/config.resources b/parm/config/config.resources index da4c7078cc..e1c6f7bdc6 100644 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -761,6 +761,10 @@ elif [ ${step} = "esfc" ]; then export nth_cycle=${nth_esfc} export npe_node_cycle=$(echo "${npe_node_max} / ${nth_cycle}" | bc) export memory_esfc="80GB" + #xjet only has 64GB of memory + if [[ ${PARTITION_BATCH} = "xjet" ]]; then + export memory_esfc="60GB" + fi elif [ ${step} = "efcs" ]; then diff --git a/parm/config/config.vrfy b/parm/config/config.vrfy index 28e7d81ec7..506bc7c84c 100644 --- a/parm/config/config.vrfy +++ b/parm/config/config.vrfy @@ -13,7 +13,11 @@ export CDUMPFCST="gdas" # Fit-to-obs with GDAS/GFS prepbufr export CDFNL="gdas" # Scores verification against GDAS/GFS analysis export MKPGB4PRCP="YES" # Make 0.25-deg pgb files in ARCDIR for precip verification -export VRFYFITS="YES" # Fit to observations +if [[ $machine == "JET" ]]; then + export VRFYFITS="NO" # Fit to observations +else + export VRFYFITS="YES" # Fit to observations +fi export VRFYRAD="YES" # Radiance data assimilation monitoring export VRFYOZN="YES" # Ozone data assimilation monitoring export VRFYMINMON="YES" # GSI minimization monitoring From 389037dce79f303363a7fef863d1294c45e286a6 Mon Sep 17 00:00:00 2001 From: David Huber Date: Fri, 3 Feb 2023 21:07:57 +0000 Subject: [PATCH 18/63] Temporarilly set supported platforms to C96/48 for Jet. #357 --- workflow/hosts/jet.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/hosts/jet.yaml b/workflow/hosts/jet.yaml index 1cb60b1080..8af40f801a 100644 --- a/workflow/hosts/jet.yaml +++ b/workflow/hosts/jet.yaml @@ -20,4 +20,4 @@ LOCALARCH: 'NO' ATARDIR: '/NCEPDEV/${HPSS_PROJECT}/1year/${USER}/${machine}/scratch/${PSLOT}' MAKE_NSSTBUFR: 'NO' MAKE_ACFTBUFR: 'NO' -SUPPORTED_RESOLUTIONS: ['C768', 'C384', 'C192', 'C96', 'C48'] +SUPPORTED_RESOLUTIONS: ['C96', 'C48'] From 1cf5192688bb3919961ffe17db9f14e0b03b8584 Mon Sep 17 00:00:00 2001 From: David Huber Date: Fri, 3 Feb 2023 21:24:05 +0000 Subject: [PATCH 19/63] Shorter hpss load. #357 --- modulefiles/module_base.jet.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modulefiles/module_base.jet.lua b/modulefiles/module_base.jet.lua index 305f8c5900..79cbcf04d6 100644 --- a/modulefiles/module_base.jet.lua +++ b/modulefiles/module_base.jet.lua @@ -9,7 +9,7 @@ load(pathJoin("hpc-intel", "18.0.5.274")) load(pathJoin("hpc-impi", "2018.4.274")) load(pathJoin("cmake", "3.20.1")) -load(pathJoin("hpss")) +load("hpss") load(pathJoin("nco", "4.9.1")) load(pathJoin("gempak", "7.4.2")) load(pathJoin("ncl", "6.6.2")) From 25175bfe00cc56179a566e718f16d2eff8b7a077 Mon Sep 17 00:00:00 2001 From: David Huber Date: Fri, 3 Feb 2023 21:24:41 +0000 Subject: [PATCH 20/63] Updated Jet environment file. #357 --- env/JET.env | 21 +++++++++++++++++---- jobs/JGDAS_ATMOS_VERFOZN | 1 + 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/env/JET.env b/env/JET.env index 7bb092e8e1..3d86abf66c 100755 --- a/env/JET.env +++ b/env/JET.env @@ -1,10 +1,11 @@ -#!/bin/ksh -x +#! /usr/bin/env bash if [[ $# -ne 1 ]]; then echo "Must specify an input argument to set runtime environment variables!" echo "argument can be any one of the following:" echo "atmanalrun atmensanalrun" + echo "aeroanlrun" echo "anal sfcanl fcst post vrfy metp" echo "eobs eupd ecen efcs epos" echo "postsnd awips gempak" @@ -80,7 +81,17 @@ elif [[ "${step}" = "aeroanlrun" ]]; then [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max} export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun}" -elif [ ${step} = "ocnanalrun" ]; then +elif [[ "${step}" = "ocnanalbmat" ]]; then + + export APRUNCFP="${launcher} -n \$ncmd --multi-prog" + + nth_max=$((npe_node_max / npe_node_ocnanalbmat)) + + export NTHREADS_OCNANAL=${nth_ocnanalbmat:-${nth_max}} + [[ ${NTHREADS_OCNANAL} -gt ${nth_max} ]] && export NTHREADS_OCNANAL=${nth_max} + export APRUN_OCNANAL="${launcher} -n ${npe_ocnanalbmat}" + +elif [[ "${step}" = "ocnanalrun" ]]; then export APRUNCFP="${launcher} -n \$ncmd --multi-prog" @@ -231,7 +242,7 @@ elif [[ "${step}" = "ecen" ]]; then export NTHREADS_ECEN=${nth_ecen:-${nth_max}} [[ ${NTHREADS_ECEN} -gt ${nth_max} ]] && export NTHREADS_ECEN=${nth_max} - export APRUN_ECEN="${launcher}" + export APRUN_ECEN="${launcher} -n ${npe_ecen}" export NTHREADS_CHGRES=${nth_chgres:-12} [[ ${NTHREADS_CHGRES} -gt ${npe_node_max} ]] && export NTHREADS_CHGRES=${npe_node_max} @@ -267,6 +278,8 @@ elif [[ "${step}" = "init" ]]; then elif [[ "${step}" = "postsnd" ]]; then + export CFP_MP="YES" + nth_max=$((npe_node_max / npe_node_postsnd)) export NTHREADS_POSTSND=${nth_postsnd:-1} @@ -275,7 +288,7 @@ elif [[ "${step}" = "postsnd" ]]; then export NTHREADS_POSTSNDCFP=${nth_postsndcfp:-1} [[ ${NTHREADS_POSTSNDCFP} -gt ${nth_max} ]] && export NTHREADS_POSTSNDCFP=${nth_max} - export APRUN_POSTSNDCFP="${launcher} -n ${npe_postsnd} ${mpmd_opt}" + export APRUN_POSTSNDCFP="${launcher} -n ${npe_postsndcfp} ${mpmd_opt}" elif [[ "${step}" = "awips" ]]; then diff --git a/jobs/JGDAS_ATMOS_VERFOZN b/jobs/JGDAS_ATMOS_VERFOZN index e16cb6d548..762a815f77 100755 --- a/jobs/JGDAS_ATMOS_VERFOZN +++ b/jobs/JGDAS_ATMOS_VERFOZN @@ -1,5 +1,6 @@ #! /usr/bin/env bash +set -x ############################################################# # Set up environment for GDAS Ozone Monitor job ############################################################# From 59bd97e413f5d085fa15737fbf6d62e31a276033 Mon Sep 17 00:00:00 2001 From: David Huber Date: Fri, 3 Feb 2023 21:28:31 +0000 Subject: [PATCH 21/63] Fixed TC tracker version. #357 --- parm/config/config.vrfy | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/parm/config/config.vrfy b/parm/config/config.vrfy index 506bc7c84c..67c0f16f71 100644 --- a/parm/config/config.vrfy +++ b/parm/config/config.vrfy @@ -109,8 +109,12 @@ fi # Cyclone genesis and cyclone track verification #------------------------------------------------- -export ens_tracker_ver=v1.1.15.4 -export HOMEens_tracker=/lfs1/NESDIS/nesdis-rdo2/David.Huber/TC_Tracker/TC_tracker.v1.1.15.4 +export ens_tracker_ver=v1.1.15.5 +export HOMEens_tracker=$BASE_GIT/TC_tracker/${ens_tracker_ver} +if [[ ${machine} = "JET" ]]; then + export ens_tracker_ver=v1.1.15.4 + export HOMEens_tracker=/lfs1/NESDIS/nesdis-rdo2/David.Huber/TC_Tracker/TC_tracker.v1.1.15.4 +fi if [ "$VRFYTRAK" = "YES" ]; then From 3b370edfb3286a22fd122686d389945a78314175 Mon Sep 17 00:00:00 2001 From: David Huber Date: Fri, 3 Feb 2023 21:30:09 +0000 Subject: [PATCH 22/63] Removed debug set -x. #357 --- jobs/JGDAS_ATMOS_VERFOZN | 1 - 1 file changed, 1 deletion(-) diff --git a/jobs/JGDAS_ATMOS_VERFOZN b/jobs/JGDAS_ATMOS_VERFOZN index 762a815f77..e16cb6d548 100755 --- a/jobs/JGDAS_ATMOS_VERFOZN +++ b/jobs/JGDAS_ATMOS_VERFOZN @@ -1,6 +1,5 @@ #! /usr/bin/env bash -set -x ############################################################# # Set up environment for GDAS Ozone Monitor job ############################################################# From 9bc234975a60c0468e82b47842b973200b2601bf Mon Sep 17 00:00:00 2001 From: David Huber Date: Tue, 14 Feb 2023 16:13:33 +0000 Subject: [PATCH 23/63] Update jet environment after C192, C384, partial C768 testing. #357 --- env/JET.env | 8 ++++---- modulefiles/module_base.jet.lua | 3 +-- parm/config/config.resources | 4 ++++ parm/config/config.vrfy | 5 ----- workflow/hosts/jet.yaml | 2 +- 5 files changed, 10 insertions(+), 12 deletions(-) diff --git a/env/JET.env b/env/JET.env index 3d86abf66c..70a7287ccd 100755 --- a/env/JET.env +++ b/env/JET.env @@ -22,7 +22,7 @@ elif [[ "${PARTITION_BATCH}" = "vjet" ]]; then elif [[ "${PARTITION_BATCH}" = "kjet" ]]; then export npe_node_max=40 fi -export launcher="srun -l --export=ALL" +export launcher="srun -l --epilog=/apps/local/bin/report-mem --export=ALL" export mpmd_opt="--multi-prog" # Configure MPI environment @@ -39,6 +39,7 @@ if [[ "${step}" = "prep" ]] || [[ "${step}" = "prepbufr" ]]; then export POE="NO" export BACK="NO" export sys_tp="JET" + export launcher_PREP="srun" elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then @@ -83,7 +84,7 @@ elif [[ "${step}" = "aeroanlrun" ]]; then elif [[ "${step}" = "ocnanalbmat" ]]; then - export APRUNCFP="${launcher} -n \$ncmd --multi-prog" + export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" nth_max=$((npe_node_max / npe_node_ocnanalbmat)) @@ -93,7 +94,7 @@ elif [[ "${step}" = "ocnanalbmat" ]]; then elif [[ "${step}" = "ocnanalrun" ]]; then - export APRUNCFP="${launcher} -n \$ncmd --multi-prog" + export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" nth_max=$((npe_node_max / npe_node_ocnanalrun)) @@ -125,7 +126,6 @@ elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then npe_cycle=${ntiles:-6} export APRUN_CYCLE="${launcher} -n ${npe_cycle}" - export NTHREADS_GAUSFCANL=1 npe_gausfcanl=${npe_gausfcanl:-1} export APRUN_GAUSFCANL="${launcher} -n ${npe_gausfcanl}" diff --git a/modulefiles/module_base.jet.lua b/modulefiles/module_base.jet.lua index 79cbcf04d6..403cbf6875 100644 --- a/modulefiles/module_base.jet.lua +++ b/modulefiles/module_base.jet.lua @@ -47,8 +47,7 @@ load(pathJoin("cdo", "1.9.5")) load(pathJoin("R", "4.0.2")) ---TODO need to port/build prepobs -prepend_path("MODULEPATH", pathJoin("/lfs1/NESDIS/nesdis-rdo2/David.Huber/prepobs/v1.0.1/modulefiles")) +prepend_path("MODULEPATH", pathJoin("/lfs4/HFIP/hfv3gfs/glopara/git/prepobs/v1.0.1/modulefiles")) load(pathJoin("prepobs", "1.0.1")) prepend_path("MODULEPATH", "/contrib/anaconda/modulefiles") diff --git a/parm/config/config.resources b/parm/config/config.resources index ffddf693ce..f61087d7a1 100644 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -310,6 +310,10 @@ elif [ ${step} = "anal" ]; then if [[ "${machine}" = "WCOSS2" ]]; then export nth_anal=8 export nth_anal_gfs=8 + elif [[ "${PARTITION_BATCH}" = "xjet" ]]; then + export wtime_anal="01:10:00" + export npe_anal=792 + export npe_anal_gfs=816 fi if [ ${CASE} = "C384" ]; then export npe_anal=160 diff --git a/parm/config/config.vrfy b/parm/config/config.vrfy index 67c0f16f71..13fd05ddac 100644 --- a/parm/config/config.vrfy +++ b/parm/config/config.vrfy @@ -53,7 +53,6 @@ if [ $VRFYFITS = "YES" ]; then export PREPQFITSH="$fitdir/subfits_orion_netcdf" elif [ $machine = "JET" ]; then export PREPQFITSH="$fitdir/subfits_jet_netcdf" - export COMINsyn="/lfs4/HFIP/hwrf-data/hwrf-input/SYNDAT-PLUS" else echo "Fit2Obs NOT supported on this machine" fi @@ -111,10 +110,6 @@ fi export ens_tracker_ver=v1.1.15.5 export HOMEens_tracker=$BASE_GIT/TC_tracker/${ens_tracker_ver} -if [[ ${machine} = "JET" ]]; then - export ens_tracker_ver=v1.1.15.4 - export HOMEens_tracker=/lfs1/NESDIS/nesdis-rdo2/David.Huber/TC_Tracker/TC_tracker.v1.1.15.4 -fi if [ "$VRFYTRAK" = "YES" ]; then diff --git a/workflow/hosts/jet.yaml b/workflow/hosts/jet.yaml index 8af40f801a..5b4f43ff4c 100644 --- a/workflow/hosts/jet.yaml +++ b/workflow/hosts/jet.yaml @@ -20,4 +20,4 @@ LOCALARCH: 'NO' ATARDIR: '/NCEPDEV/${HPSS_PROJECT}/1year/${USER}/${machine}/scratch/${PSLOT}' MAKE_NSSTBUFR: 'NO' MAKE_ACFTBUFR: 'NO' -SUPPORTED_RESOLUTIONS: ['C96', 'C48'] +SUPPORTED_RESOLUTIONS: ['C384', 'C192', 'C96', 'C48'] From 81681a3cc2b4d8b9b07cf7e389260e74e6d08209 Mon Sep 17 00:00:00 2001 From: David Huber Date: Tue, 14 Feb 2023 16:26:17 +0000 Subject: [PATCH 24/63] Address lint comments #357 --- parm/config/config.metp | 2 +- parm/config/config.resources | 2 +- parm/config/config.vrfy | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/parm/config/config.metp b/parm/config/config.metp index 2afeb9da3e..d432a52937 100644 --- a/parm/config/config.metp +++ b/parm/config/config.metp @@ -93,7 +93,7 @@ export precip1_type_list="ccpa_accum24hr" export precip1_ccpa_accum24hr_model_bucket="06" export precip1_ccpa_accum24hr_model_var="APCP" export precip1_ccpa_accum24hr_model_file_format="pgbf{lead?fmt=%2H}.${CDUMP}.{init?fmt=%Y%m%d%H}.grib2" -export precip1_ccpa_accum24hr_fhr_min=$FHMIN_GFS +export precip1_ccpa_accum24hr_fhr_min="$FHMIN_GFS" export precip1_ccpa_accum24hr_fhr_max="180" export precip1_ccpa_accum24hr_grid="G211" export precip1_ccpa_accum24hr_gather_by="VSDB" diff --git a/parm/config/config.resources b/parm/config/config.resources index f61087d7a1..9a5ae63067 100644 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -367,7 +367,7 @@ elif [ ${step} = "analcalc" ]; then export wtime_analcalc="00:10:00" export npe_analcalc=127 - export ntasks=${npe_analcalc} + export ntasks="${npe_analcalc}" export nth_analcalc=1 export nth_echgres=4 export nth_echgres_gfs=12 diff --git a/parm/config/config.vrfy b/parm/config/config.vrfy index 13fd05ddac..b871ab9f89 100644 --- a/parm/config/config.vrfy +++ b/parm/config/config.vrfy @@ -114,7 +114,7 @@ export HOMEens_tracker=$BASE_GIT/TC_tracker/${ens_tracker_ver} if [ "$VRFYTRAK" = "YES" ]; then export TRACKERSH="$HOMEgfs/jobs/JGFS_ATMOS_CYCLONE_TRACKER" - export COMINsyn=${COMINsyn:-$(compath.py ${envir}/com/gfs/${gfs_ver})/syndat} + export COMINsyn=${COMINsyn:-$(compath.py "${envir}"/com/gfs/"${gfs_ver}")/syndat} if [ "$CDUMP" = "gdas" ]; then export FHOUT_CYCLONE=3 export FHMAX_CYCLONE=$FHMAX @@ -130,14 +130,14 @@ if [[ "$VRFYGENESIS" == "YES" && "$CDUMP" == "gfs" ]]; then export GENESISSH="$HOMEgfs/jobs/JGFS_ATMOS_CYCLONE_GENESIS" fi -if [[ "$VRFYFSU" == "YES" && "$CDUMP" == "gfs" ]]; then +if [[ "${VRFYFSU}" == "YES" && "${CDUMP}" == "gfs" ]]; then - export GENESISFSU="$HOMEgfs/jobs/JGFS_ATMOS_FSU_GENESIS" + export GENESISFSU="${HOMEgfs}/jobs/JGFS_ATMOS_FSU_GENESIS" fi -if [[ "$RUNMOS" == "YES" && "$CDUMP" == "gfs" ]]; then +if [[ "${RUNMOS}" == "YES" && "${CDUMP}" == "gfs" ]]; then - if [ "$machine" = "HERA" ] ; then + if [[ "${machine}" = "HERA" ]] ; then export RUNGFSMOSSH="${HOMEgfs}/scripts/run_gfsmos_master.sh.hera" else echo "WARNING: MOS package is not enabled on ${machine}!" From ff95dff83cacf2817e68b81f6d4faa503196fa68 Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Wed, 1 Mar 2023 17:10:23 +0000 Subject: [PATCH 25/63] Fit2Obs updates for vrfy.sh and config.vrfy - Add new variables to vrfy.sh (as done in v16 package). - Update COM_PRP path variable to use "obs" instead of "atmos" after recent change to ROTDIR structure for observations. - Change "RUN_ENVIR" variable to "OUTPUT_FILETYPE" to avoid confusion with how RUN_ENVIR is used (nco vs emc) and use correct variable for defining type as "netcdf". - Update fit_ver to tentative new version (update again if needed). - Introduce some shellcheck updates to variables. - Update PREPQFITSH script names for each platform. Refs #1232 --- jobs/rocoto/vrfy.sh | 12 +++++++++--- parm/config/config.vrfy | 25 ++++++++++++++----------- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/jobs/rocoto/vrfy.sh b/jobs/rocoto/vrfy.sh index 6324eec568..b3ab7e99b2 100755 --- a/jobs/rocoto/vrfy.sh +++ b/jobs/rocoto/vrfy.sh @@ -68,12 +68,18 @@ if [ ${VRFYFITS} = "YES" -a ${CDUMP} = ${CDFNL} -a ${CDATE} != ${SDATE} ]; then xdate=$(${NDATE} -${VBACKUP_FITS} ${CDATE}) - export RUN_ENVIR_SAVE=${RUN_ENVIR} - export RUN_ENVIR="netcdf" + export vday=$(echo ${xdate} | cut -c1-8) + export vcyc=$(echo ${xdate} | cut -c9-10) + export COMDAY=${ROTDIR}/logs/${xdate} + export COM_INA=${ROTDIR}/gdas.${vday}/${vcyc}/atmos + export COM_INF='$ROTDIR/vrfyarch/gfs.$fdy/$fzz' + export COM_PRP='$ROTDIR/gdas.$pdy/$cyc/obs' + + export OUTPUT_FILETYPE_SAVE=${OUTPUT_FILETYPE} ${PREPQFITSH} ${PSLOT} ${xdate} ${ROTDIR} ${ARCDIR} ${TMPDIR} - export RUN_ENVIR=${RUN_ENVIR_SAVE} + export OUTPUT_FILETYPE=${OUTPUT_FILETYPE_SAVE} fi diff --git a/parm/config/config.vrfy b/parm/config/config.vrfy index 22ec61deff..61fa5579ec 100644 --- a/parm/config/config.vrfy +++ b/parm/config/config.vrfy @@ -28,32 +28,35 @@ export RUNMOS="NO" # whether to run entire MOS package if [ $VRFYFITS = "YES" ]; then - export fit_ver="newm.1.3" - export fitdir="$BASE_GIT/verif/global/Fit2Obs/${fit_ver}/batrun" - export PRVT=$HOMEgfs/fix/gsi/prepobs_errtable.global - export HYBLEVS=$HOMEgfs/fix/am/global_hyblev.l${LEVS}.txt - export CUE2RUN=$QUEUE + export fit_ver="newm.1.7" + export fitdir="${BASE_GIT}/verif/global/Fit2Obs/${fit_ver}/batrun" + export PRVT=${HOMEgfs}/fix/gsi/prepobs_errtable.global + export HYBLEVS=${HOMEgfs}/fix/am/global_hyblev.l${LEVS}.txt + export CUE2RUN=${QUEUE} export VBACKUP_FITS=24 + export OUTPUT_FILETYPE="netcdf" + export CONVNETC="YES" + export ACPROFit="YES" - export CONVNETC="NO" if [ ${netcdf_diag:-".false."} = ".true." ]; then export CONVNETC="YES" fi - if [ $machine = "HERA" ]; then - export PREPQFITSH="$fitdir/subfits_hera_slurm" + if [ $machine = "WCOSS2" ]; then + export PREPQFITSH="$fitdir/subfits_wcoss2" + elif [ $machine = "HERA" ]; then + export PREPQFITSH="$fitdir/subfits_hera" + elif [ $machine = "ORION" ]; then + export PREPQFITSH="$fitdir/subfits_orion" elif [ $machine = "S4" ]; then export PREPQFITSH="$fitdir/subfits_s4_slurm" - elif [ $machine = "ORION" ]; then - export PREPQFITSH="$fitdir/subfits_orion_netcdf" else echo "Fit2Obs NOT supported on this machine" fi fi - #---------------------------------------------------------- # Minimization, Radiance and Ozone Monitoring #---------------------------------------------------------- From 18d459559e4977d3f432110d2e53e3a2425512d5 Mon Sep 17 00:00:00 2001 From: David Huber Date: Mon, 13 Mar 2023 20:02:33 +0000 Subject: [PATCH 26/63] Update directory lookups for Jet. #357 --- modulefiles/module-setup.sh.inc | 2 +- ush/load_fv3gfs_modules.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modulefiles/module-setup.sh.inc b/modulefiles/module-setup.sh.inc index 201daa7b8d..e5322cbb2c 100644 --- a/modulefiles/module-setup.sh.inc +++ b/modulefiles/module-setup.sh.inc @@ -22,7 +22,7 @@ if [[ -d /lfs/f1 ]] ; then source /usr/share/lmod/lmod/init/$__ms_shell fi module reset -elif [[ -d /lfs3 ]] ; then +elif [[ -d /mnt/lfs1 ]] ; then # We are on NOAA Jet if ( ! eval module help > /dev/null 2>&1 ) ; then source /apps/lmod/lmod/init/$__ms_shell diff --git a/ush/load_fv3gfs_modules.sh b/ush/load_fv3gfs_modules.sh index 11d7d88466..dbe2e4e86c 100755 --- a/ush/load_fv3gfs_modules.sh +++ b/ush/load_fv3gfs_modules.sh @@ -18,7 +18,7 @@ module use "$HOMEgfs/modulefiles" if [[ -d /lfs/f1 ]]; then # We are on WCOSS2 (Cactus or Dogwood) module load module_base.wcoss2 -elif [[ -d /lfs3 ]] ; then +elif [[ -d /mnt/lfs1 ]] ; then # We are on NOAA Jet module load module_base.jet elif [[ -d /scratch1 ]] ; then From 70f3e73285eac244951c2fa49b56999a212b0971 Mon Sep 17 00:00:00 2001 From: David Huber Date: Mon, 13 Mar 2023 20:05:50 +0000 Subject: [PATCH 27/63] Increased arch job memory request. Found during #357 testing. --- parm/config/config.resources | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/config/config.resources b/parm/config/config.resources index f5c30038f9..b469096247 100644 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -661,7 +661,7 @@ elif [[ ${step} = "arch" || ${step} = "earc" || ${step} = "getic" ]]; then eval "export npe_${step}=1" eval "export npe_node_${step}=1" eval "export nth_${step}=1" - eval "export memory_${step}=2048M" + eval "export memory_${step}=4096M" if [[ "${machine}" = "WCOSS2" ]]; then eval "export memory_${step}=50GB" fi From b1c94fc807300d7a30bb3eb7cb464879fb59adae Mon Sep 17 00:00:00 2001 From: David Huber Date: Tue, 14 Mar 2023 18:07:24 +0000 Subject: [PATCH 28/63] Address lint warnings. #357 --- jobs/rocoto/vrfy.sh | 9 +++++---- parm/config/config.vrfy | 12 ++++++------ 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/jobs/rocoto/vrfy.sh b/jobs/rocoto/vrfy.sh index dd3f8efdf6..334eb487b3 100755 --- a/jobs/rocoto/vrfy.sh +++ b/jobs/rocoto/vrfy.sh @@ -68,12 +68,13 @@ if [ ${VRFYFITS} = "YES" -a ${CDUMP} = ${CDFNL} -a ${CDATE} != ${SDATE} ]; then xdate=$(${NDATE} -${VBACKUP_FITS} ${CDATE}) - export vday=$(echo ${xdate} | cut -c1-8) - export vcyc=$(echo ${xdate} | cut -c9-10) + vday=$(echo ${xdate} | cut -c1-8) + vcyc=$(echo ${xdate} | cut -c9-10) + export vday vcyc export COMDAY=${ROTDIR}/logs/${xdate} export COM_INA=${ROTDIR}/gdas.${vday}/${vcyc}/atmos - export COM_INF='$ROTDIR/vrfyarch/gfs.$fdy/$fzz' - export COM_PRP='$ROTDIR/gdas.$pdy/$cyc/obs' + export COM_INF="${ROTDIR}/vrfyarch/gfs.${fdy}/${fzz}" + export COM_PRP="${ROTDIR}/gdas.${pdy}/${cyc}/obs" export OUTPUT_FILETYPE_SAVE=${OUTPUT_FILETYPE} diff --git a/parm/config/config.vrfy b/parm/config/config.vrfy index f650ec2573..92ee2a48ef 100644 --- a/parm/config/config.vrfy +++ b/parm/config/config.vrfy @@ -47,15 +47,15 @@ if [ $VRFYFITS = "YES" ]; then export CONVNETC="YES" fi - if [ $machine = "WCOSS2" ]; then + if [[ $machine == "WCOSS2" ]]; then export PREPQFITSH="$fitdir/subfits_wcoss2" - elif [ $machine = "HERA" ]; then + elif [[ $machine == "HERA" ]]; then export PREPQFITSH="$fitdir/subfits_hera" - elif [ $machine = "ORION" ]; then + elif [[ $machine == "ORION" ]]; then export PREPQFITSH="$fitdir/subfits_orion" - elif [ $machine = "S4" ]; then + elif [[ $machine == "S4" ]]; then export PREPQFITSH="$fitdir/subfits_s4_slurm" - elif [ $machine = "JET" ]; then + elif [[ $machine == "JET" ]]; then export PREPQFITSH="$fitdir/subfits_jet" else echo "Fit2Obs NOT supported on this machine" @@ -67,7 +67,7 @@ fi # Minimization, Radiance and Ozone Monitoring #---------------------------------------------------------- -if [ $VRFYRAD = "YES" -o $VRFYMINMON = "YES" -o $VRFYOZN = "YES" ]; then +if [[ $VRFYRAD == "YES" || $VRFYMINMON == "YES" || $VRFYOZN == "YES" ]]; then export envir="para" export COM_IN=$ROTDIR From 4da6ab030a48e4f6c3372e62f199ab9e13f2715c Mon Sep 17 00:00:00 2001 From: David Huber Date: Tue, 21 Mar 2023 19:33:52 +0000 Subject: [PATCH 29/63] Removed xjet-specific resources. #357 Also cleaned up to address shellcheck warnings --- parm/config/config.resources | 250 +++++++++++++++++++---------------- 1 file changed, 135 insertions(+), 115 deletions(-) diff --git a/parm/config/config.resources b/parm/config/config.resources index 2dce94de7d..ee958f9035 100644 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -4,7 +4,7 @@ # Set resource information for job tasks # e.g. walltime, node, cores per node, memory etc. -if [ $# -ne 1 ]; then +if [[ $# -ne 1 ]]; then echo "Must specify an input task argument to set resource variables!" echo "argument can be any one of the following:" @@ -49,7 +49,7 @@ elif [[ ${machine} = "ORION" ]]; then export npe_node_max=40 fi -if [ ${step} = "prep" ]; then +if [[ ${step} = "prep" ]]; then export wtime_prep='00:30:00' export npe_prep=4 export npe_node_prep=2 @@ -60,99 +60,110 @@ if [ ${step} = "prep" ]; then export memory_prep="40G" fi -elif [ ${step} = "aerosol_init" ]; then +elif [[ ${step} = "aerosol_init" ]]; then export wtime_aerosol_init="00:05:00" export npe_aerosol_init=1 export nth_aerosol_init=1 - export npe_node_aerosol_init=$(echo "${npe_node_max} / ${nth_aerosol_init}" | bc) + npe_node_aerosol_init=$(echo "${npe_node_max} / ${nth_aerosol_init}" | bc) + export npe_node_aerosol_init export NTASKS=${npe_aerosol_init} export memory_aerosol_init="6G" -elif [ ${step} = "waveinit" ]; then +elif [[ ${step} = "waveinit" ]]; then export wtime_waveinit="00:10:00" export npe_waveinit=12 export nth_waveinit=1 - export npe_node_waveinit=$(echo "${npe_node_max} / ${nth_waveinit}" | bc) + npe_node_waveinit=$(echo "${npe_node_max} / ${nth_waveinit}" | bc) + export npe_node_waveinit export NTASKS=${npe_waveinit} export memory_waveinit="2GB" -elif [ ${step} = "waveprep" ]; then +elif [[ ${step} = "waveprep" ]]; then export wtime_waveprep="00:10:00" export npe_waveprep=5 export npe_waveprep_gfs=65 export nth_waveprep=1 export nth_waveprep_gfs=1 - export npe_node_waveprep=$(echo "${npe_node_max} / ${nth_waveprep}" | bc) - export npe_node_waveprep_gfs=$(echo "${npe_node_max} / ${nth_waveprep_gfs}" | bc) + npe_node_waveprep=$(echo "${npe_node_max} / ${nth_waveprep}" | bc) + export npe_node_waveprep + npe_node_waveprep_gfs=$(echo "${npe_node_max} / ${nth_waveprep_gfs}" | bc) + export npe_node_waveprep_gfs export NTASKS=${npe_waveprep} export NTASKS_gfs=${npe_waveprep_gfs} export memory_waveprep="100GB" export memory_waveprep_gfs="150GB" -elif [ ${step} = "wavepostsbs" ]; then +elif [[ ${step} = "wavepostsbs" ]]; then export wtime_wavepostsbs="00:20:00" export wtime_wavepostsbs_gfs="03:00:00" export npe_wavepostsbs=8 export nth_wavepostsbs=1 - export npe_node_wavepostsbs=$(echo "${npe_node_max} / ${nth_wavepostsbs}" | bc) + npe_node_wavepostsbs=$(echo "${npe_node_max} / ${nth_wavepostsbs}" | bc) + export npe_node_wavepostsbs export NTASKS=${npe_wavepostsbs} export memory_wavepostsbs="10GB" export memory_wavepostsbs_gfs="10GB" -elif [ ${step} = "wavepostbndpnt" ]; then +elif [[ ${step} = "wavepostbndpnt" ]]; then export wtime_wavepostbndpnt="01:00:00" export npe_wavepostbndpnt=240 export nth_wavepostbndpnt=1 - export npe_node_wavepostbndpnt=$(echo "${npe_node_max} / ${nth_wavepostbndpnt}" | bc) + npe_node_wavepostbndpnt=$(echo "${npe_node_max} / ${nth_wavepostbndpnt}" | bc) + export npe_node_wavepostbndpnt export NTASKS=${npe_wavepostbndpnt} export is_exclusive=True -elif [ ${step} = "wavepostbndpntbll" ]; then +elif [[ ${step} = "wavepostbndpntbll" ]]; then export wtime_wavepostbndpntbll="01:00:00" export npe_wavepostbndpntbll=448 export nth_wavepostbndpntbll=1 - export npe_node_wavepostbndpntbll=$(echo "${npe_node_max} / ${nth_wavepostbndpntbll}" | bc) + npe_node_wavepostbndpntbll=$(echo "${npe_node_max} / ${nth_wavepostbndpntbll}" | bc) + export npe_node_wavepostbndpntbll export NTASKS=${npe_wavepostbndpntbll} export is_exclusive=True -elif [ ${step} = "wavepostpnt" ]; then +elif [[ ${step} = "wavepostpnt" ]]; then export wtime_wavepostpnt="01:30:00" export npe_wavepostpnt=200 export nth_wavepostpnt=1 - export npe_node_wavepostpnt=$(echo "${npe_node_max} / ${nth_wavepostpnt}" | bc) + npe_node_wavepostpnt=$(echo "${npe_node_max} / ${nth_wavepostpnt}" | bc) + export npe_node_wavepostpnt export NTASKS=${npe_wavepostpnt} export is_exclusive=True -elif [ ${step} = "wavegempak" ]; then +elif [[ ${step} = "wavegempak" ]]; then export wtime_wavegempak="02:00:00" export npe_wavegempak=1 export nth_wavegempak=1 - export npe_node_wavegempak=$(echo "${npe_node_max} / ${nth_wavegempak}" | bc) + npe_node_wavegempak=$(echo "${npe_node_max} / ${nth_wavegempak}" | bc) + export npe_node_wavegempak export NTASKS=${npe_wavegempak} export memory_wavegempak="1GB" -elif [ ${step} = "waveawipsbulls" ]; then +elif [[ ${step} = "waveawipsbulls" ]]; then export wtime_waveawipsbulls="00:20:00" export npe_waveawipsbulls=1 export nth_waveawipsbulls=1 - export npe_node_waveawipsbulls=$(echo "${npe_node_max} / ${nth_waveawipsbulls}" | bc) + npe_node_waveawipsbulls=$(echo "${npe_node_max} / ${nth_waveawipsbulls}" | bc) + export npe_node_waveawipsbulls export NTASKS=${npe_waveawipsbulls} export is_exclusive=True -elif [ ${step} = "waveawipsgridded" ]; then +elif [[ ${step} = "waveawipsgridded" ]]; then export wtime_waveawipsgridded="02:00:00" export npe_waveawipsgridded=1 export nth_waveawipsgridded=1 - export npe_node_waveawipsgridded=$(echo "${npe_node_max} / ${nth_waveawipsgridded}" | bc) + npe_node_waveawipsgridded=$(echo "${npe_node_max} / ${nth_waveawipsgridded}" | bc) + export npe_node_waveawipsgridded export NTASKS=${npe_waveawipsgridded} export memory_waveawipsgridded_gfs="1GB" @@ -324,7 +335,7 @@ elif [[ "${step}" = "ocnanalpost" ]]; then npe_node_ocnanalpost=$(echo "${npe_node_max} / ${nth_ocnanalpost}" | bc) export npe_node_ocnanalpost -elif [ ${step} = "anal" ]; then +elif [[ ${step} = "anal" ]]; then export wtime_anal="00:50:00" export wtime_anal_gfs="00:40:00" @@ -335,23 +346,13 @@ elif [ ${step} = "anal" ]; then if [[ "${machine}" = "WCOSS2" ]]; then export nth_anal=8 export nth_anal_gfs=8 - elif [[ "${PARTITION_BATCH}" = "xjet" ]]; then - export wtime_anal="01:10:00" - export npe_anal=792 - export npe_anal_gfs=816 fi - if [ ${CASE} = "C384" ]; then + if [[ ${CASE} = "C384" ]]; then export npe_anal=160 export npe_anal_gfs=160 export nth_anal=10 export nth_anal_gfs=10 - if [[ ${PARTITION_BATCH} = "xjet" ]]; then - #For the analysis jobs, the number of tasks and cores used must be equal - #On the xjet partition, this is accomplished by increasing the task - #count to a multiple of 24 - export npe_anal=168 - export npe_anal_gfs=168 - elif [[ ${machine} = "S4" ]]; then + if [[ ${machine} = "S4" ]]; then #On the S4-s4 partition, this is accomplished by increasing the task #count to a multiple of 32 if [[ ${PARTITION_BATCH} = "s4" ]]; then @@ -367,10 +368,7 @@ elif [ ${step} = "anal" ]; then if [[ ${CASE} = "C192" || ${CASE} = "C96" || ${CASE} = "C48" ]]; then export npe_anal=84 export npe_anal_gfs=84 - if [[ $PARTITION_BATCH = "xjet" ]]; then - export nth_anal=4 - export nth_anal_gfs=4 - elif [[ ${machine} = "S4" ]]; then + if [[ ${machine} = "S4" ]]; then export nth_anal=4 export nth_anal_gfs=4 #Adjust job count for S4 @@ -383,12 +381,14 @@ elif [ ${step} = "anal" ]; then fi fi fi - export npe_node_anal=$(echo "${npe_node_max} / ${nth_anal}" | bc) + npe_node_anal=$(echo "${npe_node_max} / ${nth_anal}" | bc) + export npe_node_anal export nth_cycle=${nth_anal} - export npe_node_cycle=$(echo "${npe_node_max} / ${nth_cycle}" | bc) + npe_node_cycle=$(echo "${npe_node_max} / ${nth_cycle}" | bc) + export npe_node_cycle export is_exclusive=True -elif [ ${step} = "analcalc" ]; then +elif [[ ${step} = "analcalc" ]]; then export wtime_analcalc="00:10:00" export npe_analcalc=127 @@ -396,34 +396,39 @@ elif [ ${step} = "analcalc" ]; then export nth_analcalc=1 export nth_echgres=4 export nth_echgres_gfs=12 - export npe_node_analcalc=$(echo "${npe_node_max} / ${nth_analcalc}" | bc) + npe_node_analcalc=$(echo "${npe_node_max} / ${nth_analcalc}" | bc) + export npe_node_analcalc export is_exclusive=True -elif [ ${step} = "analdiag" ]; then +elif [[ ${step} = "analdiag" ]]; then export wtime_analdiag="00:15:00" export npe_analdiag=96 # Should be at least twice npe_ediag export nth_analdiag=1 - export npe_node_analdiag=$(echo "${npe_node_max} / ${nth_analdiag}" | bc) + npe_node_analdiag=$(echo "${npe_node_max} / ${nth_analdiag}" | bc) + export npe_node_analdiag export memory_analdiag="48GB" -elif [ ${step} = "sfcanl" ]; then +elif [[ ${step} = "sfcanl" ]]; then export wtime_sfcanl="00:10:00" export npe_sfcanl=6 export nth_sfcanl=1 - export npe_node_sfcanl=$(echo "${npe_node_max} / ${nth_sfcanl}" | bc) + npe_node_sfcanl=$(echo "${npe_node_max} / ${nth_sfcanl}" | bc) + export npe_node_sfcanl export is_exclusive=True -elif [ ${step} = "gldas" ]; then +elif [[ ${step} = "gldas" ]]; then export wtime_gldas="00:10:00" export npe_gldas=112 export nth_gldas=1 - export npe_node_gldas=$(echo "${npe_node_max} / ${nth_gldas}" | bc) + npe_node_gldas=$(echo "${npe_node_max} / ${nth_gldas}" | bc) + export npe_node_gldas export npe_gaussian=96 export nth_gaussian=1 - export npe_node_gaussian=$(echo "${npe_node_max} / ${nth_gaussian}" | bc) + npe_node_gaussian=$(echo "${npe_node_max} / ${nth_gaussian}" | bc) + export npe_node_gaussian export is_exclusive=True elif [[ "${step}" = "fcst" || "${step}" = "efcs" ]]; then @@ -518,25 +523,25 @@ elif [[ "${step}" = "fcst" || "${step}" = "efcs" ]]; then echo "Total PETS for ${_CDUMP} = ${NTASKS_TOT}" if [[ "${_CDUMP}" =~ "gfs" ]]; then - declare -x npe_${step}_gfs="${NTASKS_TOT}" - declare -x nth_${step}_gfs=1 # ESMF handles threading for the UFS-weather-model - declare -x npe_node_${step}_gfs="${npe_node_max}" + declare -x "npe_${step}_gfs"="${NTASKS_TOT}" + declare -x "nth_${step}_gfs"=1 # ESMF handles threading for the UFS-weather-model + declare -x "npe_node_${step}_gfs"="${npe_node_max}" else - declare -x npe_${step}="${NTASKS_TOT}" - declare -x nth_${step}=1 # ESMF handles threading for the UFS-weather-model - declare -x npe_node_${step}="${npe_node_max}" + declare -x "npe_${step}"="${NTASKS_TOT}" + declare -x "nth_${step}"=1 # ESMF handles threading for the UFS-weather-model + declare -x "npe_node_${step}"="${npe_node_max}" fi done case "${CASE}" in "C48" | "C96" | "C192") - declare -x wtime_${step}="00:30:00" - declare -x wtime_${step}_gfs="03:00:00" + declare -x wtime_"${step}"="00:30:00" + declare -x wtime_"${step}_gfs"="03:00:00" ;; "C384" | "C768" | "C1152") - declare -x wtime_${step}="01:00:00" - declare -x wtime_${step}_gfs="06:00:00" + declare -x wtime_"${step}"="01:00:00" + declare -x wtime_"${step}_gfs"="06:00:00" ;; *) echo "FATAL ERROR: Resolution ${CASE} not supported in ${step}" @@ -547,7 +552,7 @@ elif [[ "${step}" = "fcst" || "${step}" = "efcs" ]]; then unset _CDUMP _CDUMP_LIST unset NTASKS_TOT -elif [ ${step} = "ocnpost" ]; then +elif [[ ${step} = "ocnpost" ]]; then export wtime_ocnpost="00:30:00" export npe_ocnpost=1 @@ -555,12 +560,12 @@ elif [ ${step} = "ocnpost" ]; then export nth_ocnpost=1 export memory_ocnpost="96G" -elif [ ${step} = "post" ]; then +elif [[ ${step} = "post" ]]; then export wtime_post="00:12:00" export wtime_post_gfs="01:00:00" export npe_post=126 - res=$(echo ${CASE} | cut -c2-) + res=$(echo "${CASE}" | cut -c2-) if (( npe_post > res )); then export npe_post=${res} fi @@ -572,7 +577,7 @@ elif [ ${step} = "post" ]; then if [[ "${npe_node_post_gfs}" -gt "${npe_node_max}" ]]; then export npe_node_post_gfs=${npe_node_max} ; fi export is_exclusive=True -elif [ ${step} = "wafs" ]; then +elif [[ ${step} = "wafs" ]]; then export wtime_wafs="00:30:00" export npe_wafs=1 @@ -580,7 +585,7 @@ elif [ ${step} = "wafs" ]; then export nth_wafs=1 export memory_wafs="1GB" -elif [ ${step} = "wafsgcip" ]; then +elif [[ ${step} = "wafsgcip" ]]; then export wtime_wafsgcip="00:30:00" export npe_wafsgcip=2 @@ -588,39 +593,43 @@ elif [ ${step} = "wafsgcip" ]; then export npe_node_wafsgcip=1 export memory_wafsgcip="50GB" -elif [ ${step} = "wafsgrib2" ]; then +elif [[ ${step} = "wafsgrib2" ]]; then export wtime_wafsgrib2="00:30:00" export npe_wafsgrib2=18 export nth_wafsgrib2=1 - export npe_node_wafsgrib2=$(echo "${npe_node_max} / ${nth_wafsgrib2}" | bc) + npe_node_wafsgrib2=$(echo "${npe_node_max} / ${nth_wafsgrib2}" | bc) + export npe_node_wafsgrib2 export memory_wafsgrib2="80GB" -elif [ ${step} = "wafsblending" ]; then +elif [[ ${step} = "wafsblending" ]]; then export wtime_wafsblending="00:30:00" export npe_wafsblending=1 export nth_wafsblending=1 - export npe_node_wafsblending=$(echo "${npe_node_max} / ${nth_wafsblending}" | bc) + npe_node_wafsblending=$(echo "${npe_node_max} / ${nth_wafsblending}" | bc) + export npe_node_wafsblending export memory_wafsblending="15GB" -elif [ ${step} = "wafsgrib20p25" ]; then +elif [[ ${step} = "wafsgrib20p25" ]]; then export wtime_wafsgrib20p25="00:30:00" export npe_wafsgrib20p25=11 export nth_wafsgrib20p25=1 - export npe_node_wafsgrib20p25=$(echo "${npe_node_max} / ${nth_wafsgrib20p25}" | bc) + npe_node_wafsgrib20p25=$(echo "${npe_node_max} / ${nth_wafsgrib20p25}" | bc) + export npe_node_wafsgrib20p25 export memory_wafsgrib20p25="80GB" -elif [ ${step} = "wafsblending0p25" ]; then +elif [[ ${step} = "wafsblending0p25" ]]; then export wtime_wafsblending0p25="00:30:00" export npe_wafsblending0p25=1 export nth_wafsblending0p25=1 - export npe_node_wafsblending0p25=$(echo "${npe_node_max} / ${nth_wafsblending0p25}" | bc) + npe_node_wafsblending0p25=$(echo "${npe_node_max} / ${nth_wafsblending0p25}" | bc) + export npe_node_wafsblending0p25 export memory_wafsblending0p25="15GB" -elif [ ${step} = "vrfy" ]; then +elif [[ ${step} = "vrfy" ]]; then export wtime_vrfy="03:00:00" export wtime_vrfy_gfs="06:00:00" @@ -634,7 +643,7 @@ elif [ ${step} = "vrfy" ]; then fi export is_exclusive=True -elif [ ${step} = "metp" ]; then +elif [[ ${step} = "metp" ]]; then export nth_metp=1 export wtime_metp="03:00:00" @@ -645,7 +654,7 @@ elif [ ${step} = "metp" ]; then export npe_node_metp_gfs=4 export is_exclusive=True -elif [ ${step} = "echgres" ]; then +elif [[ ${step} = "echgres" ]]; then export wtime_echgres="00:10:00" export npe_echgres=3 @@ -655,7 +664,7 @@ elif [ ${step} = "echgres" ]; then export memory_echgres="200GB" fi -elif [ ${step} = "init" ]; then +elif [[ ${step} = "init" ]]; then export wtime_init="00:30:00" export npe_init=24 @@ -663,14 +672,14 @@ elif [ ${step} = "init" ]; then export npe_node_init=6 export memory_init="70G" -elif [ ${step} = "init_chem" ]; then +elif [[ ${step} = "init_chem" ]]; then export wtime_init_chem="00:30:00" export npe_init_chem=1 export npe_node_init_chem=1 export is_exclusive=True -elif [ ${step} = "mom6ic" ]; then +elif [[ ${step} = "mom6ic" ]]; then export wtime_mom6ic="00:30:00" export npe_mom6ic=24 @@ -688,7 +697,7 @@ elif [[ ${step} = "arch" || ${step} = "earc" || ${step} = "getic" ]]; then eval "export memory_${step}=50GB" fi -elif [ ${step} = "coupled_ic" ]; then +elif [[ ${step} = "coupled_ic" ]]; then export wtime_coupled_ic="00:15:00" export npe_coupled_ic=1 @@ -696,34 +705,39 @@ elif [ ${step} = "coupled_ic" ]; then export nth_coupled_ic=1 export is_exclusive=True -elif [ ${step} = "atmensanalprep" ]; then +elif [[ ${step} = "atmensanalprep" ]]; then export wtime_atmensanalprep="00:10:00" export npe_atmensanalprep=1 export nth_atmensanalprep=1 - export npe_node_atmensanalprep=$(echo "${npe_node_max} / ${nth_atmensanalprep}" | bc) + npe_node_atmensanalprep=$(echo "${npe_node_max} / ${nth_atmensanalprep}" | bc) + export npe_node_atmensanalprep export is_exclusive=True -elif [ ${step} = "atmensanalrun" ]; then +elif [[ ${step} = "atmensanalrun" ]]; then # make below case dependent later export layout_x=2 export layout_y=3 export wtime_atmensanalrun="00:30:00" - export npe_atmensanalrun=$(echo "${layout_x} * ${layout_y} * 6" | bc) - export npe_atmensanalrun_gfs=$(echo "${layout_x} * ${layout_y} * 6" | bc) + npe_atmensanalrun=$(echo "${layout_x} * ${layout_y} * 6" | bc) + export npe_atmensanalrun + npe_atmensanalrun_gfs=$(echo "${layout_x} * ${layout_y} * 6" | bc) + export npe_atmensanalrun_gfs export nth_atmensanalrun=1 export nth_atmensanalrun_gfs=${nth_atmensanalrun} export is_exclusive=True - export npe_node_atmensanalrun=$(echo "${npe_node_max} / ${nth_atmensanalrun}" | bc) + npe_node_atmensanalrun=$(echo "${npe_node_max} / ${nth_atmensanalrun}" | bc) + export npe_node_atmensanalrun -elif [ ${step} = "atmensanalpost" ]; then +elif [[ ${step} = "atmensanalpost" ]]; then export wtime_atmensanalpost="00:30:00" export npe_atmensanalpost=${npe_node_max} export nth_atmensanalpost=1 - export npe_node_atmensanalpost=$(echo "${npe_node_max} / ${nth_atmensanalpost}" | bc) + npe_node_atmensanalpost=$(echo "${npe_node_max} / ${nth_atmensanalpost}" | bc) + export npe_node_atmensanalpost export is_exclusive=True elif [[ ${step} = "eobs" || ${step} = "eomg" ]]; then @@ -740,34 +754,36 @@ elif [[ ${step} = "eobs" || ${step} = "eomg" ]]; then export npe_eomg=${npe_eobs} export nth_eobs=2 export nth_eomg=${nth_eobs} - export npe_node_eobs=$(echo "${npe_node_max} / ${nth_eobs}" | bc) + npe_node_eobs=$(echo "${npe_node_max} / ${nth_eobs}" | bc) + export npe_node_eobs export npe_node_eomg=${npe_node_eobs} export is_exclusive=True #The number of tasks and cores used must be the same for eobs - #For S4 and Jet/xjet, this is accomplished by running 10 tasks/node - if [[ ${machine} = "S4" || ${PARTITION_BATCH} = "xjet" ]]; then + #For S4, this is accomplished by running 10 tasks/node + if [[ ${machine} = "S4" ]]; then export npe_node_eobs=10 fi -elif [ ${step} = "ediag" ]; then +elif [[ ${step} = "ediag" ]]; then export wtime_ediag="00:15:00" export npe_ediag=48 export nth_ediag=1 - export npe_node_ediag=$(echo "${npe_node_max} / ${nth_ediag}" | bc) + npe_node_ediag=$(echo "${npe_node_max} / ${nth_ediag}" | bc) + export npe_node_ediag export memory_ediag="30GB" -elif [ ${step} = "eupd" ]; then +elif [[ ${step} = "eupd" ]]; then export wtime_eupd="00:30:00" - if [ ${CASE} = "C768" ]; then + if [[ ${CASE} = "C768" ]]; then export npe_eupd=480 export nth_eupd=6 if [[ "${machine}" = "WCOSS2" ]]; then export npe_eupd=315 export nth_eupd=14 fi - elif [ ${CASE} = "C384" ]; then + elif [[ ${CASE} = "C384" ]]; then export npe_eupd=270 export nth_eupd=2 if [[ "${machine}" = "WCOSS2" ]]; then @@ -786,36 +802,37 @@ elif [ ${step} = "eupd" ]; then export nth_eupd=4 fi fi - export npe_node_eupd=$(echo "${npe_node_max} / ${nth_eupd}" | bc) + npe_node_eupd=$(echo "${npe_node_max} / ${nth_eupd}" | bc) + export npe_node_eupd export is_exclusive=True -elif [ ${step} = "ecen" ]; then +elif [[ ${step} = "ecen" ]]; then export wtime_ecen="00:10:00" export npe_ecen=80 export nth_ecen=4 if [[ "${machine}" = "HERA" ]]; then export nth_ecen=6; fi if [[ ${CASE} = "C384" || ${CASE} = "C192" || ${CASE} = "C96" || ${CASE} = "C48" ]]; then export nth_ecen=2; fi - export npe_node_ecen=$(echo "${npe_node_max} / ${nth_ecen}" | bc) + npe_node_ecen=$(echo "${npe_node_max} / ${nth_ecen}" | bc) + export npe_node_ecen export nth_cycle=${nth_ecen} - export npe_node_cycle=$(echo "${npe_node_max} / ${nth_cycle}" | bc) + npe_node_cycle=$(echo "${npe_node_max} / ${nth_cycle}" | bc) + export npe_node_cycle export is_exclusive=True -elif [ ${step} = "esfc" ]; then +elif [[ ${step} = "esfc" ]]; then export wtime_esfc="00:06:00" export npe_esfc=80 export nth_esfc=1 - export npe_node_esfc=$(echo "${npe_node_max} / ${nth_esfc}" | bc) + npe_node_esfc=$(echo "${npe_node_max} / ${nth_esfc}" | bc) + export npe_node_esfc export nth_cycle=${nth_esfc} - export npe_node_cycle=$(echo "${npe_node_max} / ${nth_cycle}" | bc) + npe_node_cycle=$(echo "${npe_node_max} / ${nth_cycle}" | bc) + export npe_node_cycle export memory_esfc="80GB" - #xjet only has 64GB of memory - if [[ ${PARTITION_BATCH} = "xjet" ]]; then - export memory_esfc="60GB" - fi -elif [ ${step} = "epos" ]; then +elif [[ ${step} = "epos" ]]; then export wtime_epos="00:15:00" export npe_epos=80 @@ -823,10 +840,11 @@ elif [ ${step} = "epos" ]; then if [[ "${machine}" == "HERA" ]]; then export nth_epos=6 fi - export npe_node_epos=$(echo "${npe_node_max} / ${nth_epos}" | bc) + npe_node_epos=$(echo "${npe_node_max} / ${nth_epos}" | bc) + export npe_node_epos export is_exclusive=True -elif [ ${step} = "postsnd" ]; then +elif [[ ${step} = "postsnd" ]]; then export wtime_postsnd="02:00:00" export npe_postsnd=40 @@ -834,12 +852,14 @@ elif [ ${step} = "postsnd" ]; then export npe_node_postsnd=10 export npe_postsndcfp=9 export npe_node_postsndcfp=1 - if [[ "$(echo "${npe_node_postsnd} * ${nth_postsnd}" | bc)" -gt "${npe_node_max}" ]]; then - export npe_node_postsnd=$(echo "${npe_node_max} / ${nth_postsnd}" | bc) + postsnd_req_cores=$(echo "${npe_node_postsnd} * ${nth_postsnd}" | bc) + if [[ ${postsnd_req_cores} -gt "${npe_node_max}" ]]; then + npe_node_postsnd=$(echo "${npe_node_max} / ${nth_postsnd}" | bc) + export npe_node_postsnd fi export is_exclusive=True -elif [ ${step} = "awips" ]; then +elif [[ ${step} = "awips" ]]; then export wtime_awips="03:30:00" export npe_awips=1 @@ -847,7 +867,7 @@ elif [ ${step} = "awips" ]; then export nth_awips=1 export memory_awips="3GB" -elif [ ${step} = "gempak" ]; then +elif [[ ${step} = "gempak" ]]; then export wtime_gempak="03:00:00" export npe_gempak=2 From 822bf71cb0948f99e571252b2abd30882824695d Mon Sep 17 00:00:00 2001 From: David Huber Date: Tue, 21 Mar 2023 19:36:33 +0000 Subject: [PATCH 30/63] Set default partition to kjet on jet. #357 --- workflow/hosts/jet.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/hosts/jet.yaml b/workflow/hosts/jet.yaml index 5b4f43ff4c..903213b761 100644 --- a/workflow/hosts/jet.yaml +++ b/workflow/hosts/jet.yaml @@ -11,7 +11,7 @@ ACCOUNT: hfv3gfs SCHEDULER: slurm QUEUE: batch QUEUE_SERVICE: batch -PARTITION_BATCH: xjet +PARTITION_BATCH: kjet PARTITION_SERVICE: service CHGRP_RSTPROD: 'YES' CHGRP_CMD: 'chgrp rstprod' From 8b98e18bb5e2722c164687fb342a78c9a3b73d2d Mon Sep 17 00:00:00 2001 From: David Huber Date: Tue, 21 Mar 2023 19:43:30 +0000 Subject: [PATCH 31/63] Enable metplus and fit2obs jobs on Jet. #357 Also fix a multitude of shellcheck warnings in related scripts. --- jobs/rocoto/vrfy.sh | 70 ++++++++++++++------------- parm/config/config.metp | 42 +++++++---------- parm/config/config.vrfy | 96 +++++++++++++++++++------------------- ush/load_fv3gfs_modules.sh | 6 +-- 4 files changed, 106 insertions(+), 108 deletions(-) diff --git a/jobs/rocoto/vrfy.sh b/jobs/rocoto/vrfy.sh index dd3f8efdf6..3c3040a861 100755 --- a/jobs/rocoto/vrfy.sh +++ b/jobs/rocoto/vrfy.sh @@ -17,12 +17,14 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "vrfy" -c "base vrfy" ############################################################### export COMPONENT="atmos" -export CDATEm1=$(${NDATE} -24 ${CDATE}) -export PDYm1=$(echo ${CDATEm1} | cut -c1-8) +CDATEm1=$(${NDATE} -24 "${CDATE}") +export CDATEm1 +PDYm1=$(echo "${CDATEm1}" | cut -c1-8) +export PDYm1 -CDATEm1c=$(${NDATE} -06 ${CDATE}) -PDYm1c=$(echo ${CDATEm1c} | cut -c1-8) -pcyc=$(echo ${CDATEm1c} | cut -c9-10) +CDATEm1c=$(${NDATE} -06 "${CDATE}") +PDYm1c=$(echo "${CDATEm1c}" | cut -c1-8) +pcyc=$(echo "${CDATEm1c}" | cut -c9-10) export COMIN="${ROTDIR}/${CDUMP}.${PDY}/${cyc}/${COMPONENT}" @@ -30,20 +32,21 @@ export COMIN="${ROTDIR}/${CDUMP}.${PDY}/${cyc}/${COMPONENT}" ############################################################### echo echo "=============== START TO GENERATE QUARTER DEGREE GRIB1 FILES ===============" -if [ ${MKPGB4PRCP} = "YES" -a ${CDUMP} = "gfs" ]; then - if [ ! -d ${ARCDIR} ]; then mkdir -p ${ARCDIR} ; fi +if [[ ${MKPGB4PRCP} = "YES" && ${CDUMP} = "gfs" ]]; then + if [[ ! -d "${ARCDIR}" ]]; then mkdir -p "${ARCDIR}" ; fi nthreads_env=${OMP_NUM_THREADS:-1} # get threads set in env export OMP_NUM_THREADS=1 - cd ${COMIN} + set -e + cd "${COMIN}" fhmax=${vhr_rain:-${FHMAX_GFS}} fhr=0 - while [ ${fhr} -le ${fhmax} ]; do - fhr2=$(printf %02i ${fhr}) - fhr3=$(printf %03i ${fhr}) + while [[ ${fhr} -le ${fhmax} ]]; do + fhr2=$(printf %02i "${fhr}") + fhr3=$(printf %03i "${fhr}") fname=${CDUMP}.t${cyc}z.sfluxgrbf${fhr3}.grib2 fileout=${ARCDIR}/pgbq${fhr2}.${CDUMP}.${CDATE}.grib2 - ${WGRIB2} ${fname} -match "(:PRATE:surface:)|(:TMP:2 m above ground:)" -grib ${fileout} - (( fhr = ${fhr} + 6 )) + ${WGRIB2} "${fname}" -match "(:PRATE:surface:)|(:TMP:2 m above ground:)" -grib "${fileout}" + (( fhr = fhr + 6 )) done export OMP_NUM_THREADS=${nthreads_env} # revert to threads set in env fi @@ -52,32 +55,34 @@ fi ############################################################### echo echo "=============== START TO RUN MOS ===============" -if [ ${RUNMOS} = "YES" -a ${CDUMP} = "gfs" ]; then - ${RUNGFSMOSSH} ${PDY}${cyc} +if [[ ${RUNMOS} = "YES" && ${CDUMP} = "gfs" ]]; then + ${RUNGFSMOSSH} "${PDY}${cyc}" fi ############################################################### echo echo "=============== START TO RUN FIT2OBS VERIFICATION ===============" -if [ ${VRFYFITS} = "YES" -a ${CDUMP} = ${CDFNL} -a ${CDATE} != ${SDATE} ]; then +if [[ ${VRFYFITS} = "YES" && ${CDUMP} = "${CDFNL}" && ${CDATE} != "${SDATE}" ]]; then export CDUMPFCST=${VDUMP} export TMPDIR="${RUNDIR}/${CDATE}/${CDUMP}" - [[ ! -d ${TMPDIR} ]] && mkdir -p ${TMPDIR} + [[ ! -d ${TMPDIR} ]] && mkdir -p "${TMPDIR}" - xdate=$(${NDATE} -${VBACKUP_FITS} ${CDATE}) + xdate=$(${NDATE} -"${VBACKUP_FITS} ${CDATE}") - export vday=$(echo ${xdate} | cut -c1-8) - export vcyc=$(echo ${xdate} | cut -c9-10) + vday=$(echo "${xdate}" | cut -c1-8) + export vday + vcyc=$(echo "${xdate}" | cut -c9-10) + export vcyc export COMDAY=${ROTDIR}/logs/${xdate} export COM_INA=${ROTDIR}/gdas.${vday}/${vcyc}/atmos - export COM_INF='$ROTDIR/vrfyarch/gfs.$fdy/$fzz' - export COM_PRP='$ROTDIR/gdas.$pdy/$cyc/obs' + export COM_INF="${ROTDIR}/vrfyarch/gfs.${fdy}/${fzz}" + export COM_PRP="${ROTDIR}/gdas.${pdy}/${cyc}/obs" export OUTPUT_FILETYPE_SAVE=${OUTPUT_FILETYPE} - ${PREPQFITSH} ${PSLOT} ${xdate} ${ROTDIR} ${ARCDIR} ${TMPDIR} + ${PREPQFITSH} "${PSLOT} ${xdate} ${ROTDIR} ${ARCDIR} ${TMPDIR}" export OUTPUT_FILETYPE=${OUTPUT_FILETYPE_SAVE} @@ -87,7 +92,7 @@ fi ############################################################### echo echo "=============== START TO RUN RADMON DATA EXTRACTION ===============" -if [ ${VRFYRAD} = "YES" -a "${CDUMP}" = "${CDFNL}" -a "${CDATE}" != "${SDATE}" ]; then +if [[ ${VRFYRAD} = "YES" && ${CDUMP} = "${CDFNL}" && ${CDATE} != "${SDATE}" ]]; then export EXP=${PSLOT} export COMOUT="${ROTDIR}/${CDUMP}.${PDY}/${cyc}/${COMPONENT}" @@ -103,7 +108,7 @@ fi ############################################################### echo echo "=============== START TO RUN OZMON DATA EXTRACTION ===============" -if [ "${VRFYOZN}" = "YES" -a "${CDUMP}" = "${CDFNL}" -a "${CDATE}" != "${SDATE}" ]; then +if [[ ${VRFYOZN} = "YES" && ${CDUMP} = "${CDFNL}" && ${CDATE} != "${SDATE}" ]]; then export EXP=${PSLOT} export COMOUT="${ROTDIR}/${CDUMP}.${PDY}/${cyc}/${COMPONENT}" @@ -119,7 +124,7 @@ fi ############################################################### echo echo "=============== START TO RUN MINMON ===============" -if [ "${VRFYMINMON}" = "YES" -a "${CDATE}" != "${SDATE}" ]; then +if [[ ${VRFYMINMON} = "YES" && ${CDATE} != "${SDATE}" ]]; then export COMOUT="${ROTDIR}/${CDUMP}.${PDY}/${cyc}/${COMPONENT}" export M_TANKverfM0="${M_TANKverf}/stats/${PSLOT}/${CDUMP}.${PDY}/${cyc}" @@ -134,9 +139,10 @@ fi ################################################################################ echo echo "=============== START TO RUN CYCLONE TRACK VERIFICATION ===============" -if [ ${VRFYTRAK} = "YES" ]; then +if [[ ${VRFYTRAK} = "YES" ]]; then - export COMINsyn=${COMINsyn:-$(compath.py ${envir}/com/gfs/${gfs_ver})/syndat} + COMINsyn=${COMINsyn:-$(compath.py "${envir}/com/gfs/${gfs_ver}")/syndat} + export COMINsyn ${TRACKERSH} fi @@ -145,7 +151,7 @@ fi ################################################################################ echo echo "=============== START TO RUN CYCLONE GENESIS VERIFICATION ===============" -if [ ${VRFYGENESIS} = "YES" -a "${CDUMP}" = "gfs" ]; then +if [[ ${VRFYGENESIS} = "YES" && "${CDUMP}" = "gfs" ]]; then ${GENESISSH} fi @@ -153,15 +159,15 @@ fi ################################################################################ echo echo "=============== START TO RUN CYCLONE GENESIS VERIFICATION (FSU) ===============" -if [ ${VRFYFSU} = "YES" -a "${CDUMP}" = "gfs" ]; then +if [[ ${VRFYFSU} = "YES" && "${CDUMP}" = "gfs" ]]; then ${GENESISFSU} fi ############################################################### # Force Exit out cleanly -cd ${DATAROOT} -if [ ${KEEPDATA:-"NO"} = "NO" ] ; then rm -rf ${DATA} ; fi +cd "${DATAROOT}" +if [[ ${KEEPDATA:-"NO"} = "NO" ]] ; then rm -rf "${DATA}" ; fi exit 0 diff --git a/parm/config/config.metp b/parm/config/config.metp index d432a52937..8fb55f275f 100644 --- a/parm/config/config.metp +++ b/parm/config/config.metp @@ -6,17 +6,11 @@ echo "BEGIN: config.metp" # Get task specific resources -. $EXPDIR/config.resources metp +. "${EXPDIR}"/config.resources metp -if [[ $machine == "JET" ]]; then - export RUN_GRID2GRID_STEP1="NO" - export RUN_GRID2OBS_STEP1="NO" - export RUN_PRECIP_STEP1="NO" -else - export RUN_GRID2GRID_STEP1="YES" # Run grid-to-grid verification using METplus - export RUN_GRID2OBS_STEP1="YES" # Run grid-to-obs verification using METplus - export RUN_PRECIP_STEP1="YES" # Run precip verification using METplus -fi +export RUN_GRID2GRID_STEP1="YES" # Run grid-to-grid verification using METplus +export RUN_GRID2OBS_STEP1="YES" # Run grid-to-obs verification using METplus +export RUN_PRECIP_STEP1="YES" # Run precip verification using METplus #---------------------------------------------------------- @@ -24,15 +18,15 @@ fi #---------------------------------------------------------- ## EMC_VERIF_GLOBAL SETTINGS export HOMEverif_global=${HOMEgfs}/sorc/verif-global.fd -export VERIF_GLOBALSH=$HOMEverif_global/ush/run_verif_global_in_global_workflow.sh +export VERIF_GLOBALSH=${HOMEverif_global}/ush/run_verif_global_in_global_workflow.sh ## INPUT DATA SETTINGS -export model=$PSLOT +export model=${PSLOT} export model_file_format="pgbf{lead?fmt=%2H}.${CDUMP}.{init?fmt=%Y%m%d%H}.grib2" -export model_hpss_dir=$ATARDIR/.. +export model_hpss_dir=${ATARDIR}/.. export get_data_from_hpss="NO" export hpss_walltime="10" ## OUTPUT SETTINGS -export model_stat_dir=$ARCDIR/.. +export model_stat_dir=${ARCDIR}/.. export make_met_data_by="VALID" export SENDMETVIEWER="NO" ## DATE SETTINGS @@ -45,20 +39,20 @@ export log_MET_output_to_METplus="yes" export g2g1_type_list="anom pres sfc" export g2g1_anom_truth_name="self_anl" export g2g1_anom_truth_file_format="pgbanl.${CDUMP}.{valid?fmt=%Y%m%d%H}.grib2" -export g2g1_anom_fhr_min=$FHMIN_GFS -export g2g1_anom_fhr_max=$FHMAX_GFS +export g2g1_anom_fhr_min=${FHMIN_GFS} +export g2g1_anom_fhr_max=${FHMAX_GFS} export g2g1_anom_grid="G002" export g2g1_anom_gather_by="VSDB" export g2g1_pres_truth_name="self_anl" export g2g1_pres_truth_file_format="pgbanl.${CDUMP}.{valid?fmt=%Y%m%d%H}.grib2" -export g2g1_pres_fhr_min=$FHMIN_GFS -export g2g1_pres_fhr_max=$FHMAX_GFS +export g2g1_pres_fhr_min=${FHMIN_GFS} +export g2g1_pres_fhr_max=${FHMAX_GFS} export g2g1_pres_grid="G002" export g2g1_pres_gather_by="VSDB" export g2g1_sfc_truth_name="self_f00" export g2g1_sfc_truth_file_format="pgbf00.${CDUMP}.{valid?fmt=%Y%m%d%H}.grib2" -export g2g1_sfc_fhr_min=$FHMIN_GFS -export g2g1_sfc_fhr_max=$FHMAX_GFS +export g2g1_sfc_fhr_min=${FHMIN_GFS} +export g2g1_sfc_fhr_max=${FHMAX_GFS} export g2g1_sfc_grid="G002" export g2g1_sfc_gather_by="VSDB" export g2g1_mv_database_name="mv_${PSLOT}_grid2grid_metplus" @@ -68,19 +62,19 @@ export g2g1_mv_database_desc="Grid-to-grid METplus data for global workflow expe export g2o1_type_list="upper_air conus_sfc" export g2o1_upper_air_msg_type_list="ADPUPA" export g2o1_upper_air_vhr_list="00 06 12 18" -export g2o1_upper_air_fhr_min=$FHMIN_GFS +export g2o1_upper_air_fhr_min=${FHMIN_GFS} export g2o1_upper_air_fhr_max="240" export g2o1_upper_air_grid="G003" export g2o1_upper_air_gather_by="VSDB" export g2o1_conus_sfc_msg_type_list="ONLYSF ADPUPA" export g2o1_conus_sfc_vhr_list="00 03 06 09 12 15 18 21" -export g2o1_conus_sfc_fhr_min=$FHMIN_GFS +export g2o1_conus_sfc_fhr_min=${FHMIN_GFS} export g2o1_conus_sfc_fhr_max="240" export g2o1_conus_sfc_grid="G104" export g2o1_conus_sfc_gather_by="VSDB" export g2o1_polar_sfc_msg_type_list="IABP" export g2o1_polar_sfc_vhr_list="00 03 06 09 12 15 18 21" -export g2o1_polar_sfc_fhr_min=$FHMIN_GFS +export g2o1_polar_sfc_fhr_min=${FHMIN_GFS} export g2o1_polar_sfc_fhr_max="240" export g2o1_polar_sfc_grid="G219" export g2o1_polar_sfc_gather_by="VSDB" @@ -93,7 +87,7 @@ export precip1_type_list="ccpa_accum24hr" export precip1_ccpa_accum24hr_model_bucket="06" export precip1_ccpa_accum24hr_model_var="APCP" export precip1_ccpa_accum24hr_model_file_format="pgbf{lead?fmt=%2H}.${CDUMP}.{init?fmt=%Y%m%d%H}.grib2" -export precip1_ccpa_accum24hr_fhr_min="$FHMIN_GFS" +export precip1_ccpa_accum24hr_fhr_min=${FHMIN_GFS} export precip1_ccpa_accum24hr_fhr_max="180" export precip1_ccpa_accum24hr_grid="G211" export precip1_ccpa_accum24hr_gather_by="VSDB" diff --git a/parm/config/config.vrfy b/parm/config/config.vrfy index f650ec2573..8a921554e3 100644 --- a/parm/config/config.vrfy +++ b/parm/config/config.vrfy @@ -6,18 +6,14 @@ echo "BEGIN: config.vrfy" # Get task specific resources -. $EXPDIR/config.resources vrfy +. "${EXPDIR}/config.resources vrfy" export VDUMP="gfs" # Verifying dump export CDUMPFCST="gdas" # Fit-to-obs with GDAS/GFS prepbufr export CDFNL="gdas" # Scores verification against GDAS/GFS analysis export MKPGB4PRCP="YES" # Make 0.25-deg pgb files in ARCDIR for precip verification -if [[ $machine == "JET" ]]; then - export VRFYFITS="NO" # Fit to observations -else - export VRFYFITS="YES" # Fit to observations -fi +export VRFYFITS="YES" # Fit to observations export VRFYRAD="YES" # Radiance data assimilation monitoring export VRFYOZN="YES" # Ozone data assimilation monitoring export VRFYMINMON="YES" # GSI minimization monitoring @@ -30,7 +26,7 @@ export RUNMOS="NO" # whether to run entire MOS package # Fit to Observations #------------------------------------------------- -if [ $VRFYFITS = "YES" ]; then +if [[ ${VRFYFITS} = "YES" ]]; then export fit_ver="newm.1.7" export fitdir="${BASE_GIT}/verif/global/Fit2Obs/${fit_ver}/batrun" @@ -43,20 +39,20 @@ if [ $VRFYFITS = "YES" ]; then export CONVNETC="YES" export ACPROFit="YES" - if [ ${netcdf_diag:-".false."} = ".true." ]; then + if [[ ${netcdf_diag:-".false."} = ".true." ]]; then export CONVNETC="YES" fi - if [ $machine = "WCOSS2" ]; then - export PREPQFITSH="$fitdir/subfits_wcoss2" - elif [ $machine = "HERA" ]; then - export PREPQFITSH="$fitdir/subfits_hera" - elif [ $machine = "ORION" ]; then - export PREPQFITSH="$fitdir/subfits_orion" - elif [ $machine = "S4" ]; then - export PREPQFITSH="$fitdir/subfits_s4_slurm" - elif [ $machine = "JET" ]; then - export PREPQFITSH="$fitdir/subfits_jet" + if [[ ${machine} = "WCOSS2" ]]; then + export PREPQFITSH="${fitdir}/subfits_wcoss2" + elif [[ ${machine} = "HERA" ]]; then + export PREPQFITSH="${fitdir}/subfits_hera" + elif [[ ${machine} = "ORION" ]]; then + export PREPQFITSH="${fitdir}/subfits_orion" + elif [[ ${machine} = "S4" ]]; then + export PREPQFITSH="${fitdir}/subfits_s4_slurm" + elif [[ ${machine} = "JET" ]]; then + export PREPQFITSH="${fitdir}/subfits_jet" else echo "Fit2Obs NOT supported on this machine" fi @@ -67,40 +63,40 @@ fi # Minimization, Radiance and Ozone Monitoring #---------------------------------------------------------- -if [ $VRFYRAD = "YES" -o $VRFYMINMON = "YES" -o $VRFYOZN = "YES" ]; then +if [[ ${VRFYRAD} = "YES" || ${VRFYMINMON} = "YES" || ${VRFYOZN} = "YES" ]]; then export envir="para" - export COM_IN=$ROTDIR + export COM_IN=${ROTDIR} # Radiance Monitoring - if [[ "$VRFYRAD" == "YES" && "$CDUMP" == "$CDFNL" ]] ; then + if [[ "${VRFYRAD}" == "YES" && "${RUN}" == "${CDFNL}" ]] ; then - export RADMON_SUFFIX=$PSLOT - export TANKverf="$NOSCRUB/monitor/radmon" - export VRFYRADSH="$HOMEgfs/jobs/JGDAS_ATMOS_VERFRAD" + export RADMON_SUFFIX=${PSLOT} + export TANKverf="${NOSCRUB}/monitor/radmon" + export VRFYRADSH="${HOMEgfs}/jobs/JGDAS_ATMOS_VERFRAD" fi # Minimization Monitoring - if [[ "$VRFYMINMON" = "YES" ]] ; then - - export MINMON_SUFFIX=$PSLOT - export M_TANKverf="$NOSCRUB/monitor/minmon" - if [[ "$CDUMP" = "gdas" ]] ; then - export VRFYMINSH="$HOMEgfs/jobs/JGDAS_ATMOS_VMINMON" - elif [[ "$CDUMP" = "gfs" ]] ; then - export VRFYMINSH="$HOMEgfs/jobs/JGFS_ATMOS_VMINMON" + if [[ "${VRFYMINMON}" = "YES" ]] ; then + + export MINMON_SUFFIX=${PSLOT} + export M_TANKverf="${NOSCRUB}/monitor/minmon" + if [[ "${RUN}" = "gdas" ]] ; then + export VRFYMINSH="${HOMEgfs}/jobs/JGDAS_ATMOS_VMINMON" + elif [[ "${RUN}" = "gfs" ]] ; then + export VRFYMINSH="${HOMEgfs}/jobs/JGFS_ATMOS_VMINMON" fi fi # Ozone Monitoring - if [[ "$VRFYOZN" == "YES" && "$CDUMP" == "$CDFNL" ]] ; then + if [[ "${VRFYOZN}" == "YES" && "${RUN}" == "${CDFNL}" ]] ; then - export HOMEgfs_ozn="$HOMEgfs" - export OZNMON_SUFFIX=$PSLOT - export TANKverf_ozn="$NOSCRUB/monitor/oznmon" - export VRFYOZNSH="$HOMEgfs/jobs/JGDAS_ATMOS_VERFOZN" + export HOMEgfs_ozn="${HOMEgfs}" + export OZNMON_SUFFIX=${PSLOT} + export TANKverf_ozn="${NOSCRUB}/monitor/oznmon" + export VRFYOZNSH="${HOMEgfs}/jobs/JGDAS_ATMOS_VERFOZN" fi @@ -111,34 +107,36 @@ fi # Cyclone genesis and cyclone track verification #------------------------------------------------- -export ens_tracker_ver=v1.1.15.5 -export HOMEens_tracker=$BASE_GIT/TC_tracker/${ens_tracker_ver} +export ens_tracker_ver=v1.1.15.6 +export HOMEens_tracker=/mnt/lfs4/HFIP/hwrfv3/Jiayi.Peng/ens_tracker.${ens_tracker_ver} -if [ "$VRFYTRAK" = "YES" ]; then +if [[ "${VRFYTRAK}" = "YES" ]]; then - export TRACKERSH="$HOMEgfs/jobs/JGFS_ATMOS_CYCLONE_TRACKER" - export COMINsyn=${COMINsyn:-$(compath.py "${envir}"/com/gfs/"${gfs_ver}")/syndat} - if [ "$CDUMP" = "gdas" ]; then + export TRACKERSH="${HOMEgfs}/jobs/JGFS_ATMOS_CYCLONE_TRACKER" + COMINsyn=${COMINsyn:-$(compath.py "${envir}"/com/gfs/"${gfs_ver}")/syndat} + export COMINsyn + if [[ "${RUN}" = "gdas" ]]; then export FHOUT_CYCLONE=3 - export FHMAX_CYCLONE=$FHMAX + export FHMAX_CYCLONE=${FHMAX} else export FHOUT_CYCLONE=6 - export FHMAX_CYCLONE=$(( FHMAX_GFS<240 ? FHMAX_GFS : 240 )) + FHMAX_CYCLONE=$(( FHMAX_GFS<240 ? FHMAX_GFS : 240 )) + export FHMAX_CYCLONE fi fi -if [[ "$VRFYGENESIS" == "YES" && "$CDUMP" == "gfs" ]]; then +if [[ "${VRFYGENESIS}" == "YES" && "${RUN}" == "gfs" ]]; then - export GENESISSH="$HOMEgfs/jobs/JGFS_ATMOS_CYCLONE_GENESIS" + export GENESISSH="${HOMEgfs}/jobs/JGFS_ATMOS_CYCLONE_GENESIS" fi -if [[ "${VRFYFSU}" == "YES" && "${CDUMP}" == "gfs" ]]; then +if [[ "${VRFYFSU}" == "YES" && "${RUN}" == "gfs" ]]; then export GENESISFSU="${HOMEgfs}/jobs/JGFS_ATMOS_FSU_GENESIS" fi -if [[ "${RUNMOS}" == "YES" && "${CDUMP}" == "gfs" ]]; then +if [[ "${RUNMOS}" == "YES" && "${RUN}" == "gfs" ]]; then if [[ "${machine}" = "HERA" ]] ; then export RUNGFSMOSSH="${HOMEgfs}/scripts/run_gfsmos_master.sh.hera" diff --git a/ush/load_fv3gfs_modules.sh b/ush/load_fv3gfs_modules.sh index d2172f56d5..2899e69514 100755 --- a/ush/load_fv3gfs_modules.sh +++ b/ush/load_fv3gfs_modules.sh @@ -10,10 +10,10 @@ fi ulimit_s=$( ulimit -S -s ) # Find module command and purge: -source "$HOMEgfs/modulefiles/module-setup.sh.inc" +source "${HOMEgfs}/modulefiles/module-setup.sh.inc" # Load our modules: -module use "$HOMEgfs/modulefiles" +module use "${HOMEgfs}/modulefiles" if [[ -d /lfs/f1 ]]; then # We are on WCOSS2 (Cactus or Dogwood) @@ -43,7 +43,7 @@ fi module list # Restore stack soft limit: -ulimit -S -s "$ulimit_s" +ulimit -S -s "${ulimit_s}" unset ulimit_s set_trace From b7f82f1af4fddc5e6e4cf89daa688e93ad18141e Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Fri, 24 Mar 2023 15:39:04 +0000 Subject: [PATCH 32/63] Remove fit2obs from vrfy job - Remove the fit2obs section from jobs/rocoto/vrfy.sh - Remove fit2obs variables/settings from parm/config/config.vrfy Refs #1232, #1405 --- jobs/rocoto/vrfy.sh | 27 --------------------------- parm/config/config.vrfy | 39 --------------------------------------- 2 files changed, 66 deletions(-) diff --git a/jobs/rocoto/vrfy.sh b/jobs/rocoto/vrfy.sh index dd3f8efdf6..4230b8a62b 100755 --- a/jobs/rocoto/vrfy.sh +++ b/jobs/rocoto/vrfy.sh @@ -57,33 +57,6 @@ if [ ${RUNMOS} = "YES" -a ${CDUMP} = "gfs" ]; then fi -############################################################### -echo -echo "=============== START TO RUN FIT2OBS VERIFICATION ===============" -if [ ${VRFYFITS} = "YES" -a ${CDUMP} = ${CDFNL} -a ${CDATE} != ${SDATE} ]; then - - export CDUMPFCST=${VDUMP} - export TMPDIR="${RUNDIR}/${CDATE}/${CDUMP}" - [[ ! -d ${TMPDIR} ]] && mkdir -p ${TMPDIR} - - xdate=$(${NDATE} -${VBACKUP_FITS} ${CDATE}) - - export vday=$(echo ${xdate} | cut -c1-8) - export vcyc=$(echo ${xdate} | cut -c9-10) - export COMDAY=${ROTDIR}/logs/${xdate} - export COM_INA=${ROTDIR}/gdas.${vday}/${vcyc}/atmos - export COM_INF='$ROTDIR/vrfyarch/gfs.$fdy/$fzz' - export COM_PRP='$ROTDIR/gdas.$pdy/$cyc/obs' - - export OUTPUT_FILETYPE_SAVE=${OUTPUT_FILETYPE} - - ${PREPQFITSH} ${PSLOT} ${xdate} ${ROTDIR} ${ARCDIR} ${TMPDIR} - - export OUTPUT_FILETYPE=${OUTPUT_FILETYPE_SAVE} - -fi - - ############################################################### echo echo "=============== START TO RUN RADMON DATA EXTRACTION ===============" diff --git a/parm/config/config.vrfy b/parm/config/config.vrfy index 61fa5579ec..1cf08f97c8 100644 --- a/parm/config/config.vrfy +++ b/parm/config/config.vrfy @@ -8,12 +8,8 @@ echo "BEGIN: config.vrfy" # Get task specific resources . $EXPDIR/config.resources vrfy -export VDUMP="gfs" # Verifying dump -export CDUMPFCST="gdas" # Fit-to-obs with GDAS/GFS prepbufr export CDFNL="gdas" # Scores verification against GDAS/GFS analysis - export MKPGB4PRCP="YES" # Make 0.25-deg pgb files in ARCDIR for precip verification -export VRFYFITS="YES" # Fit to observations export VRFYRAD="YES" # Radiance data assimilation monitoring export VRFYOZN="YES" # Ozone data assimilation monitoring export VRFYMINMON="YES" # GSI minimization monitoring @@ -22,41 +18,6 @@ export VRFYGENESIS="YES" # Cyclone genesis verification export VRFYFSU="NO" # Cyclone genesis verification (FSU) export RUNMOS="NO" # whether to run entire MOS package -#------------------------------------------------- -# Fit to Observations -#------------------------------------------------- - -if [ $VRFYFITS = "YES" ]; then - - export fit_ver="newm.1.7" - export fitdir="${BASE_GIT}/verif/global/Fit2Obs/${fit_ver}/batrun" - export PRVT=${HOMEgfs}/fix/gsi/prepobs_errtable.global - export HYBLEVS=${HOMEgfs}/fix/am/global_hyblev.l${LEVS}.txt - export CUE2RUN=${QUEUE} - - export VBACKUP_FITS=24 - export OUTPUT_FILETYPE="netcdf" - export CONVNETC="YES" - export ACPROFit="YES" - - if [ ${netcdf_diag:-".false."} = ".true." ]; then - export CONVNETC="YES" - fi - - if [ $machine = "WCOSS2" ]; then - export PREPQFITSH="$fitdir/subfits_wcoss2" - elif [ $machine = "HERA" ]; then - export PREPQFITSH="$fitdir/subfits_hera" - elif [ $machine = "ORION" ]; then - export PREPQFITSH="$fitdir/subfits_orion" - elif [ $machine = "S4" ]; then - export PREPQFITSH="$fitdir/subfits_s4_slurm" - else - echo "Fit2Obs NOT supported on this machine" - fi - -fi - #---------------------------------------------------------- # Minimization, Radiance and Ozone Monitoring #---------------------------------------------------------- From e72698f2ba6087f4c5a4e39f3e80fc52e48356ff Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Fri, 24 Mar 2023 15:40:49 +0000 Subject: [PATCH 33/63] Add DO_FIT2OBS switch to config.base Refs #1405, #1232 --- parm/config/config.base.emc.dyn | 1 + 1 file changed, 1 insertion(+) diff --git a/parm/config/config.base.emc.dyn b/parm/config/config.base.emc.dyn index bb7162af7e..c58c740ca8 100644 --- a/parm/config/config.base.emc.dyn +++ b/parm/config/config.base.emc.dyn @@ -379,6 +379,7 @@ export binary_diag=".false." # Verification options export DO_METP="YES" # Run METPLUS jobs - set METPLUS settings in config.metp +export DO_FIT2OBS="YES" # Run fit to observations package # Archiving options export HPSSARCH="@HPSSARCH@" # save data to HPSS archive From b2d4ab0a1d0913f5cf631b5274c7c0fdf64a5c90 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Fri, 24 Mar 2023 15:41:53 +0000 Subject: [PATCH 34/63] Add fit2obs job to config.resources - Add "fit2obs" to list of jobs in config.resources - Add fit2obs resource block and values in config.resources Refs #1405, #1232 --- parm/config/config.resources | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/parm/config/config.resources b/parm/config/config.resources index 7f27e1e9ed..424344c5e5 100644 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -12,7 +12,7 @@ if [ $# -ne 1 ]; then echo "atmanalprep atmanalrun atmanalpost" echo "atmensanalprep atmensanalrun atmensanalpost" echo "aeroanlinit aeroanlrun aeroanlfinal" - echo "anal sfcanl analcalc analdiag gldas fcst post vrfy metp arch echgres" + echo "anal sfcanl analcalc analdiag gldas fcst post vrfy fit2obs metp arch echgres" echo "eobs ediag eomg eupd ecen esfc efcs epos earc" echo "init_chem mom6ic ocnpost" echo "waveinit waveprep wavepostsbs wavepostbndpnt wavepostbndpntbll wavepostpnt" @@ -622,6 +622,14 @@ elif [ ${step} = "vrfy" ]; then fi export is_exclusive=True +elif [ ${step} = "fit2obs" ]; then + + export wtime_fit2obs="00:20:00" + export npe_fit2obs=3 + export nth_fit2obs=1 + export npe_node_fit2obs=1 + if [[ ${machine} == "WCOSS2" ]]; then export npe_node_fit2obs=3 ; fi + elif [ ${step} = "metp" ]; then export nth_metp=1 From 7e324fb1275aa5f3990869ffad2eeca30f0ccccf Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Fri, 24 Mar 2023 15:42:52 +0000 Subject: [PATCH 35/63] Add new fit2obs job to workflow setup scripts - Add do_fit2obs switch. - Add fit2obs to config list. - Add fit2obs to task list. - Create fit2obs task for gdas CDUMP. - Add fit2obs as task dependency for arch job when do_fit2obs is true and CDUMP=gdas. Refs #1405, #1232 --- workflow/applications.py | 6 +++++- workflow/rocoto/workflow_tasks.py | 17 +++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/workflow/applications.py b/workflow/applications.py index 717940a1bd..34746e613e 100644 --- a/workflow/applications.py +++ b/workflow/applications.py @@ -105,6 +105,7 @@ def __init__(self, conf: Configuration) -> None: self.do_awips = _base.get('DO_AWIPS', False) self.do_wafs = _base.get('WAFSF', False) self.do_vrfy = _base.get('DO_VRFY', True) + self.do_fit2obs = _base.get('DO_FIT2OBS', True) self.do_metp = _base.get('DO_METP', False) self.do_jedivar = _base.get('DO_JEDIVAR', False) self.do_jediens = _base.get('DO_JEDIENS', False) @@ -186,7 +187,7 @@ def _cycled_configs(self): if self.do_ocean: configs += ['ocnpost'] - configs += ['sfcanl', 'analcalc', 'fcst', 'post', 'vrfy', 'arch'] + configs += ['sfcanl', 'analcalc', 'fcst', 'post', 'vrfy', 'fit2obs', 'arch'] if self.do_gldas: configs += ['gldas'] @@ -401,6 +402,9 @@ def _get_cycled_task_names(self): gdas_tasks += wave_bndpnt_tasks gdas_tasks += wave_post_tasks + if self.do_fit2obs: + gdas_tasks += ['fit2obs'] + gdas_tasks += gdas_gfs_common_cleanup_tasks # Collect "gfs" cycle tasks diff --git a/workflow/rocoto/workflow_tasks.py b/workflow/rocoto/workflow_tasks.py index 9572140b0c..140062e16a 100644 --- a/workflow/rocoto/workflow_tasks.py +++ b/workflow/rocoto/workflow_tasks.py @@ -993,6 +993,20 @@ def vrfy(self): return task + def fit2obs(self): + deps = [] + dep_dict = {'type': 'metatask', 'name': f'{self.cdump}post'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + + cycledef = 'gdas' if self.cdump in ['gdas'] else self.cdump + + resources = self.get_resource('fit2obs') + task = create_wf_task('fit2obs', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies, + cycledef=cycledef) + + return task + def metp(self): deps = [] dep_dict = {'type': 'metatask', 'name': f'{self.cdump}post'} @@ -1020,6 +1034,9 @@ def arch(self): if self.app_config.do_vrfy: dep_dict = {'type': 'task', 'name': f'{self.cdump}vrfy'} deps.append(rocoto.add_dependency(dep_dict)) + if self.app_config.do_fit2obs and self.cdump in ['gdas']: + dep_dict = {'type': 'task', 'name': f'{self.cdump}fit2obs'} + deps.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_metp and self.cdump in ['gfs']: dep_dict = {'type': 'metatask', 'name': f'{self.cdump}metp'} deps.append(rocoto.add_dependency(dep_dict)) From 2a708832321bf3cafd44c262bdd613e8cd5c2a33 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Fri, 24 Mar 2023 15:45:21 +0000 Subject: [PATCH 36/63] Create config.fit2obs for new fit2obs job Refs #1405, #1232 --- parm/config/config.fit2obs | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 parm/config/config.fit2obs diff --git a/parm/config/config.fit2obs b/parm/config/config.fit2obs new file mode 100644 index 0000000000..ee891233a8 --- /dev/null +++ b/parm/config/config.fit2obs @@ -0,0 +1,31 @@ +#! /usr/bin/env bash + +########## config.fit2obs ########## +# Fit to Observations + +echo "BEGIN: config.fit2obs" + +# Get task specific resources +. $EXPDIR/config.resources fit2obs + +export fit_ver="newm.1.7" +export fitdir="${BASE_GIT}/verif/global/Fit2Obs/${fit_ver}" + +export HOMEcfs=${fitdir} +export EXECcfs=${HOMEcfs}/exec +export USHcfs=${HOMEcfs}/ush + +export PRVT=${HOMEgfs}/fix/gsi/prepobs_errtable.global +export HYBLEVS=${HOMEgfs}/fix/am/global_hyblev.l${LEVS}.txt + +export CDUMPFCST="gfs" # Fit-to-obs verifying dump +export VBACKUP_FITS=24 +export OUTPUT_FILETYPE="netcdf" +export CONVNETC="YES" +export ACPROFit="YES" + +if [ ${netcdf_diag:-".false."} = ".true." ]; then + export CONVNETC="YES" +fi + +echo "END: config.fit2obs" From 5653ef278b661d2a4bb022e4d15e5ce5ac79da72 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Fri, 24 Mar 2023 15:46:10 +0000 Subject: [PATCH 37/63] Create new rocoto job script for fit2obs job - Create fit2obs.sh. - Script will run new jobs/JGDAS_FIT2OBS script. Refs #1405, #1232 --- jobs/rocoto/fit2obs.sh | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100755 jobs/rocoto/fit2obs.sh diff --git a/jobs/rocoto/fit2obs.sh b/jobs/rocoto/fit2obs.sh new file mode 100755 index 0000000000..12e3bf0ca4 --- /dev/null +++ b/jobs/rocoto/fit2obs.sh @@ -0,0 +1,23 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" + +############################################################### +echo +echo "=============== START TO SOURCE FV3GFS WORKFLOW MODULES ===============" +. ${HOMEgfs}/ush/load_fv3gfs_modules.sh +status=$? +[[ ${status} -ne 0 ]] && exit ${status} + +export job="fit2obs" +export jobid="${job}.$$" + +############################################################### +echo +echo "=============== START TO RUN FIT2OBS ===============" +# Execute the JJOB +${HOMEgfs}/jobs/JGDAS_FIT2OBS +status=$? +[[ ${status} -ne 0 ]] && exit ${status} + +exit 0 From bef316550d53aa83859d33cfd887dbeb4b6576d7 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Fri, 24 Mar 2023 15:49:19 +0000 Subject: [PATCH 38/63] Create JJOB script for new fit2obs job - Create jobs/JGDAS_FIT2OBS. - Script will run fit2obs package batrun/excfs_gdas_vrfyfits.sh.ecf script. Refs #1405, #1232 --- jobs/JGDAS_FIT2OBS | 57 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100755 jobs/JGDAS_FIT2OBS diff --git a/jobs/JGDAS_FIT2OBS b/jobs/JGDAS_FIT2OBS new file mode 100755 index 0000000000..444859a450 --- /dev/null +++ b/jobs/JGDAS_FIT2OBS @@ -0,0 +1,57 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" +source "${HOMEgfs}/ush/jjob_header.sh" -e "fit2obs" -c "base fit2obs" + + +############################################## +# Set variables used in the script +############################################## + +export CDATE=${CDATE:-${PDY}${cyc}} +export CDUMP=${CDUMP:-${RUN:-"gdas"}} + +export xdate=$(${NDATE} -${VBACKUP_FITS} ${CDATE}) +export vday=$(echo ${xdate} | cut -c1-8) +export vcyc=$(echo ${xdate} | cut -c9-10) + +export COM_INA=${ROTDIR}/gdas.${vday}/${vcyc}/atmos +export COM_INF='$ROTDIR/vrfyarch/gfs.$fdy/$fzz' +export COM_PRP='$ROTDIR/gdas.$pdy/$cyc/obs' + +export OUTPUT_FILETYPE=${OUTPUT_FILETYPE:-netcdf} + +export FIT_DIR=${ARCDIR}/fits +[[ ! -d "${FIT_DIR}" ]] && mkdir -p "${FIT_DIR}" +export HORZ_DIR=${ARCDIR}/horiz +[[ ! -d "${HORZ_DIR}" ]] && mkdir -p "${HORZ_DIR}" +export COMLOX=${DATA}/fitx +[[ ! -d "${COMLOX}" ]] && mkdir -p "${COMLOX}" + +############################################################### +# RUN FIT2OBS VERIFICATION +############################################################### + +"${fitdir}/batrun/excfs_gdas_vrfyfits.sh.ecf" +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + +############################################## +# End JOB SPECIFIC work +############################################## + +############################################## +# Final processing +############################################## +if [[ -e "${pgmout}" ]] ; then + cat "${pgmout}" +fi + + +########################################## +# Remove the Temporary working directory +########################################## +cd "${DATAROOT}" || (echo "${DATAROOT} does not exist. ABORT!"; exit 1) +[[ ${KEEPDATA} = "NO" ]] && rm -rf "${DATA}" + +exit 0 From df432f655362fd5b0ec3d286ca1cc2e1f064671a Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Tue, 28 Mar 2023 14:38:02 +0000 Subject: [PATCH 39/63] Add fit2obs job to env files Refs #1405 --- env/HERA.env | 11 +++++++++++ env/JET.env | 8 ++++++++ env/ORION.env | 9 +++++++++ env/S4.env | 9 +++++++++ env/WCOSS2.env | 8 ++++++++ 5 files changed, 45 insertions(+) diff --git a/env/HERA.env b/env/HERA.env index 08c730c47f..a42e3a0170 100755 --- a/env/HERA.env +++ b/env/HERA.env @@ -133,6 +133,7 @@ elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then export APRUN_GAUSFCANL="${launcher} -n ${npe_gausfcanl}" elif [[ "${step}" = "sfcanl" ]]; then + nth_max=$((npe_node_max / npe_node_sfcanl)) export NTHREADS_CYCLE=${nth_sfcanl:-14} @@ -291,4 +292,14 @@ elif [[ "${step}" = "gempak" ]]; then export NTHREADS_GEMPAK=${nth_gempak:-1} [[ ${NTHREADS_GEMPAK} -gt ${nth_max} ]] && export NTHREADS_GEMPAK=${nth_max} export APRUN="${launcher} -n ${npe_gempak} ${mpmd_opt}" + + +elif [[ "${step}" = "fit2obs" ]]; then + + nth_max=$((npe_node_max / npe_node_fit2obs)) + + export NTHREADS_FIT2OBS=${nth_fit2obs:-1} + [[ ${NTHREADS_FIT2OBS} -gt ${nth_max} ]] && export NTHREADS_FIT2OBS=${nth_max} + export MPIRUN="${launcher} -n ${npe_fit2obs}" + fi diff --git a/env/JET.env b/env/JET.env index bc2e0089b2..8d2f8c3725 100755 --- a/env/JET.env +++ b/env/JET.env @@ -204,4 +204,12 @@ elif [[ "${step}" = "gempak" ]]; then echo "WARNING: ${step} is not enabled on ${machine}!" +elif [[ "${step}" = "fit2obs" ]]; then + + nth_max=$((npe_node_max / npe_node_fit2obs)) + + export NTHREADS_FIT2OBS=${nth_fit2obs:-1} + [[ ${NTHREADS_FIT2OBS} -gt ${nth_max} ]] && export NTHREADS_FIT2OBS=${nth_max} + export MPIRUN="${launcher} -n ${npe_fit2obs}" + fi diff --git a/env/ORION.env b/env/ORION.env index 2f6b84ff7d..3ecf456f70 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -290,4 +290,13 @@ elif [[ "${step}" = "gempak" ]]; then export NTHREADS_GEMPAK=${nth_gempak:-1} [[ ${NTHREADS_GEMPAK} -gt ${nth_max} ]] && export NTHREADS_GEMPAK=${nth_max} export APRUN="${launcher} -n ${npe_gempak} ${mpmd_opt}" + +elif [[ "${step}" = "fit2obs" ]]; then + + nth_max=$((npe_node_max / npe_node_fit2obs)) + + export NTHREADS_FIT2OBS=${nth_fit2obs:-1} + [[ ${NTHREADS_FIT2OBS} -gt ${nth_max} ]] && export NTHREADS_FIT2OBS=${nth_max} + export MPIRUN="${launcher} -n ${npe_fit2obs}" + fi diff --git a/env/S4.env b/env/S4.env index 6292e40481..e48cde8505 100755 --- a/env/S4.env +++ b/env/S4.env @@ -260,4 +260,13 @@ elif [[ "${step}" = "awips" ]]; then elif [[ "${step}" = "gempak" ]]; then echo "WARNING: ${step} is not enabled on S4!" + +elif [[ "${step}" = "fit2obs" ]]; then + + nth_max=$((npe_node_max / npe_node_fit2obs)) + + export NTHREADS_FIT2OBS=${nth_fit2obs:-1} + [[ ${NTHREADS_FIT2OBS} -gt ${nth_max} ]] && export NTHREADS_FIT2OBS=${nth_max} + export MPIRUN="${launcher} -n ${npe_fit2obs}" + fi diff --git a/env/WCOSS2.env b/env/WCOSS2.env index 50afb8915b..44f8dbc994 100755 --- a/env/WCOSS2.env +++ b/env/WCOSS2.env @@ -290,6 +290,14 @@ elif [[ "${step}" = "gempak" ]]; then [[ ${NTHREADS_GEMPAK} -gt ${nth_max} ]] && export NTHREADS_GEMPAK=${nth_max} export APRUN_GEMPAKCFP="${launcher} -np ${npe_gempak} ${mpmd_opt}" +elif [[ "${step}" = "fit2obs" ]]; then + + nth_max=$((npe_node_max / npe_node_fit2obs)) + + export NTHREADS_FIT2OBS=${nth_fit2obs:-1} + [[ ${NTHREADS_FIT2OBS} -gt ${nth_max} ]] && export NTHREADS_FIT2OBS=${nth_max} + export MPIRUN="${launcher} -np ${npe_fit2obs}" + elif [[ "${step}" = "waveawipsbulls" ]]; then unset PERL5LIB From c8dfaa9f1a0d944bf16d48ea3503dc525a6f0d67 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Tue, 28 Mar 2023 14:45:17 +0000 Subject: [PATCH 40/63] Add fit2obs to gdas_half but don't run until xdate>=SDATE - Add fit2obs job to first half cycle to resolve gdasarch dependency. - Update JGDAS_FIT2OBS to check xdate (look-back cycle for fit2obs) against SDATE and only run when xdate>=SDATE. - Add load of netcdf/4.7.0 (needed before build is updated). Refs #1405 --- jobs/JGDAS_FIT2OBS | 12 ++++++++---- workflow/rocoto/workflow_tasks.py | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/jobs/JGDAS_FIT2OBS b/jobs/JGDAS_FIT2OBS index 444859a450..a6b032ee6b 100755 --- a/jobs/JGDAS_FIT2OBS +++ b/jobs/JGDAS_FIT2OBS @@ -32,9 +32,14 @@ export COMLOX=${DATA}/fitx # RUN FIT2OBS VERIFICATION ############################################################### -"${fitdir}/batrun/excfs_gdas_vrfyfits.sh.ecf" -status=$? -[[ ${status} -ne 0 ]] && exit "${status}" +if [[ ${xdate} -ge ${SDATE} ]]; then # Run FIT2OBS + + module load netcdf/4.7.0 + "${fitdir}/batrun/excfs_gdas_vrfyfits.sh.ecf" + status=$? + [[ ${status} -ne 0 ]] && exit "${status}" + +fi ############################################## # End JOB SPECIFIC work @@ -47,7 +52,6 @@ if [[ -e "${pgmout}" ]] ; then cat "${pgmout}" fi - ########################################## # Remove the Temporary working directory ########################################## diff --git a/workflow/rocoto/workflow_tasks.py b/workflow/rocoto/workflow_tasks.py index 140062e16a..bdb8dc09ac 100644 --- a/workflow/rocoto/workflow_tasks.py +++ b/workflow/rocoto/workflow_tasks.py @@ -999,7 +999,7 @@ def fit2obs(self): deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) - cycledef = 'gdas' if self.cdump in ['gdas'] else self.cdump + cycledef = 'gdas_half,gdas' if self.cdump in ['gdas'] else self.cdump resources = self.get_resource('fit2obs') task = create_wf_task('fit2obs', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies, From 0899ea180ce1249650153e855ca4b1bcc908d873 Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Wed, 29 Mar 2023 12:38:53 -0500 Subject: [PATCH 41/63] Update path settings in config.fit2obs and remove CDUMPFCST - Remove CDUMPFCST, it gets set in fit2obs package. - Update EXECcfs and USHcfs to include "install" in paths Refs #1405 --- parm/config/config.fit2obs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/parm/config/config.fit2obs b/parm/config/config.fit2obs index ee891233a8..575be1d06f 100644 --- a/parm/config/config.fit2obs +++ b/parm/config/config.fit2obs @@ -12,13 +12,12 @@ export fit_ver="newm.1.7" export fitdir="${BASE_GIT}/verif/global/Fit2Obs/${fit_ver}" export HOMEcfs=${fitdir} -export EXECcfs=${HOMEcfs}/exec -export USHcfs=${HOMEcfs}/ush +export EXECcfs=${HOMEcfs}/install/exec +export USHcfs=${HOMEcfs}/install/ush export PRVT=${HOMEgfs}/fix/gsi/prepobs_errtable.global export HYBLEVS=${HOMEgfs}/fix/am/global_hyblev.l${LEVS}.txt -export CDUMPFCST="gfs" # Fit-to-obs verifying dump export VBACKUP_FITS=24 export OUTPUT_FILETYPE="netcdf" export CONVNETC="YES" From 96e8dadcde439f3d9b78c619bd8eaf2af29dd7be Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Wed, 29 Mar 2023 12:40:24 -0500 Subject: [PATCH 42/63] Updates to JGDAS_FIT2OBS for functioning - Add second CDATE setting (CDATE=${xdate}) for lookback date. The fit2obs package uses CDATE and needs it set from above based on the xdate variable (lookback date). - Update COM_PRP to evaluate it up front. - Add PRPI, sig1, sfc1, and CNVS input file variables for dependency checking later in script. - Add err_chk and postmsg setup (as done in runfits in fit2obs package). - Add check of xdate against SDATE (-gt) for sufficient spinup of test. - Add check of input file existence and exit with FATAL ERROR is any are missing. - Remove temporary "module load netcdf/4.7.0" now that build is updated. - Indent section of script that's now within xdate/SDATE and input file check. Only run fit2obs package script if dependencies are met. Refs #1405 --- jobs/JGDAS_FIT2OBS | 52 +++++++++++++++++++++++++++++++++------------- 1 file changed, 37 insertions(+), 15 deletions(-) diff --git a/jobs/JGDAS_FIT2OBS b/jobs/JGDAS_FIT2OBS index a6b032ee6b..2af9791e00 100755 --- a/jobs/JGDAS_FIT2OBS +++ b/jobs/JGDAS_FIT2OBS @@ -12,12 +12,18 @@ export CDATE=${CDATE:-${PDY}${cyc}} export CDUMP=${CDUMP:-${RUN:-"gdas"}} export xdate=$(${NDATE} -${VBACKUP_FITS} ${CDATE}) +export CDATE=${xdate} # Reset CDATE to xdate for lookback export vday=$(echo ${xdate} | cut -c1-8) export vcyc=$(echo ${xdate} | cut -c9-10) export COM_INA=${ROTDIR}/gdas.${vday}/${vcyc}/atmos export COM_INF='$ROTDIR/vrfyarch/gfs.$fdy/$fzz' -export COM_PRP='$ROTDIR/gdas.$pdy/$cyc/obs' +export COM_PRP=${ROTDIR}/gdas.${vday}/${vcyc}/obs + +export PRPI=${COM_PRP}/${CDUMP}.t${vcyc}z.prepbufr +export sig1=${COM_INA}/${CDUMP}.t${vcyc}z.atmanl.nc +export sfc1=${COM_INA}/${CDUMP}.t${vcyc}z.atmanl.nc +export CNVS=${COM_INA}/${CDUMP}.t${vcyc}z.cnvstat export OUTPUT_FILETYPE=${OUTPUT_FILETYPE:-netcdf} @@ -28,28 +34,44 @@ export HORZ_DIR=${ARCDIR}/horiz export COMLOX=${DATA}/fitx [[ ! -d "${COMLOX}" ]] && mkdir -p "${COMLOX}" -############################################################### -# RUN FIT2OBS VERIFICATION -############################################################### +echo "echo err_chk">${DATA}/err_chk; chmod 755 ${DATA}/err_chk +echo "echo postmsg">${DATA}/postmsg; chmod 755 ${DATA}/postmsg + +############################################## +# Check spinup and available inputs +############################################## + +if [[ ${xdate} -gt ${SDATE} ]]; then + for file in ${PRPI} ${sig1} ${sfc1} ${CNVS}; do + if [ ! -f ${file} ]; then + echo "FATAL ERROR: FILE MISSING: ${file}" + exit 1 + fi + done -if [[ ${xdate} -ge ${SDATE} ]]; then # Run FIT2OBS + ############################################## + # RUN FIT2OBS VERIFICATION + ############################################## - module load netcdf/4.7.0 "${fitdir}/batrun/excfs_gdas_vrfyfits.sh.ecf" status=$? [[ ${status} -ne 0 ]] && exit "${status}" -fi + ############################################## + # End JOB SPECIFIC work + ############################################## -############################################## -# End JOB SPECIFIC work -############################################## + ############################################## + # Final processing + ############################################## + if [[ -e "${pgmout}" ]] ; then + cat "${pgmout}" + fi + +else + + echo "Too early for FIT2OBS to run. Exiting." -############################################## -# Final processing -############################################## -if [[ -e "${pgmout}" ]] ; then - cat "${pgmout}" fi ########################################## From ded09d519faaa854dd86678e7cb8b45d58fb12e9 Mon Sep 17 00:00:00 2001 From: David Huber Date: Fri, 31 Mar 2023 17:42:07 +0000 Subject: [PATCH 43/63] Fix quotations in the vrfy job script. #357 --- jobs/rocoto/vrfy.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/jobs/rocoto/vrfy.sh b/jobs/rocoto/vrfy.sh index e46fce2ee5..cb0f7af773 100755 --- a/jobs/rocoto/vrfy.sh +++ b/jobs/rocoto/vrfy.sh @@ -69,19 +69,20 @@ if [[ ${VRFYFITS} = "YES" && ${CDUMP} = "${CDFNL}" && ${CDATE} != "${SDATE}" ]]; export TMPDIR="${RUNDIR}/${CDATE}/${CDUMP}" [[ ! -d ${TMPDIR} ]] && mkdir -p "${TMPDIR}" - xdate=$(${NDATE} -"${VBACKUP_FITS} ${CDATE}") + xdate=$(${NDATE} -"${VBACKUP_FITS}" "${CDATE}") vday=$(echo "${xdate}" | cut -c1-8) vcyc=$(echo "${xdate}" | cut -c9-10) export vcyc + export COMDAY=${ROTDIR}/logs/${xdate} export COM_INA=${ROTDIR}/gdas.${vday}/${vcyc}/atmos - export COM_INF="${ROTDIR}/vrfyarch/gfs.${fdy}/${fzz}" - export COM_PRP="${ROTDIR}/gdas.${pdy}/${cyc}/obs" + export COM_INF='${ROTDIR}/vrfyarch/gfs.${fdy}/${fzz}' + export COM_PRP='${ROTDIR}/gdas.${pdy}/${cyc}/obs' export OUTPUT_FILETYPE_SAVE=${OUTPUT_FILETYPE} - ${PREPQFITSH} "${PSLOT} ${xdate} ${ROTDIR} ${ARCDIR} ${TMPDIR}" + ${PREPQFITSH} "${PSLOT}" "${xdate}" "${ROTDIR}" "${ARCDIR}" "${TMPDIR}" export OUTPUT_FILETYPE=${OUTPUT_FILETYPE_SAVE} From bd43c45946d6bcd6bdbc3b4dec546fd2cff2c6d1 Mon Sep 17 00:00:00 2001 From: David Huber Date: Fri, 31 Mar 2023 19:44:13 +0000 Subject: [PATCH 44/63] Disable gldas on Jet. #357 --- sorc/build_all.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sorc/build_all.sh b/sorc/build_all.sh index 14c38b93be..af15be7b1f 100755 --- a/sorc/build_all.sh +++ b/sorc/build_all.sh @@ -90,6 +90,11 @@ fi source ./partial_build.sh ${_verbose_opt} ${_partial_opt} # shellcheck disable= +# Disable gldas on Jet +if [[ ${MACHINE_ID} =~ jet.* ]]; then + Build_gldas="false" +fi + #------------------------------------ # Exception Handling Init #------------------------------------ From 085f6e2c42a06d51ba9410307d96036844e42003 Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Mon, 3 Apr 2023 13:03:52 -0500 Subject: [PATCH 45/63] Remove "/install" from EXECcfs and USHcfs paths Refs #1405 --- parm/config/config.fit2obs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parm/config/config.fit2obs b/parm/config/config.fit2obs index 575be1d06f..539e1f82a9 100644 --- a/parm/config/config.fit2obs +++ b/parm/config/config.fit2obs @@ -12,8 +12,8 @@ export fit_ver="newm.1.7" export fitdir="${BASE_GIT}/verif/global/Fit2Obs/${fit_ver}" export HOMEcfs=${fitdir} -export EXECcfs=${HOMEcfs}/install/exec -export USHcfs=${HOMEcfs}/install/ush +export EXECcfs=${HOMEcfs}/exec +export USHcfs=${HOMEcfs}/ush export PRVT=${HOMEgfs}/fix/gsi/prepobs_errtable.global export HYBLEVS=${HOMEgfs}/fix/am/global_hyblev.l${LEVS}.txt From d5ac80a67c1a28b23e6a8f0527acee9f6964482d Mon Sep 17 00:00:00 2001 From: David Huber Date: Mon, 3 Apr 2023 21:14:31 +0000 Subject: [PATCH 46/63] Fixed erroneous quotations. #357 --- parm/config/config.vrfy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/config/config.vrfy b/parm/config/config.vrfy index 8a921554e3..2cd220009b 100644 --- a/parm/config/config.vrfy +++ b/parm/config/config.vrfy @@ -6,7 +6,7 @@ echo "BEGIN: config.vrfy" # Get task specific resources -. "${EXPDIR}/config.resources vrfy" +. ${EXPDIR}/config.resources vrfy export VDUMP="gfs" # Verifying dump export CDUMPFCST="gdas" # Fit-to-obs with GDAS/GFS prepbufr From c39c5ffba565d20b165a768ec94f0f2647b98c5e Mon Sep 17 00:00:00 2001 From: David Huber Date: Mon, 3 Apr 2023 21:15:36 +0000 Subject: [PATCH 47/63] Updated detect_machine for jet (/lfs3 no longer exists) #357 --- ush/detect_machine.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/detect_machine.sh b/ush/detect_machine.sh index 9250c89888..6f3333292f 100755 --- a/ush/detect_machine.sh +++ b/ush/detect_machine.sh @@ -50,7 +50,7 @@ fi if [[ -d /lfs/f1 ]] ; then # We are on NOAA Cactus or Dogwood MACHINE_ID=wcoss2 -elif [[ -d /lfs3 ]] ; then +elif [[ -d /lfs1 ]] ; then # We are on NOAA Jet MACHINE_ID=jet elif [[ -d /scratch1 ]] ; then From 91589738bb2d86e773bc28ed8e287604e6b39ed9 Mon Sep 17 00:00:00 2001 From: David Huber Date: Thu, 6 Apr 2023 14:53:43 +0000 Subject: [PATCH 48/63] Added memory spec for Fit2Obs on Jet. #357 NOAA-EMC/Fit2Obs#5 --- env/JET.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/env/JET.env b/env/JET.env index 1e76124a3f..828149a533 100755 --- a/env/JET.env +++ b/env/JET.env @@ -281,6 +281,6 @@ elif [[ "${step}" = "fit2obs" ]]; then export NTHREADS_FIT2OBS=${nth_fit2obs:-1} [[ ${NTHREADS_FIT2OBS} -gt ${nth_max} ]] && export NTHREADS_FIT2OBS=${nth_max} - export MPIRUN="${launcher} -n ${npe_fit2obs}" + export MPIRUN="${launcher} -n ${npe_fit2obs} --mem=0" fi From 87e86d51a22d087f0f63b01df7fdd3772159ebb8 Mon Sep 17 00:00:00 2001 From: David Huber Date: Wed, 12 Apr 2023 19:24:32 +0000 Subject: [PATCH 49/63] Added Jet locations for coupled/aero ICs. #357 --- parm/config/config.aero | 3 +++ parm/config/config.coupled_ic | 2 ++ 2 files changed, 5 insertions(+) diff --git a/parm/config/config.aero b/parm/config/config.aero index 171701cd2a..1cb3bf5679 100644 --- a/parm/config/config.aero +++ b/parm/config/config.aero @@ -19,6 +19,9 @@ case $machine in "WCOSS2") AERO_INPUTS_DIR="/lfs/h2/emc/global/noscrub/emc.global/data/gocart_emissions" ;; + "JET") + AERO_INPUTS_DIR="/lfs4/HFIP/hfv3gfs/glopara/data/gocart_emissions" + ;; *) echo "FATAL ERROR: Machine $machine unsupported for aerosols" exit 2 diff --git a/parm/config/config.coupled_ic b/parm/config/config.coupled_ic index 1977c56ca4..a573f6b770 100644 --- a/parm/config/config.coupled_ic +++ b/parm/config/config.coupled_ic @@ -15,6 +15,8 @@ elif [[ "${machine}" == "ORION" ]]; then export BASE_CPLIC="/work/noaa/global/glopara/data/ICSDIR/prototype_ICs" elif [[ "${machine}" == "S4" ]]; then export BASE_CPLIC="/data/prod/glopara/coupled_ICs" +elif [[ "${machine}" == "JET" ]]; then + export BASE_CPLIC="/lfs4/HFIP/hfv3gfs/glopara/data/IC/IC" fi From faeba021c270dcc083ce8b948048682a3872c06f Mon Sep 17 00:00:00 2001 From: David Huber Date: Wed, 12 Apr 2023 19:25:21 +0000 Subject: [PATCH 50/63] Decrease ocnpost memory requirements. #357 --- parm/config/config.resources | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/config/config.resources b/parm/config/config.resources index eb9c034a31..659db2c809 100644 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -558,7 +558,7 @@ elif [[ ${step} = "ocnpost" ]]; then export npe_ocnpost=1 export npe_node_ocnpost=1 export nth_ocnpost=1 - export memory_ocnpost="96G" + export memory_ocnpost="88G" elif [[ ${step} = "post" ]]; then From 6d322de0fb9d2bc367e7f58b292ac1c5a0706aac Mon Sep 17 00:00:00 2001 From: David Huber Date: Wed, 12 Apr 2023 20:28:54 +0000 Subject: [PATCH 51/63] Adjusted quotation marks. #357 --- parm/config/config.resources | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/parm/config/config.resources b/parm/config/config.resources index 6c0e4f49e5..cb732727bf 100644 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -536,12 +536,12 @@ elif [[ "${step}" = "fcst" || "${step}" = "efcs" ]]; then case "${CASE}" in "C48" | "C96" | "C192") - declare -x wtime_"${step}"="00:30:00" - declare -x wtime_"${step}_gfs"="03:00:00" + declare -x "wtime_${step}"="00:30:00" + declare -x "wtime_${step}_gfs"="03:00:00" ;; "C384" | "C768" | "C1152") - declare -x wtime_"${step}"="01:00:00" - declare -x wtime_"${step}_gfs"="06:00:00" + declare -x "wtime_${step}"="01:00:00" + declare -x "wtime_${step}_gfs"="06:00:00" ;; *) echo "FATAL ERROR: Resolution ${CASE} not supported in ${step}" From f93731111b948d90d071bf2566c95ec51640aef7 Mon Sep 17 00:00:00 2001 From: David Huber Date: Thu, 13 Apr 2023 14:18:08 +0000 Subject: [PATCH 52/63] Restore develop TC Tracker info. #357 --- parm/config/config.vrfy | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/parm/config/config.vrfy b/parm/config/config.vrfy index ec3d7a51de..0fcb1b1268 100644 --- a/parm/config/config.vrfy +++ b/parm/config/config.vrfy @@ -66,8 +66,13 @@ fi # Cyclone genesis and cyclone track verification #------------------------------------------------- -export ens_tracker_ver=v1.1.15.6 -export HOMEens_tracker=/mnt/lfs4/HFIP/hwrfv3/Jiayi.Peng/ens_tracker.${ens_tracker_ver} +export ens_tracker_ver=v1.1.15.5 +export HOMEens_tracker=$BASE_GIT/TC_tracker/${ens_tracker_ver} + +if [[ ${machine} == "JET" ]]; then + export ens_tracker_ver=v1.1.15.6 + export HOMEens_tracker=/mnt/lfs4/HFIP/hwrfv3/Jiayi.Peng/ens_tracker.${ens_tracker_ver} +fi if [[ "${VRFYTRAK}" = "YES" ]]; then From ede4c4aae886c53944fcc9cc576151c22425224b Mon Sep 17 00:00:00 2001 From: David Huber Date: Thu, 13 Apr 2023 17:19:53 +0000 Subject: [PATCH 53/63] De-support gldas on Jet. #357 --- env/JET.env | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/env/JET.env b/env/JET.env index d8f85b5f1f..64b32bbd2d 100755 --- a/env/JET.env +++ b/env/JET.env @@ -139,24 +139,7 @@ elif [[ "${step}" = "sfcanl" ]]; then elif [[ "${step}" = "gldas" ]]; then - export USE_CFP="NO" - export CFP_MP="YES" - - nth_max=$((npe_node_max / npe_node_gldas)) - - export NTHREADS_GLDAS=${nth_gldas:-${nth_max}} - [[ ${NTHREADS_GLDAS} -gt ${nth_max} ]] && export NTHREADS_GLDAS=${nth_max} - export APRUN_GLDAS="${launcher} -n ${npe_gldas}" - - export NTHREADS_GAUSSIAN=${nth_gaussian:-1} - [[ ${NTHREADS_GAUSSIAN} -gt ${nth_max} ]] && export NTHREADS_GAUSSIAN=${nth_max} - export APRUN_GAUSSIAN="${launcher} -n ${npe_gaussian}" - -# Must run data processing with exactly the number of tasks as time -# periods being processed. - - npe_gldas_data_proc=$((gldas_spinup_hours + 12)) - export APRUN_GLDAS_DATA_PROC="${launcher} -n ${npe_gldas_data_proc} ${mpmd_opt}" + echo "WARNING: ${step} is not enabled on ${machine}!" elif [[ "${step}" = "eobs" ]]; then From 815e49fd3e87de77f0c74d47761145c386ed5a92 Mon Sep 17 00:00:00 2001 From: David Huber Date: Thu, 13 Apr 2023 17:33:39 +0000 Subject: [PATCH 54/63] Check Jet for jet with /mnt/lfs1. #357 --- ush/detect_machine.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/detect_machine.sh b/ush/detect_machine.sh index 6f3333292f..647722b7a3 100755 --- a/ush/detect_machine.sh +++ b/ush/detect_machine.sh @@ -50,7 +50,7 @@ fi if [[ -d /lfs/f1 ]] ; then # We are on NOAA Cactus or Dogwood MACHINE_ID=wcoss2 -elif [[ -d /lfs1 ]] ; then +elif [[ -d /mnt/lfs1 ]] ; then # We are on NOAA Jet MACHINE_ID=jet elif [[ -d /scratch1 ]] ; then From 2431190c7f79bd396e058ab13ac4e4f46b1e6361 Mon Sep 17 00:00:00 2001 From: David Huber Date: Thu, 13 Apr 2023 17:40:53 +0000 Subject: [PATCH 55/63] Add a node for ocnpost on Jet. #357 --- parm/config/config.resources | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/parm/config/config.resources b/parm/config/config.resources index cb732727bf..ce082b26d0 100644 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -558,7 +558,12 @@ elif [[ ${step} = "ocnpost" ]]; then export npe_ocnpost=1 export npe_node_ocnpost=1 export nth_ocnpost=1 - export memory_ocnpost="88G" + export memory_ocnpost="96G" + if [[ ${machine} == "JET" ]]; then + # JET only has 88GB of requestable memory per node + # so a second node is required to meet the requiremtn + npe_ocnpost=2 + fi elif [[ ${step} = "post" ]]; then From 5c927822ae350415fd52d158d400b2cb06cad654 Mon Sep 17 00:00:00 2001 From: David Huber Date: Thu, 13 Apr 2023 17:41:23 +0000 Subject: [PATCH 56/63] Update coupled IC location on Jet. #357 --- parm/config/config.coupled_ic | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/config/config.coupled_ic b/parm/config/config.coupled_ic index a573f6b770..50fab283b5 100644 --- a/parm/config/config.coupled_ic +++ b/parm/config/config.coupled_ic @@ -16,7 +16,7 @@ elif [[ "${machine}" == "ORION" ]]; then elif [[ "${machine}" == "S4" ]]; then export BASE_CPLIC="/data/prod/glopara/coupled_ICs" elif [[ "${machine}" == "JET" ]]; then - export BASE_CPLIC="/lfs4/HFIP/hfv3gfs/glopara/data/IC/IC" + export BASE_CPLIC="/mnt/lfs4/HFIP/hfv3gfs/glopara/data/ICSDIR/prototype_ICs" fi From a311e058e84b414c1a3cb348a53b15b3ced9da14 Mon Sep 17 00:00:00 2001 From: David Huber Date: Thu, 13 Apr 2023 17:42:01 +0000 Subject: [PATCH 57/63] Address shellcheck issue. #357 --- parm/config/config.metp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/config/config.metp b/parm/config/config.metp index 8fb55f275f..c90903f6a5 100644 --- a/parm/config/config.metp +++ b/parm/config/config.metp @@ -6,7 +6,7 @@ echo "BEGIN: config.metp" # Get task specific resources -. "${EXPDIR}"/config.resources metp +. "${EXPDIR}/config.resources" metp export RUN_GRID2GRID_STEP1="YES" # Run grid-to-grid verification using METplus export RUN_GRID2OBS_STEP1="YES" # Run grid-to-obs verification using METplus From 396a6968912eec59956d957f46dc5aa644a82d83 Mon Sep 17 00:00:00 2001 From: David Huber Date: Thu, 13 Apr 2023 17:42:52 +0000 Subject: [PATCH 58/63] Add quotes for shellcheck. #357 --- parm/config/config.vrfy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/config/config.vrfy b/parm/config/config.vrfy index 0fcb1b1268..430cd4f5af 100644 --- a/parm/config/config.vrfy +++ b/parm/config/config.vrfy @@ -6,7 +6,7 @@ echo "BEGIN: config.vrfy" # Get task specific resources -. ${EXPDIR}/config.resources vrfy +. "${EXPDIR}/config.resources" vrfy export CDFNL="gdas" # Scores verification against GDAS/GFS analysis export MKPGB4PRCP="YES" # Make 0.25-deg pgb files in ARCDIR for precip verification From 8474526ecffea37e05afcd0690593e2ff0e61bc3 Mon Sep 17 00:00:00 2001 From: David Huber Date: Thu, 13 Apr 2023 17:43:39 +0000 Subject: [PATCH 59/63] Revert TC_tracker version/location to develop. #357 --- parm/config/config.vrfy | 5 ----- 1 file changed, 5 deletions(-) diff --git a/parm/config/config.vrfy b/parm/config/config.vrfy index 430cd4f5af..7707f99f89 100644 --- a/parm/config/config.vrfy +++ b/parm/config/config.vrfy @@ -69,11 +69,6 @@ fi export ens_tracker_ver=v1.1.15.5 export HOMEens_tracker=$BASE_GIT/TC_tracker/${ens_tracker_ver} -if [[ ${machine} == "JET" ]]; then - export ens_tracker_ver=v1.1.15.6 - export HOMEens_tracker=/mnt/lfs4/HFIP/hwrfv3/Jiayi.Peng/ens_tracker.${ens_tracker_ver} -fi - if [[ "${VRFYTRAK}" = "YES" ]]; then export TRACKERSH="${HOMEgfs}/jobs/JGFS_ATMOS_CYCLONE_TRACKER" From 1cb17d6dbf285897fe3cda623f3ed6bbd9c6110d Mon Sep 17 00:00:00 2001 From: David Huber Date: Thu, 13 Apr 2023 18:16:19 +0000 Subject: [PATCH 60/63] Add Jet documentation. #357 --- docs/note_fixfield.txt | 2 ++ docs/source/components.rst | 1 + docs/source/hpc.rst | 9 ++++++--- docs/source/init.rst | 3 ++- docs/source/setup.rst | 11 +++++++++++ 5 files changed, 22 insertions(+), 4 deletions(-) diff --git a/docs/note_fixfield.txt b/docs/note_fixfield.txt index 3b22de5e13..af2539e48a 100644 --- a/docs/note_fixfield.txt +++ b/docs/note_fixfield.txt @@ -4,6 +4,8 @@ They are saved locally on all platforms Hera: /scratch1/NCEPDEV/global/glopara/fix Orion: /work/noaa/global/glopara/fix +Jet: /mnt/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix +S4: /data/prod/glopara/fix ------------------------------------------------------------------------------ 09/28/2018 diff --git a/docs/source/components.rst b/docs/source/components.rst index 6b947b3432..3ebd575a82 100644 --- a/docs/source/components.rst +++ b/docs/source/components.rst @@ -60,6 +60,7 @@ Observation data, also known as dump data, is prepared in production and then ar * Hera: /scratch1/NCEPDEV/global/glopara/dump * Orion: /work/noaa/rstprod/dump +* Jet: /mnt/lfs4/HFIP/hfv3gfs/glopara/dump * WCOSS2: /lfs/h2/emc/global/noscrub/emc.global/dump * S4: /data/prod/glopara/dump diff --git a/docs/source/hpc.rst b/docs/source/hpc.rst index 7161e2b742..da54f29521 100644 --- a/docs/source/hpc.rst +++ b/docs/source/hpc.rst @@ -22,6 +22,7 @@ HPC helpdesks * HPSS: rdhpcs.hpss.help@noaa.gov * Gaea: oar.gfdl.help@noaa.gov * S4: david.huber@noaa.gov +* Jet: rdhpcs.jet.help@noaa.gov ====================== Restricted data access @@ -76,6 +77,8 @@ It is advised to use Git v2+ when available. At the time of writing this documen +---------+----------+---------------------------------------+ | Orion | v1.8.3.1 | **module load git/2.28.0** | +---------+----------+---------------------------------------+ +| Jet | v2.18.0 | default | ++---------+----------+---------------------------------------+ | WCOSS2 | v2.26.2 | default or **module load git/2.29.0** | +---------+----------+---------------------------------------+ | S4 | v1.8.3.1 | **module load git/2.30.0** | @@ -96,9 +99,9 @@ For the manage_externals utility functioning:: Error: fatal: ssh variant 'simple' does not support setting port Fix: git config --global ssh.variant ssh -=================================== -Stacksize on R&Ds (Hera, Orion, S4) -=================================== +======================================== +Stacksize on R&Ds (Hera, Orion, Jet, S4) +======================================== Some GFS components, like the UPP, need an unlimited stacksize. Add the following setting into your appropriate .*rc file to support these components: diff --git a/docs/source/init.rst b/docs/source/init.rst index b7a86f5b62..37e2ae95e0 100644 --- a/docs/source/init.rst +++ b/docs/source/init.rst @@ -190,6 +190,7 @@ Forecast-only P8 prototype initial conditions are made available to users on sup WCOSS2: /lfs/h2/emc/global/noscrub/emc.global/IC/COUPLED HERA: /scratch1/NCEPDEV/climate/role.ufscpara/IC ORION: /work/noaa/global/glopara/data/ICSDIR/prototype_ICs + ORION: /mnt/lfs4/HFIP/hfv3gfs/glopara/data/ICSDIR/prototype_ICs S4: /data/prod/glopara/coupled_ICs These locations are known within the workflow via paths set in ``parm/config/config.coupled_ic``. @@ -217,7 +218,7 @@ Not yet supported. See :ref:`Manual Generation` section below --------------------- Forecast-only coupled --------------------- -Coupled initial conditions are currently only generated offline and copied prior to the forecast run. Prototype initial conditions will automatically be used when setting up an experiment as an S2SW app, there is no need to do anything additional. Copies of initial conditions from the prototype runs are currently maintained on Hera, Orion, and WCOSS2. The locations used are determined by ``parm/config/config.coupled_ic``. If you need prototype ICs on another machine, please contact Walter (Walter.Kolczynski@noaa.gov). +Coupled initial conditions are currently only generated offline and copied prior to the forecast run. Prototype initial conditions will automatically be used when setting up an experiment as an S2SW app, there is no need to do anything additional. Copies of initial conditions from the prototype runs are currently maintained on Hera, Orion, Jet, and WCOSS2. The locations used are determined by ``parm/config/config.coupled_ic``. If you need prototype ICs on another machine, please contact Walter (Walter.Kolczynski@noaa.gov). .. _forecastonly-atmonly: diff --git a/docs/source/setup.rst b/docs/source/setup.rst index a4e70fbfcb..eb13b4b6f3 100644 --- a/docs/source/setup.rst +++ b/docs/source/setup.rst @@ -28,6 +28,13 @@ Experiment Setup module load miniconda/3.8-s4 + * - Jet + - :: + + module use /mnt/lfs4/HFIP/hfv3gfs/role.epic/miniconda3/modulefiles + module load miniconda3/4.12.0 + conda activate ufswm + If running with Rocoto make sure to have a Rocoto module loaded before running setup scripts: .. list-table:: ROCOTO Module Load Commands @@ -54,6 +61,10 @@ If running with Rocoto make sure to have a Rocoto module loaded before running s - :: module load rocoto/1.3.4 + * - Jet + - :: + + module load rocoto/1.3.3 ^^^^^^^^^^^^^^^^^^^^^^^^ Forecast-only experiment From ba4f2eefe5250eda8572ef302fec11a8452bf569 Mon Sep 17 00:00:00 2001 From: David Huber Date: Thu, 13 Apr 2023 19:49:28 +0000 Subject: [PATCH 61/63] Changed ORION to JET. #357 --- docs/source/init.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/init.rst b/docs/source/init.rst index 37e2ae95e0..b065af2373 100644 --- a/docs/source/init.rst +++ b/docs/source/init.rst @@ -190,7 +190,7 @@ Forecast-only P8 prototype initial conditions are made available to users on sup WCOSS2: /lfs/h2/emc/global/noscrub/emc.global/IC/COUPLED HERA: /scratch1/NCEPDEV/climate/role.ufscpara/IC ORION: /work/noaa/global/glopara/data/ICSDIR/prototype_ICs - ORION: /mnt/lfs4/HFIP/hfv3gfs/glopara/data/ICSDIR/prototype_ICs + JET: /mnt/lfs4/HFIP/hfv3gfs/glopara/data/ICSDIR/prototype_ICs S4: /data/prod/glopara/coupled_ICs These locations are known within the workflow via paths set in ``parm/config/config.coupled_ic``. From 6744f69a04276f5b1079ae5e5884ee02ed327ff1 Mon Sep 17 00:00:00 2001 From: David Huber Date: Fri, 14 Apr 2023 14:29:24 +0000 Subject: [PATCH 62/63] Set a memory request for fit2obs. #357 --- env/JET.env | 2 +- parm/config/config.resources | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/env/JET.env b/env/JET.env index 64b32bbd2d..c01cd18cb8 100755 --- a/env/JET.env +++ b/env/JET.env @@ -263,6 +263,6 @@ elif [[ "${step}" = "fit2obs" ]]; then export NTHREADS_FIT2OBS=${nth_fit2obs:-1} [[ ${NTHREADS_FIT2OBS} -gt ${nth_max} ]] && export NTHREADS_FIT2OBS=${nth_max} - export MPIRUN="${launcher} -n ${npe_fit2obs} --mem=0" + export MPIRUN="${launcher} -n ${npe_fit2obs}" fi diff --git a/parm/config/config.resources b/parm/config/config.resources index ce082b26d0..31b871da61 100644 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -654,6 +654,7 @@ elif [[ "${step}" = "fit2obs" ]]; then export npe_fit2obs=3 export nth_fit2obs=1 export npe_node_fit2obs=1 + export memory_fit2obs="20G" if [[ ${machine} == "WCOSS2" ]]; then export npe_node_fit2obs=3 ; fi elif [[ "${step}" = "metp" ]]; then From c2b3c9da710a841a20b241b4befde4c9a7236eda Mon Sep 17 00:00:00 2001 From: David Huber Date: Fri, 14 Apr 2023 14:31:11 +0000 Subject: [PATCH 63/63] Increment UFS hash to include new epic stack on Jet #357 --- sorc/checkout.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/checkout.sh b/sorc/checkout.sh index 3f34d9eaa3..4d53cf2d1f 100755 --- a/sorc/checkout.sh +++ b/sorc/checkout.sh @@ -157,7 +157,7 @@ mkdir -p "${logdir}" errs=0 checkout "gfs_utils.fd" "https://github.com/NOAA-EMC/gfs-utils" "0b8ff56" ; errs=$((errs + $?)) checkout "ufs_utils.fd" "https://github.com/ufs-community/UFS_UTILS.git" "4e673bf" ; errs=$((errs + $?)) -checkout "ufs_model.fd" "https://github.com/ufs-community/ufs-weather-model" "${ufs_model_hash:-c22aaad}" ; errs=$((errs + $?)) +checkout "ufs_model.fd" "https://github.com/ufs-community/ufs-weather-model" "${ufs_model_hash:-0c8e74c}" ; errs=$((errs + $?)) checkout "verif-global.fd" "https://github.com/NOAA-EMC/EMC_verif-global.git" "c267780" ; errs=$((errs + $?)) if [[ ${checkout_gsi} == "YES" ]]; then