enable feature/coupled-crow to work on stampede2

env/STAMPEDE2.env

@@ -0,0 +1,217 @@
+#!/bin/ksh -x
+
+if [ $# -ne 1 ]; then
+
+    echo "Must specify an input argument to set runtime environment variables!"
+    echo "argument can be any one of the following:"
+    echo "anal fcst post vrfy metp"
+    echo "eobs eupd ecen efcs epos"
+    echo "postsnd awips gempak"
+    exit 1
+
+fi
+
+step=$1
+
+export npe_node_max=48
+export launcher="srun --export=ALL"
+export launcher_fcst="ibrun "
+
+# Configure MPI environment
+export I_MPI_ADJUST_ALLREDUCE=5
+export MPI_BUFS_PER_PROC=2048
+export MPI_BUFS_PER_HOST=2048
+export MPI_GROUP_MAX=256
+export MPI_MEMMAP_OFF=1
+export MP_STDOUTMODE="ORDERED"
+export KMP_AFFINITY=scatter
+export OMP_STACKSIZE=2048000
+export NTHSTACK=1024000000
+#export LD_BIND_NOW=1
+
+ulimit -s unlimited
+ulimit -a
+
+export job=${PBS_JOBNAME:-$step}
+export jobid=${job}.${PBS_JOBID:-$$}
+
+if [ $step = "prep" -o $step = "prepbufr" ]; then
+
+    nth_max=$(($npe_node_max / $npe_node_prep))
+
+    export POE="NO"
+    export BACK=${BACK:-"YES"}
+    export sys_tp="ORION"
+
+elif [ $step = "anal" ]; then
+
+    export MKL_NUM_THREADS=4
+    export MKL_CBWR=AUTO
+
+    nth_max=$(($npe_node_max / $npe_node_anal))
+
+    export NTHREADS_GSI=${nth_anal:-$nth_max}
+    [[ $NTHREADS_GSI -gt $nth_max ]] && export NTHREADS_GSI=$nth_max
+    export APRUN_GSI="$launcher"
+
+    export NTHREADS_CALCINC=${nth_calcinc:-1}
+    [[ $NTHREADS_CALCINC -gt $nth_max ]] && export NTHREADS_CALCINC=$nth_max
+    export APRUN_CALCINC="$launcher"
+
+    export NTHREADS_CYCLE=${nth_cycle:-12}
+    [[ $NTHREADS_CYCLE -gt $npe_node_max ]] && export NTHREADS_CYCLE=$npe_node_max
+    npe_cycle=${ntiles:-6}
+    export APRUN_CYCLE="$launcher -n $npe_cycle"
+
+
+    export NTHREADS_GAUSFCANL=1
+    npe_gausfcanl=${npe_gausfcanl:-1}
+
+    export APRUN_GAUSFCANL="$launcher -n $npe_gausfcanl"
+
+elif [ $step = "gldas" ]; then
+
+    nth_max=$(($npe_node_max / $npe_node_gldas))
+
+    export NTHREADS_GLDAS=${nth_gldas:-$nth_max}
+    [[ $NTHREADS_GLDAS -gt $nth_max ]] && export NTHREADS_GLDAS=$nth_max
+    export APRUN_GLDAS="$launcher -n $npe_gldas"
+
+    export NTHREADS_GAUSSIAN=${nth_gaussian:-1}
+    [[ $NTHREADS_GAUSSIAN -gt $nth_max ]] && export NTHREADS_GAUSSIAN=$nth_max
+    export APRUN_GAUSSIAN="$launcher -n $npe_gaussian"
+
+# Must run data processing with exactly the number of tasks as time
+# periods being processed.
+
+    npe_gldas_data_proc=$(($gldas_spinup_hours + 12))
+    export APRUN_GLDAS_DATA_PROC="$launcher -n $npe_gldas_data_proc --multi-prog"
+
+elif [ $step = "eobs" ]; then
+
+    export MKL_NUM_THREADS=4
+    export MKL_CBWR=AUTO
+
+    nth_max=$(($npe_node_max / $npe_node_eobs))
+
+    export NTHREADS_GSI=${nth_eobs:-$nth_max}
+    [[ $NTHREADS_GSI -gt $nth_max ]] && export NTHREADS_GSI=$nth_max
+    export APRUN_GSI="$launcher"
+
+elif [ $step = "eupd" ]; then
+
+    nth_max=$(($npe_node_max / $npe_node_eupd))
+
+    export NTHREADS_ENKF=${nth_eupd:-$nth_max}
+    [[ $NTHREADS_ENKF -gt $nth_max ]] && export NTHREADS_ENKF=$nth_max
+    export APRUN_ENKF="$launcher"
+
+elif [ $step = "fcst" ]; then
+
+    nth_max=$(($npe_node_max / $npe_node_fcst))
+
+    export NTHREADS_FV3=${nth_fv3:-$nth_max}
+    [[ $NTHREADS_FV3 -gt $nth_max ]] && export NTHREADS_FV3=$nth_max
+    export cores_per_node=$npe_node_max
+    export APRUN_FV3="$launcher_fcst"
+
+    export NTHREADS_REGRID_NEMSIO=${nth_regrid_nemsio:-1}
+    [[ $NTHREADS_REGRID_NEMSIO -gt $nth_max ]] && export NTHREADS_REGRID_NEMSIO=$nth_max
+    export APRUN_REGRID_NEMSIO="$launcher"
+
+    export NTHREADS_REMAP=${nth_remap:-2}
+    [[ $NTHREADS_REMAP -gt $nth_max ]] && export NTHREADS_REMAP=$nth_max
+    export APRUN_REMAP="$launcher"
+
+elif [ $step = "efcs" ]; then
+
+    nth_max=$(($npe_node_max / $npe_node_efcs))
+
+    export NTHREADS_FV3=${nth_efcs:-$nth_max}
+    [[ $NTHREADS_FV3 -gt $nth_max ]] && export NTHREADS_FV3=$nth_max
+    export cores_per_node=$npe_node_max
+    export APRUN_FV3="$launcher"
+
+    export NTHREADS_REGRID_NEMSIO=${nth_regrid_nemsio:-1}
+    [[ $NTHREADS_REGRID_NEMSIO -gt $nth_max ]] && export NTHREADS_REGRID_NEMSIO=$nth_max
+    export APRUN_REGRID_NEMSIO="$launcher $LEVS"
+
+elif [ $step = "post" ]; then
+
+    nth_max=$(($npe_node_max / $npe_node_post))
+
+    export NTHREADS_NP=${nth_np:-1}
+    [[ $NTHREADS_NP -gt $nth_max ]] && export NTHREADS_NP=$nth_max
+    export APRUN_NP="$launcher"
+
+    export NTHREADS_DWN=${nth_dwn:-1}
+    [[ $NTHREADS_DWN -gt $nth_max ]] && export NTHREADS_DWN=$nth_max
+    export APRUN_DWN="$launcher"
+
+elif [ $step = "ecen" ]; then
+
+    nth_max=$(($npe_node_max / $npe_node_ecen))
+
+    export NTHREADS_ECEN=${nth_ecen:-$nth_max}
+    [[ $NTHREADS_ECEN -gt $nth_max ]] && export NTHREADS_ECEN=$nth_max
+    export APRUN_ECEN="$launcher"
+
+    export NTHREADS_CHGRES=${nth_chgres:-12}
+    [[ $NTHREADS_CHGRES -gt $npe_node_max ]] && export NTHREADS_CHGRES=$npe_node_max
+    export APRUN_CHGRES="time"
+
+    export NTHREADS_CALCINC=${nth_calcinc:-1}
+    [[ $NTHREADS_CALCINC -gt $nth_max ]] && export NTHREADS_CALCINC=$nth_max
+    export APRUN_CALCINC="$launcher"
+
+elif [ $step = "esfc" ]; then
+
+    nth_max=$(($npe_node_max / $npe_node_esfc))
+
+    export NTHREADS_ESFC=${nth_esfc:-$nth_max}
+    [[ $NTHREADS_ESFC -gt $nth_max ]] && export NTHREADS_ESFC=$nth_max
+    export APRUN_ESFC="$launcher -n $npe_esfc"
+
+    export NTHREADS_CYCLE=${nth_cycle:-14}
+    [[ $NTHREADS_CYCLE -gt $npe_node_max ]] && export NTHREADS_CYCLE=$npe_node_max
+    export APRUN_CYCLE="$launcher -n $npe_esfc"
+
+elif [ $step = "epos" ]; then
+
+    nth_max=$(($npe_node_max / $npe_node_epos))
+
+    export NTHREADS_EPOS=${nth_epos:-$nth_max}
+    [[ $NTHREADS_EPOS -gt $nth_max ]] && export NTHREADS_EPOS=$nth_max
+    export APRUN_EPOS="$launcher"
+
+elif [ $step = "fv3ic" ]; then
+
+    export NTHREADS_CHGRES=${nth_chgres:-$npe_node_max}
+    [[ $NTHREADS_CHGRES -gt $npe_node_max ]] && export NTHREADS_CHGRES=$npe_node_max
+    export APRUN_CHGRES="time"
+
+elif [ $step = "waveinit" -o $step = "waveprep" -o $step = "wavepostsbs" ]; then
+
+    export mpmd="--multi-prog"
+    export CFP_MP="YES"
+
+elif [ $step = "postsnd" ]; then
+
+    nth_max=$(($npe_node_max / $npe_node_postsnd))
+
+    export NTHREADS_POSTSND=${nth_postsnd:-1}
+    [[ $NTHREADS_POSTSND -gt $nth_max ]] && export NTHREADS_POSTSND=$nth_max
+    export APRUN_POSTSND="$launcher"
+
+    export NTHREADS_POSTSNDCFP=${nth_postsndcfp:-1}
+    [[ $NTHREADS_POSTSNDCFP -gt $nth_max ]] && export NTHREADS_POSTSNDCFP=$nth_max
+    export APRUN_POSTSNDCFP="$launcher"
+
+elif [ $step = "awips" ]; then
+
+     echo "WARNING: $step is not enabled on $machine!"
+
+elif [ $step = "gempak" ]; then
+
+     echo "WARNING: $step is not enabled on $machine!"
+fi

modulefiles/module_base.orion

@@ -6,57 +6,39 @@
 ## load contrib environment
 ## load slurm utils (arbitrary.pl  layout.pl)
 ##
-module load contrib noaatools
 
-##
-## load programming environment
-## this typically includes compiler, MPI and job scheduler
-##
-module load intel/2018
-module load impi/2018
+module use /apps/contrib/NCEP/libs/hpc-stack/modulefiles/stack
 
-#For ocean post: 
-module load ncl/6.6.2
-module load nco/4.8.1
+module load hpc/1.1.0
+module load hpc-intel/2018.4
+module load hpc-impi/2018.4
+
+module load jasper/2.0.15
+module load png/1.6.35
+module load zlib/1.2.11
+
+module load hdf5/1.10.6
+module load netcdf/4.7.4
+module load netcdf/4.7.4-parallel
+module load esmf/8_1_0_beta_snapshot_21
 
-##
-## NCEP libraries (temporary version to match the CCPP requirements)
-##
-module use -a /apps/contrib/NCEPLIBS/orion/cmake/install/NCEPLIBS/modules
 module load bacio/2.4.0
-module load crtm_dev/2.3.0
-module load g2/3.4.0
-module load g2tmpl/1.9.0
 module load ip/3.3.0
-module load nceppost/dceca26
-module load nemsio/2.5.1
 module load sp/2.3.0
-module load w3emc/2.7.0
+module load g2/3.4.0
+module load g2tmpl/1.9.0
 module load w3nco/2.4.0
+module load w3emc/2.7.0
 
-module load gfsio/1.4.0
-module load sfcio/1.4.0
-module load sigio/2.3.0
+module load crtm/2.3.0
+module load nceppost/dceca26
+module load wgrib2/2.0.8
 
-module use /apps/contrib/NCEPLIBS/orion/modulefiles
-module load jasper/1.900.2
-module load png/1.2.44
-module load z/1.2.6
+#module load prod_util/develop
+module load grib_util/develop
 
+module use /apps/contrib/NCEPLIBS/orion/modulefiles
 module load prod_util/1.2.0
-##
-## load ESMF library for above compiler / MPI combination
-## use pre-compiled EMSF library for above compiler / MPI combination
-##
-module use /apps/contrib/NCEPLIBS/lib/modulefiles
-module load netcdfp/4.7.4.release
-module load esmflocal/8.1.0.27bs.release
-module load post-intel-sandybridge/8.0.5
-
-module load wgrib/2.0.8
-module load grib_util/1.2.0
-
-module load slurm/19.05.3-2
 
 ##
 ### load cmake

modulefiles/module_base.orion_bk

@@ -0,0 +1,68 @@
+#%Module######################################################################
+##
+##    NEMS FV3 Prerequisites: Orion/Intel
+
+##
+## load contrib environment
+## load slurm utils (arbitrary.pl  layout.pl)
+##
+module load contrib noaatools
+
+##
+## load programming environment
+## this typically includes compiler, MPI and job scheduler
+##
+module load intel/2018
+module load impi/2018
+
+#For ocean post: 
+module load ncl/6.6.2
+module load nco/4.8.1
+
+##
+## NCEP libraries (temporary version to match the CCPP requirements)
+##
+module use -a /apps/contrib/NCEPLIBS/orion/cmake/install/NCEPLIBS/modules
+module load bacio/2.4.0
+module load crtm_dev/2.3.0
+module load g2/3.4.0
+module load g2tmpl/1.9.0
+module load ip/3.3.0
+module load nceppost/dceca26
+module load nemsio/2.5.1
+module load sp/2.3.0
+module load w3emc/2.7.0
+module load w3nco/2.4.0
+
+module load gfsio/1.4.0
+module load sfcio/1.4.0
+module load sigio/2.3.0
+
+module use /apps/contrib/NCEPLIBS/orion/modulefiles
+module load jasper/1.900.2
+module load png/1.2.44
+module load z/1.2.6
+
+module load prod_util/1.2.0
+##
+## load ESMF library for above compiler / MPI combination
+## use pre-compiled EMSF library for above compiler / MPI combination
+##
+module use /apps/contrib/NCEPLIBS/lib/modulefiles
+module load netcdfp/4.7.4.release
+module load esmflocal/8.1.0.27bs.release
+module load post-intel-sandybridge/8.0.5
+
+module load wgrib/2.0.8
+module load grib_util/1.2.0
+
+module load slurm/19.05.3-2
+
+##
+### load cmake
+###
+module load cmake/3.15.4
+setenv CMAKE_C_COMPILER mpiicc
+setenv CMAKE_CXX_COMPILER mpiicpc
+setenv CMAKE_Fortran_COMPILER mpiifort
+setenv CMAKE_Platform orion.intel