Update WRF scripting and tutorial for derecho

models/wrf/shell_scripts/assim_advance.csh

@@ -60,15 +60,15 @@ $yyyy $mm $dd $hh $nn $ss
 mpirun.lsf ./wrf.exe
 EOF
 
-else if ( $SUPER_PLATFORM == 'cheyenne' ) then
+else if ( $SUPER_PLATFORM == 'derecho' ) then
 
 # module load openmpi
 cat >! $RUN_DIR/advance_temp${emember}/wrf.info << EOF
 ${gdatef[2]}  ${gdatef[1]}
 ${gdate[2]}   ${gdate[1]}
 $yyyy $mm $dd $hh $nn $ss
            $domains
- mpiexec_mpt dplace -s 1  ./wrf.exe
+ mpiexec -n 128 -ppn 128  ./wrf.exe
 EOF
 
 endif

models/wrf/shell_scripts/assimilate.csh

@@ -29,14 +29,13 @@ if ( $SUPER_PLATFORM == 'yellowstone' ) then
    setenv FORT_BUFFERED true
    mpirun.lsf ./filter || exit 1
 
-else if ( $SUPER_PLATFORM == 'cheyenne' ) then
+else if ( $SUPER_PLATFORM == 'derecho' ) then
 
-# TJH MPI_SHEPHERD TRUE may be a very bad thing
    setenv MPI_SHEPHERD FALSE
 
    setenv TMPDIR  /dev/shm
    limit stacksize unlimited
-   mpiexec_mpt dplace -s 1 ./filter || exit 1
+   mpiexec -n 256 -ppn 128 ./filter || exit 1
 
 endif
 

models/wrf/shell_scripts/driver.csh

@@ -82,9 +82,9 @@ while ( 1 == 1 )
       set ic_queue = caldera
       set logfile = "${RUN_DIR}/ic_gen.log"
       set sub_command = "bsub -q ${ic_queue} -W 00:05 -o ${logfile} -n 1 -P ${COMPUTER_CHARGE_ACCOUNT}"
-   else if ( $SUPER_PLATFORM == 'cheyenne' ) then
-      set ic_queue = "economy"
-      set sub_command = "qsub -l select=1:ncpus=2:mpiprocs=36:mem=5GB -l walltime=00:03:00 -q ${ic_queue} -A ${COMPUTER_CHARGE_ACCOUNT} -j oe -k eod -N icgen "
+   else if ( $SUPER_PLATFORM == 'derecho' ) then
+      set ic_queue = "main"
+      set sub_command = "qsub -l select=1:ncpus=128:mpiprocs=128:mem=5GB -l walltime=00:03:00 -q ${ic_queue} -A ${COMPUTER_CHARGE_ACCOUNT} -j oe -k eod -N icgen "
    endif
 
    echo "this platform is $SUPER_PLATFORM and the job submission command is $sub_command"
@@ -120,7 +120,7 @@ while ( 1 == 1 )
 
    set n = 1
    while ( $n <= $NUM_ENS )
-      if ( $SUPER_PLATFORM == 'cheyenne' ) then   # can't pass along arguments in the same way
+      if ( $SUPER_PLATFORM == 'derecho' ) then   # can't pass along arguments in the same way
          $sub_command -v mem_num=${n},date=${datep},domain=${domains},paramf=${paramfile} ${SHELL_SCRIPTS_DIR}/prep_ic.csh
       else
          $sub_command " ${SHELL_SCRIPTS_DIR}/prep_ic.csh ${n} ${datep} ${dn} ${paramfile} "
@@ -147,9 +147,8 @@ while ( 1 == 1 )
             @ loop++
             if ( $loop > 60 ) then    # wait 5 minutes for the ic file to be ready, else run manually
                echo "gave up on ic member $n - redo"
-               # TJH this is not the command for cheyenne, why not $sub_command from above
                ${SHELL_SCRIPTS_DIR}/prep_ic.csh ${n} ${datep} ${dn} ${paramfile}
-               # TJH the job queued above is still queued and should be killed ...
+               # If manual execution of script, shouldn't queued job be killed?
             endif
          endif
       end
@@ -241,7 +240,7 @@ while ( 1 == 1 )
       endif
       set this_filter_runtime = $FILTER_TIME
 
-   else if ( $SUPER_PLATFORM == 'cheyenne' ) then
+   else if ( $SUPER_PLATFORM == 'derecho' ) then
 
       echo "2i\"                                                                          >! script.sed
       echo "#=================================================================\"          >> script.sed
@@ -250,6 +249,7 @@ while ( 1 == 1 )
       echo "#PBS -A ${COMPUTER_CHARGE_ACCOUNT}\"                                          >> script.sed
       echo "#PBS -l walltime=${FILTER_TIME}\"                                             >> script.sed
       echo "#PBS -q ${FILTER_QUEUE}\"                                                     >> script.sed
+      echo "#PBS -l job_priority=${FILTER_PRIORITY}\"                                     >> script.sed
       echo "#PBS -m ae\"                                                                  >> script.sed
       echo "#PBS -M ${EMAIL}\"                                                            >> script.sed
       echo "#PBS -k eod\"                                                                 >> script.sed
@@ -407,7 +407,7 @@ while ( 1 == 1 )
             bsub < assim_advance_mem${n}.csh
          endif
 
-      else if ( $SUPER_PLATFORM == 'cheyenne' ) then
+      else if ( $SUPER_PLATFORM == 'derecho' ) then
 
          echo "2i\"                                                                             >! script.sed
          echo "#=================================================================\"             >> script.sed
@@ -416,6 +416,7 @@ while ( 1 == 1 )
          echo "#PBS -A ${COMPUTER_CHARGE_ACCOUNT}\"                                             >> script.sed
          echo "#PBS -l walltime=${ADVANCE_TIME}\"                                               >> script.sed
          echo "#PBS -q ${ADVANCE_QUEUE}\"                                                       >> script.sed
+         echo "#PBS -l job_priority=${ADVANCE_PRIORITY}\"                                       >> script.sed
          echo "#PBS -m a\"                                                                      >> script.sed
          echo "#PBS -M ${EMAIL}\"                                                               >> script.sed
          echo "#PBS -k eod\"                                                                    >> script.sed
@@ -470,7 +471,7 @@ while ( 1 == 1 )
 
                endif
 
-            else if ( $SUPER_PLATFORM == 'cheyenne' ) then
+            else if ( $SUPER_PLATFORM == 'derecho' ) then
 
                if ( `qstat -wa | grep assim_advance_${n} | wc -l` == 0 ) then
 
@@ -511,7 +512,7 @@ while ( 1 == 1 )
                      bsub < assim_advance_mem${n}.csh
       	          endif
 
-               else if ( $SUPER_PLATFORM == 'cheyenne' ) then
+               else if ( $SUPER_PLATFORM == 'derecho' ) then
 
                   qsub assim_advance_mem${n}.csh
 

models/wrf/shell_scripts/first_advance.csh

@@ -48,19 +48,16 @@ if ( $SUPER_PLATFORM == 'yellowstone' ) then
  mpirun.lsf ./wrf.exe
 EOF
 
-else if ( $SUPER_PLATFORM == 'cheyenne' ) then
+else if ( $SUPER_PLATFORM == 'derecho' ) then
 
-   # TJH MPI_IB_CONGESTED, MPI_LAUNCH_TIMEOUT used after cheyenne O/S change in July 2019
-   # TJH setenv MPI_IB_CONGESTED 1
-   # TJH setenv MPI_LAUNCH_TIMEOUT 40
    setenv MPI_SHEPHERD false
 
    cat >! $RUN_DIR/advance_temp${emember}/wrf.info << EOF
  ${gdatef[2]}  ${gdatef[1]}
  ${gdate[2]}   ${gdate[1]}
  $yyyy $mm $dd $hh $nn $ss
            $domains
- mpiexec_mpt dplace -s 1 ./wrf.exe
+ mpiexec -n 128 -ppn 128 ./wrf.exe
 EOF
 
 endif

models/wrf/shell_scripts/gen_retro_icbc.csh

@@ -42,7 +42,7 @@ echo "gen_retro_icbc.csh is running in `pwd`"
 
 set datea     = 2017042700
 set datefnl   = 2017042712 # set this appropriately #%%%#
-set paramfile = /glade2/scratch2/USERNAME/WORK_DIR/scripts/param.csh   # set this appropriately #%%%#
+set paramfile = /glade/derecho/scratch/USERNAME/WORK_DIR/scripts/param.csh   # set this appropriately #%%%#
 
 source $paramfile
 
@@ -171,22 +171,23 @@ EOF
      #if ( -e rsl.out.0000 )  cat rsl.out.0000 >> out.real.exe
 
       rm script.sed real_done rsl.*
-      echo "2i\"                                      >! script.sed
-      echo "#======================================\" >> script.sed
-      echo "#PBS -N run_real\"                        >> script.sed
-      echo "#PBS -A ${COMPUTER_CHARGE_ACCOUNT}\"      >> script.sed
-      echo "#PBS -l walltime=00:05:00\"               >> script.sed
-      echo "#PBS -q ${ADVANCE_QUEUE}\"                >> script.sed
-      echo "#PBS -o run_real.out\"                    >> script.sed
-      echo "#PBS -j oe\"                              >> script.sed
-      echo "#PBS -k eod\"                             >> script.sed
-      echo "#PBS -l select=3:ncpus=36:mpiprocs=36\"   >> script.sed
-      echo "#PBS -V\"                                 >> script.sed
-      echo "#======================================\" >> script.sed
-      echo "\"                                        >> script.sed
-      echo ""                                         >> script.sed
-      echo 's%${1}%'"${paramfile}%g"                  >> script.sed
-      sed -f script.sed ${SHELL_SCRIPTS_DIR}/real.csh >! real.csh
+      echo "2i\"                                       >! script.sed
+      echo "#======================================\"  >> script.sed
+      echo "#PBS -N run_real\"                         >> script.sed
+      echo "#PBS -A ${COMPUTER_CHARGE_ACCOUNT}\"       >> script.sed
+      echo "#PBS -l walltime=00:05:00\"                >> script.sed
+      echo "#PBS -q ${ADVANCE_QUEUE}\"                 >> script.sed
+      echo "#PBS -l job_priority=${ADVANCE_PRIORITY}\" >> script.sed
+      echo "#PBS -o run_real.out\"                     >> script.sed
+      echo "#PBS -j oe\"                               >> script.sed
+      echo "#PBS -k eod\"                              >> script.sed
+      echo "#PBS -l select=1:ncpus=128:mpiprocs=128\"  >> script.sed
+      echo "#PBS -V\"                                  >> script.sed
+      echo "#======================================\"  >> script.sed
+      echo "\"                                         >> script.sed
+      echo ""                                          >> script.sed
+      echo 's%${1}%'"${paramfile}%g"                   >> script.sed
+      sed -f script.sed ${SHELL_SCRIPTS_DIR}/real.csh  >! real.csh
 
       qsub real.csh
 

models/wrf/shell_scripts/init_ensemble_var.csh

@@ -83,6 +83,7 @@ EOF
 #PBS -A ${COMPUTER_CHARGE_ACCOUNT}
 #PBS -l walltime=${ADVANCE_TIME}
 #PBS -q ${ADVANCE_QUEUE}
+#PBS -l job_priority=${ADVANCE_PRIORITY}
 #PBS -m ae
 #PBS -M ${EMAIL}
 #PBS -k eod

models/wrf/shell_scripts/param.csh

@@ -9,8 +9,7 @@
 #                        ASSIM_INT_MINUTES support needs to be added to param.csh,
 #                        it is referenced in assim_advance.csh but not declared in param.csh
 
-# Set up environment. Current settings are for NCAR's Cheyenne
-module load mpt          # set this appropriately #%%%#
+# Set up environment. Current settings are for NCAR's Derecho
 module load nco          # set this appropriately #%%%#
 module load ncl/6.6.2    # set this appropriately #%%%#
 
@@ -25,7 +24,7 @@ set NUM_DOMAINS        = 1
 #  Directories where things are run
 #  IMPORTANT : Scripts provided rely on this directory structure and names relative to BASE_DIR.
 #              Do not change, otherwise tutorial will fail.    
-set BASE_DIR         = /glade2/scratch2/USER/WORK_DIR     # set this appropriately #%%%#
+set BASE_DIR         = /glade/derecho/scratch/USER/WORK_DIR     # set this appropriately #%%%#
 set RUN_DIR          = ${BASE_DIR}/rundir
 set TEMPLATE_DIR     = ${BASE_DIR}/template
 set OBSPROC_DIR      = ${BASE_DIR}/obsproc
@@ -37,15 +36,15 @@ set PERTS_DIR        = ${BASE_DIR}/perts
 
 #  Directories that can be used by many things
 set SHELL_SCRIPTS_DIR = ${BASE_DIR}/scripts
-set DART_DIR          = /glade/p/work/USER/DART_manhattan           # set this appropriately #%%%#
-set WRF_DM_SRC_DIR    = /glade/p/work/USER/WRFV3_dmpar              # set this appropriately #%%%#
-set WPS_SRC_DIR       = /glade/p/work/USER/WPS                      # set this appropriately #%%%#
-set VAR_SRC_DIR       = /glade/p/work/USER/WRFDA                    # set this appropriately #%%%#
+set DART_DIR          = /glade/work/USER/DART                     # set this appropriately #%%%#
+set WRF_DM_SRC_DIR    = /glade/work/USER/WRFV3                    # set this appropriately #%%%#
+set WPS_SRC_DIR       = /glade/work/USER/WPS                      # set this appropriately #%%%#
+set VAR_SRC_DIR       = /glade/work/USER/WRFDA                    # set this appropriately #%%%#
 
 # for generating wrf template files
-set GEO_FILES_DIR     = /glade/p/work/USER/WPS        # set this appropriately #%%%#
-set GRIB_DATA_DIR     = /glade/p/work/USER/WPS/GRIB   # set this appropriately #%%%#
-set GRIB_SRC          = 'GFS'                         # set this appropriately #%%%#
+set GEO_FILES_DIR     = /glade/u/home/wrfhelp/WPS_GEOG            # set this appropriately #%%%#
+set GRIB_DATA_DIR     = ${ICBC_DIR}/grib_data                     # set this appropriately #%%%#
+set GRIB_SRC          = 'GFS'                                     # set this appropriately #%%%#
 
 # list of variables for extraction and cycling
 set extract_vars_a   = ( U V PH T MU QVAPOR QCLOUD QRAIN QICE QSNOW QGRAUP QNICE QNRAIN \
@@ -61,31 +60,29 @@ set increment_vars_a = ( U V PH T MU QVAPOR QCLOUD QRAIN QICE QSNOW QGRAUP QNICE
 set OBS_VERIF_DAYS      = 7
 
 #  Generic queuing system parameters
-set SUPER_PLATFORM      = cheyenne
-
-# TJH consistent way of checking the SUPER_PLATFORM and injecting that
-#     header information into the scripts ... rather than have scripts
-#     that have redundant blocks in them ...
-#
+set SUPER_PLATFORM          = derecho
 set COMPUTER_CHARGE_ACCOUNT = YOUR_ACCT                  # set this appropriately #%%%#
-set EMAIL                   = YOUR_EMAIL@SOMEPLACE.COM   # set this appropriately #%%%#
+set EMAIL                   = YOUR_EMAIL                 # set this appropriately #%%%#
 
-if ( $SUPER_PLATFORM == 'cheyenne') then
-   # cheyenne values (uses 'PBS' queueing system) 
-   # set this appropriately #%%%#  ... ALL OF THESE if using PBS
-   set FILTER_QUEUE       = regular
+if ( $SUPER_PLATFORM == 'derecho') then
+   # Derecho values (uses 'PBS' queueing system) 
+   # Set these appropriately for your PBS system  #%%%#  
+   set FILTER_QUEUE       = main
+   set FILTER_PRIORITY    = premium
    set FILTER_TIME        = 0:35:00
-   set FILTER_NODES       = 10
-   set FILTER_PROCS       = 36
-   set FILTER_MPI         = 36
-   set ADVANCE_QUEUE       = regular
-   set ADVANCE_TIME        = 0:20:00
-   set ADVANCE_NODES       = 3
-   set ADVANCE_PROCS       = 36
-   set ADVANCE_MPI         = 36
+   set FILTER_NODES       = 2
+   set FILTER_PROCS       = 128
+   set FILTER_MPI         = 128
+
+   set ADVANCE_QUEUE      = main
+   set ADVANCE_PRIORITY   = premium
+   set ADVANCE_TIME       = 0:20:00
+   set ADVANCE_NODES      = 1
+   set ADVANCE_PROCS      = 128
+   set ADVANCE_MPI        = 128
 else
-   # yellowstone (uses 'LSF' queueing system)
-   # set this appropriately #%%%#  ... ALL OF THESE if using LSF
+   # 'LSF' queueing system example
+   # Set these appropriately for your LSF or Slurm system #%%%# 
    set FILTER_QUEUE        = regular
    set FILTER_TIME         = 0:25
    set FILTER_CORES        = 512
@@ -97,8 +94,6 @@ else
 endif
 
 #  System specific commands
-# TJH ... The LINK command probably should not have the force option.
-# TJH ... and if the LINK fails, should it die right there?
 setenv   REMOVE 'rm -rf'
 setenv   COPY 'cp -pfr'
 setenv   MOVE 'mv -f'

models/wrf/shell_scripts/real.csh

@@ -5,7 +5,7 @@
  source $paramfile
 
  cd ${ICBC_DIR}
- mpiexec_mpt dplace -s 1 ${RUN_DIR}/WRF_RUN/real.exe
+ mpiexec -n 128 -ppn 128 ${RUN_DIR}/WRF_RUN/real.exe
 
 #if ( `grep "Successful completion of program real.exe" ./rsl.out.0000 | wc -l ` == 1  )  touch ${ICBC_DIR}/real_done