From 15d5bed4119b067905f5a4bf47656557f2211883 Mon Sep 17 00:00:00 2001
From: "russ.treadon" <Russ.Treadon@noaa.gov>
Date: Mon, 15 Jun 2020 20:47:47 +0000
Subject: [PATCH] Issue #85:  add cfp option to chgresfcst; enable threads with
 chgresfcst

---
 env/WCOSS_DELL_P3.env                     |  4 ++++
 parm/config/config.resources              |  6 +++---
 sorc/enkf_chgres_recenter_nc.fd/setup.f90 |  6 ++++--
 ush/rocoto/setup_workflow.py              | 24 +++++++++++------------
 4 files changed, 23 insertions(+), 17 deletions(-)

diff --git a/env/WCOSS_DELL_P3.env b/env/WCOSS_DELL_P3.env
index 484b5ce73c..06e68653d3 100755
--- a/env/WCOSS_DELL_P3.env
+++ b/env/WCOSS_DELL_P3.env
@@ -66,6 +66,10 @@ elif [ $step = "anal" ]; then
     npe_gausfcanl=${npe_gausfcanl:-1}
     export APRUN_GAUSFCANL="$launcher $npe_gausfcanl"
 
+    export NTHREADS_CHGRES=${nth_chgresfcst:-14}
+    [[ $NTHREADS_CHGRES -gt $npe_node_max ]] && export NTHREADS_CHGRES=$npe_node_max
+    export APRUN_CHGRES=""
+
     if [ ${USE_CFP:-"NO"} = "YES" ]; then
         export APRUNCFP="$launcher \$ncmd $mpmd"
     fi
diff --git a/parm/config/config.resources b/parm/config/config.resources
index a01bf32e36..fba9802223 100755
--- a/parm/config/config.resources
+++ b/parm/config/config.resources
@@ -203,10 +203,10 @@ elif [ $step = "metp" ]; then
 elif [ $step = "chgresfcst" ]; then
 
     export wtime_chgresfcst="01:00:00"
-    export npe_chgresfcst=24
+    export npe_chgresfcst=3
     export nth_chgresfcst=1
-    export npe_node_chgresfcst=$npe_node_max
-    if [[ "$machine" = "WCOSS_DELL_P3" ]]; then export npe_chgresfcst=24 ; fi
+    export npe_node_chgresfcst=1
+    if [[ "$machine" = "WCOSS_DELL_P3" ]]; then export nth_chgresfcst=28 ; fi
 
 elif [ $step = "arch" -o $step = "earc" -o $step = "getic" ]; then
 
diff --git a/sorc/enkf_chgres_recenter_nc.fd/setup.f90 b/sorc/enkf_chgres_recenter_nc.fd/setup.f90
index 4bb1a0253c..1ea5b98a82 100644
--- a/sorc/enkf_chgres_recenter_nc.fd/setup.f90
+++ b/sorc/enkf_chgres_recenter_nc.fd/setup.f90
@@ -23,6 +23,7 @@ subroutine program_setup
  implicit none
 
  integer                           :: istat
+ character(len=500)                :: filenamelist
 
  namelist /chgres_setup/ i_output, j_output, input_file, output_file, &
                       terrain_file, vcoord_file, cld_amt
@@ -30,8 +31,9 @@ subroutine program_setup
  cld_amt = .false. ! default option
 
  print*
- print*,"OPEN SETUP NAMELIST."
- open(43, file="./chgres_nc_gauss.nml", iostat=istat)
+ call getarg(1,filenamelist)
+ print*,"OPEN SETUP NAMELIST ",trim(filenamelist)
+ open(43, file=filenamelist, iostat=istat)
  if (istat /= 0) then
    print*,"FATAL ERROR OPENING NAMELIST FILE. ISTAT IS: ",istat
    stop 
diff --git a/ush/rocoto/setup_workflow.py b/ush/rocoto/setup_workflow.py
index e67f7f9ec4..b2a3617821 100755
--- a/ush/rocoto/setup_workflow.py
+++ b/ush/rocoto/setup_workflow.py
@@ -54,7 +54,7 @@ def main():
     #wav_steps_awips = ['waveawipssbs', 'waveawips']
 # From gfsv16b latest
 #    gfs_steps = ['prep', 'anal', 'gldas', 'fcst', 'postsnd', 'post', 'awips', 'gempak', 'vrfy', 'metp', 'arch']
-    hyb_steps = ['eobs', 'ediag', 'eomg', 'eupd', 'ecen', 'esfc', 'efcs', 'epos', 'earc', 'chgresfcst']
+    hyb_steps = ['eobs', 'ediag', 'eomg', 'eupd', 'ecen', 'esfc', 'efcs', 'chgresfcst', 'epos', 'earc']
 
     steps = gfs_steps + hyb_steps if _base.get('DOHYBVAR', 'NO') == 'YES' else gfs_steps
     steps = steps + metp_steps if _base.get('DO_METP', 'NO') == 'YES' else steps
@@ -860,17 +860,6 @@ def get_hyb_tasks(dict_configs, cycledef='enkf'):
 
     dict_tasks['%sesfc' % cdump] = task
 
-    # chgresfcst
-    deps1 = []
-    dep_dict = {'type': 'task', 'name': '%sfcst' % cdump}
-    deps1.append(rocoto.add_dependency(dep_dict))
-    dep_dict = {'type': 'metatask', 'name': '%sefmn' % cdump}
-    deps1.append(rocoto.add_dependency(dep_dict))
-    dependencies1 = rocoto.create_dependency(dep_condition='and', dep=deps1)
-    task = wfu.create_wf_task('chgresfcst', cdump=cdump, envar=envars1, dependency=dependencies1, cycledef=cycledef)
-
-    dict_tasks['%schgresfcst' % cdump] = task
-
     # efmn, efcs
     deps1 = []
     dep_dict = {'type': 'metatask', 'name': '%secmn' % cdump}
@@ -891,6 +880,17 @@ def get_hyb_tasks(dict_configs, cycledef='enkf'):
 
     dict_tasks['%sefmn' % cdump] = task
 
+    # chgresfcst
+    deps1 = []
+    dep_dict = {'type': 'task', 'name': '%sfcst' % cdump}
+    deps1.append(rocoto.add_dependency(dep_dict))
+    dep_dict = {'type': 'metatask', 'name': '%sefmn' % cdump}
+    deps1.append(rocoto.add_dependency(dep_dict))
+    dependencies1 = rocoto.create_dependency(dep_condition='and', dep=deps1)
+    task = wfu.create_wf_task('chgresfcst', cdump=cdump, envar=envars1, dependency=dependencies1, cycledef=cycledef)
+
+    dict_tasks['%schgresfcst' % cdump] = task
+
     # epmn, epos
     deps = []
     dep_dict = {'type': 'metatask', 'name': '%sefmn' % cdump}