From 31dfb354e319c9137bb206a253826b75c9c9e1aa Mon Sep 17 00:00:00 2001
From: RussTreadon-NOAA <Russ.Treadon@noaa.gov>
Date: Sat, 11 Feb 2023 01:03:10 +0000
Subject: [PATCH 1/3] updates to regression global_4dvar test, job
 configuration, and error checking (#531)

---
 regression/global_4dvar.sh         | 12 +++++-----
 regression/regression_driver.sh    |  1 -
 regression/regression_param.sh     |  4 ++--
 regression/regression_test.sh      | 17 ++++++++++----
 regression/regression_test_enkf.sh | 37 ++++++++++++++++++++++--------
 5 files changed, 48 insertions(+), 23 deletions(-)

diff --git a/regression/global_4dvar.sh b/regression/global_4dvar.sh
index ec2a8396e0..cac0d28e6b 100755
--- a/regression/global_4dvar.sh
+++ b/regression/global_4dvar.sh
@@ -98,7 +98,7 @@ SINGLEOB=""
 #   bftab_sst= bufr table for sst ONLY needed for sst retrieval (retrieval=.true.)
 #   aeroinfo = text file with information about assimilation of aerosol data
 
-anavinfo=$fixgsi/global_anavinfo.l${LEVS}.txt
+anavinfo=$fixgsi/global_anavinfo_qlqi.l${LEVS}.txt
 berror=$fixgsi/Big_Endian/global_berror.l${LEVS}y${NLAT}.f77
 locinfo=$fixgsi/global_hybens_info.l${LEVS}.txt
 satinfo=$fixgsi/global_satinfo.txt
@@ -311,9 +311,9 @@ $gsi_namelist
 EOF
 cp gsiparm.anl gsiparm.anl.obsvr
 
-echo "run gsi now"
+echo "run gsi observer"
 eval "$APRUN $tmpdir/gsi.x < gsiparm.anl > stdout.obsvr 2>&1"
-rc=$?
+ra=$?
 
 # Run gsi identity model 4dvar under Parallel Operating Environment (poe) on NCEP IBM
 rm -f siganl sfcanl.gsi satbias_out fort.2*
@@ -331,8 +331,8 @@ cat <<EOF > gsiparm.anl
 $gsi_namelist
 EOF
 
-echo "run gsi now"
+echo "run gsi 4dvar"
 eval "$APRUN $tmpdir/gsi.x < gsiparm.anl > stdout 2>&1"
-rc=$?
-
+rb=$?
+rc=$((ra+rb))
 exit $rc
diff --git a/regression/regression_driver.sh b/regression/regression_driver.sh
index 621ccbf485..e1d3b18dc7 100755
--- a/regression/regression_driver.sh
+++ b/regression/regression_driver.sh
@@ -47,7 +47,6 @@ for jn in `seq ${RSTART} ${REND}`; do
    $scripts/regression_wait.sh ${job[$jn]} ${rcname} $check_resource
    rc=$?
    if [ $rc -ne 0 ]; then
-     rm -f ${rcname}
      exit 1
    fi
    done
diff --git a/regression/regression_param.sh b/regression/regression_param.sh
index 85f2949bfe..a2808ddfc0 100755
--- a/regression/regression_param.sh
+++ b/regression/regression_param.sh
@@ -150,8 +150,8 @@ case $regtest in
             topts[1]="0:15:00" ; popts[1]="20/1/"  ; ropts[1]="/1"
             topts[2]="0:15:00" ; popts[2]="20/2/"  ; ropts[2]="/1"
         elif [[ "$machine" = "Orion" ]]; then
-            topts[1]="0:15:00" ; popts[1]="4/4/"  ; ropts[1]="/1"
-            topts[2]="0:15:00" ; popts[2]="6/6/"  ; ropts[2]="/1"
+            topts[1]="0:15:00" ; popts[1]="20/1/"  ; ropts[1]="/1"
+            topts[2]="0:15:00" ; popts[2]="20/2/"  ; ropts[2]="/1"
         elif [[ "$machine" = "Jet" ]]; then
             topts[1]="0:15:00" ; popts[1]="4/4/"  ; ropts[1]="/1"
             topts[2]="0:15:00" ; popts[2]="6/6/"  ; ropts[2]="/1"
diff --git a/regression/regression_test.sh b/regression/regression_test.sh
index deb34ff244..0bcb9f4d90 100755
--- a/regression/regression_test.sh
+++ b/regression/regression_test.sh
@@ -576,11 +576,20 @@ mkdir -p $vfydir
 
 $ncp $output                        $vfydir/
 
+# Final check for any failed tests
+count=$(grep -i "fail" $output |wc -l)
+if [ $count -gt 0 ]; then
+    (( failed_test = $failed_test + $count ))
+fi
+
+# Remove job log files is no failures detected
 cd $scripts
-rm -f ${exp1}.out
-rm -f ${exp2}.out
-rm -f ${exp3}.out
-rm -f ${exp2_scale}.out
+if [ $count -eq 0 ]; then
+    rm -f ${exp1}.out
+    rm -f ${exp2}.out
+    rm -f ${exp3}.out
+    rm -f ${exp2_scale}.out
+fi
 
 if [[ "$clean" = ".true." ]]; then
    rm -rf $savdir
diff --git a/regression/regression_test_enkf.sh b/regression/regression_test_enkf.sh
index 213ee726da..805fa8781b 100755
--- a/regression/regression_test_enkf.sh
+++ b/regression/regression_test_enkf.sh
@@ -34,7 +34,7 @@ maxtime=1200
 maxmem=${maxmem:-3400000} # set in regression_param
 maxmem=$((${memnode:-64}*1024*1024))
 
-# Copy stdout and sanl files 
+# Copy stdout and incr files 
 # from $savdir to $tmpdir
 list="$exp1 $exp2 $exp3"
 for exp in $list; do
@@ -43,7 +43,7 @@ for exp in $list; do
    imem=1
    while [[ $imem -le $nmem ]]; do
       member="_mem"`printf %03i $imem`
-      $ncp $savdir/$exp/sanl_${global_adate}_fhr06$member $tmpdir/sanl$member.$exp
+      $ncp $savdir/$exp/incr_${global_adate}_fhr06$member $tmpdir/incr$member.$exp
       (( imem = $imem + 1 ))
    done
 done
@@ -282,9 +282,13 @@ nmem=10
 imem=1
 while [[ $imem -le $nmem ]]; do
    member="_mem"`printf %03i $imem`
-   if ! cmp -s sanl$member.${exp1} sanl$member.${exp2} 
+   ncdump incr$member.${exp1} > incr$member.${exp1}.out
+   ncdump incr$member.${exp2} > incr$member.${exp2}.out
+   if ! diff incr$member.${exp1}.out incr$member.${exp2}.out
 then
-   echo 'sanl'$member'.'${exp1}' sanl'$member'.'${exp2}' are NOT identical'
+   echo 'incr'$member'.'${exp1}' incr'$member'.'${exp2}' are NOT identical'
+else
+       rm -f incr$member.${exp1}.out incr$member.${exp2}.out
 fi
    (( imem = $imem + 1 ))
 done
@@ -379,9 +383,13 @@ else
    imem=1
    while [[ $imem -le $nmem ]]; do
       member="_mem"`printf %03i $imem`
-      if ! cmp -s sanl$member.${exp1} sanl$member.${exp3}
+      ncdump incr$member.${exp1} > incr$member.${exp1}.out
+      ncdump incr$member.${exp3} > incr$member.${exp3}.out
+      if ! diff incr$member.${exp1}.out incr$member.${exp3}.out
       then
-      echo 'sanl'$member'.'${exp1}' sanl'$member'.'${exp3}' are NOT identical'
+      echo 'incr'$member'.'${exp1}' incr'$member'.'${exp3}' are NOT identical'
+      else
+          rm -f incr$member.${exp1}.out incr$member.${exp3}.out
       fi
    (( imem = $imem + 1 ))
    done
@@ -411,11 +419,20 @@ mkdir -p $vfydir
 
 $ncp $output                        $vfydir/
 
+# Final check for any failed tests
+count=$(grep -i "fail" $output |wc -l)
+if [ $count -gt 0 ]; then
+    (( failed_test = $failed_test + $count ))
+fi
+
+# Remove job log files is no failures detected
 cd $scripts
-rm -f ${exp1}.out
-rm -f ${exp2}.out
-rm -f ${exp3}.out
-rm -f ${exp2_scale}.out
+if [ $count -eq 0 ]; then
+    rm -f ${exp1}.out
+    rm -f ${exp2}.out
+    rm -f ${exp3}.out
+    rm -f ${exp2_scale}.out
+fi
 
 if [[ "$clean" = ".true." ]]; then
    rm -rf $savdir

From 74cad42e40e88a444f0a5bd771a14c05815c05bd Mon Sep 17 00:00:00 2001
From: RussTreadon-NOAA <Russ.Treadon@noaa.gov>
Date: Sat, 11 Feb 2023 17:12:03 +0000
Subject: [PATCH 2/3] add indentation to improve readability of
 regression_test_enkf.sh (#531)

---
 regression/regression_test_enkf.sh | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/regression/regression_test_enkf.sh b/regression/regression_test_enkf.sh
index 805fa8781b..2563225ef4 100755
--- a/regression/regression_test_enkf.sh
+++ b/regression/regression_test_enkf.sh
@@ -284,12 +284,11 @@ while [[ $imem -le $nmem ]]; do
    member="_mem"`printf %03i $imem`
    ncdump incr$member.${exp1} > incr$member.${exp1}.out
    ncdump incr$member.${exp2} > incr$member.${exp2}.out
-   if ! diff incr$member.${exp1}.out incr$member.${exp2}.out
-then
-   echo 'incr'$member'.'${exp1}' incr'$member'.'${exp2}' are NOT identical'
-else
+   if [[ ! diff incr$member.${exp1}.out incr$member.${exp2}.out ]]; then
+       echo 'incr'$member'.'${exp1}' incr'$member'.'${exp2}' are NOT identical'
+   else
        rm -f incr$member.${exp1}.out incr$member.${exp2}.out
-fi
+   fi
    (( imem = $imem + 1 ))
 done
 echo
@@ -385,13 +384,12 @@ else
       member="_mem"`printf %03i $imem`
       ncdump incr$member.${exp1} > incr$member.${exp1}.out
       ncdump incr$member.${exp3} > incr$member.${exp3}.out
-      if ! diff incr$member.${exp1}.out incr$member.${exp3}.out
-      then
-      echo 'incr'$member'.'${exp1}' incr'$member'.'${exp3}' are NOT identical'
+      if [[ ! diff incr$member.${exp1}.out incr$member.${exp3}.out ]]; then
+          echo 'incr'$member'.'${exp1}' incr'$member'.'${exp3}' are NOT identical'
       else
           rm -f incr$member.${exp1}.out incr$member.${exp3}.out
       fi
-   (( imem = $imem + 1 ))
+      (( imem = $imem + 1 ))
    done
    echo
 } >> $output

From b6c56f2eea17c8265f89faa35c6c8e131bf83e35 Mon Sep 17 00:00:00 2001
From: RussTreadon-NOAA <Russ.Treadon@noaa.gov>
Date: Sat, 11 Feb 2023 18:11:28 +0000
Subject: [PATCH 3/3] correct syntax error in regression_test_enkf.sh (#531)

---
 regression/regression_test_enkf.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/regression/regression_test_enkf.sh b/regression/regression_test_enkf.sh
index 2563225ef4..f52a5d451f 100755
--- a/regression/regression_test_enkf.sh
+++ b/regression/regression_test_enkf.sh
@@ -284,7 +284,7 @@ while [[ $imem -le $nmem ]]; do
    member="_mem"`printf %03i $imem`
    ncdump incr$member.${exp1} > incr$member.${exp1}.out
    ncdump incr$member.${exp2} > incr$member.${exp2}.out
-   if [[ ! diff incr$member.${exp1}.out incr$member.${exp2}.out ]]; then
+   if [ ! diff incr$member.${exp1}.out incr$member.${exp2}.out ]; then
        echo 'incr'$member'.'${exp1}' incr'$member'.'${exp2}' are NOT identical'
    else
        rm -f incr$member.${exp1}.out incr$member.${exp2}.out
@@ -384,7 +384,7 @@ else
       member="_mem"`printf %03i $imem`
       ncdump incr$member.${exp1} > incr$member.${exp1}.out
       ncdump incr$member.${exp3} > incr$member.${exp3}.out
-      if [[ ! diff incr$member.${exp1}.out incr$member.${exp3}.out ]]; then
+      if [ ! diff incr$member.${exp1}.out incr$member.${exp3}.out ]; then
           echo 'incr'$member'.'${exp1}' incr'$member'.'${exp3}' are NOT identical'
       else
           rm -f incr$member.${exp1}.out incr$member.${exp3}.out