Merge pull request NOAA-EMC#376 from EdwardSafford-NOAA/develop

GitHub Issue NOAA-EMC#373. Compress RadMon data files.
EdwardSafford-NOAA · May 9, 2022 · 2294cfb · 2294cfb
2 parents f989a2e + 9c3fc61
commit 2294cfb
Show file tree

Hide file tree

Showing 15 changed files with 121 additions and 74 deletions.
diff --git a/util/Radiance_Monitor/data_extract/ush/radmon_copy.sh b/util/Radiance_Monitor/data_extract/ush/radmon_copy.sh
@@ -82,7 +82,7 @@ if [[ ${exit_value} -eq 0 ]]; then
 
    for type in ${type_list}; do 
 
-      file_list=`ls ${DATA_LOCATION}/${type}.*${PDATE}*ieee_d* `
+      file_list=`ls ${DATA_LOCATION}/${type}.*${PDATE}*ieee_d* ${DATA_LOCATION}/${type}*tar* `
 
       for file in ${file_list}; do
          bfile=`basename ${file}`

diff --git a/util/Radiance_Monitor/image_gen/ush/RadMon_IG_glb.sh b/util/Radiance_Monitor/image_gen/ush/RadMon_IG_glb.sh
@@ -154,7 +154,7 @@ fi
 
 if [[ ${pdate} -gt ${latest_data} ]]; then
   echo " Unable to plot, pdate is > latest_data, ${pdate}, ${latest_data}"
-  exit 4 
+  exit 5 
 else
   echo " OK to plot"
 fi
@@ -199,23 +199,29 @@ fi
 
 if [[ ! -d ${ieee_src} ]]; then
    echo "Unable to set ieee_src, aborting plot"
-   exit 5
+   exit 6
 fi
 
-#-----------------------------------------------------
+#-------------------------------------------------------------
 # check $ieee_src for data files.  If none are found
-# check contents of the radmon_angle.tar file.
+# check contents of the radmon_angle.tar file.  If both
+# a compressed and an uncompressed version of radmon_angle.tar
+# exist, flag that condition as an error.
 #
 nfile_src=`ls -l ${ieee_src}/*${PDATE}*ieee_d* | egrep -c '^-'`
 if [[ $nfile_src -le 0 ]]; then
-   if [[ -e ${ieee_src}/radmon_angle.tar ]]; then
-      nfile_src=`tar -tf ${ieee_src}/radmon_angle.tar | grep ieee_d | wc -l`
+   if [[ -e ${ieee_src}/radmon_angle.tar && -e ${ieee_src}/radmon_angle.tar.${Z} ]]; then
+      echo "Located both radmon_angle.tar and radmon_angle.tar.${Z} in ${ieee_src}.  Unable to plot."
+      exit 7
+
+   elif [[ -e ${ieee_src}/radmon_angle.tar || -e ${ieee_src}/radmon_angle.tar.${Z} ]]; then
+      nfile_src=`tar -tf ${ieee_src}/radmon_angle.tar* | grep ieee_d | wc -l`
    fi
 fi
 
 if [[ $nfile_src -le 0 ]]; then
    echo " Missing ieee_src files, nfile_src = ${nfile_src}, aborting plot"
-   exit 6
+   exit 8
 fi
 
 export PLOT_WORK_DIR=${PLOT_WORK_DIR}.${PDATE}
@@ -251,7 +257,7 @@ fi
 #
 test_list=`ls ${ieee_src}/angle.*${PDATE}.ieee_d*`
 if [[ $test_list = "" ]]; then
-   test_list=`tar -tf ${ieee_src}/radmon_angle.tar | grep ieee_d` 
+   test_list=`tar -tf ${ieee_src}/radmon_angle.tar* | grep ieee_d` 
 fi
 
 for test in ${test_list}; do

diff --git a/util/Radiance_Monitor/image_gen/ush/mk_angle_plots.sh b/util/Radiance_Monitor/image_gen/ush/mk_angle_plots.sh
@@ -80,17 +80,23 @@ for type in ${SATYPE}; do
          fi
 
          using_tar=0
-         #----------------------------------------------------
-         #  Determine if the angle files are in a tar file
-         #  and, if so, extract the ctl files for this $type.
+         #--------------------------------------------------------------
+         #  Determine if the angle files are in a tar file.  If so
+         #  extract the ctl files for this $type.  If both a compressed
+         #  and uncompressed version of the radmon_bcoef.tar file exist,
+         #  report that as an error condition.
          #
-         if [[ -s ${ieee_src}/radmon_angle.tar ]]; then
+         if [[ -e ${ieee_src}/radmon_angle.tar && -e ${ieee_src}/radmon_angle.tar.${Z} ]]; then
+            echo "Located both radmon_angle.tar and radmon_angle.tar.${Z} in ${ieee_src}.  Unable to plot."
+            exit 2
+
+         elif [[ -e ${ieee_src}/radmon_angle.tar || -e ${ieee_src}/radmon_angle.tar.${Z} ]]; then
             using_tar=1
-            ctl_list=`tar -tf ${ieee_src}/radmon_angle.tar | grep ${type} | grep ctl`
+            ctl_list=`tar -tf ${ieee_src}/radmon_angle.tar* | grep ${type} | grep ctl`
             if [[ ${ctl_list} != "" ]]; then
                cwd=`pwd`
                cd ${ieee_src}
-               tar -xf ./radmon_angle.tar ${ctl_list}
+               tar -xf ./radmon_angle.tar* ${ctl_list}
                cd ${cwd} 
             fi
          fi
@@ -139,7 +145,7 @@ done
 
 if [[ $allmissing = 1 ]]; then
    echo ERROR:  Unable to plot.  All angle control files are missing from ${TANKverf} for requested date range.
-   exit 2
+   exit 3
 fi
 
 

diff --git a/util/Radiance_Monitor/image_gen/ush/mk_bcoef_plots.sh b/util/Radiance_Monitor/image_gen/ush/mk_bcoef_plots.sh
@@ -12,10 +12,6 @@ set -ax
 date
 echo "begin mk_bcoef_plots.sh"
 
-#export NUM_CYCLES=${NUM_CYCLES:-121}
-#export CYCLE_INTERVAL=${CYCLE_INTERVAL:-6}
-echo "NUM_CYCLES, CYCLE_INTERVAL = ${NUM_CYCLES}, ${CYCLE_INTERVAL}"
-
 imgndir="${IMGNDIR}/bcoef"
 tankdir="${TANKverf}/bcoef"
 
@@ -74,19 +70,25 @@ for type in ${SATYPE}; do
          fi
 
          using_tar=0
-         #----------------------------------------------------
-         #  Determine if the bcoef files are in an tar file.
-         #  if so extract the ctl files for this $type.
+         #--------------------------------------------------------------
+         #  Determine if the bcoef files are in a tar file.  If so
+         #  extract the ctl files for this $type.  If both a compressed
+         #  and uncompressed version of the radmon_bcoef.tar file exist, 
+         #  flag that as an error condition.
          #
-         if [[ -s ${ieee_src}/radmon_bcoef.tar ]]; then
+         if [[ -e ${ieee_src}/radmon_bcoef.tar && -e ${ieee_src}/radmon_bcoef.tar.${Z} ]]; then
+            echo "Located both radmon_bcoef.tar and radmon_bcoef.tar.${Z} in ${ieee_src}.  Unable to plot."
+            exit 1
+
+         elif [[ -e ${ieee_src}/radmon_bcoef.tar || -e ${ieee_src}/radmon_bcoef.tar.${Z} ]]; then
             using_tar=1
-            ctl_list=`tar -tf ${ieee_src}/radmon_bcoef.tar | grep ${type} | grep ctl`
-	    if [[ ${ctl_list} != "" ]]; then
+            ctl_list=`tar -tf ${ieee_src}/radmon_bcoef.tar* | grep ${type} | grep ctl`
+            if [[ ${ctl_list} != "" ]]; then
                cwd=`pwd`
                cd ${ieee_src}
-               tar -xf ./radmon_bcoef.tar ${ctl_list}
+               tar -xf ./radmon_bcoef.tar* ${ctl_list}
                cd ${cwd}
-	    fi
+            fi
          fi
 
          #--------------------------------------------------

diff --git a/util/Radiance_Monitor/image_gen/ush/mk_bcor_plots.sh b/util/Radiance_Monitor/image_gen/ush/mk_bcor_plots.sh
@@ -76,21 +76,27 @@ for type in ${SATYPE}; do
          fi
 
          using_tar=0
-         #---------------------------------------------------------
-         #  Determine if the bcor files are in a tar file.
-         #  if so extract the ctl files for this $type.  
+         #---------------------------------------------------------------
+         #  Determine if the bcor files are in a tar file.  If so
+         #  extract the ctl files for this $type.  If both a compressed
+         #  and uncompressed version of the radmon_bcor.tar file exist,
+         #  flag that as an error condition.
          #
          #  Note that the ctl files are moved back to ${ieee_src}
          #  so the code block that follows will work with both 
          #  tarred and non-tarred storage schemes.
          #
-         if [[ -s ${ieee_src}/radmon_bcor.tar ]]; then
+         if [[ -e ${ieee_src}/radmon_bcor.tar && -e ${ieee_src}/radmon_bcor.tar.${Z} ]]; then
+            echo "Located both radmon_bcor.tar and radmon_bcor.tar.${Z} in ${ieee_src}.  Unable to plot."
+            exit 1
+
+         elif [[ -e ${ieee_src}/radmon_bcor.tar || -e ${ieee_src}/radmon_bcor.tar.${Z} ]]; then
             using_tar=1
-            ctl_list=`tar -tf ${ieee_src}/radmon_bcor.tar | grep ${type} | grep ctl`
+            ctl_list=`tar -tf ${ieee_src}/radmon_bcor.tar* | grep ${type} | grep ctl`
             if [[ ${ctl_list} != "" ]]; then
                cwd=`pwd`
                cd ${ieee_src}
-               tar -xf ./radmon_bcor.tar ${ctl_list}            
+               tar -xf ./radmon_bcor.tar* ${ctl_list}            
                cd ${cwd}
             fi
          fi

diff --git a/util/Radiance_Monitor/image_gen/ush/mk_time_plots.sh b/util/Radiance_Monitor/image_gen/ush/mk_time_plots.sh
@@ -79,17 +79,23 @@ for type in ${SATYPE}; do
 
          using_tar=0
          #--------------------------------------------------
-         #  Determine if the time files are in an tar file.
-         #  if so extract the ctl files for this $type.
+         #  Determine if the time files are in a tar file.  If so
+         #  extract the ctl files for this $type.  If both a compressed
+         #  and uncompressed version of the radmon_time.tar file exist,
+         #  flag that as an error condition.
          #
-         if [[ -s ${ieee_src}/radmon_time.tar ]]; then
+         if [[ -e ${ieee_src}/radmon_time.tar && -e ${ieee_src}/radmon_time.tar.${Z} ]]; then
+            echo "Located both radmon_time.tar and radmon_time.tar.${Z} in ${ieee_src}.  Unable to plot."
+            exit 1
+
+         elif [[ -e ${ieee_src}/radmon_time.tar || -e ${ieee_src}/radmon_time.tar.${Z} ]]; then
             using_tar=1
-            ctl_list=`tar -tf ${ieee_src}/radmon_time.tar | grep $type | grep ctl`
+            ctl_list=`tar -tf ${ieee_src}/radmon_time.tar* | grep $type | grep ctl`
             if [[ ${ctl_list} != "" ]]; then
                cwd=`pwd`
                cd ${ieee_src}
-               ctl_list=`tar -tf ./radmon_time.tar | grep $type | grep ctl`
-               tar -xf ${ieee_src}/radmon_time.tar ${ctl_list}            
+               ctl_list=`tar -tf ./radmon_time.tar* | grep $type | grep ctl`
+               tar -xf ${ieee_src}/radmon_time.tar* ${ctl_list}            
                cd ${cwd}
             fi
          fi

diff --git a/util/Radiance_Monitor/image_gen/ush/plot_angle.sh b/util/Radiance_Monitor/image_gen/ush/plot_angle.sh
@@ -9,10 +9,6 @@
 set -ax
 export list=$listvar
 
-echo
-echo PATH=$PATH
-echo
-
 SATYPE2=$1
 PVAR=$2
 PTYPE=$3
@@ -104,11 +100,16 @@ for type in ${SATYPE2}; do
       #  Locate the data files, first checking for a tar file,
       #  and copy them locally.
       #
-      if [[ -s ${ieee_src}/radmon_angle.tar ]]; then
-         files=`tar -tf ${ieee_src}/radmon_angle.tar | grep ${type} | grep ieee_d`
+      if [[ -e ${ieee_src}/radmon_angle.tar && -e ${ieee_src}/radmon_angle.tar.${Z} ]]; then
+         echo "Located both radmon_angle.tar and radmon_angle.tar.${Z} in ${ieee_src}.  Unable to plot."
+         exit 20
+
+      elif [[ -e ${ieee_src}/radmon_angle.tar || -e ${ieee_src}/radmon_angle.tar.${Z} ]]; then
+         files=`tar -tf ${ieee_src}/radmon_angle.tar* | grep ${type} | grep ieee_d`
          if [[ ${files} != "" ]]; then
-            tar -xf ${ieee_src}/radmon_angle.tar ${files}
+            tar -xf ${ieee_src}/radmon_angle.tar* ${files}
          fi
+
       else				
          files=`ls ${ieee_src}/angle.*${type}*ieee_d*`
          for f in ${files}; do

diff --git a/util/Radiance_Monitor/image_gen/ush/plot_bcoef.sh b/util/Radiance_Monitor/image_gen/ush/plot_bcoef.sh
@@ -81,11 +81,16 @@ for type in ${SATYPE}; do
       #  Locate the ieee_d files, first checking for a tar file,
       #  and copy them locally.
       #
-      if [[ -s ${ieee_src}/radmon_bcoef.tar ]]; then
-         files=`tar -tf ${ieee_src}/radmon_bcoef.tar | grep ${type} | grep ieee_d`
+      if [[ -e ${ieee_src}/radmon_bcoef.tar && -e ${ieee_src}/radmon_bcoef.tar.${Z} ]]; then
+         echo "Located both radmon_bcoef.tar and radmon_bcoef.tar.${Z} in ${ieee_src}.  Unable to plot."
+         exit 21
+
+      elif [[ -e ${ieee_src}/radmon_bcoef.tar || -e ${ieee_src}/radmon_bcoef.tar.${Z} ]]; then
+         files=`tar -tf ${ieee_src}/radmon_bcoef.tar* | grep ${type} | grep ieee_d`
          if [[ ${files} != "" ]]; then
-            tar -xf ${ieee_src}/radmon_bcoef.tar ${files}
+            tar -xf ${ieee_src}/radmon_bcoef.tar* ${files}
          fi
+
       else				
          files=`ls ${ieee_src}/bcoef.*${type}*ieee_d*`
          for f in ${files}; do

diff --git a/util/Radiance_Monitor/image_gen/ush/plot_bcor.sh b/util/Radiance_Monitor/image_gen/ush/plot_bcor.sh
@@ -91,9 +91,13 @@ for type in ${SATYPE2}; do
       #  Now locate this cycle's data files, first checking for 
       #  a tar file, and copy them locally.
       #
-      if [[ -s ${ieee_src}/radmon_bcor.tar ]]; then
-         files=`tar -tf ${ieee_src}/radmon_bcor.tar | grep ${type} | grep ieee_d`
-         tar -xf ${ieee_src}/radmon_bcor.tar ${files}
+      if [[ -e ${ieee_src}/radmon_bcor.tar && -e ${ieee_src}/radmon_bcor.tar.${Z} ]]; then
+         echo "Located both radmon_bcor.tar and radmon_bcor.tar.${Z} in ${ieee_src}.  Unable to plot."
+         exit 22
+
+      elif [[ -e ${ieee_src}/radmon_bcor.tar || -e ${ieee_src}/radmon_bcor.tar.${Z} ]]; then
+         files=`tar -tf ${ieee_src}/radmon_bcor.tar* | grep ${type} | grep ieee_d`
+         tar -xf ${ieee_src}/radmon_bcor.tar* ${files}
 
       else				
          files=`ls ${ieee_src}/bcor.*${type}*ieee_d*`

diff --git a/util/Radiance_Monitor/image_gen/ush/plot_summary.sh b/util/Radiance_Monitor/image_gen/ush/plot_summary.sh
@@ -12,7 +12,6 @@ SATYPE2=$SATYPE
 
 echo "Start plot_summary.sh"
 
-
 #------------------------------------------------------------------
 # Set environment variables.
 tmpdir=${PLOT_WORK_DIR}/plot_summary_${RADMON_SUFFIX}
@@ -76,17 +75,20 @@ for type in ${SATYPE2}; do
          ieee_src=${TANKverf}/${RUN}.${day}
       fi
 
-      echo "rgnHH, rgnTM = $rgnHH, $rgnTM"
-
       #-----------------------------------------------------------
       #  Locate the data files, first checking for a tar file,
       #  and copy them locally.
       #
-      if [[ -s ${ieee_src}/radmon_time.tar ]]; then
-         files=`tar -tf ${ieee_src}/radmon_time.tar | grep ${type} | grep ieee_d`
+      if [[ -e ${ieee_src}/radmon_time.tar && -e ${ieee_src}/radmon_time.tar.${Z} ]]; then
+         echo "Located both radmon_time.tar and radmon_time.tar.${Z} in ${ieee_src}.  Unable to plot."
+         exit 23
+
+      elif [[ -e ${ieee_src}/radmon_time.tar || -e ${ieee_src}/radmon_time.tar.${Z} ]]; then
+         files=`tar -tf ${ieee_src}/radmon_time.tar* | grep ${type} | grep ieee_d`
 	 if [[ ${files} != "" ]]; then 
-            tar -xf ${ieee_src}/radmon_time.tar ${files}
+            tar -xf ${ieee_src}/radmon_time.tar* ${files}
          fi
+
       else				
          files=`ls ${ieee_src}/time.*${type}*ieee_d*`
          for f in ${files}; do

diff --git a/util/Radiance_Monitor/image_gen/ush/plot_time.sh b/util/Radiance_Monitor/image_gen/ush/plot_time.sh
@@ -89,11 +89,16 @@ for type in ${SATYPE2}; do
       #  Locate the data files, first checking for a tar file,
       #  and copy them locally.
       #
-      if [[ -s ${ieee_src}/radmon_time.tar ]]; then
-         files=`tar -tf ${ieee_src}/radmon_time.tar | grep ${type} | grep ieee_d`
+      if [[ -e ${ieee_src}/radmon_time.tar && -e ${ieee_src}/radmon_time.tar.${Z} ]]; then
+         echo "Located both radmon_time.tar and radmon_time.tar.${Z} in ${ieee_src}.  Unable to plot."
+         exit 24
+
+      elif [[ -e ${ieee_src}/radmon_time.tar || -e ${ieee_src}/radmon_time.tar.${Z} ]]; then
+         files=`tar -tf ${ieee_src}/radmon_time.tar* | grep ${type} | grep ieee_d`
          if [[ ${files} != "" ]]; then
-            tar -xf ${ieee_src}/radmon_time.tar ${files}
+            tar -xf ${ieee_src}/radmon_time.tar* ${files}
          fi
+
       else
          files=`ls ${ieee_src}/time.*${type}*ieee_d*`
          for f in ${files}; do

diff --git a/util/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_verf_angle.sh b/util/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_verf_angle.sh
@@ -229,13 +229,14 @@ EOF
 
    tar_file=radmon_angle.tar 
    tar -cf $tar_file angle*.ieee_d* angle*.ctl*
-   mv $tar_file ${TANKverf_rad}
+   ${COMPRESS} ${tar_file}
+   mv $tar_file.${Z} ${TANKverf_rad}/.
 
    if [[ $RAD_AREA = "rgn" ]]; then
       cwd=`pwd`
       cd ${TANKverf_rad}
-      tar -xf ${tar_file}
-      rm ${tar_file}
+      tar -xf ${tar_file}.${Z}
+      rm ${tar_file}.${Z}
       cd ${cwd}
    fi   
 

diff --git a/util/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_verf_bcoef.sh b/util/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_verf_bcoef.sh
@@ -214,13 +214,14 @@ EOF
 
    tar_file=radmon_bcoef.tar
    tar -cf $tar_file bcoef*.ieee_d* bcoef*.ctl*
-   mv $tar_file ${TANKverf_rad}
+   ${COMPRESS} ${tar_file}
+   mv $tar_file.${Z} ${TANKverf_rad}
 
    if [[ $RAD_AREA = "rgn" ]]; then
       cwd=`pwd`
       cd ${TANKverf_rad}
-      tar -xf ${tar_file}
-      rm ${tar_file}
+      tar -xf ${tar_file}.${Z}
+      rm ${tar_file}.${Z}
       cd ${cwd}
    fi
 

diff --git a/util/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_verf_bcor.sh b/util/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_verf_bcor.sh
@@ -207,13 +207,14 @@ EOF
    tar_file=radmon_bcor.tar
 
    tar -cf $tar_file bcor*.ieee_d* bcor*.ctl*
-   mv $tar_file ${TANKverf_rad}
+   ${COMPRESS} ${tar_file}
+   mv $tar_file.${Z} ${TANKverf_rad}/.
 
    if [[ $RAD_AREA = "rgn" ]]; then
       cwd=`pwd`
       cd ${TANKverf_rad}
-      tar -xf ${tar_file}
-      rm ${tar_file}
+      tar -xf ${tar_file}.${Z}
+      rm ${tar_file}.${Z}
       cd ${cwd}
    fi