Syncing develop into BRIDGE_FROM_DEVELOP_TO_MASTER (#188)

* fix gnu debug (#178) * checkpoint files are geneareted by GEOSldas (#182) * checkpoint files are generated by GEOSldas ( not MAPL) when on cubed-sphere tile space * create year/month directories for ldas_ObsFcstAna from within GEOSldas executable * change system() calls to Fortran2008 standard Execute_command_line() * subdaily2daily concatenation; SLES12 sbatch fix - Added sbatch submission for pre-processing of restarts to comply with SLES12 requirements. - Subdaily-to-daily nc4 file concatenation now processes before the month is complete. * pass in optional account through ldas_setup (#184) * updating to MAPL 2.1.1 (updated ESMA_env and cmake) * Resurrecting SMAP L1C Tb fore-minus-aft check
GEOS-ESM · Apr 23, 2020 · 4e8d4cf · 4e8d4cf
1 parent f424f43
commit 4e8d4cf
Show file tree

Hide file tree

Showing 10 changed files with 166 additions and 87 deletions.
diff --git a/Externals.cfg b/Externals.cfg
@@ -2,14 +2,14 @@
 required = True
 repo_url = git@github.com:GEOS-ESM/ESMA_env.git
 local_path = ./@env
-tag = v2.0.2
+tag = v2.1.1 
 protocol = git
 
 [ESMA_cmake]
 required = True
 repo_url = git@github.com:GEOS-ESM/ESMA_cmake.git
 local_path = ./@cmake
-tag = v2.1.2
+tag = v3.0.1
 externals = Externals.cfg
 protocol = git
 
@@ -25,7 +25,7 @@ sparse = ../../../config/GMAO_Shared.sparse
 required = True
 repo_url = git@github.com:GEOS-ESM/MAPL.git
 local_path = ./src/Shared/@MAPL
-tag = v2.0.1
+tag = v2.1.1
 protocol = git
 
 [GEOSgcm_GridComp]

diff --git a/src/Applications/LDAS_App/ldas_setup b/src/Applications/LDAS_App/ldas_setup
@@ -389,6 +389,8 @@ class LDASsetup:
         assert self.rqdRmInp['rm_name'].upper() == 'SLURM'
         ## account and walltime should exist
         assert self.rqdRmInp['account']
+        if cmdLineArgs['account'] != 'None':
+           self.rqdRmInp['account'] = cmdLineArgs['account']
         assert self.rqdRmInp['walltime']
         ## ntasks is a +ve integer
         _ntasks = int(self.rqdRmInp['ntasks'])
@@ -1180,7 +1182,9 @@ class LDASsetup:
                 valn = self.catch+ensid+'_internal_checkpoint'
                 ldasrcInp[keyn]= valn
 
-                if((self.has_ldassa_pert or self.has_geos_pert) and _perturb == 1):
+                # for lat/lon and EASE tile space, specify LANDPERT checkpoint file here (via MAPL);
+                # for cube-sphere tile space, Landpert GC will set up LANDPERT checkpoint file 
+                if('-CF' not in self.rqdExeInp['GRIDNAME'] and _perturb == 1):
                     keyn = 'LANDPERT_INTERNAL_CHECKPOINT_FILE'
                     valn = 'landpert'+ensid+'_internal_checkpoint'
                     ldasrcInp[keyn]= valn
@@ -1643,6 +1647,11 @@ def parseCmdLine():
         help='model run (no assimilation)',
         action='store_true',
         )
+    p_setup.add_argument(
+        '--account',
+        help='replace the account in batinpfile)',
+        type=str, default='None'
+        )
     #p_setup.add_argument(
     #    '--ForceReuseDir',
     #    help='force re-use existing exp dir',

diff --git a/src/Applications/LDAS_App/lenkf.j.template b/src/Applications/LDAS_App/lenkf.j.template
@@ -395,55 +395,23 @@ while ( $counter <= ${NUM_SGMT} )
 
           # if monthly exists, move on to the next collection
           if (-f $EXPID.${ThisCol}.monthly.$YYYY$MM.nc4) continue
-
-          set time_steps = `ls -1 $EXPID.$ThisCol.${YYYY}${MM}* | rev | cut -d'.' -f2 | cut -d'.' -f1 | rev`
-          set tstep2 = \"`echo $time_steps | sed 's/\ /\","/g'`\"
-          set LEN = `echo $#time_steps`
-          # no file? move on  
-          if ($LEN == 0) continue
-
-          set dayl = `echo $time_steps[$LEN] | cut -c1-8`
-          set day1 = `echo $time_steps[1] | cut -c1-8`
-          @ NAVAIL = ($dayl - $day1) + 1
-
-          # not enough days? move on to the next collection
-          if($NAVAIL != $NDAYS) continue
-
-          set time_steps = `ls -1 $EXPID.$ThisCol.${YYYY}${MM}01* | rev | cut -d'.' -f2 | cut -d'.' -f1 | rev`
-          set LEN_SUB = `echo $#time_steps`
-          @ LEN_AVAIL = $LEN_SUB * $NDAYS
-
-          # not enough sub-daylies? move on to the next collection
-          if ($LEN != $LEN_AVAIL) continue
-
-          # create the monly average
-          #ncrcat -h $EXPID.$ThisCol.${YYYY}${MM}* ${EXPID}.${ThisCol}.$YYYY$MM.nc4
-          #ncra -h $EXPID.${ThisCol}.$YYYY$MM.nc4 $EXPID.${ThisCol}.monthly.$YYYY$MM.nc4
-
-          #/bin/rm $EXPID.${ThisCol}.$YYYY$MM.nc4
-          ncra -h $EXPID.$ThisCol.${YYYY}${MM}??_*z.nc4 ${EXPID}.${ThisCol}.monthly.$YYYY$MM.nc4
-
-          # don't want a daily? delete the daily and sub-dailies and continue
-          #
-          if($NODAILIES == 2) then
-             /bin/rm $EXPID.${ThisCol}.${YYYY}${MM}*
-             continue
-          endif
-
+
           # create daily and remove the sub-daily
           # ------------------------------------------------------------------
           set day=1
           while ($day <= $NDAYS && $LEN_SUB > 1)
              if ( $day < 10  ) set DD=0${day}
              if ( $day >= 10 ) set DD=${day}
-
-             set time_steps = `ls -1 $EXPID.$ThisCol.${YYYY}${MM}${DD}_* | rev | cut -d'.' -f2 | cut -d'.' -f1 | rev`
+             @ day++    
+             set time_steps = `ls -1 $EXPID.$ThisCol.${YYYY}${MM}${DD}_* | rev | cut -d'.' -f2 | rev`
+             set LEN_SUB = `echo $#time_steps`
+             if ($LEN_SUB <= 1) continue
              set tstep2 = \"`echo $time_steps | sed 's/\ /\","/g'`\"
 
 # ----------------------------------------------------------------------------
 #
 # WARNING: The following block MUST begin in column 1!!!  Do NOT indent!!!
-
+   
 cat << EOF > timestamp.cdl
 netcdf timestamp {
 dimensions:
@@ -458,9 +426,7 @@ time_stamp =
 DATAVALUES;
 }      
 EOF
-
-# ----------------------------------------------------------------------------
-
+
              sed -i -e "s/NT/$LEN_SUB/g" timestamp.cdl
              sed -i -e "s/DATAVALUES/$tstep2/g" timestamp.cdl
              ncgen -k4 -o timestamp.nc4 timestamp.cdl
@@ -469,8 +435,43 @@ EOF
              /bin/rm timestamp.cdl
              /bin/rm timestamp.nc4
              /bin/rm $EXPID.${ThisCol}.${YYYY}${MM}${DD}_*.nc4 
-             @ day++ 
+
           end # concatenate for each day 
+
+          set time_steps = `ls -1 $EXPID.$ThisCol.${YYYY}${MM}* | rev | cut -d'.' -f2 | rev`
+          set tstep2 = \"`echo $time_steps | sed 's/\ /\","/g'`\"
+          set LEN = `echo $#time_steps`
+          # no file? move on  
+          if ($LEN == 0) continue
+
+          set dayl = `echo $time_steps[$LEN] | cut -c1-8`
+          set day1 = `echo $time_steps[1] | cut -c1-8`
+          @ NAVAIL = ($dayl - $day1) + 1
+
+          # not enough days? move on to the next collection
+          if($NAVAIL != $NDAYS) continue
+
+          set time_steps = `ls -1 $EXPID.$ThisCol.${YYYY}${MM}01* | rev | cut -d'.' -f2 | rev`
+          set LEN_SUB = `echo $#time_steps`
+          @ LEN_AVAIL = $LEN_SUB * $NDAYS
+
+          # not enough sub-daylies? move on to the next collection
+          if ($LEN != $LEN_AVAIL) continue
+
+          # create the monly average
+          #ncrcat -h $EXPID.$ThisCol.${YYYY}${MM}* ${EXPID}.${ThisCol}.$YYYY$MM.nc4
+          #ncra -h $EXPID.${ThisCol}.$YYYY$MM.nc4 $EXPID.${ThisCol}.monthly.$YYYY$MM.nc4
+
+          #/bin/rm $EXPID.${ThisCol}.$YYYY$MM.nc4
+          ncra -h $EXPID.$ThisCol.${YYYY}${MM}??.nc4 ${EXPID}.${ThisCol}.monthly.$YYYY$MM.nc4
+
+          # don't want a daily? delete the daily and sub-dailies and continue
+          #
+          if($NODAILIES == 2) then
+             /bin/rm $EXPID.${ThisCol}.${YYYY}${MM}*
+             continue
+          endif
+
        end # each collection
      end # each month
      cd $PWD

diff --git a/src/Applications/LDAS_App/process_rst.csh b/src/Applications/LDAS_App/process_rst.csh
@@ -153,13 +153,20 @@ case [1]:
 
         cd $EXPDIR/$EXPID/mk_restarts/
         echo '#\!/bin/csh -f ' > this.file
+
+        echo "#SBATCH --account=${SPONSORID}">> this.file   
+        echo "#SBATCH --time=1:00:00"        >> this.file  
+        echo "#SBATCH --ntasks=$NUMENS"      >> this.file     
+        echo "#SBATCH --job-name=mkRst"     >> this.file    
+        echo "#SBATCH --qos=debug"           >> this.file    
+        echo "#SBATCH --output=mkRst.o"     >> this.file 
+        echo "#SBATCH --error=mkRst.e"      >> this.file  
         echo 'source $INSTDIR/bin/g5_modules' >> this.file
         echo 'if ( -e /etc/os-release ) then' >> this.file
         echo '    module load nco/4.8.1'      >> this.file
         echo 'else'                           >> this.file
         echo '    module load other/nco-4.6.8-gcc-5.3-sp3 ' >> this.file
         echo 'endif' >> this.file
-        #echo 'setenv OMPI_MCA_shmem_mmap_enable_nfs_warning 0' >> this.file
         echo 'setenv LD_LIBRARY_PATH ${LD_LIBRARY_PATH}:${BASEDIR}/Linux/lib' >> this.file
 
         set mpi_mpmd = "${INSTDIR}/bin/esma_mpirun -np 1 bin/mk_GEOSldasRestarts.x -b ${BCSDIR} -d ${YYYYMMDD} -e ${RESTART_ID} -k 0000 -l ${RESTART_short} -m ${MODEL} -s ${SURFLAY} -r Y -t ${TILFILE}"
@@ -178,11 +185,8 @@ case [1]:
            @ j++
         end
         echo 'wait' >> this.file
-
-        chmod +x this.file
-        ./this.file
-        rm -f this.file
-        echo DONE > done_rst_file
+        echo 'echo DONE > done_rst_file' >> this.file
+        sbatch this.file
         cd $PWD
         sleep 1
      else

diff --git a/src/Components/GEOSldas_GridComp/GEOSens_GridComp/GEOS_EnsGridComp.F90 b/src/Components/GEOSldas_GridComp/GEOSens_GridComp/GEOS_EnsGridComp.F90
@@ -3549,8 +3549,8 @@ subroutine Finalize(gc, import, export, clock, rc)
     VERIFY_(status)
     Iam = trim(comp_name) // "::Finalize"
 
-    ! Get MAPL obj
-    call MAPL_GetObjectFromGC(gc, MAPL, rc=status)
+   ! Call Finalize for every child
+    call MAPL_GenericFinalize(gc, import, export, clock, rc=status)
     VERIFY_(status)
 
     ! End

diff --git a/src/Components/GEOSldas_GridComp/GEOSlandassim_GridComp/clsm_ensupd_enkf_update.F90 b/src/Components/GEOSldas_GridComp/GEOSlandassim_GridComp/clsm_ensupd_enkf_update.F90
@@ -1416,7 +1416,7 @@ subroutine output_ObsFcstAna(date_time, work_path, exp_id, &
     integer,        dimension(numprocs)       :: N_obsl_vec, tmp_low_ind
 
     character(300)                            :: fname
-
+    integer                                   :: i
 #ifdef LDAS_MPI
 
     integer                                   :: this_species, ind_tmp, j
@@ -1559,7 +1559,10 @@ subroutine output_ObsFcstAna(date_time, work_path, exp_id, &
 
        fname = get_io_filename( work_path, exp_id, file_tag, date_time=date_time, &
             dir_name=dir_name, ens_id=-1 )
+       i = index(fname, '/', .true.) 
 
+       if( i >0) call Execute_command_line('mkdir -p '//fname(1:i))
+
        open( 10, file=fname, form='unformatted', action='write')
 
        ! write header