Skip to content

Commit

Permalink
Tb HISTORY output, bug fixes, and cleanup (#231)
Browse files Browse the repository at this point in the history 
enhancements:
* output of L-band Tb via HISTORY

nodata value decision:
* everything from HISTORY uses MAPL_UNDEF (1.e15)
* continue to use -9999 for LDASsa-heritage output Collections ("ObsFcstAna" and "smapL4SMaup")

bug fixes:
*before the fix, ApplyPrognPert was executed too late and the current time step's prognostics perturbations were missed by the land analysis and by HISTORY
* no-data-handling in computation of ensemble average for surface temperature components

cleanup:
* avoid redundant entries in LDAS.rc
* setup for NUM_ENSEMBLE=1 with PERTURBATION=1
* added stop if output_smapL4SMaup==.true. and NUM_ENSEMBLE<=1
* removed obsolete LDASsa *_get_command_line() subroutines

documentation:
* improved help and log messages for setup and configuration
* updated README.md
* clarified comments in GEOS_LdasGridComp.F90
  • Loading branch information
@gmao-rreichle
gmao-rreichle committed May 29, 2020
1 parent 350abdd commit 4086d5e
Show file tree
Hide file tree
Showing 23 changed files with 367 additions and 1,078 deletions.
15 changes: 3 additions & 12 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,16 +63,7 @@ See below for how to build the model in multiple steps.

## How to Set Up and Run GEOSldas

a) Obtain an interactive _compute_ node:

```
xalloc --nodes=1
```

The GEOSldas setup script uses MPI and **must** be run on a compute node. (For NCCS SLES11, a login node also works.)


b) On the _compute_ node, set up the job as follows:
a) Set up the job as follows:

```
cd (build_path)/GEOSldas/install/bin
Expand Down Expand Up @@ -113,9 +104,9 @@ ldas_setup sample -h
ldas_setup setup -h
```

Configure the experiment output by editing the ```HISTORY.rc``` file.
b) Configure the experiment output by editing the ```./run/HISTORY.rc``` file as needed.

c) Finally, run the job:
c) Run the job:
```
cd [exp_path]/[exp_name]/run/
sbatch lenkf.j
Expand Down
17 changes: 15 additions & 2 deletions src/Applications/LDAS_App/GEOSldas_HIST.rc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ COLLECTIONS:
#EASE 'tavg24_1d_lnd_Nt'
#CUBE 'tavg24_2d_lnd_Nx'
#ASSIM 'SMAP_L4_SM_gph'
# 'inst1_1d_lnr_Nt'
# 'catch_progn_incr'
::

Expand Down Expand Up @@ -323,10 +324,22 @@ COLLECTIONS:
'TA' , 'ENSAVG' , 'temp_lowatmmodlay' ,
'QA' , 'ENSAVG' , 'specific_humidity_lowatmmodlay' ,
'UU' , 'ENSAVG' , 'windspeed_lowatmmodlay' ,
'GRN' , 'VEGDYN' , 'vegetation_greenness_fraction' ,
'LAI' , 'VEGDYN' , 'leaf_area_index' ,
'GRN' , 'VEGDYN' , 'vegetation_greenness_fraction' ,
'LAI' , 'VEGDYN' , 'leaf_area_index' ,
::

inst1_1d_lnr_Nt.descr: 'Tile-space,1-Hourly,Instantaneous,Single-Level,Assimilation,Land Nature Run Diagnostics',
inst1_1d_lnr_Nt.template: '%y4%m2%d2_%h2%n2z.bin' ,
inst1_1d_lnr_Nt.mode: 'instantaneous' ,
inst1_1d_lnr_Nt.frequency: 010000 ,
inst1_1d_lnr_Nt.ref_time: 000000,
inst1_1d_lnr_Nt.fields: 'TPSURF' , 'ENSAVG' , 'surface_temp' ,
'TSOIL1TILE' , 'ENSAVG' , 'soil_temp_layer1' ,
'TPSNOW' , 'ENSAVG' , 'snow_temp_layer1' ,
'TB_LAND_1410MHZ_40DEG_HPOL' , 'LANDASSIM' , 'tb_h' ,
'TB_LAND_1410MHZ_40DEG_VPOL' , 'LANDASSIM' , 'tb_v' ,
::

catch_progn_incr.descr: 'Tile-space,3-Hourly,Instantaneous,Single-Level,Assimilation,Ensemble-Average Land Prognostics Increments',
catch_progn_incr.template: '%y4%m2%d2_%h2%n2z.bin',
catch_progn_incr.mode: 'instantaneous',
Expand Down
12 changes: 4 additions & 8 deletions src/Applications/LDAS_App/GEOSldas_LDAS.rc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,15 +38,15 @@ LSM_CHOICE: 1
#
# Specify extremities of lat/lon rectangle:
# Max lat/lon range: lon=-180:180, lat=-90:90.
# If only whitelist should be used, specify dummy valuessuch that:
# If only whitelist should be used, specify dummy values such that:
# MINLON > MAXLON and MINLAT > MAXLAT.
#
# MINLON: -180.
# MAXLON: 180.
# MINLAT: -90.
# MAXLAT: 90.
#
# Specify path and filenames for blacklist and whitelist files:
# Path and filenames for blacklist and whitelist files.
# (May leave blank.)
#
# BLACK_FILE: ''
Expand All @@ -63,8 +63,8 @@ MET_HINTERP: 1

# ---- Specify if running model only or data assimilation
#
# NO : model only (DEFAULT; with --runmodel option)
# YES : assimilation (without --runmodel option)
# NO : model only, with or without perturbations (default)
# YES : assimilation (full land analysis or just processing of obs for "innovations" output)
#
LAND_ASSIM: NO

Expand Down Expand Up @@ -164,10 +164,6 @@ SURFRC: LDAS.rc
#
DYCORE: none

# ---- Only one surface level
#
LM: 1

# ---- For MAPL_RestartOptional
#
MAPL_ENABLE_BOOTSTRAP: YES
76 changes: 37 additions & 39 deletions src/Applications/LDAS_App/ldas_setup
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ class LDASsetup:
'RESTART_DOMAIN','RESTART_ID','BCS_PATH','TILING_FILE','GRN_FILE','LAI_FILE','NIRDF_FILE',
'VISDF_FILE','CATCH_DEF_FILE','NDVI_FILE',
'NML_INPUT_PATH','HISTRC_FILE','RST_FROM_GLOBAL','JOB_SGMT','NUM_SGMT','POSTPROC_HIST',
'MINLON','MAXLON','MINLAT','MAXLAT','BLACK_FILE','WHITE_FILE','MWRTM_FILE']
'MINLON','MAXLON','MINLAT','MAXLAT','BLACK_FILE','WHITE_FILE','MWRTM_FILE','GRIDNAME']


# ------
Expand Down Expand Up @@ -104,6 +104,7 @@ class LDASsetup:
self.has_geos_pert = False
self.has_ldassa_pert = False
self.nSegments = 1
self.perturb = 0
# ------
# Read exe input file which is required to set up the dir
# ------
Expand Down Expand Up @@ -142,6 +143,9 @@ class LDASsetup:
assert not os.path.isdir(_mydir), 'Dir [%s] already exists!' % _mydir
_mydir = None
_first_ens_id = int(self.rqdExeInp.get('FIRST_ENS_ID',0))
self.perturb = int(self.rqdExeInp.get('PERTURBATIONS',0))
if self.nens > 1:
self.perturb = 1
self.ensdirs = ['ens%04d'%iens for iens in range(_first_ens_id, self.nens + _first_ens_id)]
self.ensids = ['%04d'%iens for iens in range(_first_ens_id, self.nens + _first_ens_id)]
if (self.nens == 1) :
Expand Down Expand Up @@ -275,8 +279,6 @@ class LDASsetup:
if 'GRIDNAME' not in self.rqdExeInp :
tmptile =self.rqdExeInp['TILING_FILE']
self.rqdExeInp['GRIDNAME'] = linecache.getline(tmptile, 3).strip()
if 'RESOLUTION' not in self.rqdExeInp :
self.rqdExeInp['RESOLUTION']= os.path.split(os.path.split(self.rqdExeInp['BCS_PATH'])[0])[1]

if 'LSM_CHOICE' not in self.rqdExeInp:
self.rqdExeInp['LSM_CHOICE'] = 1
Expand Down Expand Up @@ -772,7 +774,7 @@ class LDASsetup:
catchRstFile = self.exphome+'/'+exp_id+'/mk_restarts/'+self.catch+'_internal_rst.'+YYYYMMDD

# catchment restart file
print 'catchRstFile1: ' + catchRstFile
print 'catchRstFile: ' + catchRstFile
if os.path.isfile(catchRstFile) :

catchLocal = self.rstdir+ens +'/'+ y4m2+'/'+self.rqdExeInp['EXP_ID']+'.'+self.catch+'_internal_rst.'+y4m2d2_h2m2
Expand Down Expand Up @@ -809,14 +811,14 @@ class LDASsetup:
vegdynRstFile0 = vegdynRstFile
else :
vegdynRstFile = vegdynRstFile0
_perturb = 1 if self.nens > 1 else 0
if (self.has_geos_pert and _perturb == 1) :

if (self.has_geos_pert and self.perturb == 1) :
pertRstFile = rstpath+ens +'/'+ y4m2+'/'+self.rqdExeInp['RESTART_ID']+'.landpert_internal_rst.'+y4m2d2_h2m2
pertLocal = self.rstdir+ens +'/'+ y4m2+'/'+self.rqdExeInp['EXP_ID']+'.landpert_internal_rst.'+y4m2d2_h2m2
shutil.copy(pertRstFile,pertLocal)
pertRstFile = pertLocal

if (self.has_ldassa_pert and _perturb == 1 ) :
if (self.has_ldassa_pert and self.perturb == 1 ) :
pertRstFile = rstpath+ens +'/'+ y4m2+'/'+self.rqdExeInp['RESTART_ID']+'.'+ens+'.pert_ldas_rst.'+y4m2d2_h2m2+'z.bin'
pertLocal = self.rstdir+ens +'/'+ y4m2+'/'+self.rqdExeInp['EXP_ID']+'.landpert_internal_rst.'+y4m2d2_h2m2
print "Convert LDASsa pert " + ensid + " rst to GEOSldas rst"
Expand All @@ -828,7 +830,7 @@ class LDASsetup:

os.symlink(catchRstFile, myCatchRst)
os.symlink(vegdynRstFile, myVegRst)
if ( (self.has_geos_pert or self.has_ldassa_pert) and _perturb == 1 ):
if ( (self.has_geos_pert or self.has_ldassa_pert) and self.perturb == 1 ):
os.symlink(pertRstFile, myPertRst)

# catch_param restar file
Expand Down Expand Up @@ -896,18 +898,17 @@ class LDASsetup:
etcdir = self.blddirLn + '/etc'

#defalt nml
default_nml = glob.glob(etcdir+'/LDASsa_DEFAULT*.nml')
default_nml = glob.glob(etcdir+'/LDASsa_DEFAULT_inputs_*.nml')
for nmlfile in default_nml:
shortfile=self.rundir+'/'+nmlfile.split('/')[-1]
shutil.copy2(nmlfile, shortfile)
# special nml
special_nml=[]
if 'NML_INPUT_PATH' in self.rqdExeInp :
special_nml = glob.glob(self.rqdExeInp['NML_INPUT_PATH']+'/*SPECIAL*nml')
special_nml = glob.glob(self.rqdExeInp['NML_INPUT_PATH']+'/LDASsa_SPECIAL_inputs_*.nml')
for nmlfile in special_nml:
if (_nens > 1) :
shortfile=nmlfile.split('/')[-1]
shutil.copy2(nmlfile, self.rundir+'/'+shortfile)
shortfile=nmlfile.split('/')[-1]
shutil.copy2(nmlfile, self.rundir+'/'+shortfile)

# get optimzed NX and IMS
if os.path.isfile('optimized_distribution'):
Expand Down Expand Up @@ -958,9 +959,8 @@ class LDASsetup:
if '-CF' in self.rqdExeInp['GRIDNAME'] :
GRID ='CUBE ' + self.rqdExeInp['GRIDNAME'] + ' ' +tmprcfile
_assim = '1' if self.assim else '0'
_perturb = '1' if self.nens > 1 else '0'
cmd ='./process_hist.csh '+ str(self.rqdExeInp['LSM_CHOICE']) + ' ' + str(self.rqdExeInp['AEROSOL_DEPOSITION']) + \
' ' + GRID + ' ' + str(self.rqdExeInp['RUN_IRRIG']) + ' ' + _assim + ' '+ _perturb
' ' + GRID + ' ' + str(self.rqdExeInp['RUN_IRRIG']) + ' ' + _assim + ' '+ str(self.nens)
print(cmd)
os.system(cmd)
#sp.call(cmd)
Expand Down Expand Up @@ -1009,8 +1009,9 @@ class LDASsetup:
# create BC in rc file
tmpl_ = ''
if self.nens >1 :
ldasrcInp['PERTURBATIONS'] ='1'
tmpl_='%s'
if self.perturb == 1:
ldasrcInp['PERTURBATIONS'] ='1'
bcval=['../input/green','../input/lai','../input/ndvi','../input/nirdf','../input/visdf']
bckey=['GREEN','LAI','NDVI','NIRDF','VISDF']
for key, val in zip(bckey,bcval):
Expand All @@ -1032,8 +1033,7 @@ class LDASsetup:

rstkey=[catch_,'VEGDYN']
rstval=[self.catch,'vegdyn']
_perturb = 1 if self.nens > 1 else 0
if((self.has_ldassa_pert or self.has_geos_pert) and _perturb == 1) :
if((self.has_ldassa_pert or self.has_geos_pert) and self.perturb == 1) :
rstkey=[catch_,'VEGDYN','LANDPERT']
rstval=[self.catch,'vegdyn','landpert']

Expand All @@ -1042,38 +1042,36 @@ class LDASsetup:
valn='../input/restart/mwrtm_param_rst'
ldasrcInp[keyn]= valn

if self.nens > 1 :
keyn='ENS_ID_WIDTH'
valn='4'
ldasrcInp[keyn]= valn

if self.has_landassim_seed and self.assim :
keyn='LANDASSIM_OBSPERTRSEED_RESTART_FILE'
valn='../input/restart/landassim_obspertrseed%s_rst'
valn='../input/restart/landassim_obspertrseed'+tmpl_+'_rst'
ldasrcInp[keyn]= valn

if self.assim:
keyn='LANDASSIM_OBSPERTRSEED_CHECKPOINT_FILE'
valn='landassim_obspertrseed%s_checkpoint'
ldasrcInp[keyn]= valn

ensid =""
if self.nens > 1 :
ensid ="%s"
keyn='ENS_ID_WIDTH'
valn='4'
valn='landassim_obspertrseed'+tmpl_+'_checkpoint'
ldasrcInp[keyn]= valn

for key,val in zip(rstkey,rstval) :
keyn = key+ '_INTERNAL_RESTART_FILE'
valn = '../input/restart/'+val+ensid+'_internal_rst'
valn = '../input/restart/'+val+tmpl_+'_internal_rst'
ldasrcInp[keyn]= valn

# checkpoint file and its type
keyn = catch_ + '_INTERNAL_CHECKPOINT_FILE'
valn = self.catch+ensid+'_internal_checkpoint'
valn = self.catch+tmpl_+'_internal_checkpoint'
ldasrcInp[keyn]= valn

# for lat/lon and EASE tile space, specify LANDPERT checkpoint file here (via MAPL);
# for cube-sphere tile space, Landpert GC will set up LANDPERT checkpoint file
if('-CF' not in self.rqdExeInp['GRIDNAME'] and _perturb == 1):
if('-CF' not in self.rqdExeInp['GRIDNAME'] and self.perturb == 1):
keyn = 'LANDPERT_INTERNAL_CHECKPOINT_FILE'
valn = 'landpert'+ensid+'_internal_checkpoint'
valn = 'landpert'+tmpl_+'_internal_checkpoint'
ldasrcInp[keyn]= valn


Expand Down Expand Up @@ -1535,28 +1533,28 @@ def parseCmdLine():
'batinpfile',
help='input file with arguments for SLURM',
)
p_setup.add_argument(
'--runmodel',
help='obsolete, no effect any more',
action='store_true',
)
p_setup.add_argument(
'--account',
help='replace the account in batinpfile)',
help='replace computing/sponsor account in batinp file',
type=str, default='None'
)
p_setup.add_argument(
'--runmodel',
help='Obsolete.',
action='store_true',
)
spltgrp = p_setup.add_mutually_exclusive_group()
spltgrp.add_argument(
'--daysperjob',
type=int,
metavar='N',
help='This option is no longer available. Use NUM_SGMT and JOB_SGMT in exeinp file.',
help='Obsolete. Use NUM_SGMT and JOB_SGMT in exeinp file.',
)
spltgrp.add_argument(
'--monthsperjob',
type=int,
metavar='N',
help='This option is no longer available. Use NUM_SGMT and JOB_SGMT in exeinp file.',
help='Obsolete. Use NUM_SGMT and JOB_SGMT in exeinp file.',
)

return p.parse_args()
Expand Down
2 changes: 1 addition & 1 deletion src/Applications/LDAS_App/lenkf.j.template
View file
Original file line number Diff line number Diff line change
Expand Up @@ -502,7 +502,7 @@ EOF
set dayl = `echo $time_steps[$LEN] | cut -c1-8`
set day1 = `echo $time_steps[1] | cut -c1-8`
@ NAVAIL = ($dayl - $day1) + 1
else if( $LEN_ !=0 ) then
else if( $LEN_ != 0 ) then
set dayl = `echo $time_steps_[$LEN] | cut -c1-8`
set day1 = `echo $time_steps_[1] | cut -c1-8`
@ NAVAIL = ($dayl - $day1) + 1
Expand Down
44 changes: 22 additions & 22 deletions src/Applications/LDAS_App/mwrtm_bin2nc4.F90
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,30 +20,30 @@ PROGRAM mwrtm_bin2nc4
character(len=:),allocatable :: shnms(:)
type(mwRTM_param_type), allocatable :: mwp(:)
integer :: unitnum
logical :: is_nodata
logical :: mwp_nodata


nVars = 18
shnms =[ &
'MWRTM_VEGCLS ',&
'MWRTM_SOILCLS ',&
'MWRTM_SAND ',&
'MWRTM_CLAY ',&
'MWRTM_POROS ',&
'MWRTM_WANGWT ',&
'MWRTM_WANGWP ',&
'MWRTM_RGHHMIN ',&
'MWRTM_RGHHMAX ',&
'MWRTM_RGHWMIN ',&
'MWRTM_RGHWMAX ',&
'MWRTM_RGHNRH ',&
'MWRTM_RGHNRV ',&
'MWRTM_RGHPOLMIX',&
'MWRTM_OMEGA ',&
'MWRTM_BH ',&
'MWRTM_BV ',&
'MWRTM_LEWT ']

shnms = [ &
'MWRTM_VEGCLS ',&
'MWRTM_SOILCLS ',&
'MWRTM_SAND ',&
'MWRTM_CLAY ',&
'MWRTM_POROS ',&
'MWRTM_WANGWT ',&
'MWRTM_WANGWP ',&
'MWRTM_RGHHMIN ',&
'MWRTM_RGHHMAX ',&
'MWRTM_RGHWMIN ',&
'MWRTM_RGHWMAX ',&
'MWRTM_RGHNRH ',&
'MWRTM_RGHNRV ',&
'MWRTM_RGHPOLMIX',&
'MWRTM_OMEGA ',&
'MWRTM_BH ',&
'MWRTM_BV ',&
'MWRTM_LEWT ']
! processing command line agruments
I = iargc()

Expand Down Expand Up @@ -123,7 +123,7 @@ PROGRAM mwrtm_bin2nc4
read (unitnum) VAR; mwp(1:NTILES)%lewt = VAR(1:NTILES)

do i = 1, NTILES
call mwRTM_param_nodata_check( mwp(i), is_nodata )
call mwRTM_param_nodata_check( mwp(i), mwp_nodata )
enddo

VAR = real(mwp(1:NTILES)%vegcls)
Expand Down
Loading

0 comments on commit 4086d5e

Please sign in to comment.