diff --git a/CMakeLists.txt b/CMakeLists.txt index eec710da5..393067c99 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -68,15 +68,15 @@ if(BUILD_GDASBUNDLE) # Build GSI-B option(BUILD_GSIBEC "Build GSI-B" OFF) if(BUILD_GSIBEC) - ecbuild_bundle( PROJECT gsibec GIT "https://github.com/GEOS-ESM/GSIbec.git" BRANCH develop ) + ecbuild_bundle( PROJECT gsibec GIT "https://github.com/GEOS-ESM/GSIbec.git" TAG 1.0.7 ) endif() # Core JEDI repositories - ecbuild_bundle( PROJECT oops GIT "https://github.com/jcsda/oops.git" TAG c5d4b39 ) - ecbuild_bundle( PROJECT vader GIT "https://github.com/jcsda/vader.git" TAG 8d11002 ) - ecbuild_bundle( PROJECT saber GIT "https://github.com/jcsda/saber.git" TAG 2555221 ) - ecbuild_bundle( PROJECT ioda GIT "https://github.com/jcsda/ioda.git" TAG 433daa1 ) - ecbuild_bundle( PROJECT ufo GIT "https://github.com/jcsda/ufo.git" TAG d308177 ) + ecbuild_bundle( PROJECT oops GIT "https://github.com/jcsda/oops.git" TAG e2b9dc6 ) + ecbuild_bundle( PROJECT vader GIT "https://github.com/jcsda/vader.git" TAG de52190 ) + ecbuild_bundle( PROJECT saber GIT "https://github.com/jcsda/saber.git" TAG 4c79b8c ) + ecbuild_bundle( PROJECT ioda GIT "https://github.com/jcsda/ioda.git" TAG f27bd1c ) + ecbuild_bundle( PROJECT ufo GIT "https://github.com/jcsda/ufo.git" TAG 1a67b67 ) # FMS and FV3 dynamical core ecbuild_bundle( PROJECT fms GIT "https://github.com/jcsda/FMS.git" TAG 1f73914 ) @@ -85,22 +85,22 @@ if(BUILD_GDASBUNDLE) # fv3-jedi and associated repositories ecbuild_bundle( PROJECT femps GIT "https://github.com/jcsda/femps.git" TAG cb39681 ) ecbuild_bundle( PROJECT fv3-jedi-lm GIT "https://github.com/jcsda/fv3-jedi-linearmodel.git" TAG a934a7b ) - ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda/fv3-jedi.git" TAG 53e8a75 ) + ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda/fv3-jedi.git" TAG 07b1a28 ) # SOCA associated repositories ecbuild_bundle( PROJECT gsw GIT "https://github.com/jcsda-internal/GSW-Fortran.git" TAG 1a02eba ) ecbuild_bundle( PROJECT mom6 GIT "https://github.com/jcsda-internal/MOM6.git" TAG 51ec489 RECURSIVE ) - ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" TAG 1c5e0ef ) + ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" TAG 5d1a7ad ) # Build IODA converters option(BUILD_IODA_CONVERTERS "Build IODA Converters" ON) if(BUILD_IODA_CONVERTERS) - ecbuild_bundle( PROJECT iodaconv GIT "https://github.com/JCSDA-internal/ioda-converters.git" TAG 4c7db32 ) + ecbuild_bundle( PROJECT iodaconv GIT "https://github.com/JCSDA-internal/ioda-converters.git" TAG c27c1df ) endif() # Land associated repositories ecbuild_bundle( PROJECT land-imsproc GIT "https://github.com/NOAA-PSL/land-IMS_proc.git" TAG 6373819 ) - ecbuild_bundle( PROJECT land-jediincr GIT "https://github.com/NOAA-PSL/land-apply_jedi_incr.git" TAG 80b19b8 ) + ecbuild_bundle( PROJECT land-jediincr GIT "https://github.com/NOAA-PSL/land-apply_jedi_incr.git" TAG ced6576 ) # ioda, ufo, fv3-jedi, and saber test data #--------------------------------- diff --git a/ci/validation/oblist_gfsv16p3.txt b/ci/validation/oblist_gfsv16p3.txt index fc05bd9a9..cf3f0874d 100644 --- a/ci/validation/oblist_gfsv16p3.txt +++ b/ci/validation/oblist_gfsv16p3.txt @@ -17,10 +17,10 @@ amsua_n18 amsua_n19 atms_n20 atms_npp -avhrr_metop-b -avhrr_metop-c -avhrr_n18 -avhrr_n19 +avhrr3_metop-b +avhrr3_metop-c +avhrr3_n18 +avhrr3_n19 cris-fsr_n20 cris-fsr_npp gmi_gpm @@ -41,4 +41,4 @@ ompslp_npp ompsnp_n20 ompsnp_npp ompstc8_n20 -ompstc8_npp \ No newline at end of file +ompstc8_npp diff --git a/parm/atm/berror/hybvar_bump.yaml b/parm/atm/berror/hybvar_bump.yaml index ba39a78c0..879441e46 100644 --- a/parm/atm/berror/hybvar_bump.yaml +++ b/parm/atm/berror/hybvar_bump.yaml @@ -10,7 +10,7 @@ components: active variables: &active_vars [psi,chi,t,ps,rh,liq_wat,o3mr] bump: io: - data directory: !ENV &staticb_dir ${DATA}/berror + data directory: &staticb_dir $(DATA)/berror drivers: multivariate strategy: specific_univariate read local nicas: true @@ -95,17 +95,17 @@ components: covariance model: ensemble members from template: template: - datetime: '{{BKG_ISOTIME}}' + datetime: '{{ current_cycle | to_isotime }}' filetype: fms restart state variables: *3dvars datapath: ens/mem%mem%/ - filename_core: '{{BKG_YYYYmmddHHMMSS}}.fv_core.res.nc' - filename_trcr: '{{BKG_YYYYmmddHHMMSS}}.fv_tracer.res.nc' - filename_sfcd: '{{BKG_YYYYmmddHHMMSS}}.sfc_data.nc' - filename_sfcw: '{{BKG_YYYYmmddHHMMSS}}.fv_srf_wnd.res.nc' - filename_cplr: '{{BKG_YYYYmmddHHMMSS}}.coupler.res' + filename_core: '{{ current_cycle | to_fv3time }}.fv_core.res.nc' + filename_trcr: '{{ current_cycle | to_fv3time }}.fv_tracer.res.nc' + filename_sfcd: '{{ current_cycle | to_fv3time }}.sfc_data.nc' + filename_sfcw: '{{ current_cycle | to_fv3time }}.fv_srf_wnd.res.nc' + filename_cplr: '{{ current_cycle | to_fv3time }}.coupler.res' pattern: '%mem%' - nmembers: !ENV ${NMEM_ENKF} + nmembers: $(NMEM_ENKF) zero padding: 3 # localization: # localization method: SABER diff --git a/parm/atm/berror/hybvar_gsibec.yaml b/parm/atm/berror/hybvar_gsibec.yaml new file mode 100644 index 000000000..33df8ca16 --- /dev/null +++ b/parm/atm/berror/hybvar_gsibec.yaml @@ -0,0 +1,67 @@ +covariance model: hybrid +components: +- covariance: + covariance model: SABER + saber central block: + saber block name: gsi covariance + input variables: &control_vars [eastward_wind,northward_wind,air_temperature,surface_pressure, + specific_humidity,cloud_liquid_ice,cloud_liquid_water, + mole_fraction_of_ozone_in_air] + output variables: *control_vars + gsi akbk: &akbk $(DATA)/fv3jedi/akbk.nc4 + gsi error covariance file: &gsiberr $(DATA)/berror/gsi-coeffs-gfs-global.nc4 +# gsi error covariance file: &gsiberr $(DATA)/berror/global_berror.f77 + gsi berror namelist file: &gsibnml $(DATA)/berror/gfs_gsi_global.nml + processor layout x direction: &layout_gsib_x 3 + processor layout y direction: &layout_gsib_y 2 + debugging mode: false + saber outer blocks: + - saber block name: gsi interpolation to model grid + input variables: *control_vars + output variables: *control_vars + gsi akbk: *akbk + gsi error covariance file: *gsiberr + gsi berror namelist file: *gsibnml + processor layout x direction: *layout_gsib_x + processor layout y direction: *layout_gsib_y + debugging mode: false + linear variable change: + linear variable change name: Control2Analysis + input variables: *control_vars + output variables: &3dvars_anal [ua,va,t,ps,sphum,ice_wat,liq_wat,o3mr] + weight: + value: 0.125 +- covariance: + covariance model: ensemble + members from template: + template: + datetime: '{{ current_cycle | to_isotime }}' + filetype: fms restart + state variables: *3dvars_anal + datapath: ens/mem%mem% + filename_core: '{{ current_cycle | to_fv3time }}.fv_core.res.nc' + filename_trcr: '{{ current_cycle | to_fv3time }}.fv_tracer.res.nc' + filename_sfcd: '{{ current_cycle | to_fv3time }}.sfc_data.nc' + filename_sfcw: '{{ current_cycle | to_fv3time }}.fv_srf_wnd.res.nc' + filename_cplr: '{{ current_cycle | to_fv3time }}.coupler.res' + pattern: '%mem%' + nmembers: $(NMEM_ENKF) + zero padding: 3 +# localization: +# localization method: SABER +# saber block: +# - saber block name: BUMP_NICAS +# input variables: *control_vars +# output variables: *control_vars +# active variables: *active_vars +# bump: +# datadir: *staticb_dir +# verbosity: main +# strategy: specific_univariate +# method: loc +# load_nicas_local: true +# grids: +# - prefix: nicas/nicas_3D_gfs +# variables: [stream_function,velocity_potential,air_temperature,relative_humidity,cloud_liquid_water,ozone_mass_mixing_ratio] + weight: + value: 0.875 diff --git a/parm/atm/berror/staticb_bump.yaml b/parm/atm/berror/staticb_bump.yaml index 27c207d1d..8cabf7060 100644 --- a/parm/atm/berror/staticb_bump.yaml +++ b/parm/atm/berror/staticb_bump.yaml @@ -8,7 +8,7 @@ saber blocks: active variables: &active_vars [psi,chi,t,ps,rh,liq_wat,o3mr] bump: io: - data directory: !ENV &staticb_dir ${DATA}/berror + data directory: &staticb_dir $(DATA)/berror drivers: multivariate strategy: specific_univariate read local nicas: true diff --git a/parm/atm/berror/staticb_gsibec.yaml b/parm/atm/berror/staticb_gsibec.yaml index 66b98b535..ab82efe43 100644 --- a/parm/atm/berror/staticb_gsibec.yaml +++ b/parm/atm/berror/staticb_gsibec.yaml @@ -6,10 +6,10 @@ saber central block: specific_humidity,cloud_liquid_ice,cloud_liquid_water, mole_fraction_of_ozone_in_air] output variables: *control_vars - gsi akbk: !ENV &akbk ${DATA}/fv3jedi/akbk.nc4 - gsi error covariance file: !ENV &gsiberr ${DATA}/berror/gsi-coeffs-gfs-global.nc4 -# gsi error covariance file: !ENV &gsiberr ${DATA}/berror/global_berror.f77 - gsi berror namelist file: !ENV &gsibnml ${DATA}/berror/gfs_gsi_global.nml + gsi akbk: &akbk $(DATA)/fv3jedi/akbk.nc4 + gsi error covariance file: &gsiberr $(DATA)/berror/gsi-coeffs-gfs-global.nc4 +# gsi error covariance file: &gsiberr $(DATA)/berror/global_berror.f77 + gsi berror namelist file: &gsibnml $(DATA)/berror/gfs_gsi_global.nml processor layout x direction: &layout_gsib_x 3 processor layout y direction: &layout_gsib_y 2 debugging mode: false diff --git a/parm/atm/hofx/hofx4d.yaml b/parm/atm/hofx/hofx4d.yaml index 0bd231398..14b6c0e66 100644 --- a/parm/atm/hofx/hofx4d.yaml +++ b/parm/atm/hofx/hofx4d.yaml @@ -1,35 +1,35 @@ -window begin: '{{ATM_WINDOW_BEGIN}}' +window begin: '{{ ATM_WINDOW_BEGIN | to_isotime }}' window length: $(ATM_WINDOW_LENGTH) forecast length: $(ATM_WINDOW_LENGTH) geometry: fms initialization: - namelist filename: !ENV ${DATA}/fv3jedi/fmsmpp.nml - field table filename: !ENV ${DATA}/fv3jedi/field_table - akbk: !ENV ${DATA}/fv3jedi/akbk.nc4 + namelist filename: $(DATA)/fv3jedi/fmsmpp.nml + field table filename: $(DATA)/fv3jedi/field_table + akbk: $(DATA)/fv3jedi/akbk.nc4 layout: - - !ENV ${layout_x} - - !ENV ${layout_y} + - $(layout_x) + - $(layout_y) npx: $(npx_ges) npy: $(npy_ges) npz: $(npz_ges) - field metadata override: !ENV ${DATA}/fv3jedi/gfs-restart.yaml + field metadata override: $(DATA)/fv3jedi/gfs-restart.yaml initial condition: - datapath: !ENV ${DATA}/bkg + datapath: $(DATA)/bkg filetype: fms restart - datetime: '{{ATM_WINDOW_BEGIN}}' - filename_core: '{{ATM_BEGIN_YYYYmmddHHMMSS}}.fv_core.res.nc' - filename_trcr: '{{ATM_BEGIN_YYYYmmddHHMMSS}}.fv_tracer.res.nc' - filename_sfcd: '{{ATM_BEGIN_YYYYmmddHHMMSS}}.sfc_data.nc' - filename_sfcw: '{{ATM_BEGIN_YYYYmmddHHMMSS}}.fv_srf_wnd.res.nc' - filename_cplr: '{{ATM_BEGIN_YYYYmmddHHMMSS}}.coupler.res' + datetime: '{{ ATM_WINDOW_BEGIN | to_isotime }}' + filename_core: '{{ ATM_WINDOW_BEGIN | to_fv3time }}.fv_core.res.nc' + filename_trcr: '{{ ATM_WINDOW_BEGIN | to_fv3time }}.fv_tracer.res.nc' + filename_sfcd: '{{ ATM_WINDOW_BEGIN | to_fv3time }}.sfc_data.nc' + filename_sfcw: '{{ ATM_WINDOW_BEGIN | to_fv3time }}.fv_srf_wnd.res.nc' + filename_cplr: '{{ ATM_WINDOW_BEGIN | to_fv3time }}.coupler.res' state variables: [ua,va,t,delp,sphum,ice_wat,liq_wat,o3mr,phis, slmsk,sheleg,tsea,vtype,stype,vfrac,stc,smc,snwdph, u_srf,v_srf,f10m] model: name: PSEUDO - datapath: !ENV ${DATA}/bkg + datapath: $(DATA)/bkg filetype: fms restart - datetime: '{{ATM_WINDOW_BEGIN}}' + datetime: '{{ ATM_WINDOW_BEGIN | to_isotime }}' filename is datetime templated: true filename_core: '%yyyy%mm%dd.%hh%MM%ss.fv_core.res.nc' filename_trcr: '%yyyy%mm%dd.%hh%MM%ss.fv_tracer.res.nc' diff --git a/parm/atm/hofx/hofx_nomodel.yaml b/parm/atm/hofx/hofx_nomodel.yaml index b50d23417..f980d53be 100644 --- a/parm/atm/hofx/hofx_nomodel.yaml +++ b/parm/atm/hofx/hofx_nomodel.yaml @@ -1,26 +1,26 @@ -window begin: '{{ATM_WINDOW_BEGIN}}' +window begin: '{{ ATM_WINDOW_BEGIN | to_isotime }}' window length: $(ATM_WINDOW_LENGTH) geometry: fms initialization: - namelist filename: !ENV ${DATA}/fv3jedi/fmsmpp.nml - field table filename: !ENV ${DATA}/fv3jedi/field_table - akbk: !ENV ${DATA}/fv3jedi/akbk.nc4 + namelist filename: $(DATA)/fv3jedi/fmsmpp.nml + field table filename: $(DATA)/fv3jedi/field_table + akbk: $(DATA)/fv3jedi/akbk.nc4 layout: - - !ENV ${layout_x} - - !ENV ${layout_y} + - $(layout_x) + - $(layout_y) npx: $(npx_ges) npy: $(npy_ges) npz: $(npz_ges) - field metadata override: !ENV ${DATA}/fv3jedi/gfs-restart.yaml + field metadata override: $(DATA)/fv3jedi/gfs-restart.yaml state: - datapath: !ENV ${DATA}/bkg + datapath: $(DATA)/bkg filetype: fms restart - datetime: '{{BKG_ISOTIME}}' - filename_core: '{{BKG_YYYYmmddHHMMSS}}.fv_core.res.nc' - filename_trcr: '{{BKG_YYYYmmddHHMMSS}}.fv_tracer.res.nc' - filename_sfcd: '{{BKG_YYYYmmddHHMMSS}}.sfc_data.nc' - filename_sfcw: '{{BKG_YYYYmmddHHMMSS}}.fv_srf_wnd.res.nc' - filename_cplr: '{{BKG_YYYYmmddHHMMSS}}.coupler.res' + datetime: '{{ current_cycle | to_isotime }}' + filename_core: '{{ current_cycle | to_fv3time }}.fv_core.res.nc' + filename_trcr: '{{ current_cycle | to_fv3time }}.fv_tracer.res.nc' + filename_sfcd: '{{ current_cycle | to_fv3time }}.sfc_data.nc' + filename_sfcw: '{{ current_cycle | to_fv3time }}.fv_srf_wnd.res.nc' + filename_cplr: '{{ current_cycle | to_fv3time }}.coupler.res' state variables: [ua,va,t,delp,sphum,ice_wat,liq_wat,o3mr,phis, slmsk,sheleg,tsea,vtype,stype,vfrac,stc,smc,snwdph, u_srf,v_srf,f10m] diff --git a/parm/atm/lgetkf/lgetkf.yaml b/parm/atm/lgetkf/lgetkf.yaml index 1869c15d9..b8a6f9886 100644 --- a/parm/atm/lgetkf/lgetkf.yaml +++ b/parm/atm/lgetkf/lgetkf.yaml @@ -4,32 +4,33 @@ geometry: field table filename: ./fv3jedi/field_table akbk: ./fv3jedi/akbk.nc4 layout: - - !ENV ${layout_x} - - !ENV ${layout_y} + - $(layout_x) + - $(layout_y) npx: $(npx_ges) npy: $(npy_ges) npz: $(npz_ges) field metadata override: ./fv3jedi/gfs-restart.yaml -window begin: '$(ATM_WINDOW_BEGIN)' -window length: PT6H +window begin: '{{ ATM_WINDOW_BEGIN | to_isotime }}' +window length: $(ATM_WINDOW_LENGTH) +increment variables: [ua,va,DZ,delp,t,ps,sphum,ice_wat,liq_wat,o3mr] background: - datetime: '{{BKG_ISOTIME}}' + datetime: '{{ current_cycle | to_isotime }}' members from template: template: - datetime: '{{BKG_ISOTIME}}' + datetime: '{{ current_cycle | to_isotime }}' filetype: fms restart state variables: [ua,va,t,DZ,delp,ps,sphum,ice_wat,liq_wat,o3mr,phis, slmsk,sheleg,tsea,vtype,stype,vfrac,stc,smc,snwdph, u_srf,v_srf,f10m] - datapath: bkg/mem%mem%/RESTART - filename_core: '{{BKG_YYYYmmddHHMMSS}}.fv_core.res.nc' - filename_trcr: '{{BKG_YYYYmmddHHMMSS}}.fv_tracer.res.nc' - filename_sfcd: '{{BKG_YYYYmmddHHMMSS}}.sfc_data.nc' - filename_sfcw: '{{BKG_YYYYmmddHHMMSS}}.fv_srf_wnd.res.nc' - filename_cplr: '{{BKG_YYYYmmddHHMMSS}}.coupler.res' + datapath: bkg/mem%mem% + filename_core: '{{ current_cycle | to_fv3time }}.fv_core.res.nc' + filename_trcr: '{{ current_cycle | to_fv3time }}.fv_tracer.res.nc' + filename_sfcd: '{{ current_cycle | to_fv3time }}.sfc_data.nc' + filename_sfcw: '{{ current_cycle | to_fv3time }}.fv_srf_wnd.res.nc' + filename_cplr: '{{ current_cycle | to_fv3time }}.coupler.res' pattern: '%mem%' - nmembers: !ENV ${NMEM_ENKF} + nmembers: $(NMEM_ENKF) zero padding: 3 observations: !INC ${OBS_LIST} @@ -41,6 +42,7 @@ driver: save prior mean: false save posterior mean increment: false save posterior ensemble increments: true + do posterior observer: false local ensemble DA: solver: GETKF diff --git a/parm/atm/obs/config/aircraft.yaml b/parm/atm/obs/config/aircraft.yaml index 460af86b9..4920a90db 100644 --- a/parm/atm/obs/config/aircraft.yaml +++ b/parm/atm/obs/config/aircraft.yaml @@ -3,7 +3,7 @@ obs space: obsdatain: engine: type: H5File - obsfile: !ENV ${DATA}/obs/${OPREFIX}aircraft.${CDATE}.nc4 + obsfile: $(DATA)/obs/$(OPREFIX)aircraft.${{ current_cycle | to_YMDH }}.nc4 obsgrouping: group variables: ["stationIdentification"] sort variable: "pressure" @@ -11,7 +11,7 @@ obs space: obsdataout: engine: type: H5File - obsfile: !ENV ${DATA}/diags/diag_aircraft_${CDATE}.nc4 + obsfile: $(DATA)/diags/diag_aircraft_${{ current_cycle | to_YMDH }}.nc4 io pool: max pool size: 1 simulated variables: [windEastward, windNorthward, airTemperature, specificHumidity] diff --git a/parm/atm/obs/config/amsua_n19.yaml b/parm/atm/obs/config/amsua_n19.yaml index 0096f2fd0..df6c38528 100644 --- a/parm/atm/obs/config/amsua_n19.yaml +++ b/parm/atm/obs/config/amsua_n19.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: !ENV ${DATA}/obs/${OPREFIX}amsua_n19.${PDY}${cyc}.nc4 + obsfile: $(DATA)/obs/$(OPREFIX)amsua_n19.{{ current_cycle | to_YMDH }}.nc4 obsdataout: engine: type: H5File - obsfile: !ENV ${DATA}/diags/diag_amsua_n19_${PDY}${cyc}.nc4 + obsfile: $(DATA)/diags/diag_amsua_n19_{{ current_cycle | to_YMDH }}.nc4 io pool: max pool size: 1 simulated variables: [brightnessTemperature] @@ -20,16 +20,16 @@ obs operator: obs options: Sensor_ID: amsua_n19 EndianType: little_endian - CoefficientPath: !ENV ${DATA}/crtm/ + CoefficientPath: $(DATA)/crtm/ obs bias: - input file: !ENV ${DATA}/obs/${GPREFIX}amsua_n19.satbias.${gPDY}${gcyc}.nc4 - output file: !ENV ${DATA}/bc/${APREFIX}amsua_n19.satbias.${PDY}${cyc}.nc4 + input file: $(DATA)/obs/$(GPREFIX)amsua_n19.satbias.nc4 + output file: $(DATA)/bc/$(APREFIX)amsua_n19.satbias.nc4 variational bc: predictors: - name: constant - name: lapse_rate order: 2 - tlapse: &amsua_n19_tlapse !ENV ${DATA}/obs/${GPREFIX}amsua_n19.tlapse.${gPDY}${gcyc}.txt + tlapse: &amsua_n19_tlapse $(DATA)/obs/$(GPREFIX)amsua_n19.tlapse.txt - name: lapse_rate tlapse: *amsua_n19_tlapse - name: emissivity @@ -46,11 +46,11 @@ obs bias: step size: 1.0e-4 largest analysis variance: 10000.0 prior: - input file: !ENV ${DATA}/obs/${GPREFIX}amsua_n19.satbias_cov.${gPDY}${gcyc}.nc4 + input file: $(DATA)/obs/$(GPREFIX)amsua_n19.satbias_cov.nc4 inflation: ratio: 1.1 ratio for small dataset: 2.0 - output file: !ENV ${DATA}/bc/${APREFIX}amsua_n19.satbias_cov.${PDY}${cyc}.nc4 + output file: $(DATA)/bc/$(APREFIX)amsua_n19.satbias_cov.nc4 obs filters: - filter: BlackList filter variables: diff --git a/parm/atm/obs/config/atms_n20.yaml b/parm/atm/obs/config/atms_n20.yaml index 0d9e863d5..28b19e692 100644 --- a/parm/atm/obs/config/atms_n20.yaml +++ b/parm/atm/obs/config/atms_n20.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: !ENV ${DATA}/obs/${OPREFIX}atms_n20.${CDATE}.nc4 + obsfile: $(DATA)/obs/$(OPREFIX)atms_n20.${{ current_cycle | to_YMDH }}.nc4 obsdataout: engine: type: H5File - obsfile: !ENV ${DATA}/diags/diag_atms_n20_${CDATE}.nc4 + obsfile: $(DATA)/diags/diag_atms_n20_${{ current_cycle | to_YMDH }}.nc4 io pool: max pool size: 1 simulated variables: [brightness_temperature] @@ -20,16 +20,16 @@ obs operator: obs options: Sensor_ID: atms_n20 EndianType: little_endian - CoefficientPath: !ENV ${DATA}/crtm/ + CoefficientPath: $(DATA)/crtm/ obs bias: - input file: !ENV ${DATA}/obs/${GPREFIX}atms_n20.satbias.${GDATE}.nc4 - output file: !ENV ${DATA}/bc/${APREFIX}atms_n20.satbias.${CDATE}.nc4 + input file: $(DATA)/obs/$(GPREFIX)atms_n20.satbias.nc4 + output file: $(DATA)/bc/$(APREFIX)atms_n20.satbias.nc4 variational bc: predictors: - name: constant - name: lapse_rate order: 2 - tlapse: &atms_n20_tlapse !ENV ${DATA}/obs/${GPREFIX}atms_n20.tlapse.${GDATE}.txt + tlapse: &atms_n20_tlapse $(DATA)/obs/$(GPREFIX)atms_n20.tlapse.txt - name: lapse_rate tlapse: *atms_n20_tlapse - name: emissivity @@ -46,11 +46,11 @@ obs bias: step size: 1.0e-4 largest analysis variance: 10000.0 prior: - input file: !ENV ${DATA}/obs/${GPREFIX}atms_n20.satbias_cov.${GDATE}.nc4 + input file: $(DATA)/obs/$(GPREFIX)atms_n20.satbias_cov.nc4 inflation: ratio: 1.1 ratio for small dataset: 2.0 - output file: !ENV ${DATA}/bc/${APREFIX}atms_n20.satbias_cov.${CDATE}.nc4 + output file: $(DATA)/bc/$(APREFIX)atms_n20.satbias_cov.nc4 obs filters: - filter: BlackList filter variables: diff --git a/parm/atm/obs/config/cris-fsr_n20.yaml b/parm/atm/obs/config/cris-fsr_n20.yaml index 3d2a43313..579439862 100644 --- a/parm/atm/obs/config/cris-fsr_n20.yaml +++ b/parm/atm/obs/config/cris-fsr_n20.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: !ENV ${DATA}/obs/${OPREFIX}cris-fsr_n20.${CDATE}.nc4 + obsfile: $(DATA)/obs/$(OPREFIX)cris-fsr_n20.${{ current_cycle | to_YMDH }}.nc4 obsdataout: engine: type: H5File - obsfile: !ENV ${DATA}/diags/diag_cris-fsr_n20_${CDATE}.nc4 + obsfile: $(DATA)/diags/diag_cris-fsr_n20_${{ current_cycle | to_YMDH }}.nc4 io pool: max pool size: 1 simulated variables: [brightness_temperature] @@ -51,16 +51,16 @@ obs operator: obs options: Sensor_ID: cris-fsr_n20 EndianType: little_endian - CoefficientPath: !ENV ${DATA}/crtm/ + CoefficientPath: $(DATA)/crtm/ obs bias: - input file: !ENV ${DATA}/obs/${GPREFIX}cris-fsr_n20.satbias.${GDATE}.nc4 - output file: !ENV ${DATA}/bc/${APREFIX}cris-fsr_n20.satbias.${CDATE}.nc4 + input file: $(DATA)/obs/$(GPREFIX)cris-fsr_n20.satbias.nc4 + output file: $(DATA)/bc/$(APREFIX)cris-fsr_n20.satbias.nc4 variational bc: predictors: - name: constant - name: lapse_rate order: 2 - tlapse: &cris-fsr_n20_tlapse !ENV ${DATA}/obs/${GPREFIX}cris-fsr_n20.tlapse.${GDATE}.txt + tlapse: &cris-fsr_n20_tlapse $(DATA)/obs/$(GPREFIX)cris-fsr_n20.tlapse.txt - name: lapse_rate tlapse: *cris-fsr_n20_tlapse - name: emissivity @@ -77,11 +77,11 @@ obs bias: step size: 1.0e-4 largest analysis variance: 10000.0 prior: - input file: !ENV ${DATA}/obs/${GPREFIX}cris-fsr_n20.satbias_cov.${GDATE}.nc4 + input file: $(DATA)/obs/$(GPREFIX)cris-fsr_n20.satbias_cov.nc4 inflation: ratio: 1.1 ratio for small dataset: 2.0 - output file: !ENV ${DATA}/bc/${APREFIX}cris-fsr_n20.satbias_cov.${CDATE}.nc4 + output file: $(DATA)/bc/$(APREFIX)cris-fsr_n20.satbias_cov.nc4 # #obs filters: ## Wavenumber Check diff --git a/parm/atm/obs/config/cris-fsr_npp.yaml b/parm/atm/obs/config/cris-fsr_npp.yaml index a3943264d..55409f454 100644 --- a/parm/atm/obs/config/cris-fsr_npp.yaml +++ b/parm/atm/obs/config/cris-fsr_npp.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: !ENV ${DATA}/obs/${OPREFIX}cris-fsr_npp.${CDATE}.nc4 + obsfile: $(DATA)/obs/$(OPREFIX)cris-fsr_npp.${{ current_cycle | to_YMDH }}.nc4 obsdataout: engine: type: H5File - obsfile: !ENV ${DATA}/diags/diag_cris-fsr_npp_${CDATE}.nc4 + obsfile: $(DATA)/diags/diag_cris-fsr_npp_${{ current_cycle | to_YMDH }}.nc4 io pool: max pool size: 1 simulated variables: [brightness_temperature] @@ -51,16 +51,16 @@ obs operator: obs options: Sensor_ID: cris-fsr_npp EndianType: little_endian - CoefficientPath: !ENV ${DATA}/crtm/ + CoefficientPath: $(DATA)/crtm/ obs bias: - input file: !ENV ${DATA}/obs/${GPREFIX}cris-fsr_npp.satbias.${GDATE}.nc4 - output file: !ENV ${DATA}/bc/${APREFIX}cris-fsr_npp.satbias.${CDATE}.nc4 + input file: $(DATA)/obs/$(GPREFIX)cris-fsr_npp.satbias.nc4 + output file: $(DATA)/bc/$(APREFIX)cris-fsr_npp.satbias.nc4 variational bc: predictors: - name: constant - name: lapse_rate order: 2 - tlapse: &cris-fsr_npp_tlapse !ENV ${DATA}/obs/${GPREFIX}cris-fsr_npp.tlapse.${GDATE}.txt + tlapse: &cris-fsr_npp_tlapse $(DATA)/obs/$(GPREFIX)cris-fsr_npp.tlapse.txt - name: lapse_rate tlapse: *cris-fsr_npp_tlapse - name: emissivity @@ -77,11 +77,11 @@ obs bias: step size: 1.0e-4 largest analysis variance: 10000.0 prior: - input file: !ENV ${DATA}/obs/${GPREFIX}cris-fsr_npp.satbias_cov.${GDATE}.nc4 + input file: $(DATA)/obs/$(GPREFIX)cris-fsr_npp.satbias_cov.nc4 inflation: ratio: 1.1 ratio for small dataset: 2.0 - output file: !ENV ${DATA}/bc/${APREFIX}cris-fsr_npp.satbias_cov.${CDATE}.nc4 + output file: $(DATA)/bc/$(APREFIX)cris-fsr_npp.satbias_cov.nc4 #obs filters: ## Wavenumber Check #- filter: BlackList diff --git a/parm/atm/obs/config/lgetkf_amsua_n19.yaml b/parm/atm/obs/config/lgetkf_amsua_n19.yaml index 77dff58ae..d5fe9691a 100644 --- a/parm/atm/obs/config/lgetkf_amsua_n19.yaml +++ b/parm/atm/obs/config/lgetkf_amsua_n19.yaml @@ -6,11 +6,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: !ENV ./obs/${OPREFIX}amsua_n19.${PDY}${cyc}.nc4 + obsfile: ./obs/$(OPREFIX)amsua_n19.{{ current_cycle | to_YMDH }}.nc4 obsdataout: engine: type: H5File - obsfile: !ENV ./diags/diag_amsua_n19_${PDY}${cyc}.nc4 + obsfile: ./diags/diag_amsua_n19_{{ current_cycle | to_YMDH }}.nc4 io pool: max pool size: 1 simulated variables: [brightnessTemperature] @@ -27,14 +27,14 @@ obs operator: obs error: covariance model: diagonal obs bias: - input file: !ENV ${DATA}/obs/${GPREFIX}amsua_n19.satbias.${gPDY}${gcyc}.nc4 - output file: !ENV ${DATA}/bc/${APREFIX}amsua_n19.satbias.${PDY}${cyc}.nc4 + input file: $(DATA)/obs/$(GPREFIX)amsua_n19.satbias.nc4 + output file: $(DATA)/bc/$(APREFIX)amsua_n19.satbias.nc4 variational bc: predictors: - name: constant - name: lapse_rate order: 2 - tlapse: &amsua_n19_tlapse !ENV ${DATA}/obs/${GPREFIX}amsua_n19.tlapse.${gPDY}${gcyc}.txt + tlapse: &amsua_n19_tlapse $(DATA)/obs/$(GPREFIX)amsua_n19.tlapse.txt - name: lapse_rate tlapse: *amsua_n19_tlapse - name: emissivity @@ -51,11 +51,11 @@ obs bias: step size: 1.0e-4 largest analysis variance: 10000.0 prior: - input file: !ENV ${DATA}/obs/${GPREFIX}amsua_n19.satbias_cov.${gPDY}${gcyc}.nc4 + input file: $(DATA)/obs/$(GPREFIX)amsua_n19.satbias_cov.nc4 inflation: ratio: 1.1 ratio for small dataset: 2.0 - output file: !ENV ${DATA}/bc/${APREFIX}amsua_n19.satbias_cov.${PDY}${cyc}.nc4 + output file: $(DATA)/bc/$(APREFIX)amsua_n19.satbias_cov.nc4 obs filters: - filter: Bounds Check filter variables: diff --git a/parm/atm/obs/config/lgetkf_sondes.yaml b/parm/atm/obs/config/lgetkf_sondes.yaml index 837b54a48..4f2358097 100644 --- a/parm/atm/obs/config/lgetkf_sondes.yaml +++ b/parm/atm/obs/config/lgetkf_sondes.yaml @@ -6,11 +6,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: !ENV ./obs/${OPREFIX}sondes.${PDY}${cyc}.nc4 + obsfile: ./obs/$(OPREFIX)sondes.{{ current_cycle | to_YMDH }}.nc4 obsdataout: engine: type: H5File - obsfile: !ENV ./diags/diag_sondes_lgetkf_${PDY}${cyc}.nc4 + obsfile: ./diags/diag_sondes_lgetkf_{{ current_cycle | to_YMDH }}.nc4 io pool: max pool size: 1 simulated variables: [windEastward, windNorthward, airTemperature] diff --git a/parm/atm/obs/config/omi_aura.yaml b/parm/atm/obs/config/omi_aura.yaml index e5346e322..79973780c 100644 --- a/parm/atm/obs/config/omi_aura.yaml +++ b/parm/atm/obs/config/omi_aura.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: !ENV ${DATA}/obs/${OPREFIX}omi_aura.${CDATE}.nc4 + obsfile: $(DATA)/obs/$(OPREFIX)omi_aura.${{ current_cycle | to_YMDH }}.nc4 obsdataout: engine: type: H5File - obsfile: !ENV ${DATA}/diags/diag_omi_aura_${CDATE}.nc4 + obsfile: $(DATA)/diags/diag_omi_aura_${{ current_cycle | to_YMDH }}.nc4 io pool: max pool size: 1 simulated variables: [ozoneTotal] diff --git a/parm/atm/obs/config/ompsnp_npp.yaml b/parm/atm/obs/config/ompsnp_npp.yaml index cae446044..5ba772fb3 100644 --- a/parm/atm/obs/config/ompsnp_npp.yaml +++ b/parm/atm/obs/config/ompsnp_npp.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: !ENV ${DATA}/obs/${OPREFIX}ompsnp_npp.${CDATE}.nc4 + obsfile: $(DATA)/obs/$(OPREFIX)ompsnp_npp.${{ current_cycle | to_YMDH }}.nc4 obsdataout: engine: type: H5File - obsfile: !ENV ${DATA}/diags/diag_ompsnp_npp_${CDATE}.nc4 + obsfile: $(DATA)/diags/diag_ompsnp_npp_${{ current_cycle | to_YMDH }}.nc4 io pool: max pool size: 1 simulated variables: [ozoneTotal] diff --git a/parm/atm/obs/config/ompstc8_npp.yaml b/parm/atm/obs/config/ompstc8_npp.yaml index c3aee74f4..e3b11ff64 100644 --- a/parm/atm/obs/config/ompstc8_npp.yaml +++ b/parm/atm/obs/config/ompstc8_npp.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: !ENV ${DATA}/obs/${OPREFIX}ompstc8_npp.${CDATE}.nc4 + obsfile: $(DATA)/obs/$(OPREFIX)ompstc8_npp.${{ current_cycle | to_YMDH }}.nc4 obsdataout: engine: type: H5File - obsfile: !ENV ${DATA}/diags/diag_ompstc8_npp_${CDATE}.nc4 + obsfile: $(DATA)/diags/diag_ompstc8_npp_${{ current_cycle | to_YMDH }}.nc4 io pool: max pool size: 1 simulated variables: [ozoneTotal] diff --git a/parm/atm/obs/config/satwind.yaml b/parm/atm/obs/config/satwind.yaml index ebef654b4..853cbce77 100644 --- a/parm/atm/obs/config/satwind.yaml +++ b/parm/atm/obs/config/satwind.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: !ENV ${DATA}/obs/${OPREFIX}satwind.${CDATE}.nc4 + obsfile: $(DATA)/obs/$(OPREFIX)satwind.${{ current_cycle | to_YMDH }}.nc4 obsdataout: engine: type: H5File - obsfile: !ENV ${DATA}/diags/diag_satwind_${CDATE}.nc4 + obsfile: $(DATA)/diags/diag_satwind_${{ current_cycle | to_YMDH }}.nc4 io pool: max pool size: 1 simulated variables: [windEastward, windNorthward] diff --git a/parm/atm/obs/config/sfc.yaml b/parm/atm/obs/config/sfc.yaml index 32a3f1acb..31fd74c0c 100644 --- a/parm/atm/obs/config/sfc.yaml +++ b/parm/atm/obs/config/sfc.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: !ENV ${DATA}/obs/${OPREFIX}sfc.${CDATE}.nc4 + obsfile: $(DATA)/obs/$(OPREFIX)sfc.${{ current_cycle | to_YMDH }}.nc4 obsdataout: engine: type: H5File - obsfile: !ENV ${DATA}/diags/diag_sfc_${CDATE}.nc4 + obsfile: $(DATA)/diags/diag_sfc_${{ current_cycle | to_YMDH }}.nc4 io pool: max pool size: 1 simulated variables: [stationPressure] diff --git a/parm/atm/obs/config/sfcship.yaml b/parm/atm/obs/config/sfcship.yaml index c5bf1b304..c3530f404 100644 --- a/parm/atm/obs/config/sfcship.yaml +++ b/parm/atm/obs/config/sfcship.yaml @@ -3,12 +3,12 @@ obs space: obsdatain: engine: type: H5File - obsfile: !ENV ${DATA}/obs/${OPREFIX}sfcship.${CDATE}.nc4 + obsfile: $(DATA)/obs/$(OPREFIX)sfcship.${{ current_cycle | to_YMDH }}.nc4 obsdataout: engine: type: H5File overwrite: true - obsfile: !ENV ${DATA}/diags/diag_sfcship_${CDATE}.nc4 + obsfile: $(DATA)/diags/diag_sfcship_${{ current_cycle | to_YMDH }}.nc4 io pool: max pool size: 1 simulated variables: [stationPressure, airTemperature, specificHumidity] diff --git a/parm/atm/obs/config/sondes.yaml b/parm/atm/obs/config/sondes.yaml index b1b6cf9e9..e9331f7bd 100644 --- a/parm/atm/obs/config/sondes.yaml +++ b/parm/atm/obs/config/sondes.yaml @@ -3,7 +3,7 @@ obs space: obsdatain: engine: type: H5File - obsfile: !ENV ${DATA}/obs/${OPREFIX}sondes.${PDY}${cyc}.nc4 + obsfile: $(DATA)/obs/$(OPREFIX)sondes.{{ current_cycle | to_YMDH }}.nc4 obsgrouping: group variables: ["stationIdentification"] sort variable: "pressure" @@ -11,7 +11,7 @@ obs space: obsdataout: engine: type: H5File - obsfile: !ENV ${DATA}/diags/diag_sondes_${PDY}${cyc}.nc4 + obsfile: $(DATA)/diags/diag_sondes_{{ current_cycle | to_YMDH }}.nc4 io pool: max pool size: 1 simulated variables: [stationPressure, airTemperature, windEastward, windNorthward, specificHumidity] @@ -26,7 +26,7 @@ obs operator: - name: specificHumidity - name: SfcPCorrected da_psfc_scheme: GSI - geovar_sfc_geomz: surface_geometric_height + geovar_sfc_geomz: surface_geopotential_height geovar_geomz: geopotential_height variables: - name: stationPressure @@ -92,7 +92,7 @@ obs filters: error_min: 100.0 # 1 mb error_max: 300.0 # 3 mb geovar_geomz: geopotential_height - geovar_sfc_geomz: surface_geometric_height + geovar_sfc_geomz: surface_geopotential_height # # Gross error check with (O - B) / ObsError greater than threshold - filter: Background Check diff --git a/parm/atm/obs/config/ssmis_f17.yaml b/parm/atm/obs/config/ssmis_f17.yaml index 12d96ee47..5054951ef 100644 --- a/parm/atm/obs/config/ssmis_f17.yaml +++ b/parm/atm/obs/config/ssmis_f17.yaml @@ -4,22 +4,22 @@ obs operator: obs options: Sensor_ID: ssmis_f17 EndianType: little_endian - CoefficientPath: !ENV ${DATA}/crtm/ + CoefficientPath: $(DATA)/crtm/ obs space: name: ssmis_f17 obsdatain: engine: type: H5File - obsfile: !ENV ${DATA}/obs/${OPREFIX}ssmis_f17.${CDATE}.nc4 + obsfile: $(DATA)/obs/$(OPREFIX)ssmis_f17.${{ current_cycle | to_YMDH }}.nc4 obsdataout: engine: type: H5File - obsfile: !ENV ${DATA}/diags/diag_ssmis_f17_${CDATE}.nc4 + obsfile: $(DATA)/diags/diag_ssmis_f17_${{ current_cycle | to_YMDH }}.nc4 simulated variables: [brightness_temperature] channels: 1-24 obs bias: - input file: !ENV ${DATA}/obs/${GPREFIX}ssmis_f17.satbias.${GDATE}.nc4 - output file: !ENV ${DATA}/bc/${APREFIX}ssmis_f17.satbias.${CDATE}.nc4 + input file: $(DATA)/obs/$(GPREFIX)ssmis_f17.satbias.nc4 + output file: $(DATA)/bc/$(APREFIX)ssmis_f17.satbias.nc4 variational bc: predictors: - name: constant @@ -36,7 +36,7 @@ obs bias: - name: sine_of_latitude - name: lapse_rate order: 2 - tlapse: &ssmis_f17_tlapse !ENV ${DATA}/obs/${GPREFIX}ssmis_f17.tlapse.${GDATE}.txt + tlapse: &ssmis_f17_tlapse $(DATA)/obs/$(GPREFIX)ssmis_f17.tlapse.txt - name: lapse_rate tlapse: *ssmis_f17_tlapse - name: emissivity diff --git a/parm/atm/obs/config/ssmis_f18.yaml b/parm/atm/obs/config/ssmis_f18.yaml index 41bdf450a..2c757268a 100644 --- a/parm/atm/obs/config/ssmis_f18.yaml +++ b/parm/atm/obs/config/ssmis_f18.yaml @@ -4,22 +4,22 @@ obs operator: obs options: Sensor_ID: ssmis_f18 EndianType: little_endian - CoefficientPath: !ENV ${DATA}/crtm/ + CoefficientPath: $(DATA)/crtm/ obs space: name: ssmis_f18 obsdatain: engine: type: H5File - obsfile: !ENV ${DATA}/obs/${OPREFIX}ssmis_f18.${CDATE}.nc4 + obsfile: $(DATA)/obs/$(OPREFIX)ssmis_f18.${{ current_cycle | to_YMDH }}.nc4 obsdataout: engine: type: H5File - obsfile: !ENV ${DATA}/diags/diag_ssmis_f18_${CDATE}.nc4 + obsfile: $(DATA)/diags/diag_ssmis_f18_${{ current_cycle | to_YMDH }}.nc4 simulated variables: [brightness_temperature] channels: 1-24 obs bias: - input file: !ENV ${DATA}/obs/${GPREFIX}ssmis_f18.satbias.${GDATE}.nc4 - output file: !ENV ${DATA}/bc/${APREFIX}ssmis_f18.satbias.${CDATE}.nc4 + input file: $(DATA)/obs/$(GPREFIX)ssmis_f18.satbias.nc4 + output file: $(DATA)/bc/$(APREFIX)ssmis_f18.satbias.nc4 variational bc: predictors: - name: constant @@ -36,7 +36,7 @@ obs bias: - name: sine_of_latitude - name: lapse_rate order: 2 - tlapse: &ssmis_f18_tlapse !ENV ${DATA}/obs/${GPREFIX}ssmis_f18.tlapse.${GDATE}.txt + tlapse: &ssmis_f18_tlapse $(DATA)/obs/$(GPREFIX)ssmis_f18.tlapse.txt - name: lapse_rate tlapse: *ssmis_f18_tlapse - name: emissivity diff --git a/parm/atm/obs/testing/omi_aura.yaml b/parm/atm/obs/testing/omi_aura.yaml new file mode 100644 index 000000000..92b233702 --- /dev/null +++ b/parm/atm/obs/testing/omi_aura.yaml @@ -0,0 +1,106 @@ +obs space: + name: omi_aura + obsdatain: + engine: + type: H5File + obsfile: !ENV omi_aura_obs_${CDATE}.nc4 + obsdataout: + engine: + type: H5File + obsfile: !ENV omi_aura_diag_${CDATE}.nc4 + io pool: + max pool size: 1 + simulated variables: [ozoneTotal] +geovals: + filename: !ENV omi_aura_geoval_${CDATE}.nc4 +obs operator: + name: AtmVertInterpLay + geovals: [mole_fraction_of_ozone_in_air] + coefficients: [0.007886131] # convert from ppmv to DU + nlevels: [1] + +obs filters: +# range sanity check +- filter: Bounds Check + filter variables: + - name: ozoneTotal + minvalue: 0 + maxvalue: 1000 + action: + name: reject + +# Toss observations for which the profile ozone error code is neither 0 nor 1 +- filter: RejectList + filter variables: + - name: ozoneTotal + where: + - variable: + name: MetaData/total_ozone_error_flag + minvalue: 0.1 + action: + name: reject +# +- filter: AcceptList + filter variables: + - name: ozoneTotal + where: + - variable: + name: MetaData/total_ozone_error_flag + minvalue: 0.9 + maxvalue: 1.1 + action: + name: accept + +#exclude afbo=3 or 13 +- filter: RejectList + filter variables: + - name: ozoneTotal + where: + - variable: + name: MetaData/bestOzoneAlgorithmFlag + minvalue: 2.9 + maxvalue: 3.1 + action: + name: reject +- filter: RejectList + filter variables: + - name: ozoneTotal + where: + - variable: + name: MetaData/bestOzoneAlgorithmFlag + minvalue: 12.9 + maxvalue: 13.1 + action: + name: reject + +#Do not use the data if row anomaly (bit 10)is 1 +- filter: RejectList + filter variables: + - name: ozoneTotal + where: + - variable: + name: MetaData/row_anomaly_index + minvalue: 0.9 + maxvalue: 1.1 + action: + name: reject + +#scan position check +- filter: RejectList + filter variables: + - name: ozoneTotal + where: + - variable: + name: MetaData/sensorScanPosition + minvalue: 25 + action: + name: reject + + # Gross check +- filter: Background Check + filter variables: + - name: ozoneTotal + threshold: 5.0 + action: + name: reject + diff --git a/parm/atm/obs/testing/ompsnp_npp.yaml b/parm/atm/obs/testing/ompsnp_npp.yaml new file mode 100644 index 000000000..15a9d31db --- /dev/null +++ b/parm/atm/obs/testing/ompsnp_npp.yaml @@ -0,0 +1,274 @@ +obs space: + name: ompsnp_npp + obsdatain: + engine: + type: H5File + obsfile: !ENV ompsnp_npp_obs_${CDATE}.nc4 + obsgrouping: + group variables: ["latitude"] + sort variable: "pressure" + sort order: "ascending" + obsdataout: + engine: + type: H5File + obsfile: !ENV ompsnp_npp_diag_${CDATE}.nc4 + io pool: + max pool size: 1 + simulated variables: [ozoneLayer] +geovals: + filename: !ENV ompsnp_npp_geoval_${CDATE}.nc4 +obs operator: + name: AtmVertInterpLay + geovals: [mole_fraction_of_ozone_in_air] + coefficients: [0.007886131] # convert from ppmv to DU + nlevels: [22] + +obs filters: +# range sanity check +- filter: Bounds Check + filter variables: + - name: ozoneLayer + minvalue: 0 + maxvalue: 1000 + action: + name: reject + +# Accept total_ozone_error_flag values of 0 and 2, but not any others. +- filter: RejectList + filter variables: + - name: ozoneLayer + where: + - variable: + name: MetaData/total_ozone_error_flag + minvalue: 0.1 + action: + name: reject +# +- filter: AcceptList + filter variables: + - name: ozoneLayer + where: + - variable: + name: MetaData/total_ozone_error_flag + minvalue: 1.9 + maxvalue: 2.1 + action: + name: accept + +# +# Accept profile_ozone_error_flag values of 0, 1, and 7, but not any others. +- filter: RejectList + filter variables: + - name: ozoneLayer + where: + - variable: + name: MetaData/profile_ozone_error_flag + minvalue: 1.1 + action: + name: reject +# +- filter: AcceptList + filter variables: + - name: ozoneLayer + where: + - variable: + name: MetaData/profile_ozone_error_flag + minvalue: 6.9 + maxvalue: 7.1 + action: + name: accept +# +# Reject all the pressure levels higher in height than pressure of 0.64hPa +- filter: RejectList + filter variables: + - name: ozoneLayer + where: + - variable: + name: MetaData/pressure + maxvalue: 64.0 + action: + name: reject +# +# Assign initial error value based on pressure +- filter: Perform Action + filter variables: + - name: ozoneLayer + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [0.001, 10.1325, 16.00935, 25.43258, 40.32735, 63.93607, 101.325, 160.0935, 254.3257, 403.2735, 639.3608, 1013.25, 1600.935, 2543.258, 4032.735, 6393.607, 10132.5, 16009.35, 25432.57, 40327.35, 63936.07, 101325] + errors: [7.7236, 0.04, 0.02, 0.015, 0.03, 0.15, 0.056, 0.125, 0.2, 0.299, 0.587, 0.864, 1.547, 2.718, 3.893, 4.353, 3.971, 4.407, 4.428, 3.312, 2.198, 2.285] + +# Gross error check with (O - B) / ObsError greater than threshold. +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 100 + action: + name: reject + where: + - variable: + name: MetaData/pressure + maxvalue: 0.001 +# +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 30 + action: + name: reject + where: + - variable: + name: MetaData/pressure + minvalue: 30000.0 + maxvalue: 110000.0 +# +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 40 + action: + name: reject + where: + - variable: + name: MetaData/pressure + minvalue: 20000.0 + maxvalue: 30000.0 +# +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 44.42 + action: + name: reject + where: + - variable: + name: MetaData/pressure + minvalue: 10100.0 + maxvalue: 20000.0 +# +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 57.52 + action: + name: reject + where: + - variable: + name: MetaData/pressure + minvalue: 6400.0 + maxvalue: 10100.0 + +# +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 69.4 + action: + name: reject + where: + - variable: + name: MetaData/pressure + minvalue: 4000.0 + maxvalue: 6400.0 +# +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 70 + action: + name: reject + where: + - variable: + name: MetaData/pressure + minvalue: 2600.0 + maxvalue: 4000.0 +# +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 62.73 + action: + name: reject + where: + - variable: + name: MetaData/pressure + minvalue: 1600.0 + maxvalue: 2600.0 +# +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 50.52 + action: + name: reject + where: + - variable: + name: MetaData/pressure + minvalue: 1100.0 + maxvalue: 1600.0 +# +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 35.9 + action: + name: reject + where: + - variable: + name: MetaData/pressure + minvalue: 700.0 + maxvalue: 1100.0 +# +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 26.41 + action: + name: reject + where: + - variable: + name: MetaData/pressure + minvalue: 400.0 + maxvalue: 700.0 +# +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 20.51 + action: + name: reject + where: + - variable: + name: MetaData/pressure + minvalue: 300.0 + maxvalue: 400.0 +# +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 12.82 + action: + name: reject + where: + - variable: + name: MetaData/pressure + minvalue: 200.0 + maxvalue: 300.0 +# +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 10 + action: + name: reject + where: + - variable: + name: MetaData/pressure + maxvalue: 200.0 + +passedBenchmark: 4133 diff --git a/parm/atm/obs/testing/ompstc8_npp.yaml b/parm/atm/obs/testing/ompstc8_npp.yaml new file mode 100644 index 000000000..cbefd1cfd --- /dev/null +++ b/parm/atm/obs/testing/ompstc8_npp.yaml @@ -0,0 +1,93 @@ +obs space: + name: ompstc8_npp + obsdatain: + engine: + type: H5File + obsfile: !ENV ompstc8_npp_obs_${CDATE}.nc4 + obsgrouping: + group variables: ["latitude"] + sort variable: "pressure" + sort order: "ascending" + obsdataout: + engine: + type: H5File + obsfile: !ENV ompstc8_npp_diag_${CDATE}.nc4 + io pool: + max pool size: 1 + simulated variables: [ozoneTotal] +geovals: + filename: !ENV ompstc8_npp_geoval_${CDATE}.nc4 +obs operator: + name: AtmVertInterpLay + geovals: [mole_fraction_of_ozone_in_air] + coefficients: [0.007886131] # convert from ppmv to DU + nlevels: [1] + +obs filters: +# range sanity check +- filter: Bounds Check + filter variables: + - name: ozoneTotal + minvalue: 0 + maxvalue: 1000 + action: + name: reject + +# Accept total_ozone_error_flag values of 0 and 1, but not any others. +- filter: RejectList + filter variables: + - name: ozoneTotal + where: + - variable: + name: MetaData/total_ozone_error_flag + minvalue: 0.1 + action: + name: reject +# +- filter: AcceptList + filter variables: + - name: ozoneTotal + where: + - variable: + name: MetaData/total_ozone_error_flag + minvalue: 0.9 + maxvalue: 1.1 + action: + name: accept + +#exclude afbo=3 or 13 +- filter: RejectList + filter variables: + - name: ozoneTotal + where: + - variable: + name: MetaData/bestOzoneAlgorithmFlag + minvalue: 2.9 + maxvalue: 3.1 + action: + name: reject +- filter: RejectList + filter variables: + - name: ozoneTotal + where: + - variable: + name: MetaData/bestOzoneAlgorithmFlag + minvalue: 12.9 + maxvalue: 13.1 + action: + name: reject + +#scan position check +- filter: RejectList + filter variables: + - name: ozoneTotal + where: + - variable: + name: MetaData/sensorScanPosition + minvalue: 1 + maxvalue: 4 + - variable: + name: MetaData/latitude + minvalue: 50.0 + action: + name: reject diff --git a/parm/atm/obs/testing/satwind.yaml b/parm/atm/obs/testing/satwind.yaml index 9f9d8297d..d8c913b53 100644 --- a/parm/atm/obs/testing/satwind.yaml +++ b/parm/atm/obs/testing/satwind.yaml @@ -478,15 +478,17 @@ obs prior filters: name: reject # Reject obs with qualityInformationWithoutForecast < 85 # CLEARED - - filter: Perform Action + - filter: Bounds Check filter variables: - name: windEastward - name: windNorthward where: - variable: MetaData/satelliteIdentifier is_in: 50-79 - - variable: MetaData/qualityInformationWithoutForecast - is_not_in: 85-100 + test variables: + - name: MetaData/qualityInformationWithoutForecast + minvalue: 85. + maxvalue: 100. action: name: reject # JMA: satelliteIdentifier [100–199] (>99, <200) @@ -519,15 +521,17 @@ obs prior filters: name: reject # Reject obs with qualityInformationWithoutForecast < 85. # CLEARED - - filter: Perform Action + - filter: Bounds Check filter variables: - name: windEastward - name: windNorthward where: - variable: MetaData/satelliteIdentifier is_in: 100-199 - - variable: MetaData/qualityInformationWithoutForecast - is_not_in: 85-100 + test variables: + - name: MetaData/qualityInformationWithoutForecast + minvalue: 85. + maxvalue: 100. action: name: reject # NESDIS: satelliteIdentifier [250–299] (>249, <300) @@ -547,15 +551,17 @@ obs prior filters: name: reject # Reject obs with qualityInformationWithoutForecast < 90. OR > 100. # CLEARED - - filter: Perform Action + - filter: Bounds Check filter variables: - name: windEastward - name: windNorthward where: - variable: MetaData/satelliteIdentifier is_in: 250-299 - - variable: MetaData/qualityInformationWithoutForecast - is_not_in: 90-100 + test variables: + - name: MetaData/qualityInformationWithoutForecast + minvalue: 90. + maxvalue: 100. action: name: reject # Reject obs with pressure < 15000. diff --git a/parm/atm/variational/3dvar_dripcg.yaml b/parm/atm/variational/3dvar_dripcg.yaml index 2ea6542b0..4563f45ac 100644 --- a/parm/atm/variational/3dvar_dripcg.yaml +++ b/parm/atm/variational/3dvar_dripcg.yaml @@ -1,6 +1,6 @@ cost function: cost type: 3D-Var - window begin: '{{ATM_WINDOW_BEGIN}}' + window begin: '{{ ATM_WINDOW_BEGIN | to_isotime }}' window length: $(ATM_WINDOW_LENGTH) analysis variables: &3dvars [ua,va,t,ps,sphum,ice_wat,liq_wat,o3mr] geometry: @@ -9,8 +9,8 @@ cost function: field table filename: ./fv3jedi/field_table akbk: ./fv3jedi/akbk.nc4 layout: - - !ENV ${layout_x} - - !ENV ${layout_y} + - $(layout_x) + - $(layout_y) npx: $(npx_ges) npy: $(npy_ges) npz: $(npz_ges) @@ -18,12 +18,12 @@ cost function: background: datapath: ./bkg filetype: fms restart - datetime: '{{BKG_ISOTIME}}' - filename_core: '{{BKG_YYYYmmddHHMMSS}}.fv_core.res.nc' - filename_trcr: '{{BKG_YYYYmmddHHMMSS}}.fv_tracer.res.nc' - filename_sfcd: '{{BKG_YYYYmmddHHMMSS}}.sfc_data.nc' - filename_sfcw: '{{BKG_YYYYmmddHHMMSS}}.fv_srf_wnd.res.nc' - filename_cplr: '{{BKG_YYYYmmddHHMMSS}}.coupler.res' + datetime: '{{ current_cycle | to_isotime }}' + filename_core: '{{ current_cycle | to_fv3time }}.fv_core.res.nc' + filename_trcr: '{{ current_cycle | to_fv3time }}.fv_tracer.res.nc' + filename_sfcd: '{{ current_cycle | to_fv3time }}.sfc_data.nc' + filename_sfcw: '{{ current_cycle | to_fv3time }}.fv_srf_wnd.res.nc' + filename_cplr: '{{ current_cycle | to_fv3time }}.coupler.res' state variables: [ua,va,t,DZ,delp,ps,sphum,ice_wat,liq_wat,o3mr,phis, slmsk,sheleg,tsea,vtype,stype,vfrac,stc,smc,snwdph, u_srf,v_srf,f10m] @@ -42,8 +42,8 @@ variational: field table filename: ./fv3jedi/field_table akbk: ./fv3jedi/akbk.nc4 layout: - - !ENV ${layout_x} - - !ENV ${layout_y} + - $(layout_x) + - $(layout_y) npx: $(npx_anl) npy: $(npy_anl) npz: $(npz_anl) @@ -59,8 +59,8 @@ variational: field table filename: ./fv3jedi/field_table akbk: ./fv3jedi/akbk.nc4 layout: - - !ENV ${layout_x} - - !ENV ${layout_y} + - $(layout_x) + - $(layout_y) npx: $(npx_anl) npy: $(npy_anl) npz: $(npz_anl) @@ -81,8 +81,8 @@ final: field table filename: ./fv3jedi/field_table akbk: ./fv3jedi/akbk.nc4 layout: - - !ENV ${layout_x} - - !ENV ${layout_y} + - $(layout_x) + - $(layout_y) npx: $(npx_anl) npy: $(npy_anl) npz: $(npz_anl) diff --git a/parm/land/hofx/hofx_nomodel.yaml b/parm/land/hofx/hofx_nomodel.yaml index 28b06e5bc..a767a0e28 100644 --- a/parm/land/hofx/hofx_nomodel.yaml +++ b/parm/land/hofx/hofx_nomodel.yaml @@ -1,30 +1,30 @@ -window begin: '{{LAND_WINDOW_BEGIN}}' +window begin: '{{ LAND_WINDOW_BEGIN | to_isotime }}' window length: $(LAND_WINDOW_LENGTH) geometry: fms initialization: - namelist filename: !ENV ${DATA}/fv3jedi/fmsmpp.nml - field table filename: !ENV ${DATA}/fv3jedi/field_table - akbk: !ENV ${DATA}/fv3jedi/akbk.nc4 + namelist filename: $(DATA)/fv3jedi/fmsmpp.nml + field table filename: $(DATA)/fv3jedi/field_table + akbk: $(DATA)/fv3jedi/akbk.nc4 layout: - - !ENV ${layout_x} - - !ENV ${layout_y} + - $(layout_x) + - $(layout_y) npx: $(npx_ges) npy: $(npy_ges) npz: $(npz_ges) - field metadata override: !ENV ${DATA}/fv3jedi/gfs-land.yaml + field metadata override: $(DATA)/fv3jedi/gfs-land.yaml time invariant fields: state fields: - datetime: '{{LAND_WINDOW_BEGIN}}' + datetime: '{{ LAND_WINDOW_BEGIN | to_isotime }}' filetype: fms restart skip coupler file: true state variables: [orog_filt] - datapath: !ENV ${FIXgfs}/fix_orog/${CASE}/ - filename_orog: !ENV ${CASE}_oro_data.nc + datapath: $(FIXgfs)/fix_orog/${CASE}/ + filename_orog: $(CASE)_oro_data.nc state: - datapath: !ENV ${DATA}/bkg + datapath: $(DATA)/bkg filetype: fms restart - datetime: '{{LAND_BKG_ISOTIME}}' - filename_sfcd: '{{LAND_BKG_YYYYmmddHHMMSS}}.sfc_data.nc' - filename_cplr: '{{LAND_BKG_YYYYmmddHHMMSS}}.coupler.res' + datetime: '{{ current_cycle | to_isotime }}' + filename_sfcd: '{{ current_cycle | to_fv3time }}.sfc_data.nc' + filename_cplr: '{{ current_cycle | to_fv3time }}.coupler.res' state variables: [snwdph,vtype,slmsk] observations: !INC ${OBS_LIST} diff --git a/parm/land/letkfoi/letkfoi.yaml b/parm/land/letkfoi/letkfoi.yaml index 294ccaf1c..0dc764978 100644 --- a/parm/land/letkfoi/letkfoi.yaml +++ b/parm/land/letkfoi/letkfoi.yaml @@ -1,37 +1,37 @@ geometry: fms initialization: - namelist filename: !ENV ${DATA}/fv3jedi/fmsmpp.nml - field table filename: !ENV ${DATA}/fv3jedi/field_table - akbk: !ENV ${DATA}/fv3jedi/akbk.nc4 + namelist filename: $(DATA)/fv3jedi/fmsmpp.nml + field table filename: $(DATA)/fv3jedi/field_table + akbk: $(DATA)/fv3jedi/akbk.nc4 layout: - - !ENV ${layout_x} - - !ENV ${layout_y} + - $(layout_x) + - $(layout_y) npx: $(npx_ges) npy: $(npy_ges) npz: $(npz_ges) - field metadata override: !ENV ${DATA}/fv3jedi/gfs-land.yaml + field metadata override: $(DATA)/fv3jedi/gfs-land.yaml time invariant fields: state fields: - datetime: '{{LAND_WINDOW_BEGIN}}' + datetime: '{{ LAND_WINDOW_BEGIN | to_isotime }}' filetype: fms restart skip coupler file: true state variables: [orog_filt] - datapath: !ENV ${FIXgfs}/orog/${CASE}/ - filename_orog: !ENV ${CASE}_oro_data.nc + datapath: $(FIXgfs)/orog/${CASE}/ + filename_orog: $(CASE)_oro_data.nc -window begin: '{{LAND_WINDOW_BEGIN}}' +window begin: '{{ LAND_WINDOW_BEGIN | to_isotime }}' window length: $(LAND_WINDOW_LENGTH) background: - datetime: '{{LAND_BKG_ISOTIME}}' + datetime: '{{ current_cycle | to_isotime }}' members from template: template: - datetime: '{{LAND_BKG_ISOTIME}}' + datetime: '{{ current_cycle | to_isotime }}' filetype: fms restart state variables: [snwdph,vtype,slmsk] - datapath: !ENV ${DATA}/bkg/mem%mem%/RESTART - filename_sfcd: '{{LAND_BKG_YYYYmmddHHMMSS}}.sfc_data.nc' - filename_cplr: '{{LAND_BKG_YYYYmmddHHMMSS}}.coupler.res' + datapath: $(DATA)/bkg/mem%mem%/RESTART + filename_sfcd: '{{ current_cycle | to_fv3time }}.sfc_data.nc' + filename_cplr: '{{ current_cycle | to_fv3time }}.coupler.res' pattern: '%mem%' nmembers: 2 zero padding: 3 @@ -51,10 +51,10 @@ local ensemble DA: mult: 1.0 output increment: - datapath: !ENV ${DATA}/anl + datapath: $(DATA)/anl prefix: landinc filetype: fms restart - filename_sfcd: '{{LAND_BKG_YYYYmmddHHMMSS}}.sfc_data.nc' - filename_cplr: '{{LAND_BKG_YYYYmmddHHMMSS}}.coupler.res' + filename_sfcd: '{{ current_cycle | to_fv3time }}.sfc_data.nc' + filename_cplr: '{{ current_cycle | to_fv3time }}.coupler.res' state variables: [snwdph,vtype,slmsk] diff --git a/parm/land/obs/config/adpsfc_snow.yaml b/parm/land/obs/config/adpsfc_snow.yaml index c26b6f965..935b8ab5f 100644 --- a/parm/land/obs/config/adpsfc_snow.yaml +++ b/parm/land/obs/config/adpsfc_snow.yaml @@ -6,11 +6,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: !ENV ${DATA}/obs/${OPREFIX}adpsfc_snow_${CDATE}.nc4 + obsfile: $(DATA)/obs/$(OPREFIX)adpsfc_snow_{{ current_cycle | to_YMDH }}.nc4 obsdataout: engine: type: H5File - obsfile: !ENV ${DATA}/diags/diag_adpsfc_snow_${CDATE}.nc4 + obsfile: $(DATA)/diags/diag_adpsfc_snow_{{ current_cycle | to_YMDH }}.nc4 simulated variables: [totalSnowDepth] obs operator: name: Composite diff --git a/parm/land/obs/config/ghcn_snow.yaml b/parm/land/obs/config/ghcn_snow.yaml index eee16c5dd..60dc82312 100644 --- a/parm/land/obs/config/ghcn_snow.yaml +++ b/parm/land/obs/config/ghcn_snow.yaml @@ -6,11 +6,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: !ENV ${DATA}/obs/${OPREFIX}ghcn_snow_${CDATE}.nc4 + obsfile: $(DATA)/obs/$(OPREFIX)ghcn_snow_{{ current_cycle | to_YMDH }}.nc4 obsdataout: engine: type: H5File - obsfile: !ENV ${DATA}/diags/diag_ghcn_snow_${CDATE}.nc4 + obsfile: $(DATA)/diags/diag_ghcn_snow_{{ current_cycle | to_YMDH }}.nc4 obs operator: name: Identity diff --git a/parm/land/obs/config/ims_snow.yaml b/parm/land/obs/config/ims_snow.yaml index 4e0cdde6a..3840144d0 100644 --- a/parm/land/obs/config/ims_snow.yaml +++ b/parm/land/obs/config/ims_snow.yaml @@ -6,11 +6,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: !ENV ${DATA}/obs/${OPREFIX}ims_snow_${CDATE}.nc4 + obsfile: $(DATA)/obs/$(OPREFIX)ims_snow_{{ current_cycle | to_YMDH }}.nc4 obsdataout: engine: type: H5File - obsfile: !ENV ${DATA}/diags/diag_ims_snow_${CDATE}.nc4 + obsfile: $(DATA)/diags/diag_ims_snow_{{ current_cycle | to_YMDH }}.nc4 simulated variables: [totalSnowDepth] obs operator: name: Identity diff --git a/parm/soca/obs/config/adt_3a_egm2008.yaml b/parm/soca/obs/config/adt_3a_egm2008.yaml index e53bf0c93..4abc3f775 100644 --- a/parm/soca/obs/config/adt_3a_egm2008.yaml +++ b/parm/soca/obs/config/adt_3a_egm2008.yaml @@ -8,7 +8,7 @@ obs space: engine: type: H5File obsfile: !ENV ${DATA}/diags/adt_3a_egm2008.${PDY}${cyc}.nc4 - simulated variables: [absolute_dynamic_topography] + simulated variables: [absoluteDynamicTopography] io pool: max pool size: 1 obs operator: @@ -18,53 +18,53 @@ obs error: obs filters: - filter: Domain Check where: - - variable: {name: sea_area_fraction@GeoVaLs} + - variable: {name: GeoVaLs/sea_area_fraction} minvalue: 0.9 - filter: Domain Check where: - - variable: { name: sea_surface_temperature@GeoVaLs} + - variable: { name: GeoVaLs/sea_surface_temperature} minvalue: 15.0 - filter: Background Check absolute threshold: 0.2 - filter: Domain Check where: - - variable: {name: sea_floor_depth_below_sea_surface@GeoVaLs} + - variable: {name: GeoVaLs/sea_floor_depth_below_sea_surface} minvalue: 500 - filter: Perform Action action: name: assign error error function: - name: LinearCombination@ObsFunction + name: ObsFunction/LinearCombination options: - variables: [mesoscale_representation_error@GeoVaLs, - absolute_dynamic_topography@ObsError] + variables: [GeoVaLs/mesoscale_representation_error, + ObsError/absoluteDynamicTopography] coefs: [0.1, 0.5] - filter: BlackList where: - variable: - name: latitude@MetaData + name: MetaData/latitude minvalue: -65 maxvalue: -30 - variable: - name: longitude@MetaData + name: MetaData/longitude minvalue: -125 maxvalue: -90 - filter: BlackList where: - variable: - name: latitude@MetaData + name: MetaData/latitude minvalue: -65 maxvalue: -30 - variable: - name: longitude@MetaData + name: MetaData/longitude minvalue: 60 maxvalue: 110 - filter: Domain Check where: - - variable: { name: sea_ice_area_fraction@GeoVaLs} + - variable: { name: GeoVaLs/sea_ice_area_fraction} maxvalue: 0.00001 - filter: Domain Check where: - - variable: {name: distance_from_coast@GeoVaLs} + - variable: {name: GeoVaLs/distance_from_coast} minvalue: 100e3 diff --git a/parm/soca/obs/config/adt_3b_egm2008.yaml b/parm/soca/obs/config/adt_3b_egm2008.yaml index 7d34067a5..53bb60a61 100644 --- a/parm/soca/obs/config/adt_3b_egm2008.yaml +++ b/parm/soca/obs/config/adt_3b_egm2008.yaml @@ -8,7 +8,7 @@ obs space: engine: type: H5File obsfile: !ENV ${DATA}/diags/adt_3b_egm2008.${PDY}${cyc}.nc4 - simulated variables: [absolute_dynamic_topography] + simulated variables: [absoluteDynamicTopography] io pool: max pool size: 1 obs operator: @@ -18,53 +18,53 @@ obs error: obs filters: - filter: Domain Check where: - - variable: {name: sea_area_fraction@GeoVaLs} + - variable: {name: GeoVaLs/sea_area_fraction} minvalue: 0.9 - filter: Domain Check where: - - variable: { name: sea_surface_temperature@GeoVaLs} + - variable: { name: GeoVaLs/sea_surface_temperature} minvalue: 15.0 - filter: Background Check absolute threshold: 0.2 - filter: Domain Check where: - - variable: {name: sea_floor_depth_below_sea_surface@GeoVaLs} + - variable: {name: GeoVaLs/sea_floor_depth_below_sea_surface} minvalue: 500 - filter: Perform Action action: name: assign error error function: - name: LinearCombination@ObsFunction + name: ObsFunction/LinearCombination options: - variables: [mesoscale_representation_error@GeoVaLs, - absolute_dynamic_topography@ObsError] + variables: [GeoVaLs/mesoscale_representation_error, + ObsError/absoluteDynamicTopography] coefs: [0.1, 0.5] - filter: BlackList where: - variable: - name: latitude@MetaData + name: MetaData/latitude minvalue: -65 maxvalue: -30 - variable: - name: longitude@MetaData + name: MetaData/longitude minvalue: -125 maxvalue: -90 - filter: BlackList where: - variable: - name: latitude@MetaData + name: MetaData/latitude minvalue: -65 maxvalue: -30 - variable: - name: longitude@MetaData + name: MetaData/longitude minvalue: 60 maxvalue: 110 - filter: Domain Check where: - - variable: { name: sea_ice_area_fraction@GeoVaLs} + - variable: { name: GeoVaLs/sea_ice_area_fraction} maxvalue: 0.00001 - filter: Domain Check where: - - variable: {name: distance_from_coast@GeoVaLs} + - variable: {name: GeoVaLs/distance_from_coast} minvalue: 100e3 diff --git a/parm/soca/obs/config/adt_6a_egm2008.yaml b/parm/soca/obs/config/adt_6a_egm2008.yaml index 5a3908686..cb9484a4b 100644 --- a/parm/soca/obs/config/adt_6a_egm2008.yaml +++ b/parm/soca/obs/config/adt_6a_egm2008.yaml @@ -8,7 +8,7 @@ obs space: engine: type: H5File obsfile: !ENV ${DATA}/diags/adt_6a_egm2008.${PDY}${cyc}.nc4 - simulated variables: [absolute_dynamic_topography] + simulated variables: [absoluteDynamicTopography] io pool: max pool size: 1 obs operator: @@ -18,53 +18,53 @@ obs error: obs filters: - filter: Domain Check where: - - variable: {name: sea_area_fraction@GeoVaLs} + - variable: {name: GeoVaLs/sea_area_fraction} minvalue: 0.9 - filter: Domain Check where: - - variable: { name: sea_surface_temperature@GeoVaLs} + - variable: { name: GeoVaLs/sea_surface_temperature} minvalue: 15.0 - filter: Background Check absolute threshold: 0.2 - filter: Domain Check where: - - variable: {name: sea_floor_depth_below_sea_surface@GeoVaLs} + - variable: {name: GeoVaLs/sea_floor_depth_below_sea_surface} minvalue: 500 - filter: Perform Action action: name: assign error error function: - name: LinearCombination@ObsFunction + name: ObsFunction/LinearCombination options: - variables: [mesoscale_representation_error@GeoVaLs, - absolute_dynamic_topography@ObsError] + variables: [GeoVaLs/mesoscale_representation_error, + ObsError/absoluteDynamicTopography] coefs: [0.1, 0.5] - filter: BlackList where: - variable: - name: latitude@MetaData + name: MetaData/latitude minvalue: -65 maxvalue: -30 - variable: - name: longitude@MetaData + name: MetaData/longitude minvalue: -125 maxvalue: -90 - filter: BlackList where: - variable: - name: latitude@MetaData + name: MetaData/latitude minvalue: -65 maxvalue: -30 - variable: - name: longitude@MetaData + name: MetaData/longitude minvalue: 60 maxvalue: 110 - filter: Domain Check where: - - variable: { name: sea_ice_area_fraction@GeoVaLs} + - variable: { name: GeoVaLs/sea_ice_area_fraction} maxvalue: 0.00001 - filter: Domain Check where: - - variable: {name: distance_from_coast@GeoVaLs} + - variable: {name: GeoVaLs/distance_from_coast} minvalue: 100e3 diff --git a/parm/soca/obs/config/adt_c2_egm2008.yaml b/parm/soca/obs/config/adt_c2_egm2008.yaml index a74511ac3..5c54f67d7 100644 --- a/parm/soca/obs/config/adt_c2_egm2008.yaml +++ b/parm/soca/obs/config/adt_c2_egm2008.yaml @@ -8,7 +8,7 @@ obs space: engine: type: H5File obsfile: !ENV ${DATA}/diags/adt_c2_egm2008.${PDY}${cyc}.nc4 - simulated variables: [absolute_dynamic_topography] + simulated variables: [absoluteDynamicTopography] io pool: max pool size: 1 obs operator: @@ -18,49 +18,53 @@ obs error: obs filters: - filter: Domain Check where: - - variable: {name: sea_area_fraction@GeoVaLs} + - variable: {name: GeoVaLs/sea_area_fraction} minvalue: 0.9 - filter: Domain Check where: - - variable: { name: sea_surface_temperature@GeoVaLs} + - variable: { name: GeoVaLs/sea_surface_temperature} minvalue: 15.0 - filter: Background Check absolute threshold: 0.2 - filter: Domain Check where: - - variable: {name: sea_floor_depth_below_sea_surface@GeoVaLs} + - variable: {name: GeoVaLs/sea_floor_depth_below_sea_surface} minvalue: 500 - filter: Perform Action action: name: assign error error function: - name: LinearCombination@ObsFunction + name: ObsFunction/LinearCombination options: - variables: [mesoscale_representation_error@GeoVaLs, - absolute_dynamic_topography@ObsError] + variables: [GeoVaLs/mesoscale_representation_error, + ObsError/absoluteDynamicTopography] coefs: [0.1, 0.5] - filter: BlackList where: - variable: - name: latitude@MetaData + name: MetaData/latitude minvalue: -65 maxvalue: -30 - variable: - name: longitude@MetaData + name: MetaData/longitude minvalue: -125 maxvalue: -90 - filter: BlackList where: - variable: - name: latitude@MetaData + name: MetaData/latitude minvalue: -65 maxvalue: -30 - variable: - name: longitude@MetaData + name: MetaData/longitude minvalue: 60 maxvalue: 110 - filter: Domain Check where: - - variable: { name: sea_ice_area_fraction@GeoVaLs} + - variable: { name: GeoVaLs/sea_ice_area_fraction} maxvalue: 0.00001 +- filter: Domain Check + where: + - variable: {name: GeoVaLs/distance_from_coast} + minvalue: 100e3 diff --git a/parm/soca/obs/config/adt_coperl4.yaml b/parm/soca/obs/config/adt_coperl4.yaml index 865202253..80229284c 100644 --- a/parm/soca/obs/config/adt_coperl4.yaml +++ b/parm/soca/obs/config/adt_coperl4.yaml @@ -8,7 +8,7 @@ obs space: engine: type: H5File obsfile: !ENV ${DATA}/diags/adt_coperl4.${PDY}${cyc}.nc4 - simulated variables: [absolute_dynamic_topography] + simulated variables: [absoluteDynamicTopography] io pool: max pool size: 1 obs operator: @@ -18,21 +18,21 @@ obs error: obs filters: - filter: Domain Check where: - - variable: {name: sea_area_fraction@GeoVaLs} + - variable: {name: GeoVaLs/sea_area_fraction} minvalue: 0.9 # Reject obs where ocean fraction is < 90% - filter: Domain Check action: name: reject where: - - variable: {name: sea_area_fraction@GeoVaLs} + - variable: {name: GeoVaLs/sea_area_fraction} minvalue: 0.9 # Passivate obs where ocean fraction is > 90% - filter: Domain Check action: name: passivate where: - - variable: {name: sea_area_fraction@GeoVaLs} + - variable: {name: GeoVaLs/sea_area_fraction} maxvalue: 0.9 # Reject obs outside of [-4.0m,4.0m] - filter: Bounds Check @@ -42,5 +42,5 @@ obs filters: maxvalue: 4.0 - filter: Domain Check where: - - variable: {name: distance_from_coast@GeoVaLs} + - variable: {name: GeoVaLs/distance_from_coast} minvalue: 100e3 diff --git a/parm/soca/obs/config/adt_j2_egm2008.yaml b/parm/soca/obs/config/adt_j2_egm2008.yaml index 9fc1167ef..e41d2e7b2 100644 --- a/parm/soca/obs/config/adt_j2_egm2008.yaml +++ b/parm/soca/obs/config/adt_j2_egm2008.yaml @@ -8,7 +8,7 @@ obs space: engine: type: H5File obsfile: !ENV ${DATA}/diags/adt_j2_egm2008.${PDY}${cyc}.nc4 - simulated variables: [absolute_dynamic_topography] + simulated variables: [absoluteDynamicTopography] io pool: max pool size: 1 obs operator: @@ -18,53 +18,53 @@ obs error: obs filters: - filter: Domain Check where: - - variable: {name: sea_area_fraction@GeoVaLs} + - variable: {name: GeoVaLs/sea_area_fraction} minvalue: 0.9 - filter: Domain Check where: - - variable: { name: sea_surface_temperature@GeoVaLs} + - variable: { name: GeoVaLs/sea_surface_temperature} minvalue: 15.0 - filter: Background Check absolute threshold: 0.2 - filter: Domain Check where: - - variable: {name: sea_floor_depth_below_sea_surface@GeoVaLs} + - variable: {name: GeoVaLs/sea_floor_depth_below_sea_surface} minvalue: 500 - filter: Perform Action action: name: assign error error function: - name: LinearCombination@ObsFunction + name: ObsFunction/LinearCombination options: - variables: [mesoscale_representation_error@GeoVaLs, - absolute_dynamic_topography@ObsError] + variables: [GeoVaLs/mesoscale_representation_error, + ObsError/absoluteDynamicTopography] coefs: [0.1, 0.5] - filter: BlackList where: - variable: - name: latitude@MetaData + name: MetaData/latitude minvalue: -65 maxvalue: -30 - variable: - name: longitude@MetaData + name: MetaData/longitude minvalue: -125 maxvalue: -90 - filter: BlackList where: - variable: - name: latitude@MetaData + name: MetaData/latitude minvalue: -65 maxvalue: -30 - variable: - name: longitude@MetaData + name: MetaData/longitude minvalue: 60 maxvalue: 110 - filter: Domain Check where: - - variable: { name: sea_ice_area_fraction@GeoVaLs} + - variable: { name: GeoVaLs/sea_ice_area_fraction} maxvalue: 0.00001 - filter: Domain Check where: - - variable: {name: distance_from_coast@GeoVaLs} + - variable: {name: GeoVaLs/distance_from_coast} minvalue: 100e3 diff --git a/parm/soca/obs/config/adt_j3_egm2008.yaml b/parm/soca/obs/config/adt_j3_egm2008.yaml index a754af743..c15727c95 100644 --- a/parm/soca/obs/config/adt_j3_egm2008.yaml +++ b/parm/soca/obs/config/adt_j3_egm2008.yaml @@ -15,4 +15,56 @@ obs operator: name: ADT obs error: covariance model: diagonal - +obs filters: +- filter: Domain Check + where: + - variable: {name: GeoVaLs/sea_area_fraction} + minvalue: 0.9 +- filter: Domain Check + where: + - variable: { name: GeoVaLs/sea_surface_temperature} + minvalue: 15.0 +- filter: Background Check + absolute threshold: 0.2 +- filter: Domain Check + where: + - variable: {name: GeoVaLs/sea_floor_depth_below_sea_surface} + minvalue: 500 +- filter: Perform Action + action: + name: assign error + error function: + name: ObsFunction/LinearCombination + options: + variables: [GeoVaLs/mesoscale_representation_error, + ObsError/absoluteDynamicTopography] + coefs: [0.1, + 0.5] +- filter: BlackList + where: + - variable: + name: MetaData/latitude + minvalue: -65 + maxvalue: -30 + - variable: + name: MetaData/longitude + minvalue: -125 + maxvalue: -90 +- filter: BlackList + where: + - variable: + name: MetaData/latitude + minvalue: -65 + maxvalue: -30 + - variable: + name: MetaData/longitude + minvalue: 60 + maxvalue: 110 +- filter: Domain Check + where: + - variable: { name: GeoVaLs/sea_ice_area_fraction} + maxvalue: 0.00001 +- filter: Domain Check + where: + - variable: {name: GeoVaLs/distance_from_coast} + minvalue: 100e3 diff --git a/parm/soca/obs/config/adt_sa_egm2008.yaml b/parm/soca/obs/config/adt_sa_egm2008.yaml index 019f67926..c853d0846 100644 --- a/parm/soca/obs/config/adt_sa_egm2008.yaml +++ b/parm/soca/obs/config/adt_sa_egm2008.yaml @@ -8,7 +8,7 @@ obs space: engine: type: H5File obsfile: !ENV ${DATA}/diags/adt_sa_egm2008.${PDY}${cyc}.nc4 - simulated variables: [absolute_dynamic_topography] + simulated variables: [absoluteDynamicTopography] io pool: max pool size: 1 obs operator: @@ -18,53 +18,53 @@ obs error: obs filters: - filter: Domain Check where: - - variable: {name: sea_area_fraction@GeoVaLs} + - variable: {name: GeoVaLs/sea_area_fraction} minvalue: 0.9 - filter: Domain Check where: - - variable: { name: sea_surface_temperature@GeoVaLs} + - variable: { name: GeoVaLs/sea_surface_temperature} minvalue: 15.0 - filter: Background Check absolute threshold: 0.2 - filter: Domain Check where: - - variable: {name: sea_floor_depth_below_sea_surface@GeoVaLs} + - variable: {name: GeoVaLs/sea_floor_depth_below_sea_surface} minvalue: 500 - filter: Perform Action action: name: assign error error function: - name: LinearCombination@ObsFunction + name: ObsFunction/LinearCombination options: - variables: [mesoscale_representation_error@GeoVaLs, - absolute_dynamic_topography@ObsError] + variables: [GeoVaLs/mesoscale_representation_error, + ObsError/absoluteDynamicTopography] coefs: [0.1, 0.5] - filter: BlackList where: - variable: - name: latitude@MetaData + name: MetaData/latitude minvalue: -65 maxvalue: -30 - variable: - name: longitude@MetaData + name: MetaData/longitude minvalue: -125 maxvalue: -90 - filter: BlackList where: - variable: - name: latitude@MetaData + name: MetaData/latitude minvalue: -65 maxvalue: -30 - variable: - name: longitude@MetaData + name: MetaData/longitude minvalue: 60 maxvalue: 110 - filter: Domain Check where: - - variable: { name: sea_ice_area_fraction@GeoVaLs} + - variable: { name: GeoVaLs/sea_ice_area_fraction} maxvalue: 0.00001 - filter: Domain Check where: - - variable: {name: distance_from_coast@GeoVaLs} + - variable: {name: GeoVaLs/distance_from_coast} minvalue: 100e3 diff --git a/parm/soca/obs/config/icec_amsr2_north.yaml b/parm/soca/obs/config/icec_amsr2_north.yaml new file mode 100644 index 000000000..5819b770e --- /dev/null +++ b/parm/soca/obs/config/icec_amsr2_north.yaml @@ -0,0 +1,43 @@ +obs space: + name: icec_ssmis_f17_north + obsdatain: + engine: + type: H5File + obsfile: !ENV ${DATA}/obs/${OPREFIX}icec_amsr2_north.${PDY}${cyc}.nc4 + obsdataout: + engine: + type: H5File + obsfile: !ENV ${DATA}/diags/icec_amsr2_north.${PDY}${cyc}.nc4 + simulated variables: [seaIceFraction] + io pool: + max pool size: 1 +obs operator: + name: Identity + observation alias file: obsop_name_map.yaml +obs error: + covariance model: diagonal +obs filters: +- filter: Domain Check + where: + - variable: {name: GeoVaLs/sea_area_fraction} + minvalue: 0.9 +- filter: Bounds Check + minvalue: 0.0 + maxvalue: 1.0 +#- filter: Background Check +# threshold: 5.0 +- filter: Domain Check + where: + - variable: { name: GeoVaLs/sea_surface_temperature} + maxvalue: 2.0 +- filter: Domain Check + where: + - variable: { name: GeoVaLs/sea_surface_temperature} + maxvalue: 0.0 + action: + name: inflate error + inflation factor: 2.0 +- filter: Domain Check + where: + - variable: {name: GeoVaLs/distance_from_coast} + minvalue: 100e3 diff --git a/parm/soca/obs/config/icec_amsr2_south.yaml b/parm/soca/obs/config/icec_amsr2_south.yaml new file mode 100644 index 000000000..89d95122b --- /dev/null +++ b/parm/soca/obs/config/icec_amsr2_south.yaml @@ -0,0 +1,43 @@ +obs space: + name: icec_ssmis_f17_south + obsdatain: + engine: + type: H5File + obsfile: !ENV ${DATA}/obs/${OPREFIX}icec_amsr2_south.${PDY}${cyc}.nc4 + obsdataout: + engine: + type: H5File + obsfile: !ENV ${DATA}/diags/icec_amsr2_south.${PDY}${cyc}.nc4 + simulated variables: [seaIceFraction] + io pool: + max pool size: 1 +obs operator: + name: Identity + observation alias file: obsop_name_map.yaml +obs error: + covariance model: diagonal +obs filters: +- filter: Domain Check + where: + - variable: {name: GeoVaLs/sea_area_fraction} + minvalue: 0.9 +- filter: Bounds Check + minvalue: 0.0 + maxvalue: 1.0 +#- filter: Background Check +# threshold: 5.0 +- filter: Domain Check + where: + - variable: { name: GeoVaLs/sea_surface_temperature} + maxvalue: 2.0 +- filter: Domain Check + where: + - variable: { name: GeoVaLs/sea_surface_temperature} + maxvalue: 0.0 + action: + name: inflate error + inflation factor: 2.0 +- filter: Domain Check + where: + - variable: {name: GeoVaLs/distance_from_coast} + minvalue: 100e3 diff --git a/parm/soca/obs/config/icec_ssmis_f17_north.yaml b/parm/soca/obs/config/icec_ssmis_f17_north.yaml index 49f4bd604..ba2d00075 100644 --- a/parm/soca/obs/config/icec_ssmis_f17_north.yaml +++ b/parm/soca/obs/config/icec_ssmis_f17_north.yaml @@ -8,35 +8,36 @@ obs space: engine: type: H5File obsfile: !ENV ${DATA}/diags/icec_ssmis_f17_north.${PDY}${cyc}.nc4 - simulated variables: [sea_ice_area_fraction] + simulated variables: [seaIceFraction] io pool: max pool size: 1 obs operator: name: Identity + observation alias file: obsop_name_map.yaml obs error: covariance model: diagonal obs filters: - filter: Domain Check where: - - variable: {name: sea_area_fraction@GeoVaLs} + - variable: {name: GeoVaLs/sea_area_fraction} minvalue: 0.9 - filter: Bounds Check minvalue: 0.0 maxvalue: 1.0 -- filter: Background Check - threshold: 5.0 +#- filter: Background Check +# threshold: 5.0 - filter: Domain Check where: - - variable: { name: sea_surface_temperature@GeoVaLs} + - variable: { name: GeoVaLs/sea_surface_temperature} maxvalue: 2.0 - filter: Domain Check where: - - variable: { name: sea_surface_temperature@GeoVaLs} + - variable: { name: GeoVaLs/sea_surface_temperature} maxvalue: 0.0 action: name: inflate error inflation factor: 2.0 - filter: Domain Check where: - - variable: {name: distance_from_coast@GeoVaLs} + - variable: {name: GeoVaLs/distance_from_coast} minvalue: 100e3 diff --git a/parm/soca/obs/config/icec_ssmis_f17_south.yaml b/parm/soca/obs/config/icec_ssmis_f17_south.yaml index d4eac7799..c31e1203c 100644 --- a/parm/soca/obs/config/icec_ssmis_f17_south.yaml +++ b/parm/soca/obs/config/icec_ssmis_f17_south.yaml @@ -8,36 +8,36 @@ obs space: engine: type: H5File obsfile: !ENV ${DATA}/diags/icec_ssmis_f17_south.${PDY}${cyc}.nc4 - simulated variables: [sea_ice_area_fraction] + simulated variables: [seaIceFraction] io pool: max pool size: 1 obs operator: name: Identity + observation alias file: obsop_name_map.yaml obs error: covariance model: diagonal obs filters: - filter: Domain Check where: - - variable: {name: sea_area_fraction@GeoVaLs} + - variable: {name: GeoVaLs/sea_area_fraction} minvalue: 0.9 - filter: Bounds Check minvalue: 0.0 maxvalue: 1.0 -- filter: Background Check - threshold: 5.0 +#- filter: Background Check +# threshold: 5.0 - filter: Domain Check where: - - variable: { name: sea_surface_temperature@GeoVaLs} + - variable: { name: GeoVaLs/sea_surface_temperature} maxvalue: 2.0 - filter: Domain Check where: - - variable: { name: sea_surface_temperature@GeoVaLs} + - variable: { name: GeoVaLs/sea_surface_temperature} maxvalue: 0.0 action: name: inflate error inflation factor: 2.0 - filter: Domain Check where: - - variable: {name: distance_from_coast@GeoVaLs} + - variable: {name: GeoVaLs/distance_from_coast} minvalue: 100e3 - diff --git a/parm/soca/obs/config/icec_ssmis_f18_north.yaml b/parm/soca/obs/config/icec_ssmis_f18_north.yaml index d4a217ab7..fe9fc6851 100644 --- a/parm/soca/obs/config/icec_ssmis_f18_north.yaml +++ b/parm/soca/obs/config/icec_ssmis_f18_north.yaml @@ -8,35 +8,36 @@ obs space: engine: type: H5File obsfile: !ENV ${DATA}/diags/icec_ssmis_f18_north.${PDY}${cyc}.nc4 - simulated variables: [sea_ice_area_fraction] + simulated variables: [seaIceFraction] io pool: max pool size: 1 obs operator: name: Identity + observation alias file: obsop_name_map.yaml obs error: covariance model: diagonal obs filters: - filter: Domain Check where: - - variable: {name: sea_area_fraction@GeoVaLs} + - variable: {name: GeoVaLs/sea_area_fraction} minvalue: 0.9 - filter: Bounds Check minvalue: 0.0 maxvalue: 1.0 -- filter: Background Check - threshold: 5.0 +#- filter: Background Check +# threshold: 5.0 - filter: Domain Check where: - - variable: { name: sea_surface_temperature@GeoVaLs} + - variable: { name: GeoVaLs/sea_surface_temperature} maxvalue: 2.0 - filter: Domain Check where: - - variable: { name: sea_surface_temperature@GeoVaLs} + - variable: { name: GeoVaLs/sea_surface_temperature} maxvalue: 0.0 action: name: inflate error inflation factor: 2.0 - filter: Domain Check where: - - variable: {name: distance_from_coast@GeoVaLs} + - variable: {name: GeoVaLs/distance_from_coast} minvalue: 100e3 diff --git a/parm/soca/obs/config/icec_ssmis_f18_south.yaml b/parm/soca/obs/config/icec_ssmis_f18_south.yaml index 4f8e7e69f..9b6c527a3 100644 --- a/parm/soca/obs/config/icec_ssmis_f18_south.yaml +++ b/parm/soca/obs/config/icec_ssmis_f18_south.yaml @@ -8,35 +8,36 @@ obs space: engine: type: H5File obsfile: !ENV ${DATA}/diags/icec_ssmis_f18_south.${PDY}${cyc}.nc4 - simulated variables: [sea_ice_area_fraction] + simulated variables: [seaIceFraction] io pool: max pool size: 1 obs operator: name: Identity + observation alias file: obsop_name_map.yaml obs error: covariance model: diagonal obs filters: - filter: Domain Check where: - - variable: {name: sea_area_fraction@GeoVaLs} + - variable: {name: GeoVaLs/sea_area_fraction} minvalue: 0.9 - filter: Bounds Check minvalue: 0.0 maxvalue: 1.0 -- filter: Background Check - threshold: 5.0 +#- filter: Background Check +# threshold: 5.0 - filter: Domain Check where: - - variable: { name: sea_surface_temperature@GeoVaLs} + - variable: { name: GeoVaLs/sea_surface_temperature} maxvalue: 2.0 - filter: Domain Check where: - - variable: { name: sea_surface_temperature@GeoVaLs} + - variable: { name: GeoVaLs/sea_surface_temperature} maxvalue: 0.0 action: name: inflate error inflation factor: 2.0 - filter: Domain Check where: - - variable: {name: distance_from_coast@GeoVaLs} + - variable: {name: GeoVaLs/distance_from_coast} minvalue: 100e3 diff --git a/parm/soca/obs/config/sst_drifter.yaml b/parm/soca/obs/config/sst_drifter.yaml index 22346bd2e..da45c8156 100644 --- a/parm/soca/obs/config/sst_drifter.yaml +++ b/parm/soca/obs/config/sst_drifter.yaml @@ -8,7 +8,7 @@ obs space: engine: type: H5File obsfile: !ENV ${DATA}/diags/sst_drifter.${PDY}${cyc}.nc4 - simulated variables: [sea_surface_temperature] + simulated variables: [seaSurfaceTemperature] io pool: max pool size: 1 obs operator: @@ -18,21 +18,21 @@ obs error: obs filters: - filter: Domain Check where: - - variable: {name: sea_area_fraction@GeoVaLs} + - variable: {name: GeoVaLs/sea_area_fraction} minvalue: 0.9 # Reject obs where ocean fraction is < 90% - filter: Domain Check action: name: reject where: - - variable: {name: sea_area_fraction@GeoVaLs} + - variable: {name: GeoVaLs/sea_area_fraction} minvalue: 0.9 # Passivate obs where ocean fraction is > 90% - filter: Domain Check action: name: passivate where: - - variable: {name: sea_area_fraction@GeoVaLs} + - variable: {name: GeoVaLs/sea_area_fraction} maxvalue: 0.9 # Reject obs outside of [-2.0C,36.0C] - filter: Bounds Check @@ -42,5 +42,5 @@ obs filters: maxvalue: 36.0 - filter: Domain Check where: - - variable: {name: distance_from_coast@GeoVaLs} + - variable: {name: GeoVaLs/distance_from_coast} minvalue: 100e3 diff --git a/parm/soca/obs/config/sst_metopa_l3u_so025.yaml b/parm/soca/obs/config/sst_metopa_l3u_so025.yaml index bd19d48a3..c1bdcd8df 100644 --- a/parm/soca/obs/config/sst_metopa_l3u_so025.yaml +++ b/parm/soca/obs/config/sst_metopa_l3u_so025.yaml @@ -8,36 +8,37 @@ obs space: engine: type: H5File obsfile: !ENV ${DATA}/diags/sst_metopa_l3u_so025.${PDY}${cyc}.nc4 - simulated variables: [sea_surface_temperature] + simulated variables: [seaSurfaceTemperature] io pool: max pool size: 1 obs operator: name: Identity + observation alias file: obsop_name_map.yaml obs error: covariance model: diagonal obs filters: - filter: Domain Check where: - - variable: {name: sea_area_fraction@GeoVaLs} + - variable: {name: GeoVaLs/sea_area_fraction} minvalue: 0.9 - filter: Bounds Check minvalue: 1.0 - maxvalue: 36.0 + maxvalue: 41.0 - filter: Background Check threshold: 5.0 - filter: Domain Check where: - - variable: {name: sea_surface_temperature@ObsError} + - variable: {name: ObsError/seaSurfaceTemperature} minvalue: 0.001 - filter: Domain Check where: - - variable: { name: sea_ice_area_fraction@GeoVaLs} + - variable: { name: GeoVaLs/sea_ice_area_fraction} maxvalue: 0.00001 - filter: Domain Check where: - - variable: {name: sea_surface_temperature@GeoVaLs} - minvalue: 3.0 + - variable: {name: GeoVaLs/sea_surface_temperature} + minvalue: 5.0 - filter: Domain Check where: - - variable: {name: distance_from_coast@GeoVaLs} + - variable: {name: GeoVaLs/distance_from_coast} minvalue: 100e3 diff --git a/parm/soca/obs/config/sst_metopb_l3u_so025.yaml b/parm/soca/obs/config/sst_metopb_l3u_so025.yaml index b8dff6619..9826b585c 100644 --- a/parm/soca/obs/config/sst_metopb_l3u_so025.yaml +++ b/parm/soca/obs/config/sst_metopb_l3u_so025.yaml @@ -8,36 +8,37 @@ obs space: engine: type: H5File obsfile: !ENV ${DATA}/diags/sst_metopb_l3u_so025.${PDY}${cyc}.nc4 - simulated variables: [sea_surface_temperature] + simulated variables: [seaSurfaceTemperature] io pool: max pool size: 1 obs operator: name: Identity + observation alias file: obsop_name_map.yaml obs error: covariance model: diagonal obs filters: - filter: Domain Check where: - - variable: {name: sea_area_fraction@GeoVaLs} + - variable: {name: GeoVaLs/sea_area_fraction} minvalue: 0.9 - filter: Bounds Check minvalue: 1.0 - maxvalue: 36.0 + maxvalue: 41.0 - filter: Background Check threshold: 5.0 - filter: Domain Check where: - - variable: {name: sea_surface_temperature@ObsError} + - variable: {name: ObsError/seaSurfaceTemperature} minvalue: 0.001 - filter: Domain Check where: - - variable: { name: sea_ice_area_fraction@GeoVaLs} + - variable: { name: GeoVaLs/sea_ice_area_fraction} maxvalue: 0.00001 - filter: Domain Check where: - - variable: {name: sea_surface_temperature@GeoVaLs} - minvalue: 3.0 + - variable: {name: GeoVaLs/sea_surface_temperature} + minvalue: 5.0 - filter: Domain Check where: - - variable: {name: distance_from_coast@GeoVaLs} + - variable: {name: GeoVaLs/distance_from_coast} minvalue: 100e3 diff --git a/parm/soca/obs/config/sst_metopc_l3u_so025.yaml b/parm/soca/obs/config/sst_metopc_l3u_so025.yaml index b90e53c73..9f1a22f0b 100644 --- a/parm/soca/obs/config/sst_metopc_l3u_so025.yaml +++ b/parm/soca/obs/config/sst_metopc_l3u_so025.yaml @@ -8,36 +8,37 @@ obs space: engine: type: H5File obsfile: !ENV ${DATA}/diags/sst_metopc_l3u_so025.${PDY}${cyc}.nc4 - simulated variables: [sea_surface_temperature] + simulated variables: [seaSurfaceTemperature] io pool: max pool size: 1 obs operator: name: Identity + observation alias file: obsop_name_map.yaml obs error: covariance model: diagonal obs filters: - filter: Domain Check where: - - variable: {name: sea_area_fraction@GeoVaLs} + - variable: {name: GeoVaLs/sea_area_fraction} minvalue: 0.9 - filter: Bounds Check minvalue: 1.0 - maxvalue: 36.0 + maxvalue: 41.0 - filter: Background Check threshold: 5.0 - filter: Domain Check where: - - variable: {name: sea_surface_temperature@ObsError} + - variable: {name: ObsError/seaSurfaceTemperature} minvalue: 0.001 - filter: Domain Check where: - - variable: { name: sea_ice_area_fraction@GeoVaLs} + - variable: { name: GeoVaLs/sea_ice_area_fraction} maxvalue: 0.00001 - filter: Domain Check where: - - variable: {name: sea_surface_temperature@GeoVaLs} - minvalue: 3.0 + - variable: {name: GeoVaLs/sea_surface_temperature} + minvalue: 5.0 - filter: Domain Check where: - - variable: {name: distance_from_coast@GeoVaLs} + - variable: {name: GeoVaLs/distance_from_coast} minvalue: 100e3 diff --git a/parm/soca/obs/config/sst_noaa18_l3u_so025.yaml b/parm/soca/obs/config/sst_noaa18_l3u_so025.yaml index b14deff65..01a960034 100644 --- a/parm/soca/obs/config/sst_noaa18_l3u_so025.yaml +++ b/parm/soca/obs/config/sst_noaa18_l3u_so025.yaml @@ -8,36 +8,37 @@ obs space: engine: type: H5File obsfile: !ENV ${DATA}/diags/sst_noaa18_l3u_so025.${PDY}${cyc}.nc4 - simulated variables: [sea_surface_temperature] + simulated variables: [seaSurfaceTemperature] io pool: max pool size: 1 obs operator: name: Identity + observation alias file: obsop_name_map.yaml obs error: covariance model: diagonal obs filters: - filter: Domain Check where: - - variable: {name: sea_area_fraction@GeoVaLs} + - variable: {name: GeoVaLs/sea_area_fraction} minvalue: 0.9 - filter: Bounds Check minvalue: 1.0 - maxvalue: 36.0 + maxvalue: 41.0 - filter: Background Check threshold: 5.0 - filter: Domain Check where: - - variable: {name: sea_surface_temperature@ObsError} + - variable: {name: ObsError/seaSurfaceTemperature} minvalue: 0.001 - filter: Domain Check where: - - variable: { name: sea_ice_area_fraction@GeoVaLs} + - variable: { name: GeoVaLs/sea_ice_area_fraction} maxvalue: 0.00001 - filter: Domain Check where: - - variable: {name: sea_surface_temperature@GeoVaLs} - minvalue: 3.0 + - variable: {name: GeoVaLs/sea_surface_temperature} + minvalue: 5.0 - filter: Domain Check where: - - variable: {name: distance_from_coast@GeoVaLs} + - variable: {name: GeoVaLs/distance_from_coast} minvalue: 100e3 diff --git a/parm/soca/obs/config/sst_noaa19_l3u_so025.yaml b/parm/soca/obs/config/sst_noaa19_l3u_so025.yaml index 264acc504..7a51edb62 100644 --- a/parm/soca/obs/config/sst_noaa19_l3u_so025.yaml +++ b/parm/soca/obs/config/sst_noaa19_l3u_so025.yaml @@ -8,34 +8,36 @@ obs space: engine: type: H5File obsfile: !ENV ${DATA}/diags/sst_noaa19_l3u_so025.${PDY}${cyc}.nc4 - simulated variables: [sea_surface_temperature] + simulated variables: [seaSurfaceTemperature] io pool: max pool size: 1 obs operator: name: Identity +obs error: + covariance model: diagonal obs filters: - filter: Domain Check where: - - variable: {name: sea_area_fraction@GeoVaLs} + - variable: {name: GeoVaLs/sea_area_fraction} minvalue: 0.9 - filter: Bounds Check minvalue: 1.0 - maxvalue: 36.0 + maxvalue: 41.0 - filter: Background Check threshold: 5.0 - filter: Domain Check where: - - variable: {name: sea_surface_temperature@ObsError} + - variable: {name: ObsError/seaSurfaceTemperature} minvalue: 0.001 - filter: Domain Check where: - - variable: { name: sea_ice_area_fraction@GeoVaLs} + - variable: { name: GeoVaLs/sea_ice_area_fraction} maxvalue: 0.00001 - filter: Domain Check where: - - variable: {name: sea_surface_temperature@GeoVaLs} - minvalue: 3.0 + - variable: {name: GeoVaLs/sea_surface_temperature} + minvalue: 5.0 - filter: Domain Check where: - - variable: {name: distance_from_coast@GeoVaLs} + - variable: {name: GeoVaLs/distance_from_coast} minvalue: 100e3 diff --git a/parm/soca/obs/config/sst_viirs_n20_l3u_so025.yaml b/parm/soca/obs/config/sst_viirs_n20_l3u_so025.yaml index 3b91c7359..530e7db5a 100644 --- a/parm/soca/obs/config/sst_viirs_n20_l3u_so025.yaml +++ b/parm/soca/obs/config/sst_viirs_n20_l3u_so025.yaml @@ -8,36 +8,37 @@ obs space: engine: type: H5File obsfile: !ENV ${DATA}/diags/sst_viirs_n20_l3u_so025.${PDY}${cyc}.nc4 - simulated variables: [sea_surface_temperature] + simulated variables: [seaSurfaceTemperature] io pool: max pool size: 1 obs operator: name: Identity + observation alias file: obsop_name_map.yaml obs error: covariance model: diagonal obs filters: - filter: Domain Check where: - - variable: {name: sea_area_fraction@GeoVaLs} + - variable: {name: GeoVaLs/sea_area_fraction} minvalue: 0.9 - filter: Bounds Check minvalue: 1.0 - maxvalue: 36.0 + maxvalue: 41.0 - filter: Background Check threshold: 5.0 - filter: Domain Check where: - - variable: {name: sea_surface_temperature@ObsError} + - variable: {name: ObsError/seaSurfaceTemperature} minvalue: 0.001 - filter: Domain Check where: - - variable: { name: sea_ice_area_fraction@GeoVaLs} + - variable: { name: GeoVaLs/sea_ice_area_fraction} maxvalue: 0.00001 - filter: Domain Check where: - - variable: {name: sea_surface_temperature@GeoVaLs} - minvalue: 3.0 + - variable: {name: GeoVaLs/sea_surface_temperature} + minvalue: 5.0 - filter: Domain Check where: - - variable: {name: distance_from_coast@GeoVaLs} + - variable: {name: GeoVaLs/distance_from_coast} minvalue: 100e3 diff --git a/parm/soca/obs/config/sst_viirs_npp_l3u_so025.yaml b/parm/soca/obs/config/sst_viirs_npp_l3u_so025.yaml index e3e41d820..2b2bbcafb 100644 --- a/parm/soca/obs/config/sst_viirs_npp_l3u_so025.yaml +++ b/parm/soca/obs/config/sst_viirs_npp_l3u_so025.yaml @@ -8,36 +8,37 @@ obs space: engine: type: H5File obsfile: !ENV ${DATA}/diags/sst_viirs_npp_l3u_so025.${PDY}${cyc}.nc4 - simulated variables: [sea_surface_temperature] + simulated variables: [seaSurfaceTemperature] io pool: max pool size: 1 obs operator: name: Identity + observation alias file: obsop_name_map.yaml obs error: covariance model: diagonal obs filters: - filter: Domain Check where: - - variable: {name: sea_area_fraction@GeoVaLs} + - variable: {name: GeoVaLs/sea_area_fraction} minvalue: 0.9 - filter: Bounds Check minvalue: 1.0 - maxvalue: 36.0 + maxvalue: 41.0 - filter: Background Check threshold: 5.0 - filter: Domain Check where: - - variable: {name: sea_surface_temperature@ObsError} + - variable: {name: ObsError/seaSurfaceTemperature} minvalue: 0.001 - filter: Domain Check where: - - variable: { name: sea_ice_area_fraction@GeoVaLs} + - variable: { name: GeoVaLs/sea_ice_area_fraction} maxvalue: 0.00001 - filter: Domain Check where: - - variable: {name: sea_surface_temperature@GeoVaLs} - minvalue: 3.0 + - variable: {name: GeoVaLs/sea_surface_temperature} + minvalue: 5.0 - filter: Domain Check where: - - variable: {name: distance_from_coast@GeoVaLs} + - variable: {name: GeoVaLs/distance_from_coast} minvalue: 100e3 diff --git a/test/atm/CMakeLists.txt b/test/atm/CMakeLists.txt index 55e1dbc4c..117e2ea8e 100644 --- a/test/atm/CMakeLists.txt +++ b/test/atm/CMakeLists.txt @@ -25,21 +25,6 @@ if (BUILD_GDASBUNDLE) ENVIRONMENT "PYTHONPATH=${PROJECT_BINARY_DIR}/ush:$ENV{PYTHONPATH}" ) - # test for ush/run_jedi_exe.py 3dhofx - add_test(NAME test_gdasapp_run_jedi_exe_3dhofx - COMMAND ${PROJECT_SOURCE_DIR}/test/atm/run_jedi_exe_3dhofx.sh ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} - WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/) - - # test for ush/run_jedi_exe.py 3dvar - add_test(NAME test_gdasapp_run_jedi_exe_3dvar - COMMAND ${PROJECT_SOURCE_DIR}/test/atm/run_jedi_exe_3dvar.sh ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} - WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/) - - # test for ush/run_jedi_exe.py letkf - add_test(NAME test_gdasapp_run_jedi_exe_letkf - COMMAND ${PROJECT_SOURCE_DIR}/test/atm/run_jedi_exe_letkf.sh ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} - WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/) - # Test exgdas scripts from the global-worflow if (WORKFLOW_TESTS) add_subdirectory(global-workflow) diff --git a/test/atm/global-workflow/CMakeLists.txt b/test/atm/global-workflow/CMakeLists.txt index c31571c58..850983f97 100644 --- a/test/atm/global-workflow/CMakeLists.txt +++ b/test/atm/global-workflow/CMakeLists.txt @@ -1,13 +1,37 @@ # test for creating an experiment directory within the global-workflow file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun) +# test for ush/run_jedi_exe.py 3dhofx +add_test(NAME test_gdasapp_run_jedi_exe_3dhofx + COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/run_jedi_exe_3dhofx.sh + ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} + WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/) + +# test for ush/run_jedi_exe.py 3dvar +add_test(NAME test_gdasapp_run_jedi_exe_3dvar + COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/run_jedi_exe_3dvar.sh + ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} + WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/) + +# test for ush/run_jedi_exe.py 3denvar +add_test(NAME test_gdasapp_run_jedi_exe_3denvar + COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/run_jedi_exe_3denvar.sh + ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} + WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/) + +# test for ush/run_jedi_exe.py letkf +add_test(NAME test_gdasapp_run_jedi_exe_letkf + COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/run_jedi_exe_letkf.sh + ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} + WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/) + add_test(NAME test_gdasapp_setup_atm_cycled_exp COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/setup_workflow_exp.sh ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun) -add_test(NAME test_gdasapp_atm_jjob_var_prep - COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/jjob_var_prep.sh +add_test(NAME test_gdasapp_atm_jjob_var_init + COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/jjob_var_init.sh ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun) @@ -16,13 +40,13 @@ add_test(NAME test_gdasapp_atm_jjob_var_run ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun) -add_test(NAME test_gdasapp_atm_jjob_var_post - COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/jjob_var_post.sh +add_test(NAME test_gdasapp_atm_jjob_var_final + COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/jjob_var_final.sh ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun) -add_test(NAME test_gdasapp_atm_jjob_ens_prep - COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/jjob_ens_prep.sh +add_test(NAME test_gdasapp_atm_jjob_ens_init + COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/jjob_ens_init.sh ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun) @@ -31,7 +55,7 @@ add_test(NAME test_gdasapp_atm_jjob_ens_run ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun) -add_test(NAME test_gdasapp_atm_jjob_ens_post - COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/jjob_ens_post.sh +add_test(NAME test_gdasapp_atm_jjob_ens_final + COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/jjob_ens_final.sh ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun) diff --git a/test/atm/global-workflow/config.atmanal b/test/atm/global-workflow/config.atmanl similarity index 76% rename from test/atm/global-workflow/config.atmanal rename to test/atm/global-workflow/config.atmanl index 0ee19fb54..d7039cd8d 100755 --- a/test/atm/global-workflow/config.atmanal +++ b/test/atm/global-workflow/config.atmanl @@ -1,9 +1,9 @@ #!/bin/bash -x -########## config.atmanal ########## +########## config.atmanl ########## # configuration common to all atm analysis tasks -echo "BEGIN: config.atmanal" +echo "BEGIN: config.atmanl" export OBS_YAML_DIR=${HOMEgfs}/sorc/gdas.cd/parm/atm/obs/config/ export OBS_LIST=@OBS_LIST@ @@ -19,7 +19,13 @@ export R2D2_BC_DUMP='oper_gdas' export R2D2_ARCH_DB='local' export INTERP_METHOD='barycentric' +export layout_x=1 +export layout_y=1 + export io_layout_x=1 export io_layout_y=1 -echo "END: config.atmanal" +export JEDIEXE=${HOMEgfs}/exec/fv3jedi_var.x +export crtm_VERSION="2.3.0" + +echo "END: config.atmanl" diff --git a/test/atm/global-workflow/config.base.emc.dyn b/test/atm/global-workflow/config.base.emc.dyn index 47d8bbc96..1c7239931 100755 --- a/test/atm/global-workflow/config.base.emc.dyn +++ b/test/atm/global-workflow/config.base.emc.dyn @@ -40,7 +40,7 @@ export FIXreg2grb2=$HOMEgfs/fix/reg2grb2 export PACKAGEROOT="@PACKAGEROOT@" # TODO: set via prod_envir in Ops export COMROOT="@COMROOT@" # TODO: set via prod_envir in Ops export COMINsyn="@COMINsyn@" -export DMPDIR="@DMPDIR@" +export DMPDIR=@DUMPDIR@ # USER specific paths export HOMEDIR="@HOMEDIR@" @@ -109,7 +109,7 @@ export DUMP_SUFFIX="" if [[ "$CDATE" -ge "2019092100" && "$CDATE" -le "2019110700" ]]; then export DUMP_SUFFIX="p" # Use dumps from NCO GFS v15.3 parallel fi -export DATAROOT="@DATAPATH@/RUNDIR/${jobid}" # TODO: set via prod_envir in Ops +export DATAROOT="@DATAPATH@/RUNDIR" # TODO: set via prod_envir in Ops export RUNDIR="${DATAROOT}" # TODO: Should be removed; use DATAROOT instead export ARCDIR="$NOSCRUB/archive/$PSLOT" export ICSDIR="@ICSDIR@" @@ -119,8 +119,12 @@ export ATARDIR="@ATARDIR@" export envir=${envir:-"prod"} export NET="gfs" export RUN=${RUN:-${CDUMP:-"gfs"}} -export COMIN_OBS=${DMPDIR}/${CDUMP}.${PDY}/$cyc/atmos -export COMIN_GES_OBS=${DMPDIR}/${CDUMP}.${PDY}/$cyc/atmos +CDUMP_OBS="${CDUMP}" +if [ "${CDUMP_OBS}" = "enkfgdas" ]; then + CDUMP_OBS="gdas" +fi +export COMIN_OBS=${DMPDIR}/${CDUMP_OBS}.${PDY}/$cyc/atmos +export COMIN_GES_OBS=${DMPDIR}/${CDUMP_OBS}.${PDY}/$cyc/atmos export COMINatmos=${ROTDIR}/${CDUMP}.${PDY}/${cyc}/atmos export COMOUTatmos=${ROTDIR}/${CDUMP}.${PDY}/${cyc}/atmos export COMINwave=${ROTDIR}/${CDUMP}.${PDY}/${cyc}/wave diff --git a/test/atm/global-workflow/config.yaml b/test/atm/global-workflow/config.yaml index b406303a7..f3c699f8b 100644 --- a/test/atm/global-workflow/config.yaml +++ b/test/atm/global-workflow/config.yaml @@ -4,5 +4,6 @@ base: DO_JEDIENS: "YES" UTILROOT: /apps/contrib/NCEP/libs/hpc-stack/intel-2018.4/prod_util/1.2.2 DATAPATH: "@bindir@/test/atm/global-workflow/testrun" -atmanal: + DUMPDIR: "@dumpdir@" +atmanl: OBS_LIST: "@srcdir@/test/atm/global-workflow/gdas_prototype.yaml" diff --git a/test/atm/global-workflow/gdas_prototype.yaml b/test/atm/global-workflow/gdas_prototype.yaml index fd867bcd0..18d64e54e 100644 --- a/test/atm/global-workflow/gdas_prototype.yaml +++ b/test/atm/global-workflow/gdas_prototype.yaml @@ -1,6 +1,6 @@ observers: - !INC ${OBS_YAML_DIR}/amsua_n19.yaml -##- !INC ${OBS_YAML_DIR}/sondes.yaml +- !INC ${OBS_YAML_DIR}/sondes.yaml ##- !INC ${OBS_YAML_DIR}/atms_n20.yaml ##- !INC ${OBS_YAML_DIR}/aircraft.yaml ##- !INC ${OBS_YAML_DIR}/satwind.yaml diff --git a/test/atm/global-workflow/jjob_var_post.sh b/test/atm/global-workflow/jjob_ens_final.sh similarity index 65% rename from test/atm/global-workflow/jjob_var_post.sh rename to test/atm/global-workflow/jjob_ens_final.sh index 815220e88..b1197631d 100755 --- a/test/atm/global-workflow/jjob_var_post.sh +++ b/test/atm/global-workflow/jjob_ens_final.sh @@ -12,19 +12,27 @@ export PDY=20210323 export cyc=18 export CDATE=${PDY}${cyc} export ROTDIR=$bindir/test/atm/global-workflow/testrun/ROTDIRS -export CDUMP=gdas +export RUN=enkfgdas +export CDUMP=enkfgdas export DATAROOT=$bindir/test/atm/global-workflow/testrun/RUNDIR export COMIN_GES=${bindir}/test/atm/bkg export pid=${pid:-$$} export jobid=$pid export COMROOT=$DATAROOT +export NMEM_ENKF=3 export ACCOUNT=da-cpu +export DO_JEDIENS="YES" + +# setup python path for workflow utilities and tasks +pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src" +PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}" +export PYTHONPATH # detemine machine from config.base machine=$(echo `grep 'machine=' $EXPDIR/config.base | cut -d"=" -f2` | tr -d '"') if [ $machine != 'HERA' ]; then - ${HOMEgfs}/jobs/JGDAS_GLOBAL_ATMOS_ANALYSIS_POST + ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE else - sbatch -n 1 --account=$ACCOUNT --qos=debug --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGDAS_GLOBAL_ATMOS_ANALYSIS_POST + sbatch -n 1 --account=$ACCOUNT --qos=debug --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE fi diff --git a/test/atm/global-workflow/jjob_ens_prep.sh b/test/atm/global-workflow/jjob_ens_init.sh similarity index 61% rename from test/atm/global-workflow/jjob_ens_prep.sh rename to test/atm/global-workflow/jjob_ens_init.sh index b1b0a3c3f..fd27980be 100755 --- a/test/atm/global-workflow/jjob_ens_prep.sh +++ b/test/atm/global-workflow/jjob_ens_init.sh @@ -12,7 +12,8 @@ export PDY=20210323 export cyc=18 export CDATE=${PDY}${cyc} export ROTDIR=$bindir/test/atm/global-workflow/testrun/ROTDIRS -export CDUMP=gdas +export RUN=enkfgdas +export CDUMP=enkfgdas export DATAROOT=$bindir/test/atm/global-workflow/testrun/RUNDIR export COMIN_GES=${bindir}/test/atm/bkg export pid=${pid:-$$} @@ -21,6 +22,11 @@ export COMROOT=$DATAROOT export NMEM_ENKF=3 export ACCOUNT=da-cpu +# setup python path for workflow utilities and tasks +pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src" +PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}" +export PYTHONPATH + # detemine machine from config.base machine=$(echo `grep 'machine=' $EXPDIR/config.base | cut -d"=" -f2` | tr -d '"') @@ -28,23 +34,29 @@ machine=$(echo `grep 'machine=' $EXPDIR/config.base | cut -d"=" -f2` | tr -d '"' GDATE=`date +%Y%m%d%H -d "${CDATE:0:8} ${CDATE:8:2} - 6 hours"` gPDY=$(echo $GDATE | cut -c1-8) gcyc=$(echo $GDATE | cut -c9-10) -mkdir -p $ROTDIR/gdas_test/enkfgdas.$gPDY/$gcyc/atmos/ + +mkdir -p $ROTDIR/gdas_test/gdas.$gPDY/$gcyc/atmos/ +flist="satbias tlapse" +for file in $flist; do + cp -r $GDASAPP_TESTDATA/lowres/gdas.$gPDY/$gcyc/atmos/*${file}* $ROTDIR/gdas_test/gdas.$gPDY/$gcyc/atmos/ +done # Copy tiled ges and atmf006 files to ROTDIR +mkdir -p $ROTDIR/gdas_test/enkfgdas.$gPDY/$gcyc/atmos/ for imem in $(seq 1 $NMEM_ENKF); do memchar="mem"$(printf %03i $imem) - source=$GDASAPP_TESTDATA/lowres/enkfgdas.$gPDY/$gcyc/atmos/$memchar - target=$ROTDIR/gdas_test/enkfgdas.$gPDY/$gcyc/atmos/$memchar + source=$GDASAPP_TESTDATA/lowres/enkfgdas.$gPDY/$gcyc/$memchar/atmos + target=$ROTDIR/gdas_test/enkfgdas.$gPDY/$gcyc/$memchar/atmos mkdir -p $target rm -rf $target/RESTART ln -fs $source/RESTART $target/ - ln -fs $source/gdas.t${gcyc}z.atmf006.nc $target/ + ln -fs $source/enkfgdas.t${gcyc}z.atmf006.nc $target/ done if [ $machine != 'HERA' ]; then - ${HOMEgfs}/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL_PREP + ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE else - sbatch -n 1 --account=$ACCOUNT --qos=debug --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL_PREP + sbatch -n 1 --account=$ACCOUNT --qos=debug --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE fi diff --git a/test/atm/global-workflow/jjob_ens_run.sh b/test/atm/global-workflow/jjob_ens_run.sh index 35f7d5ec7..d3a5899ab 100755 --- a/test/atm/global-workflow/jjob_ens_run.sh +++ b/test/atm/global-workflow/jjob_ens_run.sh @@ -15,7 +15,8 @@ export gPDY=20210323 export gcyc=12 export GDATE=${gPDY}${gcyc} export ROTDIR=$bindir/test/atm/global-workflow/testrun/ROTDIRS -export CDUMP=gdas +export RUN=enkfgdas +export CDUMP=enkfgdas export DATAROOT=$bindir/test/atm/global-workflow/testrun/RUNDIR export COMIN_GES=${bindir}/test/atm/bkg export pid=${pid:-$$} @@ -25,13 +26,18 @@ export NMEM_ENKF=3 export ACCOUNT=da-cpu export DO_JEDIENS="YES" +# setup python path for workflow utilities and tasks +pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src" +PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}" +export PYTHONPATH + # detemine machine from config.base machine=$(echo `grep 'machine=' $EXPDIR/config.base | cut -d"=" -f2` | tr -d '"') if [ $machine = 'HERA' -o $machine = 'ORION' ]; then - sbatch --nodes=1 --ntasks=36 --account=$ACCOUNT --qos=debug --time=00:30:00 --export=ALL --wait ${HOMEgfs}/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL_RUN + sbatch --nodes=1 --ntasks=36 --account=$ACCOUNT --qos=debug --time=00:30:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN else - ${HOMEgfs}/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL_RUN + ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN fi diff --git a/test/atm/global-workflow/jjob_ens_post.sh b/test/atm/global-workflow/jjob_var_final.sh similarity index 70% rename from test/atm/global-workflow/jjob_ens_post.sh rename to test/atm/global-workflow/jjob_var_final.sh index 06c8b81b3..c3e61e7a0 100755 --- a/test/atm/global-workflow/jjob_ens_post.sh +++ b/test/atm/global-workflow/jjob_var_final.sh @@ -12,21 +12,25 @@ export PDY=20210323 export cyc=18 export CDATE=${PDY}${cyc} export ROTDIR=$bindir/test/atm/global-workflow/testrun/ROTDIRS +export RUN=gdas export CDUMP=gdas export DATAROOT=$bindir/test/atm/global-workflow/testrun/RUNDIR export COMIN_GES=${bindir}/test/atm/bkg export pid=${pid:-$$} export jobid=$pid export COMROOT=$DATAROOT -export NMEM_ENKF=3 export ACCOUNT=da-cpu -export DO_JEDIENS="YES" + +# setup python path for workflow utilities and tasks +pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src" +PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}" +export PYTHONPATH # detemine machine from config.base machine=$(echo `grep 'machine=' $EXPDIR/config.base | cut -d"=" -f2` | tr -d '"') if [ $machine != 'HERA' ]; then - ${HOMEgfs}/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL_POST + ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_FINALIZE else - sbatch -n 1 --account=$ACCOUNT --qos=debug --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL_POST + sbatch -n 1 --account=$ACCOUNT --qos=debug --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_FINALIZE fi diff --git a/test/atm/global-workflow/jjob_var_prep.sh b/test/atm/global-workflow/jjob_var_init.sh similarity index 75% rename from test/atm/global-workflow/jjob_var_prep.sh rename to test/atm/global-workflow/jjob_var_init.sh index c4d2a89ee..c0b875502 100755 --- a/test/atm/global-workflow/jjob_var_prep.sh +++ b/test/atm/global-workflow/jjob_var_init.sh @@ -12,6 +12,7 @@ export PDY=20210323 export cyc=18 export CDATE=${PDY}${cyc} export ROTDIR=$bindir/test/atm/global-workflow/testrun/ROTDIRS +export RUN=gdas export CDUMP=gdas export DATAROOT=$bindir/test/atm/global-workflow/testrun/RUNDIR export COMIN_GES=${bindir}/test/atm/bkg @@ -20,6 +21,11 @@ export jobid=$pid export COMROOT=$DATAROOT export ACCOUNT=da-cpu +# setup python path for workflow utilities and tasks +pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src" +PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}" +export PYTHONPATH + # detemine machine from config.base machine=$(echo `grep 'machine=' $EXPDIR/config.base | cut -d"=" -f2` | tr -d '"') @@ -29,13 +35,13 @@ gPDY=$(echo $GDATE | cut -c1-8) gcyc=$(echo $GDATE | cut -c9-10) mkdir -p $ROTDIR/gdas_test/gdas.$gPDY/$gcyc/atmos/ -flist="abias atmf006 RESTART" +flist="abias atmf006 satbias tlapse RESTART" for file in $flist; do cp -r $GDASAPP_TESTDATA/lowres/gdas.$gPDY/$gcyc/atmos/*${file}* $ROTDIR/gdas_test/gdas.$gPDY/$gcyc/atmos/ done if [ $machine != 'HERA' ]; then - ${HOMEgfs}/jobs/JGDAS_GLOBAL_ATMOS_ANALYSIS_PREP + ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_INITIALIZE else - sbatch -n 1 --account=$ACCOUNT --qos=debug --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGDAS_GLOBAL_ATMOS_ANALYSIS_PREP + sbatch -n 1 --account=$ACCOUNT --qos=debug --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_INITIALIZE fi diff --git a/test/atm/global-workflow/jjob_var_run.sh b/test/atm/global-workflow/jjob_var_run.sh index 30da673b8..eb44fa707 100755 --- a/test/atm/global-workflow/jjob_var_run.sh +++ b/test/atm/global-workflow/jjob_var_run.sh @@ -15,6 +15,7 @@ export gPDY=20210323 export gcyc=12 export GDATE=${gPDY}${gcyc} export ROTDIR=$bindir/test/atm/global-workflow/testrun/ROTDIRS +export RUN=gdas export CDUMP=gdas export DATAROOT=$bindir/test/atm/global-workflow/testrun/RUNDIR export COMIN_GES=${bindir}/test/atm/bkg @@ -23,11 +24,16 @@ export jobid=$pid export COMROOT=$DATAROOT export ACCOUNT=da-cpu +# setup python path for workflow utilities and tasks +pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src" +PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}" +export PYTHONPATH + # detemine machine from config.base machine=$(echo `grep 'machine=' $EXPDIR/config.base | cut -d"=" -f2` | tr -d '"') if [ $machine != 'HERA' ]; then - ${HOMEgfs}/jobs/JGDAS_GLOBAL_ATMOS_ANALYSIS_RUN + ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_RUN else - sbatch -n 6 --account=$ACCOUNT --qos=debug --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGDAS_GLOBAL_ATMOS_ANALYSIS_RUN + sbatch -n 6 --account=$ACCOUNT --qos=debug --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_RUN fi diff --git a/test/atm/global-workflow/run_jedi_exe_3denvar.sh b/test/atm/global-workflow/run_jedi_exe_3denvar.sh new file mode 100755 index 000000000..faa6d09dc --- /dev/null +++ b/test/atm/global-workflow/run_jedi_exe_3denvar.sh @@ -0,0 +1,124 @@ +#!/bin/bash +set -x + +bindir=$1 +srcdir=$2 + +# Identify machine +if [[ -d /scratch1 ]] ; then + machine="hera" +elif [[ -d /work ]] ; then + machine="orion" +else + echo "UNSUPPORTED MACHINE. ABORT" + exit 99 +fi + +# Load modules +set +x +module use ${srcdir}/modulefiles +module load GDAS/${machine} +set -x +module list + +# Set machine dependent variables +if [ "$machine" = "hera" ] ; then + partition="hera" + gdasfix="/scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/fix" +elif [ "$machine" = "orion" ]; then + partition="debug" + gdasfix="/work2/noaa/da/cmartin/GDASApp/fix" +fi + +# Setup python path for workflow utilities and tasks +export HOMEgfs=$srcdir/../../ # TODO: HOMEgfs had to be hard-coded in config +echo $HOMEgfs +pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src" +PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}" +export PYTHONPATH + +# Create test run directory +mkdir -p ${bindir}/test/atm/global-workflow/testrun/gdas_single_test_3denvar +cd ${bindir}/test/atm/global-workflow/testrun/gdas_single_test_3denvar + +# Create input yaml +cat > ./3denvar_example.yaml << EOF +working directory: ./ +GDASApp home: ${srcdir} +GDASApp mode: variational +template: ${srcdir}/parm/atm/variational/3dvar_dripcg.yaml +config: + berror_yaml: ${srcdir}/parm/atm/berror/hybvar_gsibec.yaml + obs_dir: obs + diag_dir: diags + crtm_coeff_dir: crtm + bias_in_dir: obs + bias_out_dir: bc + obs_yaml_dir: ${srcdir}/parm/atm/obs/config + executable: ${bindir}/bin/fv3jedi_var.x + obs_list: ${srcdir}/parm/atm/obs/lists/gdas_prototype_3d.yaml + gdas_fix_root: ${gdasfix} + atm: true + layout_x: 1 + layout_y: 1 + atm_window_length: PT6H + valid_time: 2021-12-21T06:00:00Z + dump: gdas + case: C96 + case_anl: C48 + case_enkf: C48 + staticb_type: gsibec + dohybvar: true + levs: 128 + nmem: 10 + interp_method: barycentric +job options: + machine: ${machine} + account: da-cpu + queue: debug + partition: ${partition} + walltime: '30:00' + ntasks: 6 + modulepath: ${srcdir}/modulefiles +EOF + +# Execute run_jedi_exe.py +if [ -e stdout.txt ]; then + rm -f stdout.txt +fi +${srcdir}/ush/run_jedi_exe.py -c ./3denvar_example.yaml > stdout.txt 2>&1 +rc=$? +if [ $rc -ne 0 ]; then + exit $rc +fi + +# Wait and ensure buffers flushed to disk +sleep 5 +sync stdout.txt + +# Check for job submission error +error=$(grep "sbatch: error" stdout.txt | wc -l) +if [ $error -ne 0 ]; then + rc=$error + exit $rc +fi + +# Cancel submitted job +jobid=$(grep "Submitted" stdout.txt | awk -F' ' '{print $4}') +scancel $jobid +rc=$? +if [ $rc -ne 0 ]; then + exit $rc +fi + +# Check for valid yaml files +ylist="3denvar_example.yaml gdas_variational.yaml" +for yfile in $ylist; do + python3 -c 'import yaml, sys; yaml.safe_load(sys.stdin)' < $yfile + rc=$? + if [ $rc -ne 0 ]; then + exit $rc + fi +done + +exit $rc diff --git a/test/atm/run_jedi_exe_3dhofx.sh b/test/atm/global-workflow/run_jedi_exe_3dhofx.sh similarity index 88% rename from test/atm/run_jedi_exe_3dhofx.sh rename to test/atm/global-workflow/run_jedi_exe_3dhofx.sh index 84a9b057a..f55fb6415 100755 --- a/test/atm/run_jedi_exe_3dhofx.sh +++ b/test/atm/global-workflow/run_jedi_exe_3dhofx.sh @@ -30,6 +30,13 @@ elif [ "$machine" = "orion" ]; then gdasfix="/work2/noaa/da/cmartin/GDASApp/fix" fi +# Setup python path for workflow utilities and tasks +export HOMEgfs=$srcdir/../../ # TODO: HOMEgfs had to be hard-coded in config +echo $HOMEgfs +pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src" +PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}" +export PYTHONPATH + # Create test run directory mkdir -p ${bindir}/test/atm/global-workflow/testrun/gdas_single_test_hofx3d cd ${bindir}/test/atm/global-workflow/testrun/gdas_single_test_hofx3d diff --git a/test/atm/run_jedi_exe_3dvar.sh b/test/atm/global-workflow/run_jedi_exe_3dvar.sh similarity index 89% rename from test/atm/run_jedi_exe_3dvar.sh rename to test/atm/global-workflow/run_jedi_exe_3dvar.sh index ffc50816e..d333db3cc 100755 --- a/test/atm/run_jedi_exe_3dvar.sh +++ b/test/atm/global-workflow/run_jedi_exe_3dvar.sh @@ -30,6 +30,13 @@ elif [ "$machine" = "orion" ]; then gdasfix="/work2/noaa/da/cmartin/GDASApp/fix" fi +# Setup python path for workflow utilities and tasks +export HOMEgfs=$srcdir/../../ # TODO: HOMEgfs had to be hard-coded in config +echo $HOMEgfs +pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src" +PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}" +export PYTHONPATH + # Create test run directory mkdir -p ${bindir}/test/atm/global-workflow/testrun/gdas_single_test_3dvar cd ${bindir}/test/atm/global-workflow/testrun/gdas_single_test_3dvar diff --git a/test/atm/run_jedi_exe_letkf.sh b/test/atm/global-workflow/run_jedi_exe_letkf.sh similarity index 89% rename from test/atm/run_jedi_exe_letkf.sh rename to test/atm/global-workflow/run_jedi_exe_letkf.sh index 4db4b5d3a..287b67641 100755 --- a/test/atm/run_jedi_exe_letkf.sh +++ b/test/atm/global-workflow/run_jedi_exe_letkf.sh @@ -23,15 +23,20 @@ module list # Set machine dependent variables if [ "$machine" = "hera" ] ; then - cominges="/scratch1/NCEPDEV/da/Russ.Treadon/GDASApp/cases" partition="hera" gdasfix="/scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/fix" elif [ "$machine" = "orion" ]; then - cominges="/work2/noaa/da/rtreadon/GDASApp/cases" partition="debug" gdasfix="/work2/noaa/da/cmartin/GDASApp/fix" fi +# Setup python path for workflow utilities and tasks +export HOMEgfs=$srcdir/../../ # TODO: HOMEgfs had to be hard-coded in config +echo $HOMEgfs +pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src" +PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}" +export PYTHONPATH + # Create test run directory mkdir -p ${bindir}/test/atm/global-workflow/testrun/gdas_single_test_letkf cd ${bindir}/test/atm/global-workflow/testrun/gdas_single_test_letkf @@ -64,7 +69,6 @@ config: dohybvar: no levs: 128 nmem: 5 - comin_ges: ${cominges} interp_method: barycentric job options: machine: ${machine} diff --git a/test/atm/global-workflow/setup_workflow_exp.sh b/test/atm/global-workflow/setup_workflow_exp.sh index bedd46ab8..8d18fc6e5 100755 --- a/test/atm/global-workflow/setup_workflow_exp.sh +++ b/test/atm/global-workflow/setup_workflow_exp.sh @@ -24,12 +24,13 @@ rm -rf $comrot $expdir config # copy config.yaml to local config cp -r $configdir config cp $srcdir/test/atm/global-workflow/config.base.emc.dyn config/ -cp $srcdir/test/atm/global-workflow/config.atmanal config/ +cp $srcdir/test/atm/global-workflow/config.atmanl config/ cp $srcdir/test/atm/global-workflow/config.yaml . # update paths in config.yaml sed -i -e "s~@bindir@~${bindir}~g" config.yaml sed -i -e "s~@srcdir@~${srcdir}~g" config.yaml +sed -i -e "s~@dumpdir@~${GDASAPP_TESTDATA}/lowres~g" config.yaml # run the script ln -sf $srcdir/../../workflow/setup_expt.py . @@ -50,4 +51,8 @@ $srcdir/../../workflow/setup_expt.py cycled --idate $idate \ --expdir $expdir \ --yaml $expdir/../config.yaml +echo " " +echo "$expdir/../config.yaml is" +cat $expdir/../config.yaml + exit $? diff --git a/test/genYAML_prep.sh b/test/genYAML_prep.sh index aaa57ee24..b0255ab39 100755 --- a/test/genYAML_prep.sh +++ b/test/genYAML_prep.sh @@ -19,6 +19,6 @@ config: valid_time: '2022-03-30T00:00:00Z' atm_window_length: PT6H CASE: 'C48' - CASE_ENKF: 'C48' + CASE_ANL: 'C48' LEVS: '128' EOF diff --git a/test/genYAML_prep_aero.sh b/test/genYAML_prep_aero.sh index f599455d3..0ed964196 100755 --- a/test/genYAML_prep_aero.sh +++ b/test/genYAML_prep_aero.sh @@ -19,6 +19,6 @@ config: valid_time: '2019-06-14T06:00:00Z' aero_window_length: PT6H CASE: 'C96' - CASE_ENKF: 'C48' + CASE_ANL: 'C48' LEVS: '128' EOF diff --git a/test/genYAML_prep_land.sh b/test/genYAML_prep_land.sh index 7b8f04206..242619ae3 100755 --- a/test/genYAML_prep_land.sh +++ b/test/genYAML_prep_land.sh @@ -20,6 +20,6 @@ config: atm_window_length: PT6H land_window_length: PT6H CASE: 'C48' - CASE_ENKF: 'C48' + CASE_ANL: 'C48' LEVS: '128' EOF diff --git a/test/soca/CMakeLists.txt b/test/soca/CMakeLists.txt index c9e3efe66..cbbdd3bb6 100644 --- a/test/soca/CMakeLists.txt +++ b/test/soca/CMakeLists.txt @@ -87,7 +87,7 @@ add_test(NAME test_gdasapp_soca_ana_prep WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/3dvar) set_tests_properties(test_gdasapp_soca_ana_prep PROPERTIES - ENVIRONMENT "PYTHONPATH=${PROJECT_BINARY_DIR}/ush:$ENV{PYTHONPATH}") + ENVIRONMENT "PYTHONPATH=${PROJECT_BINARY_DIR}/ush:${PROJECT_SOURCE_DIR}/../../ush/python/pygw/src:$ENV{PYTHONPATH}") # Test exgdas scripts from the global-worflow if (WORKFLOW_TESTS) diff --git a/ush/eva/marine_eva_post.py b/ush/eva/marine_eva_post.py index 483d259da..bf5634f45 100755 --- a/ush/eva/marine_eva_post.py +++ b/ush/eva/marine_eva_post.py @@ -7,6 +7,13 @@ import yaml import glob +# sets the cmap vmin/vmax for each variable +# TODO: this should probably be in a yaml or something +vminmax = {'seaSurfaceTemperature': {'vmin': -2.0, 'vmax': 2.0}, + 'seaIceFraction': {'vmin': -0.2, 'vmax': 0.2}, + 'seaSurfaceSalinity': {'vmin': -0.2, 'vmax': 0.2}, # TODO: this should be changed + 'absoluteDynamicTopography': {'vmin': -0.2, 'vmax': 0.2}} + def marine_eva_post(inputyaml, outputdir, diagdir): logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', level=logging.INFO, datefmt='%Y-%m-%d %H:%M:%S') @@ -23,6 +30,15 @@ def marine_eva_post(inputyaml, outputdir, diagdir): newfilename = os.path.join(diagdir, os.path.basename(filename)) newfilenames.append(newfilename) dataset['filenames'] = newfilenames + for graphic in diagnostic['graphics']: + # this assumes that there is only one variable, or that the + # variables are all the same + variable = graphic['batch figure']['variables'][0] + for plot in graphic['plots']: + for layer in plot['layers']: + if layer['type'] == 'MapScatter': + layer['vmin'] = vminmax[variable]['vmin'] + layer['vmax'] = vminmax[variable]['vmax'] # first, let us prepend some comments that tell someone this output YAML was generated now = datetime.datetime.now() diff --git a/ush/eva/marine_gdas_plots.yaml b/ush/eva/marine_gdas_plots.yaml index e4cdf4aaf..47f04205e 100644 --- a/ush/eva/marine_gdas_plots.yaml +++ b/ush/eva/marine_gdas_plots.yaml @@ -29,18 +29,39 @@ diagnostics: # Generate omb that passed QC for JEDI - transform: accept where - new name: experiment::PassedQC::${variable} + new name: experiment::OmBQC::${variable} starting field: experiment::ombg::${variable} where: - - experiment::PreQC::${variable} == 0 + - experiment::EffectiveQC0::${variable} == 0 for: variable: *variables + # Generate oma that passed QC for JEDI + - transform: accept where + new name: experiment::OmAQC::${variable} + starting field: experiment::oman::${variable} + where: + - experiment::EffectiveQC0::${variable} == 0 + for: + variable: *variables + + # Generate oma that passed QC for JEDI + - transform: accept where + new name: experiment::hofxQC::${variable} + starting field: experiment::hofx0::${variable} + where: + - experiment::EffectiveQC0::${variable} == 0 + for: + variable: *variables + + + + graphics: # ---------- Map Plots ---------- - # Map plot of ObsValue + # Map plot of OmBQC # -------- - batch figure: @@ -48,12 +69,12 @@ diagnostics: @CHANNELSKEY@ dynamic options: - type: vminvmaxcmap - data variable: experiment::ObsValue::${variable} + data variable: experiment::OmBQC::${variable} figure: layout: [1,1] figure size: [11,5] - title: ' @NAME@ | ${variable_title}' - output name: map_plots/@NAME@/${variable}/@CHANNELVAR@/@NAME@_${variable}@CHANNELVAR@.png + title: 'OmB post QC | @NAME@ @CYCLE@ | ${variable_title}' + output name: map_plots/@NAME@/${variable}/@CHANNELVAR@/@NAME@_${variable}@CHANNELVAR@OmBQC.png tight_layout: true plots: - mapping: @@ -70,12 +91,131 @@ diagnostics: latitude: variable: experiment::MetaData::latitude data: - variable: experiment::PassedQC::${variable} + variable: experiment::OmBQC::${variable} @CHANNELKEY@ markersize: 2 label: '$(variable)' colorbar: true # below may need to be edited/removed cmap: ${dynamic_cmap} - vmin: 0 #${dynamic_vmin} + vmin: ${dynamic_vmin} vmax: ${dynamic_vmax} + + + # Histogram plots + # --------------- + + # OmA pre and post QC + - batch figure: + variables: *variables + figure: + layout: [1,1] + title: 'OmA pre- and post QC | @NAME@ | ${variable_title}' + output name: histograms/@NAME@/${variable}/oma_pre_post_qc_${variable}.png + plots: + - add_xlabel: 'Observation minus anl pre- and post-QC' + add_ylabel: 'Count' + add_legend: + loc: 'upper left' + statistics: + fields: + - field_name: experiment::OmAQC::${variable} + xloc: 0.5 + yloc: -0.10 + kwargs: + fontsize: 6 + statistics_variables: + - n + - min + - mean + - max + - std + layers: + - type: Histogram + data: + variable: experiment::OmAQC::${variable} + color: 'blue' + label: 'OmA (post QC)' + bins: 100 + alpha: 0.5 + - type: Histogram + data: + variable: experiment::oman::${variable} + color: 'red' + label: 'OmA (pre QC)' + bins: 100 + alpha: 0.5 + + # diff between OmA and OmB + - batch figure: + variables: *variables + figure: + layout: [1,1] + title: 'OmA and OmB, post-QC | @NAME@ | ${variable_title}' + output name: histograms/@NAME@/${variable}/oma_omb_histogram_${variable}.png + plots: + - add_xlabel: 'OmA and OmB post-QC' + add_ylabel: 'Count' + add_legend: + loc: 'upper left' + statistics: + fields: + - field_name: experiment::OmAQC::${variable} + xloc: 0.5 + yloc: -0.10 + kwargs: + fontsize: 6 + statistics_variables: + - n + - min + - mean + - max + - std + layers: + - type: Histogram + data: + variable: experiment::OmAQC::${variable} + color: 'blue' + label: 'OmA (post QC)' + bins: 100 + alpha: 0.5 + - type: Histogram + data: + variable: experiment::OmBQC::${variable} + color: 'red' + label: 'OmB (post QC)' + bins: 100 + alpha: 0.5 + + - batch figure: + variables: *variables + @CHANNELSKEY@ + figure: + layout: [1,1] + title: 'Observations vs. JEDI h(x) | @NAME@ @CYCLE@ | ${variable_title}' + output name: observation_scatter_plots/jedi_hofx_vs_obs_@CYCLE@_@NAME@_${variable}@CHANNELVAR@.png + plots: + - add_xlabel: 'Observation Value' + add_ylabel: 'JEDI h(x)' + add_grid: + add_legend: + loc: 'upper left' + layers: + - type: Scatter + x: + variable: experiment::ObsValue::${variable} + y: + variable: experiment::hofx0::${variable} + @CHANNELKEY@ + markersize: 5 + color: 'black' + label: 'JEDI h(x) versus obs (all obs)' + - type: Scatter + x: + variable: experiment::ObsValue::${variable} + y: + variable: experiment::hofxQC::${variable} + @CHANNELKEY@ + markersize: 5 + color: 'red' + label: 'JEDI h(x) versus obs (passed QC in JEDI)' diff --git a/ush/examples/run_jedi_exe/3denvar_hera.yaml b/ush/examples/run_jedi_exe/3denvar_hera.yaml new file mode 100644 index 000000000..363c6235d --- /dev/null +++ b/ush/examples/run_jedi_exe/3denvar_hera.yaml @@ -0,0 +1,37 @@ +working directory: /scratch2/NCEPDEV/stmp1/Cory.R.Martin/gdas_single_test_3denvar +GDASApp home: /scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/work/GDASApp +GDASApp mode: variational +template: /scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/work/GDASApp/parm/atm/variational/3dvar_dripcg.yaml +config: + berror_yaml: /scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/work/GDASApp/parm/atm/berror/hybvar_gsibec.yaml + obs_dir: obs + diag_dir: diags + crtm_coeff_dir: crtm + bias_in_dir: obs + bias_out_dir: bc + obs_yaml_dir: /scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/work/GDASApp/parm/atm/obs/config + executable: /scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/work/GDASApp/build/bin/fv3jedi_var.x + obs_list: /scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/work/GDASApp/parm/atm/obs/lists/gdas_prototype_3d.yaml + gdas_fix_root: /scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/fix + atm: true + layout_x: 1 + layout_y: 1 + atm_window_length: PT6H + valid_time: 2021-12-21T06:00:00Z + dump: gdas + case: C96 + case_anl: C48 + case_enkf: C48 + staticb_type: gsibec + dohybvar: true + levs: 128 + nmem: 10 + interp_method: barycentric +job options: + machine: hera + account: da-cpu + queue: debug + partition: hera + walltime: '30:00' + ntasks: 6 + modulepath: /scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/work/GDASApp/modulefiles diff --git a/ush/examples/run_jedi_exe/3denvar_orion.yaml b/ush/examples/run_jedi_exe/3denvar_orion.yaml new file mode 100644 index 000000000..8c196bfaa --- /dev/null +++ b/ush/examples/run_jedi_exe/3denvar_orion.yaml @@ -0,0 +1,37 @@ +working directory: /work2/noaa/stmp/cmartin/gdas_single_test_3denvar +GDASApp home: /work2/noaa/da/cmartin/GDASApp/work/GDASApp +GDASApp mode: variational +template: /work2/noaa/da/cmartin/GDASApp/work/GDASApp/parm/atm/variational/3dvar_dripcg.yaml +config: + berror_yaml: /work2/noaa/da/cmartin/GDASApp/work/GDASApp/parm/atm/berror/hybvar_gsibec.yaml + obs_dir: obs + diag_dir: diags + crtm_coeff_dir: crtm + bias_in_dir: obs + bias_out_dir: bc + obs_yaml_dir: /work2/noaa/da/cmartin/GDASApp/work/GDASApp/parm/atm/obs/config + executable: /work2/noaa/da/cmartin/GDASApp/work/GDASApp/build/bin/fv3jedi_var.x + obs_list: /work2/noaa/da/cmartin/GDASApp/work/GDASApp/parm/atm/obs/lists/gdas_prototype_3d.yaml + gdas_fix_root: /work2/noaa/da/cmartin/GDASApp/fix + atm: true + layout_x: 1 + layout_y: 1 + atm_window_length: PT6H + valid_time: 2021-12-21T06:00:00Z + dump: gdas + case: C96 + case_anl: C48 + case_enkf: C48 + staticb_type: gsibec + dohybvar: true + levs: 128 + nmem: 10 + interp_method: barycentric +job options: + machine: orion + account: da-cpu + queue: debug + partition: debug + walltime: '30:00' + ntasks: 6 + modulepath: /work2/noaa/da/cmartin/GDASApp/work/GDASApp/modulefiles diff --git a/ush/examples/run_jedi_exe/4dhofx_hera.yaml b/ush/examples/run_jedi_exe/4dhofx_hera.yaml index 749e879bf..08baaf08d 100644 --- a/ush/examples/run_jedi_exe/4dhofx_hera.yaml +++ b/ush/examples/run_jedi_exe/4dhofx_hera.yaml @@ -24,5 +24,5 @@ job options: partition: hera walltime: '30:00' ntasks: 6 - ntasks-per-node: 2 + ntasks-per-node: 1 modulepath: /scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/work/GDASApp/modulefiles diff --git a/ush/examples/run_jedi_exe/4dhofx_orion.yaml b/ush/examples/run_jedi_exe/4dhofx_orion.yaml index 9025bcff5..cabc2fde8 100644 --- a/ush/examples/run_jedi_exe/4dhofx_orion.yaml +++ b/ush/examples/run_jedi_exe/4dhofx_orion.yaml @@ -24,5 +24,5 @@ job options: partition: debug walltime: '30:00' ntasks: 6 - ntasks-per-node: 2 + ntasks-per-node: 1 modulepath: /work2/noaa/da/cmartin/GDASApp/work/GDASApp/modulefiles diff --git a/ush/examples/run_jedi_exe/letkf_hera.yaml b/ush/examples/run_jedi_exe/letkf_hera.yaml index 9c869a3b7..24eb2d609 100644 --- a/ush/examples/run_jedi_exe/letkf_hera.yaml +++ b/ush/examples/run_jedi_exe/letkf_hera.yaml @@ -19,13 +19,9 @@ config: valid_time: 2021-12-21T06:00:00Z dump: gdas case: C48 - case_anl: C48 case_enkf: C48 - staticb_type: gsibec - dohybvar: no levs: 128 nmem: 10 - comin_ges: /scratch1/NCEPDEV/da/Russ.Treadon/GDASApp/cases interp_method: barycentric job options: machine: hera diff --git a/ush/examples/run_jedi_exe/letkf_orion.yaml b/ush/examples/run_jedi_exe/letkf_orion.yaml index e10d6930c..58a78c2d0 100644 --- a/ush/examples/run_jedi_exe/letkf_orion.yaml +++ b/ush/examples/run_jedi_exe/letkf_orion.yaml @@ -19,19 +19,15 @@ config: valid_time: 2021-12-21T06:00:00Z dump: gdas case: C48 - case_anl: C48 case_enkf: C48 - staticb_type: gsibec - dohybvar: no levs: 128 nmem: 10 - comin_ges: /work2/noaa/da/rtreadon/GDASApp/cases interp_method: barycentric job options: - machine: hera + machine: orion account: da-cpu queue: debug - partition: hera + partition: debug walltime: '30:00' ntasks: 6 modulepath: /work2/noaa/da/cmartin/GDASApp/work/GDASApp/modulefiles diff --git a/ush/run_jedi_exe.py b/ush/run_jedi_exe.py index b5fe91d74..4c4693f6e 100755 --- a/ush/run_jedi_exe.py +++ b/ush/run_jedi_exe.py @@ -7,8 +7,8 @@ import subprocess import sys import yaml - -from ufsda.genYAML import genYAML +import pygw +from pygw.yaml_file import parse_j2yaml, save_as_yaml def export_envar(yamlfile, bashout): @@ -59,6 +59,7 @@ def run_jedi_exe(yamlconfig): import ufsda from ufsda.misc_utils import calc_fcst_steps from ufsda.stage import gdas_single_cycle, gdas_fix, background_ens, atm_obs, bias_obs + from ufsda.genYAML import genYAML # compute config for YAML for executable executable_subconfig = all_config_dict['config'] @@ -76,8 +77,10 @@ def run_jedi_exe(yamlconfig): os.environ['assim_freq'] = str(assim_freq) oprefix = executable_subconfig['dump'] + ".t" + str(cyc) + "z." gprefix = executable_subconfig['dump'] + ".t" + str(gcyc) + "z." - comin = executable_subconfig.get('comin_ges', './') - comin_ges_ens = os.path.join(comin, 'enkfgdas.' + str(gPDY), str(gcyc), 'atmos') + comin = executable_subconfig.get('gdas_fix_root', './') + comin_ges_ens = os.path.join(comin, 'cases', 'enkfgdas.' + str(gPDY), str(gcyc)) + dump = executable_subconfig['dump'] + output_file = os.path.join(workdir, f"gdas_{app_mode}.yaml") single_exec = True var_config = { @@ -113,6 +116,8 @@ def run_jedi_exe(yamlconfig): 'valid_time': f"{valid_time.strftime('%Y-%m-%dT%H:%M:%SZ')}", 'window_begin': f"{window_begin.strftime('%Y-%m-%dT%H:%M:%SZ')}", 'prev_valid_time': f"{prev_cycle.strftime('%Y-%m-%dT%H:%M:%SZ')}", + 'previous_cycle': f"{prev_cycle}", + 'current_cycle': f"{valid_time}", 'atm_window_length': executable_subconfig['atm_window_length'], 'ATM_WINDOW_LENGTH': f"PT{assim_freq}H", 'ATM_WINDOW_BEGIN': f"{window_begin.strftime('%Y-%m-%dT%H:%M:%SZ')}", @@ -128,24 +133,52 @@ def run_jedi_exe(yamlconfig): executable_subconfig['atm_window_length']), 'BKG_TSTEP': executable_subconfig.get('forecast_step', 'PT6H'), 'INTERP_METHOD': executable_subconfig.get('interp_method', 'barycentric'), + 'output_file': os.path.join(workdir, f"gdas_{app_mode}.yaml"), + 'dump': f"{dump}", + 'output_file': f"{output_file}", } - output_file = os.path.join(workdir, f"gdas_{app_mode}.yaml") + # set some environment variables os.environ['PARMgfs'] = os.path.join(all_config_dict['GDASApp home'], 'parm') for key, value in var_config.items(): os.environ[key] = str(value) # generate YAML for executable based on input config logging.info(f'Using yamlconfig {yamlconfig}') - genYAML(yamlconfig, output=output_file) + + local_dict = { + 'npx_ges': f"{int(os.environ['CASE'][1:]) + 1}", + 'npy_ges': f"{int(os.environ['CASE'][1:]) + 1}", + 'npz_ges': f"{int(os.environ['LEVS']) - 1}", + 'npz': f"{int(os.environ['LEVS']) - 1}", + 'npx_anl': f"{int(os.environ['CASE_ANL'][1:]) + 1}", + 'npy_anl': f"{int(os.environ['CASE_ANL'][1:]) + 1}", + 'npz_anl': f"{int(os.environ['LEVS']) - 1}", + 'NMEM_ENKF': f"{int(os.environ['NMEM_ENKF'])}", + 'ATM_WINDOW_BEGIN': window_begin, + 'ATM_WINDOW_LENGTH': f"PT{assim_freq}H", + 'BKG_TSTEP': executable_subconfig.get('forecast_step', 'PT6H'), + 'OPREFIX': f"{dump}.t{cyc}z.", # TODO: CDUMP is being replaced by RUN + 'APREFIX': f"{dump}.t{cyc}z.", # TODO: CDUMP is being replaced by RUN + 'GPREFIX': f"gdas.t{gcyc}z.", + 'DATA': os.path.join(workdir), + 'layout_x': str(executable_subconfig['layout_x']), + 'layout_y': str(executable_subconfig['layout_y']), + 'previous_cycle': prev_cycle, + 'current_cycle': valid_time, + } + + varda_yaml = parse_j2yaml(all_config_dict['template'], local_dict) + save_as_yaml(varda_yaml, output_file) + logging.info(f'Wrote YAML file to {output_file}') # use R2D2 to stage backgrounds, obs, bias correction files, etc. if app_mode in ['variational', 'hofx']: - ufsda.stage.gdas_single_cycle(var_config) + ufsda.stage.gdas_single_cycle(var_config, local_dict) # stage ensemble backgrouns for letkf if app_mode in ['letkf']: ufsda.stage.background_ens(var_config) - ufsda.stage.atm_obs(var_config) - ufsda.stage.bias_obs(var_config) + ufsda.stage.atm_obs(var_config, local_dict) + ufsda.stage.bias_obs(var_config, local_dict) # link additional fix files needed (CRTM, fieldmetadata, etc.) gdasfix = executable_subconfig['gdas_fix_root'] diff --git a/ush/ufoeval/run_ufo_hofx_test.sh b/ush/ufoeval/run_ufo_hofx_test.sh index 07e13e19f..8ec802038 100755 --- a/ush/ufoeval/run_ufo_hofx_test.sh +++ b/ush/ufoeval/run_ufo_hofx_test.sh @@ -121,7 +121,7 @@ export cyc=${CDATE:8:2} export gPDY=${GDATE:0:8} export gcyc=${GDATE:8:2} export CASE="C768" -export CASE_ENKF="C384" +export CASE_ANL="C384" export LEVS="128" # Load Modules for GDASApp diff --git a/ush/ufsda/genYAML.py b/ush/ufsda/genYAML.py index 8d0134458..485ce3f0f 100644 --- a/ush/ufsda/genYAML.py +++ b/ush/ufsda/genYAML.py @@ -115,8 +115,8 @@ def get_runtime_config(config_dict): npx_ges_var: f"{int(os.environ['CASE'][1:]) + 1}", npy_ges_var: f"{int(os.environ['CASE'][1:]) + 1}", npz_ges_var: f"{int(os.environ['LEVS']) - 1}", - npx_anl_var: f"{int(os.environ['CASE_ENKF'][1:]) + 1}", - npy_anl_var: f"{int(os.environ['CASE_ENKF'][1:]) + 1}", + npx_anl_var: f"{int(os.environ['CASE_ANL'][1:]) + 1}", + npy_anl_var: f"{int(os.environ['CASE_ANL'][1:]) + 1}", npz_anl_var: f"{int(os.environ['LEVS']) - 1}", } diff --git a/ush/ufsda/misc_utils.py b/ush/ufsda/misc_utils.py index 55a2f85ce..519cdc11d 100644 --- a/ush/ufsda/misc_utils.py +++ b/ush/ufsda/misc_utils.py @@ -154,8 +154,8 @@ def get_env_config(component='atm'): config['npx_ges'] = int(os.environ['CASE'][1:]) + 1 config['npy_ges'] = int(os.environ['CASE'][1:]) + 1 config['npz_ges'] = int(os.environ['LEVS']) - 1 - config['npx_anl'] = int(os.environ['CASE_ENKF'][1:]) + 1 - config['npy_anl'] = int(os.environ['CASE_ENKF'][1:]) + 1 + config['npx_anl'] = int(os.environ['CASE_ANL'][1:]) + 1 + config['npy_anl'] = int(os.environ['CASE_ANL'][1:]) + 1 config['npz_anl'] = int(os.environ['LEVS']) - 1 return config diff --git a/ush/ufsda/stage.py b/ush/ufsda/stage.py index 16793c107..0be2e9562 100644 --- a/ush/ufsda/stage.py +++ b/ush/ufsda/stage.py @@ -1,7 +1,6 @@ from r2d2 import fetch from solo.basic_files import mkdir from solo.date import Hour, DateIncrement, date_sequence -from solo.logger import Logger from solo.stage import Stage from solo.configuration import Configuration from solo.nice_dict import NiceDict @@ -13,7 +12,7 @@ import logging import glob import numpy as np -from pygw.yaml_file import YAMLFile +from pygw.yaml_file import YAMLFile, parse_j2yaml import ufsda.soca_utils __all__ = ['atm_background', 'atm_obs', 'bias_obs', 'background', 'background_ens', 'fv3jedi', 'obs', 'berror', 'gdas_fix', 'gdas_single_cycle'] @@ -33,20 +32,23 @@ def gdas_fix(input_fix_dir, working_dir, config): # create output directories ufsda.disk_utils.mkdir(config['fv3jedi_fieldmetadata_dir']) ufsda.disk_utils.mkdir(config['fv3jedi_fix_dir']) - # figure out analysis resolution - if config['DOHYBVAR']: - case_anl = config['CASE_ENKF'] - else: - case_anl = config['CASE_ANL'] + + # error checking + dohybvar = config['DOHYBVAR'] + case = config['CASE'] + case_enkf = config['CASE_ENKF'] + case_anl = config['CASE_ANL'] + if dohybvar and not case_enkf == case_anl: + raise ValueError(f"dohybvar is '{dohybvar}' but case_enkf= '{case_enkf}' does not equal case_anl= '{case_anl}'") + + # set layers layers = int(config['LEVS'])-1 # figure out staticb source staticb_source = config.get('STATICB_TYPE', 'gsibec') - - # link staticb - if staticb_source in ['bump', 'gsibec']: - ufsda.disk_utils.symlink(os.path.join(input_fix_dir, staticb_source, case_anl), - config['fv3jedi_staticb_dir']) + case_berror = case if staticb_source in ['gsibec'] else case_anl + ufsda.disk_utils.symlink(os.path.join(input_fix_dir, staticb_source, case_berror), + config['fv3jedi_staticb_dir']) # link akbk file ufsda.disk_utils.symlink(os.path.join(input_fix_dir, 'fv3jedi', @@ -144,7 +146,7 @@ def atm_background(config): ufsda.r2d2.fetch(r2d2_config) -def atm_obs(config): +def atm_obs(config, local_dict): # fetch atm analysis obs r2d2_config = { 'start': config['prev_valid_time'], @@ -157,22 +159,25 @@ def atm_obs(config): r2d2_config = NiceDict(r2d2_config) # get list of obs to process and their output files obs_list_yaml = config['OBS_LIST'] - obs_list_config = YAMLFile(path=obs_list_yaml) + obs_list_config = Configuration(obs_list_yaml) + obs_list_config = ufsda.yamltools.iter_config(config, obs_list_config) + for ob in obs_list_config['observers']: + ob_config = parse_j2yaml(ob, local_dict) # first get obs r2d2_config.pop('file_type', None) r2d2_config['type'] = 'ob' r2d2_config['provider'] = 'ncdiag' r2d2_config['start'] = config['ATM_WINDOW_BEGIN'] r2d2_config['end'] = r2d2_config['start'] - ob_basename = os.path.basename(ob['obs space']['obsdatain']['engine']['obsfile']) + ob_basename = os.path.basename(ob_config['obs space']['obsdatain']['engine']['obsfile']) target_file = os.path.join(os.environ['COMOUT'], ob_basename) r2d2_config['target_file_fmt'] = target_file - r2d2_config['obs_types'] = [ob['obs space']['name']] + r2d2_config['obs_types'] = [ob_config['obs space']['name']] ufsda.r2d2.fetch(r2d2_config) -def bias_obs(config): +def bias_obs(config, local_dict): # fetch bias files r2d2_config = { 'start': config['prev_valid_time'], @@ -185,12 +190,15 @@ def bias_obs(config): r2d2_config = NiceDict(r2d2_config) # get list of obs to process and their output files obs_list_yaml = config['OBS_LIST'] - obs_list_config = YAMLFile(path=obs_list_yaml) + obs_list_config = Configuration(obs_list_yaml) + obs_list_config = ufsda.yamltools.iter_config(config, obs_list_config) + for ob in obs_list_config['observers']: + ob_config = parse_j2yaml(ob, local_dict) r2d2_config.pop('file_type', None) - r2d2_config['obs_types'] = [ob['obs space']['name']] + r2d2_config['obs_types'] = [ob_config['obs space']['name']] # get bias files if needed - if 'obs bias' in ob.keys(): + if 'obs bias' in ob_config.keys(): r2d2_config['type'] = 'bc' r2d2_config['provider'] = 'gsi' r2d2_config['start'] = config['prev_valid_time'] @@ -198,7 +206,7 @@ def bias_obs(config): # fetch satbias r2d2_config['file_type'] = 'satbias' - target_file = ob['obs bias']['input file'] + target_file = ob_config['obs bias']['input file'] ob_basename = os.path.basename(target_file) target_file = os.path.join(os.environ['COMOUT'], ob_basename) r2d2_config['target_file_fmt'] = target_file @@ -228,7 +236,7 @@ def bias_obs(config): ufsda.r2d2.fetch(r2d2_config) -def gdas_single_cycle(config): +def gdas_single_cycle(config, local_dict): # grab backgrounds first r2d2_config = { 'start': config['prev_valid_time'], @@ -265,8 +273,9 @@ def gdas_single_cycle(config): obs_list_yaml = config['OBS_LIST'] obs_list_config = Configuration(obs_list_yaml) obs_list_config = ufsda.yamltools.iter_config(config, obs_list_config) + for ob in obs_list_config['observers']: - ob_config = YAMLFile(path=ob) + ob_config = parse_j2yaml(ob, local_dict) # first get obs r2d2_config.pop('file_type', None) r2d2_config['type'] = 'ob' @@ -303,32 +312,26 @@ def gdas_single_cycle(config): target_file = target_file.replace('nc4', 'txt') r2d2_config['target_file_fmt'] = target_file ufsda.r2d2.fetch(r2d2_config) + # if hybvar, link ensemble members to DATA + if config['DOHYBVAR']: + background_ens(config) def background(config): """ - Stage backgrounds and create links for analysis + Stage backgrounds and create analysis directory This involves: - - cp RESTART to RESTART_GES - - ln RESTART_GES to analysis/bkg - - mkdir analysis/anl + - ln RESTART to bkg_dir + - mkdir anl """ rst_dir = os.path.join(config['background_dir'], 'RESTART') - ges_dir = os.path.join(config['background_dir'], 'RESTART_GES') jedi_bkg_dir = os.path.join(config['DATA'], 'bkg') jedi_anl_dir = os.path.join(config['DATA'], 'anl') - - # copy RESTART to RESTART_GES try: - shutil.copytree(rst_dir, ges_dir) - except FileExistsError: - shutil.rmtree(ges_dir) - shutil.copytree(rst_dir, ges_dir) - try: - os.symlink(ges_dir, jedi_bkg_dir) + os.symlink(rst_dir, jedi_bkg_dir) except FileExistsError: os.remove(jedi_bkg_dir) - os.symlink(ges_dir, jedi_bkg_dir) + os.symlink(rst_dir, jedi_bkg_dir) mkdir(jedi_anl_dir) @@ -415,32 +418,31 @@ def berror(config): def background_ens(config): """ - Stage backgrounds and create links for analysis + Stage backgrounds and optionally create analysis directories This involves: - - cp RESTART to RESTART_GES - - ln RESTART_GES to analysis/bkg - - mkdir analysis/anl + - ln member RESTART to bkg_mem + - optionally mkdir anl_mem """ + + # set background directory keyword based on dohybvar + bkgdir = 'ens' + if not config['DOHYBVAR']: + bkgdir = 'bkg' + for imem in range(1, config['NMEM_ENKF']+1): memchar = f"mem{imem:03d}" - print(f"background_ens: stage {memchar}") - rst_dir = os.path.join(config['COMIN_GES_ENS'], memchar, 'RESTART') - ges_dir = os.path.join(config['COMIN_GES_ENS'], memchar, 'RESTART_GES') - jedi_bkg_mem = os.path.join(config['DATA'], 'bkg', memchar) - jedi_bkg_dir = os.path.join(config['DATA'], 'bkg', memchar, 'RESTART') - jedi_anl_dir = os.path.join(config['DATA'], 'anl', memchar) - mkdir(jedi_bkg_mem) - - # copy RESTART to RESTART_GES - if not os.path.exists(ges_dir): - try: - shutil.copytree(rst_dir, ges_dir) - except FileExistsError: - shutil.rmtree(ges_dir) - shutil.copytree(rst_dir, ges_dir) + logging.info(f'Stage background_ens {memchar}') + rst_dir = os.path.join(config['COMIN_GES_ENS'], memchar, 'atmos', 'RESTART') + jedi_bkg_dir = os.path.join(config['DATA'], bkgdir) + jedi_bkg_mem = os.path.join(config['DATA'], bkgdir, memchar) + jedi_anl_mem = os.path.join(config['DATA'], 'anl', memchar) + mkdir(jedi_bkg_dir) try: - os.symlink(ges_dir, jedi_bkg_dir) + os.symlink(rst_dir, jedi_bkg_mem) except FileExistsError: - os.remove(jedi_bkg_dir) - os.symlink(ges_dir, jedi_bkg_dir) - mkdir(jedi_anl_dir) + os.remove(jedi_bkg_mem) + os.symlink(rst_dir, jedi_bkg_mem) + + # do not create member analysis directories for dohybvar + if not config['DOHYBVAR']: + mkdir(jedi_anl_mem) diff --git a/ush/ufsda/ufs_yaml.py b/ush/ufsda/ufs_yaml.py index b2a0f61f2..aebc8bd7a 100644 --- a/ush/ufsda/ufs_yaml.py +++ b/ush/ufsda/ufs_yaml.py @@ -108,7 +108,7 @@ def get_cycle_vars(): def get_exp_vars(): # variables computed from shell variables but not time dependent exp_dict = {} - npx_anl = str(int(os.getenv('CASE_ENKF', os.environ['CASE'])[1:]) + 1) + npx_anl = str(int(os.getenv('CASE_ANL', os.environ['CASE'])[1:]) + 1) npx = str(int(os.environ['CASE'][1:]) + 1) exp_dict['npx_anl'] = npx_anl exp_dict['npy_anl'] = npx_anl