diff --git a/parm/aero/berror/staticb_bump.yaml b/parm/aero/berror/staticb_bump.yaml deleted file mode 100644 index 1e6ea4603..000000000 --- a/parm/aero/berror/staticb_bump.yaml +++ /dev/null @@ -1,65 +0,0 @@ -covariance model: SABER -saber central block: - saber block name: BUMP_NICAS - read: - io: - data directory: &staticb_aero_dir $(DATA)/berror - files prefix: 'nicas_aero' - alias: - - in code: mass_fraction_of_sulfate_in_air - in file: mass_fraction_of_sulfate_in_air-mass_fraction_of_sulfate_in_air - - in code: mass_fraction_of_hydrophobic_black_carbon_in_air - in file: mass_fraction_of_hydrophobic_black_carbon_in_air-mass_fraction_of_hydrophobic_black_carbon_in_air - - in code: mass_fraction_of_hydrophilic_black_carbon_in_air - in file: mass_fraction_of_hydrophilic_black_carbon_in_air-mass_fraction_of_hydrophilic_black_carbon_in_air - - in code: mass_fraction_of_hydrophobic_organic_carbon_in_air - in file: mass_fraction_of_hydrophobic_organic_carbon_in_air-mass_fraction_of_hydrophobic_organic_carbon_in_air - - in code: mass_fraction_of_hydrophilic_organic_carbon_in_air - in file: mass_fraction_of_hydrophilic_organic_carbon_in_air-mass_fraction_of_hydrophilic_organic_carbon_in_air - - in code: mass_fraction_of_dust001_in_air - in file: mass_fraction_of_dust001_in_air-mass_fraction_of_dust001_in_air - - in code: mass_fraction_of_dust002_in_air - in file: mass_fraction_of_dust002_in_air-mass_fraction_of_dust002_in_air - - in code: mass_fraction_of_dust003_in_air - in file: mass_fraction_of_dust003_in_air-mass_fraction_of_dust003_in_air - - in code: mass_fraction_of_dust004_in_air - in file: mass_fraction_of_dust004_in_air-mass_fraction_of_dust004_in_air - - in code: mass_fraction_of_dust005_in_air - in file: mass_fraction_of_dust005_in_air-mass_fraction_of_dust005_in_air - - in code: mass_fraction_of_sea_salt001_in_air - in file: mass_fraction_of_sea_salt001_in_air-mass_fraction_of_sea_salt001_in_air - - in code: mass_fraction_of_sea_salt002_in_air - in file: mass_fraction_of_sea_salt002_in_air-mass_fraction_of_sea_salt002_in_air - - in code: mass_fraction_of_sea_salt003_in_air - in file: mass_fraction_of_sea_salt003_in_air-mass_fraction_of_sea_salt003_in_air - - in code: mass_fraction_of_sea_salt004_in_air - in file: mass_fraction_of_sea_salt004_in_air-mass_fraction_of_sea_salt004_in_air - drivers: - multivariate strategy: univariate - read local nicas: true - input fields: - - parameter: universe radius - file: - filetype: fms restart - datetime: '2016-06-30T00:00:00Z' - set datetime on read: true - psinfile: true - datapath: *staticb_aero_dir - prefix: '20160630.000000' - filename_core: cor_rh.fv_core.res.nc - filename_trcr: cor_rh.fv_tracer.res.nc - filename_cplr: cor_rh.coupler.res - saber outer blocks: - - saber block name: StdDev - input fields: - - parameter: StdDev - file: - filetype: fms restart - datetime: '2016-06-30T00:00:00Z' - set datetime on read: true - psinfile: true - datapath: *staticb_aero_dir - prefix: '20160630.000000' - filename_core: stddev.fv_core.res.nc - filename_trcr: stddev.fv_tracer.res.nc - filename_cplr: stddev.coupler.res diff --git a/parm/aero/berror/staticb_bump_aero.yaml b/parm/aero/berror/staticb_bump_aero.yaml new file mode 100644 index 000000000..ac746b6c9 --- /dev/null +++ b/parm/aero/berror/staticb_bump_aero.yaml @@ -0,0 +1,23 @@ +covariance model: SABER +saber central block: + saber block name: BUMP_NICAS + read: + io: + data directory: &staticb_aero_dir $(DATA)/berror + files prefix: 'nicas_aero' + drivers: + multivariate strategy: univariate + read local nicas: true + saber outer blocks: + - saber block name: StdDev + read: + model file: + datetime: '2016-06-30T00:00:00Z' + set datetime on read: true + filetype: fms restart + psinfile: true + datapath: Data/staticb_aero/ + filename_core: 20160630.000000.stddev.fv_core.res.nc + filename_trcr: 20160630.000000.stddev.fv_tracer.res.nc + filename_cplr: 20160630.000000.stddev.coupler.res + date: '2016-06-30T00:00:00Z' diff --git a/parm/land/letkfoi/apply_incr_nml.j2 b/parm/land/letkfoi/apply_incr_nml.j2 index 2dac08157..39765a5d3 100644 --- a/parm/land/letkfoi/apply_incr_nml.j2 +++ b/parm/land/letkfoi/apply_incr_nml.j2 @@ -2,7 +2,7 @@ date_str = "{{ current_cycle | to_YMD }}", hour_str = "{{ current_cycle | strftime('%H') }}", res = {{ CASE[1:] }}, - frac_grid = {{ FRAC_GRID | to_f90bool }}, + frac_grid = .true., rst_path = "{{ DATA }}/anl", inc_path = "{{ DATA }}/anl", orog_path = "{{ HOMEgfs }}/fix/orog/{{ CASE }}", diff --git a/parm/land/letkfoi/letkfoi.yaml b/parm/land/letkfoi/letkfoi.yaml index 0dc764978..11db7a94d 100644 --- a/parm/land/letkfoi/letkfoi.yaml +++ b/parm/land/letkfoi/letkfoi.yaml @@ -28,7 +28,7 @@ background: template: datetime: '{{ current_cycle | to_isotime }}' filetype: fms restart - state variables: [snwdph,vtype,slmsk] + state variables: [snodl,vtype,slmsk] datapath: $(DATA)/bkg/mem%mem%/RESTART filename_sfcd: '{{ current_cycle | to_fv3time }}.sfc_data.nc' filename_cplr: '{{ current_cycle | to_fv3time }}.coupler.res' @@ -56,5 +56,5 @@ output increment: filetype: fms restart filename_sfcd: '{{ current_cycle | to_fv3time }}.sfc_data.nc' filename_cplr: '{{ current_cycle | to_fv3time }}.coupler.res' - state variables: [snwdph,vtype,slmsk] + state variables: [snodl,vtype,slmsk] diff --git a/parm/land/obs/config/adpsfc_snow.yaml b/parm/land/obs/config/adpsfc_snow.yaml index a7c69752e..a7c08eb74 100644 --- a/parm/land/obs/config/adpsfc_snow.yaml +++ b/parm/land/obs/config/adpsfc_snow.yaml @@ -86,6 +86,12 @@ obs filters: damping_factor_1: 1.0 damping_factor_2: 1.0 background_error_group: BkgError + - filter: Perform Action + filter variables: + - name: totalSnowDepth + action: + name: assign error + error parameter: 40.0 - filter: Variable Assignment assignments: - name: GrossErrorProbability/totalSnowDepth diff --git a/scripts/exgdas_global_marine_analysis_bmat.sh b/scripts/exgdas_global_marine_analysis_bmat.sh index 4fcd1375c..ed22b655d 100755 --- a/scripts/exgdas_global_marine_analysis_bmat.sh +++ b/scripts/exgdas_global_marine_analysis_bmat.sh @@ -255,7 +255,7 @@ if false; then # TODO: resurect this section when making use of bump 3D in the static B, skip for now yaml_bump2d=soca_bump2d.yaml clean_yaml $yaml_bump2d - $APRUN_OCNANAL $JEDI_BIN/soca_error_covariance_training.x $yaml_bump2d 2>$yaml_bump2d.err + $APRUN_OCNANAL $JEDI_BIN/soca_error_covariance_toolbox.x $yaml_bump2d 2>$yaml_bump2d.err export err=$?; err_chk if [ $err -gt 0 ]; then exit $err @@ -265,7 +265,7 @@ if false; then yaml_list=`ls soca_bump3d_*.yaml` for yaml in $yaml_list; do clean_yaml $yaml - $APRUN_OCNANAL $JEDI_BIN/soca_error_covariance_training.x $yaml 2>$yaml.err + $APRUN_OCNANAL $JEDI_BIN/soca_error_covariance_toolbox.x $yaml 2>$yaml.err export err=$?; err_chk if [ $err -gt 0 ]; then exit $err @@ -277,7 +277,7 @@ fi ################################################################################ # Compute convolution coefs for L clean_yaml soca_bump_loc.yaml -$APRUN_OCNANAL $JEDI_BIN/soca_error_covariance_training.x soca_bump_loc.yaml +$APRUN_OCNANAL $JEDI_BIN/soca_error_covariance_toolbox.x soca_bump_loc.yaml export err=$?; err_chk if [ $err -gt 0 ]; then exit $err diff --git a/test/atm/global-workflow/CMakeLists.txt b/test/atm/global-workflow/CMakeLists.txt index 850983f97..60c500ca5 100644 --- a/test/atm/global-workflow/CMakeLists.txt +++ b/test/atm/global-workflow/CMakeLists.txt @@ -59,3 +59,14 @@ add_test(NAME test_gdasapp_atm_jjob_ens_final COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/jjob_ens_final.sh ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun) + +# python fix + set_tests_properties( + test_gdasapp_run_jedi_exe_3dhofx + test_gdasapp_run_jedi_exe_3dvar + test_gdasapp_run_jedi_exe_3denvar + test_gdasapp_run_jedi_exe_letkf + PROPERTIES + ENVIRONMENT "PYTHONPATH=${PROJECT_BINARY_DIR}/lib/python${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}:$ENV{PYTHONPATH}" + ) + diff --git a/test/genYAML_prep_aero.sh b/test/genYAML_prep_aero.sh index 0ed964196..dd5fa3d02 100755 --- a/test/genYAML_prep_aero.sh +++ b/test/genYAML_prep_aero.sh @@ -6,7 +6,7 @@ cat > testoutput/genYAML_example_aero.yaml << EOF template: ${srcdir}/parm/aero/variational/3dvar_gfs_aero.yaml output: ${bindir}/test/testoutput/genYAML_output_3dvar_gfs_aero.yaml config: - BERROR_YAML: \${PARMgfs}/aero/berror/staticb_bump.yaml + BERROR_YAML: \${PARMgfs}/aero/berror/staticb_bump_aero.yaml OBS_YAML_DIR: \${PARMgfs}/aero/obs/config COMPONENT: chem layout_x: '8' diff --git a/test/land/letkfoi_snowda.sh b/test/land/letkfoi_snowda.sh index 8747241c0..aecf2f62c 100755 --- a/test/land/letkfoi_snowda.sh +++ b/test/land/letkfoi_snowda.sh @@ -19,7 +19,7 @@ WORKDIR=$project_binary_dir/test/land/letkfoi_snowda RSTDIR=$GDASAPP_TESTDATA/lowres/gdas.$GYMD/$GHR/model_data/atmos/restart export OBSDIR=$GDASAPP_TESTDATA/land -GFSv17=${GFSv17:-"NO"} +GFSv17=${GFSv17:-"YES"} DAtype=letkfoi_snow if [ $GFSv17 == "YES" ]; then diff --git a/test/land/test_imsproc.sh b/test/land/test_imsproc.sh index 5d79db264..91a1ce00b 100755 --- a/test/land/test_imsproc.sh +++ b/test/land/test_imsproc.sh @@ -56,7 +56,7 @@ done ulimit -Ss unlimited ${EXECDIR}/calcfIMS.exe -export PYTHONPATH=$PYTHONPATH:${project_source_dir}/iodaconv/src/:${project_source_dir}/build/lib/python${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}/pyioda/ +export PYTHONPATH=$PYTHONPATH:${project_source_dir}/iodaconv/src/:${project_source_dir}/build/lib/python${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR} IMS_IODA=${EXECDIR}/imsfv3_scf2ioda.py echo 'do_landDA: calling ioda converter' diff --git a/test/soca/gw/CMakeLists.txt b/test/soca/gw/CMakeLists.txt index 5b4de13a2..50916accd 100644 --- a/test/soca/gw/CMakeLists.txt +++ b/test/soca/gw/CMakeLists.txt @@ -27,7 +27,7 @@ add_test(NAME test_gdasapp_soca_concatioda WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/gw/concatioda) set_tests_properties(test_gdasapp_soca_concatioda PROPERTIES - ENVIRONMENT "PYTHONPATH=${PROJECT_BINARY_DIR}/ush:$ENV{PYTHONPATH}") + ENVIRONMENT "PYTHONPATH=${PROJECT_BINARY_DIR}/ush:${PROJECT_BINARY_DIR}/lib/python${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}:$ENV{PYTHONPATH}") # Clean-up add_test(NAME test_gdasapp_soca_run_clean @@ -58,6 +58,10 @@ foreach(jjob ${jjob_list}) -y ${PROJECT_BINARY_DIR}/test/soca/gw/testrun/run_jjobs_${jjob}.yaml ${setup} WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/gw/testrun) + set_tests_properties(${test_name} + PROPERTIES + ENVIRONMENT "PYTHONPATH=${PROJECT_BINARY_DIR}/lib/python${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}:$ENV{PYTHONPATH}") + set(setup "--skip") # Only run the setup of the first test, if not, it will hang # waiting for standard input from setup_expt.py diff --git a/ush/eva/jedi_gsi_compare_conv.yaml b/ush/eva/jedi_gsi_compare_conv.yaml index 502865385..02293a7b0 100644 --- a/ush/eva/jedi_gsi_compare_conv.yaml +++ b/ush/eva/jedi_gsi_compare_conv.yaml @@ -1,464 +1,462 @@ # template YAML file to create EVA YAML files # based on obs spaces listed in JEDI YAML files -diagnostics: -- data: - datasets: - - name: experiment - type: IodaObsSpace - filenames: - - @FILENAME@ - @CHANNELSKEY@ - groups: - - name: ObsValue - variables: &variables @VARIABLES@ - - name: GsiHofXBc - - name: hofx - - name: EffectiveQC - - name: MetaData - - name: GsiEffectiveQC - - name: GsiFinalObsError - - name: EffectiveError - transforms: +datasets: + - name: experiment + type: IodaObsSpace + filenames: + - @FILENAME@ + @CHANNELSKEY@ + groups: + - name: ObsValue + variables: &variables @VARIABLES@ + - name: GsiHofXBc + - name: hofx + - name: EffectiveQC + - name: MetaData + - name: GsiEffectiveQC + - name: GsiFinalObsError + - name: EffectiveError +transforms: - # difference of hofx - - transform: arithmetic - new name: experiment::HofXDiff::${variable} - equals: experiment::hofx::${variable}-experiment::GsiHofXBc::${variable} - for: - variable: *variables + # difference of hofx + - transform: arithmetic + new name: experiment::HofXDiff::${variable} + equals: experiment::hofx::${variable}-experiment::GsiHofXBc::${variable} + for: + variable: *variables - # difference of effective error - - transform: arithmetic - new name: experiment::ErrDiff::${variable} - equals: experiment::EffectiveError::${variable}-experiment::GsiFinalObsError::${variable} - for: - variable: *variables + # difference of effective error + - transform: arithmetic + new name: experiment::ErrDiff::${variable} + equals: experiment::EffectiveError::${variable}-experiment::GsiFinalObsError::${variable} + for: + variable: *variables - # difference of QC - - transform: arithmetic - new name: experiment::QCDiff::${variable} - equals: experiment::EffectiveQC::${variable}-experiment::GsiEffectiveQC::${variable} - for: - variable: *variables + # difference of QC + - transform: arithmetic + new name: experiment::QCDiff::${variable} + equals: experiment::EffectiveQC::${variable}-experiment::GsiEffectiveQC::${variable} + for: + variable: *variables - # Generate omb for GSI - - transform: arithmetic - new name: experiment::ObsValueMinusGsiHofXBc::${variable} - equals: experiment::ObsValue::${variable}-experiment::GsiHofXBc::${variable} - for: - variable: *variables + # Generate omb for GSI + - transform: arithmetic + new name: experiment::ObsValueMinusGsiHofXBc::${variable} + equals: experiment::ObsValue::${variable}-experiment::GsiHofXBc::${variable} + for: + variable: *variables - # Generate omb for JEDI - - transform: arithmetic - new name: experiment::ObsValueMinusHofx::${variable} - equals: experiment::ObsValue::${variable}-experiment::hofx::${variable} - for: - variable: *variables + # Generate omb for JEDI + - transform: arithmetic + new name: experiment::ObsValueMinusHofx::${variable} + equals: experiment::ObsValue::${variable}-experiment::hofx::${variable} + for: + variable: *variables - # Generate hofx that passed QC for JEDI - - transform: accept where - new name: experiment::hofxPassedQc::${variable} - starting field: experiment::hofx::${variable} - where: - - experiment::EffectiveQC::${variable} == 0 - for: - variable: *variables + # Generate hofx that passed QC for JEDI + - transform: accept where + new name: experiment::hofxPassedQc::${variable} + starting field: experiment::hofx::${variable} + where: + - experiment::EffectiveQC::${variable} == 0 + for: + variable: *variables - # Generate GSI hofx that passed JEDI QC - - transform: accept where - new name: experiment::GsiHofXBcPassedQc::${variable} - starting field: experiment::GsiHofXBc::${variable} - where: - - experiment::EffectiveQC::${variable} == 0 - for: - variable: *variables + # Generate GSI hofx that passed JEDI QC + - transform: accept where + new name: experiment::GsiHofXBcPassedQc::${variable} + starting field: experiment::GsiHofXBc::${variable} + where: + - experiment::EffectiveQC::${variable} == 0 + for: + variable: *variables - # Generate omb that passed QC for JEDI - - transform: accept where - new name: experiment::ObsValueMinushofxPassedQc::${variable} - starting field: experiment::ObsValueMinusHofx::${variable} - where: - - experiment::EffectiveQC::${variable} == 0 - for: - variable: *variables + # Generate omb that passed QC for JEDI + - transform: accept where + new name: experiment::ObsValueMinushofxPassedQc::${variable} + starting field: experiment::ObsValueMinusHofx::${variable} + where: + - experiment::EffectiveQC::${variable} == 0 + for: + variable: *variables - # Generate omb that passed QC for GSI - - transform: accept where - new name: experiment::ObsValueMinusGsiHofXBcPassedQc::${variable} - starting field: experiment::ObsValueMinusGsiHofXBc::${variable} - where: - - experiment::EffectiveQC::${variable} == 0 - for: - variable: *variables + # Generate omb that passed QC for GSI + - transform: accept where + new name: experiment::ObsValueMinusGsiHofXBcPassedQc::${variable} + starting field: experiment::ObsValueMinusGsiHofXBc::${variable} + where: + - experiment::EffectiveQC::${variable} == 0 + for: + variable: *variables - # hofxdiff that passed QC for JEDI - - transform: accept where - new name: experiment::HofXDiffPassedJediQc::${variable} - starting field: experiment::HofXDiff::${variable} - where: - - experiment::EffectiveQC::${variable} == 0 - for: - variable: *variables + # hofxdiff that passed QC for JEDI + - transform: accept where + new name: experiment::HofXDiffPassedJediQc::${variable} + starting field: experiment::HofXDiff::${variable} + where: + - experiment::EffectiveQC::${variable} == 0 + for: + variable: *variables - # hofxdiff that passed QC for GSI - - transform: accept where - new name: experiment::HofXDiffPassedGsiQc::${variable} - starting field: experiment::HofXDiff::${variable} - where: - - experiment::GsiEffectiveQC::${variable} == 0 - for: - variable: *variables + # hofxdiff that passed QC for GSI + - transform: accept where + new name: experiment::HofXDiffPassedGsiQc::${variable} + starting field: experiment::HofXDiff::${variable} + where: + - experiment::GsiEffectiveQC::${variable} == 0 + for: + variable: *variables - graphics: +graphics: - # ---------- Scatter Plots ---------- - # JEDI h(x) vs Observations - # ------------------------- - - batch figure: - variables: *variables - @CHANNELSKEY@ - figure: - layout: [1,1] - title: 'Observations vs. JEDI h(x) | @NAME@ @CYCLE@ | ${variable_title}' - output name: observation_scatter_plots/jedi_hofx_vs_obs_@CYCLE@_@NAME@_${variable}@CHANNELVAR@.png - plots: - - add_xlabel: 'Observation Value' - add_ylabel: 'JEDI h(x)' - add_grid: - add_legend: - loc: 'upper left' - layers: - - type: Scatter - x: - variable: experiment::ObsValue::${variable} - y: - variable: experiment::hofx::${variable} - @CHANNELKEY@ - markersize: 5 - color: 'black' - label: 'JEDI h(x) versus obs (all obs)' - - type: Scatter - x: - variable: experiment::ObsValue::${variable} - y: - variable: experiment::hofxPassedQc::${variable} - @CHANNELKEY@ - markersize: 5 - color: 'red' - label: 'JEDI h(x) versus obs (passed QC in JEDI)' + # ---------- Scatter Plots ---------- + # JEDI h(x) vs Observations + # ------------------------- + - batch figure: + variables: *variables + @CHANNELSKEY@ + figure: + layout: [1,1] + title: 'Observations vs. JEDI h(x) | @NAME@ @CYCLE@ | ${variable_title}' + output name: observation_scatter_plots/jedi_hofx_vs_obs_@CYCLE@_@NAME@_${variable}@CHANNELVAR@.png + plots: + - add_xlabel: 'Observation Value' + add_ylabel: 'JEDI h(x)' + add_grid: + add_legend: + loc: 'upper left' + layers: + - type: Scatter + x: + variable: experiment::ObsValue::${variable} + y: + variable: experiment::hofx::${variable} + @CHANNELKEY@ + markersize: 5 + color: 'black' + label: 'JEDI h(x) versus obs (all obs)' + - type: Scatter + x: + variable: experiment::ObsValue::${variable} + y: + variable: experiment::hofxPassedQc::${variable} + @CHANNELKEY@ + markersize: 5 + color: 'red' + label: 'JEDI h(x) versus obs (passed QC in JEDI)' - # GSI h(x) vs Observations - # ------------------------- - - batch figure: - variables: *variables - @CHANNELSKEY@ - figure: - layout: [1,1] - title: 'Observations vs. GSI h(x) | @NAME@ @CYCLE@ | ${variable_title}' - output name: observation_scatter_plots/gsi_hofx_vs_obs_@CYCLE@_@NAME@_${variable}@CHANNELVAR@.png - plots: - - add_xlabel: 'Observation Value' - add_ylabel: 'GSI h(x)' - add_grid: - add_legend: - loc: 'upper left' - layers: - - type: Scatter - x: - variable: experiment::ObsValue::${variable} - y: - variable: experiment::GsiHofXBc::${variable} - @CHANNELKEY@ - markersize: 5 - color: 'black' - label: 'GSI h(x) versus obs (all obs)' - - type: Scatter - x: - variable: experiment::ObsValue::${variable} - y: - variable: experiment::GsiHofXBcPassedQc::${variable} - @CHANNELKEY@ - markersize: 5 - color: 'red' - label: 'GSI h(x) versus obs (passed QC in JEDI)' + # GSI h(x) vs Observations + # ------------------------- + - batch figure: + variables: *variables + @CHANNELSKEY@ + figure: + layout: [1,1] + title: 'Observations vs. GSI h(x) | @NAME@ @CYCLE@ | ${variable_title}' + output name: observation_scatter_plots/gsi_hofx_vs_obs_@CYCLE@_@NAME@_${variable}@CHANNELVAR@.png + plots: + - add_xlabel: 'Observation Value' + add_ylabel: 'GSI h(x)' + add_grid: + add_legend: + loc: 'upper left' + layers: + - type: Scatter + x: + variable: experiment::ObsValue::${variable} + y: + variable: experiment::GsiHofXBc::${variable} + @CHANNELKEY@ + markersize: 5 + color: 'black' + label: 'GSI h(x) versus obs (all obs)' + - type: Scatter + x: + variable: experiment::ObsValue::${variable} + y: + variable: experiment::GsiHofXBcPassedQc::${variable} + @CHANNELKEY@ + markersize: 5 + color: 'red' + label: 'GSI h(x) versus obs (passed QC in JEDI)' - # JEDI h(x) vs GSI h(x) - # --------------------- + # JEDI h(x) vs GSI h(x) + # --------------------- - - batch figure: - variables: *variables - @CHANNELSKEY@ - figure: - layout: [1,1] - title: 'JEDI h(x) vs. GSI h(x) | @NAME@ @CYCLE@ | ${variable_title}' - output name: observation_scatter_plots/gsi_hofx_vs_jedi_hofx_@CYCLE@_@NAME@_${variable}@CHANNELVAR@.png - plots: - - add_xlabel: 'GSI h(x)' - add_ylabel: 'JEDI h(x)' - add_grid: - add_legend: - loc: 'lower left' - layers: - - type: Scatter - x: - variable: experiment::GsiHofXBc::${variable} - y: - variable: experiment::hofx::${variable} - @CHANNELKEY@ - markersize: 5 - color: 'black' - label: 'JEDI h(x) versus GSI h(x)' - - type: Scatter - x: - variable: experiment::GsiHofXBcPassedQc::${variable} - y: - variable: experiment::hofxPassedQc::${variable} - @CHANNELKEY@ - markersize: 5 - color: 'red' - label: 'JEDI h(x) versus GSI h(x) (passed QC in JEDI)' + - batch figure: + variables: *variables + @CHANNELSKEY@ + figure: + layout: [1,1] + title: 'JEDI h(x) vs. GSI h(x) | @NAME@ @CYCLE@ | ${variable_title}' + output name: observation_scatter_plots/gsi_hofx_vs_jedi_hofx_@CYCLE@_@NAME@_${variable}@CHANNELVAR@.png + plots: + - add_xlabel: 'GSI h(x)' + add_ylabel: 'JEDI h(x)' + add_grid: + add_legend: + loc: 'lower left' + layers: + - type: Scatter + x: + variable: experiment::GsiHofXBc::${variable} + y: + variable: experiment::hofx::${variable} + @CHANNELKEY@ + markersize: 5 + color: 'black' + label: 'JEDI h(x) versus GSI h(x)' + - type: Scatter + x: + variable: experiment::GsiHofXBcPassedQc::${variable} + y: + variable: experiment::hofxPassedQc::${variable} + @CHANNELKEY@ + markersize: 5 + color: 'red' + label: 'JEDI h(x) versus GSI h(x) (passed QC in JEDI)' - # JEDI omb vs GSI omb - # --------------------- + # JEDI omb vs GSI omb + # --------------------- - - batch figure: - variables: *variables - @CHANNELSKEY@ - figure: - layout: [1,1] - title: 'JEDI omb vs. GSI omb | @NAME@ @CYCLE@ | ${variable_title}' - output name: observation_scatter_plots/gsi_omb_vs_jedi_omb_@CYCLE@_@NAME@_${variable}@CHANNELVAR@.png - plots: - - add_xlabel: 'GSI observation minus h(x)' - add_ylabel: 'JEDI observation minus h(x)' - add_grid: - add_legend: - loc: 'upper left' - layers: - - type: Scatter - x: - variable: experiment::ObsValueMinusGsiHofXBc::${variable} - y: - variable: experiment::ObsValueMinusHofx::${variable} - @CHANNELKEY@ - markersize: 5 - color: 'black' - label: 'GSI omb vs JEDI omb (all obs)' - - type: Scatter - x: - variable: experiment::ObsValueMinusGsiHofXBcPassedQc::${variable} - y: - variable: experiment::ObsValueMinushofxPassedQc::${variable} - @CHANNELKEY@ - markersize: 5 - color: 'red' - label: 'GSI omb vs JEDI omb (passed QC in JEDI)' + - batch figure: + variables: *variables + @CHANNELSKEY@ + figure: + layout: [1,1] + title: 'JEDI omb vs. GSI omb | @NAME@ @CYCLE@ | ${variable_title}' + output name: observation_scatter_plots/gsi_omb_vs_jedi_omb_@CYCLE@_@NAME@_${variable}@CHANNELVAR@.png + plots: + - add_xlabel: 'GSI observation minus h(x)' + add_ylabel: 'JEDI observation minus h(x)' + add_grid: + add_legend: + loc: 'upper left' + layers: + - type: Scatter + x: + variable: experiment::ObsValueMinusGsiHofXBc::${variable} + y: + variable: experiment::ObsValueMinusHofx::${variable} + @CHANNELKEY@ + markersize: 5 + color: 'black' + label: 'GSI omb vs JEDI omb (all obs)' + - type: Scatter + x: + variable: experiment::ObsValueMinusGsiHofXBcPassedQc::${variable} + y: + variable: experiment::ObsValueMinushofxPassedQc::${variable} + @CHANNELKEY@ + markersize: 5 + color: 'red' + label: 'GSI omb vs JEDI omb (passed QC in JEDI)' - # H(x) difference as a function of pressure - - batch figure: - variables: *variables - @CHANNELSKEY@ - figure: - layout: [1,1] - title: 'H(x) JEDI-GSI vs Ob P. | @NAME@ @CYCLE@ | ${variable_title}' - output name: observation_scatter_plots/hofxdiff_vs_pressure_@CYCLE@_@NAME@_${variable}@CHANNELVAR@.png - plots: - - add_xlabel: 'h(x) JEDI - h(x) GSI' - add_ylabel: 'Observation Pressure' - add_grid: - add_legend: - loc: 'lower left' - layers: - - type: Scatter - x: - variable: experiment::HofXDiff::${variable} - y: - variable: experiment::MetaData::pressure - @CHANNELKEY@ - markersize: 5 - color: 'red' - label: 'hofxdiff vs pressure' - statistics: - fields: - - field_name: experiment::HofXDiff::${variable} - @CHANNELKEY@ - xloc: 0.5 - yloc: -0.10 - kwargs: - fontsize: 6 - statistics_variables: - - n - - min - - mean - - max - - std + # H(x) difference as a function of pressure + - batch figure: + variables: *variables + @CHANNELSKEY@ + figure: + layout: [1,1] + title: 'H(x) JEDI-GSI vs Ob P. | @NAME@ @CYCLE@ | ${variable_title}' + output name: observation_scatter_plots/hofxdiff_vs_pressure_@CYCLE@_@NAME@_${variable}@CHANNELVAR@.png + plots: + - add_xlabel: 'h(x) JEDI - h(x) GSI' + add_ylabel: 'Observation Pressure' + add_grid: + add_legend: + loc: 'lower left' + layers: + - type: Scatter + x: + variable: experiment::HofXDiff::${variable} + y: + variable: experiment::MetaData::pressure + @CHANNELKEY@ + markersize: 5 + color: 'red' + label: 'hofxdiff vs pressure' + statistics: + fields: + - field_name: experiment::HofXDiff::${variable} + @CHANNELKEY@ + xloc: 0.5 + yloc: -0.10 + kwargs: + fontsize: 6 + statistics_variables: + - n + - min + - mean + - max + - std - # QC difference as a function of H(x) difference - - batch figure: - variables: *variables - @CHANNELSKEY@ - figure: - layout: [1,1] - title: 'H(x) JEDI-GSI vs QC JEDI-GSI. | @NAME@ @CYCLE@ | ${variable_title}' - output name: observation_scatter_plots/hofxdiff_vs_qcdiff_@CYCLE@_@NAME@_${variable}@CHANNELVAR@.png - plots: - - add_xlabel: 'h(x) JEDI - h(x) GSI' - add_ylabel: 'h(x) JEDI - h(x) GSI' - add_grid: - add_legend: - loc: 'lower left' - layers: - - type: Scatter - x: - variable: experiment::QCDiff::${variable} - y: - variable: experiment::HofXDiff::${variable} - @CHANNELKEY@ - markersize: 5 - color: 'red' - label: 'hofxdiff vs qcdiff' - statistics: - fields: - - field_name: experiment::QCDiff::${variable} - @CHANNELKEY@ - xloc: 0.5 - yloc: -0.10 - kwargs: - fontsize: 6 - statistics_variables: - - n - - min - - mean - - max - - std + # QC difference as a function of H(x) difference + - batch figure: + variables: *variables + @CHANNELSKEY@ + figure: + layout: [1,1] + title: 'H(x) JEDI-GSI vs QC JEDI-GSI. | @NAME@ @CYCLE@ | ${variable_title}' + output name: observation_scatter_plots/hofxdiff_vs_qcdiff_@CYCLE@_@NAME@_${variable}@CHANNELVAR@.png + plots: + - add_xlabel: 'h(x) JEDI - h(x) GSI' + add_ylabel: 'h(x) JEDI - h(x) GSI' + add_grid: + add_legend: + loc: 'lower left' + layers: + - type: Scatter + x: + variable: experiment::QCDiff::${variable} + y: + variable: experiment::HofXDiff::${variable} + @CHANNELKEY@ + markersize: 5 + color: 'red' + label: 'hofxdiff vs qcdiff' + statistics: + fields: + - field_name: experiment::QCDiff::${variable} + @CHANNELKEY@ + xloc: 0.5 + yloc: -0.10 + kwargs: + fontsize: 6 + statistics_variables: + - n + - min + - mean + - max + - std - # Error difference as a function of pressure - - batch figure: - variables: *variables - @CHANNELSKEY@ - figure: - layout: [1,1] - title: 'Error JEDI-GSI vs Ob P. | @NAME@ @CYCLE@ | ${variable_title}' - output name: observation_scatter_plots/errordiff_vs_pressure_@CYCLE@_@NAME@_${variable}@CHANNELVAR@.png - plots: - - add_xlabel: 'Error JEDI - Error GSI' - add_ylabel: 'Observation Pressure' - add_grid: - add_legend: - loc: 'lower left' - layers: - - type: Scatter - x: - variable: experiment::ErrDiff::${variable} - y: - variable: experiment::MetaData::pressure - @CHANNELKEY@ - markersize: 5 - color: 'red' - label: 'errordiff vs pressure' + # Error difference as a function of pressure + - batch figure: + variables: *variables + @CHANNELSKEY@ + figure: + layout: [1,1] + title: 'Error JEDI-GSI vs Ob P. | @NAME@ @CYCLE@ | ${variable_title}' + output name: observation_scatter_plots/errordiff_vs_pressure_@CYCLE@_@NAME@_${variable}@CHANNELVAR@.png + plots: + - add_xlabel: 'Error JEDI - Error GSI' + add_ylabel: 'Observation Pressure' + add_grid: + add_legend: + loc: 'lower left' + layers: + - type: Scatter + x: + variable: experiment::ErrDiff::${variable} + y: + variable: experiment::MetaData::pressure + @CHANNELKEY@ + markersize: 5 + color: 'red' + label: 'errordiff vs pressure' - # ---------- Histograms ---------- - # Histogram of h(x) difference - - batch figure: - variables: *variables - @CHANNELSKEY@ - figure: - layout: [1,1] - title: 'Difference of h(x) JEDI-GSI | @NAME@ @CYCLE@ | ${variable_title}' - output name: histograms/hofxdiff_histogram_@CYCLE@_@NAME@_${variable}@CHANNELVAR@.png - plots: - - add_xlabel: 'h(x) JEDI - h(x) GSI' - add_ylabel: 'Count' - statistics: - fields: - - field_name: experiment::HofXDiff::${variable} - @CHANNELKEY@ - xloc: 0.5 - yloc: -0.10 - kwargs: - fontsize: 6 - statistics_variables: - - n - - min - - mean - - max - - std - layers: - - type: Histogram - data: - variable: experiment::HofXDiff::${variable} + # ---------- Histograms ---------- + # Histogram of h(x) difference + - batch figure: + variables: *variables + @CHANNELSKEY@ + figure: + layout: [1,1] + title: 'Difference of h(x) JEDI-GSI | @NAME@ @CYCLE@ | ${variable_title}' + output name: histograms/hofxdiff_histogram_@CYCLE@_@NAME@_${variable}@CHANNELVAR@.png + plots: + - add_xlabel: 'h(x) JEDI - h(x) GSI' + add_ylabel: 'Count' + statistics: + fields: + - field_name: experiment::HofXDiff::${variable} @CHANNELKEY@ - color: 'blue' - label: 'h(x) JEDI - h(x) GSI' - bins: 100 + xloc: 0.5 + yloc: -0.10 + kwargs: + fontsize: 6 + statistics_variables: + - n + - min + - mean + - max + - std + layers: + - type: Histogram + data: + variable: experiment::HofXDiff::${variable} + @CHANNELKEY@ + color: 'blue' + label: 'h(x) JEDI - h(x) GSI' + bins: 100 - # Histogram of obs error difference - - batch figure: - variables: *variables - @CHANNELSKEY@ - figure: - layout: [1,1] - title: 'Difference of final obs error JEDI-GSI | @NAME@ @CYCLE@ | ${variable_title}' - output name: histograms/errordiff_histogram_@CYCLE@_@NAME@_${variable}@CHANNELVAR@.png - plots: - - add_xlabel: 'error JEDI - error GSI' - add_ylabel: 'Count' - statistics: - fields: - - field_name: experiment::ErrDiff::${variable} - @CHANNELKEY@ - xloc: 0.5 - yloc: -0.10 - kwargs: - fontsize: 6 - statistics_variables: - - n - - min - - mean - - max - - std - layers: - - type: Histogram - data: - variable: experiment::ErrDiff::${variable} + # Histogram of obs error difference + - batch figure: + variables: *variables + @CHANNELSKEY@ + figure: + layout: [1,1] + title: 'Difference of final obs error JEDI-GSI | @NAME@ @CYCLE@ | ${variable_title}' + output name: histograms/errordiff_histogram_@CYCLE@_@NAME@_${variable}@CHANNELVAR@.png + plots: + - add_xlabel: 'error JEDI - error GSI' + add_ylabel: 'Count' + statistics: + fields: + - field_name: experiment::ErrDiff::${variable} @CHANNELKEY@ - color: 'blue' - label: 'error JEDI - error GSI' - bins: 100 + xloc: 0.5 + yloc: -0.10 + kwargs: + fontsize: 6 + statistics_variables: + - n + - min + - mean + - max + - std + layers: + - type: Histogram + data: + variable: experiment::ErrDiff::${variable} + @CHANNELKEY@ + color: 'blue' + label: 'error JEDI - error GSI' + bins: 100 - # ---------- Map Plots ---------- - # Map plot of h(x) difference - # -------- + # ---------- Map Plots ---------- + # Map plot of h(x) difference + # -------- - - batch figure: - variables: *variables - @CHANNELSKEY@ - figure: - layout: [1,1] - figure size: [11,5] - title: 'Difference of h(x) JEDI-GSI | @NAME@ @CYCLE@ | ${variable_title}' - output name: map_plots/hofxdiff_map_@CYCLE@_@NAME@_${variable}@CHANNELVAR@.png - tight_layout: true - plots: - - mapping: - projection: plcarr - domain: global - add_map_features: ['coastline'] - add_grid: - add_colorbar: - layers: - - type: MapScatter - longitude: - variable: experiment::MetaData::longitude - latitude: - variable: experiment::MetaData::latitude - data: - variable: experiment::HofXDiff::${variable} - @CHANNELKEY@ - markersize: 2 - label: '$(variable)' - colorbar: true - # below may need to be edited/removed - cmap: 'bwr' - vmin: -0.5 - vmax: 0.5 + - batch figure: + variables: *variables + @CHANNELSKEY@ + figure: + layout: [1,1] + figure size: [11,5] + title: 'Difference of h(x) JEDI-GSI | @NAME@ @CYCLE@ | ${variable_title}' + output name: map_plots/hofxdiff_map_@CYCLE@_@NAME@_${variable}@CHANNELVAR@.png + tight_layout: true + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_grid: + add_colorbar: + layers: + - type: MapScatter + longitude: + variable: experiment::MetaData::longitude + latitude: + variable: experiment::MetaData::latitude + data: + variable: experiment::HofXDiff::${variable} + @CHANNELKEY@ + markersize: 2 + label: '$(variable)' + colorbar: true + # below may need to be edited/removed + cmap: 'bwr' + vmin: -0.5 + vmax: 0.5 diff --git a/ush/eva/jedi_gsi_compare_rad.yaml b/ush/eva/jedi_gsi_compare_rad.yaml index 619fe2fdd..b7bd1be89 100644 --- a/ush/eva/jedi_gsi_compare_rad.yaml +++ b/ush/eva/jedi_gsi_compare_rad.yaml @@ -1,642 +1,640 @@ # template YAML file to create EVA YAML files # based on obs spaces listed in JEDI YAML files -diagnostics: -- data: - datasets: - - name: experiment - type: IodaObsSpace - filenames: - - @FILENAME@ - @CHANNELSKEY@ - groups: - - name: ObsValue - variables: &variables @VARIABLES@ - - name: GsiHofXBc - - name: hofx - - name: EffectiveQC - - name: MetaData - - name: GsiEffectiveQC - - name: GsiFinalObsError - - name: EffectiveError - transforms: - - # difference of hofx - - transform: arithmetic - new name: experiment::HofXDiff::${variable} - equals: experiment::hofx::${variable}-experiment::GsiHofXBc::${variable} - for: - variable: *variables - - # difference of effective error - - transform: arithmetic - new name: experiment::ErrDiff::${variable} - equals: experiment::EffectiveError::${variable}-experiment::GsiFinalObsError::${variable} - for: - variable: *variables - - # difference of QC - - transform: arithmetic - new name: experiment::QCDiff::${variable} - equals: experiment::EffectiveQC::${variable}-experiment::GsiEffectiveQC::${variable} - for: - variable: *variables - - # Generate omb for GSI - - transform: arithmetic - new name: experiment::ObsValueMinusGsiHofXBc::${variable} - equals: experiment::ObsValue::${variable}-experiment::GsiHofXBc::${variable} - for: - variable: *variables - - # Generate omb for JEDI - - transform: arithmetic - new name: experiment::ObsValueMinusHofx::${variable} - equals: experiment::ObsValue::${variable}-experiment::hofx::${variable} - for: - variable: *variables - - # Generate hofx that passed QC for JEDI - - transform: accept where - new name: experiment::hofxPassedQc::${variable} - starting field: experiment::hofx::${variable} - where: - - experiment::EffectiveQC::${variable} == 0 - for: - variable: *variables - - # Generate GSI hofx that passed JEDI QC - - transform: accept where - new name: experiment::GsiHofXBcPassedQc::${variable} - starting field: experiment::GsiHofXBc::${variable} - where: - - experiment::EffectiveQC::${variable} == 0 - for: - variable: *variables - - # Generate omb that passed QC for JEDI - - transform: accept where - new name: experiment::ObsValueMinushofxPassedQc::${variable} - starting field: experiment::ObsValueMinusHofx::${variable} - where: - - experiment::EffectiveQC::${variable} == 0 - for: - variable: *variables - - # Generate omb that passed QC for GSI - - transform: accept where - new name: experiment::ObsValueMinusGsiHofXBcPassedQc::${variable} - starting field: experiment::ObsValueMinusGsiHofXBc::${variable} - where: - - experiment::GsiEffectiveQC::${variable} == 0 - for: - variable: *variables - - # hofxdiff that passed QC for JEDI - - transform: accept where - new name: experiment::HofXDiffPassedJediQc::${variable} - starting field: experiment::HofXDiff::${variable} - where: - - experiment::EffectiveQC::${variable} == 0 - for: - variable: *variables - - # hofxdiff that passed QC for GSI - - transform: accept where - new name: experiment::HofXDiffPassedGsiQc::${variable} - starting field: experiment::HofXDiff::${variable} - where: - - experiment::GsiEffectiveQC::${variable} == 0 - for: - variable: *variables - - # Stats for hofx - - transform: channel_stats - variable_name: experiment::HofXDiff::${variable} - for: - variable: *variables - - # Stats for omb that passed QC for GSI - - transform: channel_stats - variable_name: experiment::ObsValueMinusGsiHofXBcPassedQc::${variable} - for: - variable: *variables - - # Stats for omb that passed QC for JEDI - - transform: channel_stats - variable_name: experiment::ObsValueMinushofxPassedQc::${variable} - for: - variable: *variables - - # Stats for Obs Error Difference - - transform: channel_stats - variable_name: experiment::ErrDiff::${variable} - for: - variable: *variables - - # Generate QC Counts Difference - - transform: arithmetic - new name: experiment::QCCountsDiff::${variable} - equals: experiment::ObsValueMinushofxPassedQcCount::brightnessTemperature-experiment::ObsValueMinusGsiHofXBcPassedQcCount::brightnessTemperature - for: - variable: *variables - - # Because of some issues with passive IR channels, the following four transforms create QC counts just for channels with non-zero JEDI counts - - transform: accept where - new name: experiment::PassedQc_Used::${variable} - starting field: experiment::ObsValueMinushofxPassedQcCount::${variable} - where: - - experiment::ObsValueMinushofxPassedQcCount::${variable} > 0 - for: - variable: *variables - - - transform: accept where - new name: experiment::GsiPassedQc_Used::${variable} - starting field: experiment::ObsValueMinusGsiHofXBcPassedQcCount::${variable} - where: - - experiment::ObsValueMinushofxPassedQcCount::${variable} > 0 - for: - variable: *variables - - - transform: arithmetic - new name: experiment::QCCountsDiff_Used::${variable} - equals: experiment::PassedQc_Used::brightnessTemperature-experiment::GsiPassedQc_Used::brightnessTemperature - for: - variable: *variables - - - transform: accept where - new name: experiment::MetaData::sensorChannelNumber_Used - starting field: experiment::MetaData::sensorChannelNumber - where: - - experiment::ObsValueMinushofxPassedQcCount::${variable} > 0 - for: - variable: *variables - - graphics: - - figure: - layout: [1,1] - title: 'JEDI HofX - GSI HofX vs Channel | @NAME@ @CYCLE@' - output name: HofX_Difference_@CYCLE@_@NAME@.png - plots: - - add_xlabel: 'Channel' - add_ylabel: 'HofX Difference' - add_grid: - add_legend: - loc: 'upper left' - layers: - - type: Scatter - x: - variable: experiment::MetaData::sensorChannelNumber - y: - variable: experiment::HofXDiffMean::brightnessTemperature - markersize: 5 - color: 'red' - label: 'Mean' - do_linear_regression: False - - type: Scatter - x: - variable: experiment::MetaData::sensorChannelNumber - y: - variable: experiment::HofXDiffMax::brightnessTemperature - markersize: 5 - color: 'green' - label: 'Maximum' - do_linear_regression: False - - type: Scatter - x: - variable: experiment::MetaData::sensorChannelNumber - y: - variable: experiment::HofXDiffMin::brightnessTemperature - markersize: 5 - color: 'blue' - label: 'Minimum' - do_linear_regression: False - - - figure: - layout: [1,1] - title: 'Difference in Number of Obs Passing QC | @NAME@ @CYCLE@' - output name: QCCount_Diff_@CYCLE@_@NAME@.png - plots: - - add_xlabel: 'Channel' - add_ylabel: 'Counts' - add_grid: - add_legend: - loc: 'upper left' - layers: - - type: Scatter - x: - variable: experiment::MetaData::sensorChannelNumber - y: - variable: experiment::QCCountsDiff::brightnessTemperature - markersize: 5 - color: 'red' - label: 'JEDI-GSI Counts Difference' - do_linear_regression: False - - - - figure: - layout: [1,1] - title: 'Difference in Number of Obs Passing QC - Used Channels Only | @NAME@ @CYCLE@' - output name: QCCount_Diff_Used_@CYCLE@_@NAME@.png - plots: - - add_xlabel: 'Channel' - add_ylabel: 'Counts' - add_grid: - add_legend: - loc: 'upper left' - layers: - - type: Scatter - x: - variable: experiment::MetaData::sensorChannelNumber_Used - y: - variable: experiment::QCCountsDiff_Used::brightnessTemperature - markersize: 5 - color: 'red' - label: 'JEDI-GSI Counts Difference' - do_linear_regression: False - - - figure: - layout: [1,1] - title: 'Number of Obs Passing QC | @NAME@ @CYCLE@' - output name: PassedQC_Counts@CYCLE@_@NAME@.png - plots: - - add_xlabel: 'Channel' - add_ylabel: 'Counts' - add_grid: - add_legend: - loc: 'upper left' - layers: - - type: Scatter - x: - variable: experiment::MetaData::sensorChannelNumber - y: - variable: experiment::ObsValueMinushofxPassedQcCount::brightnessTemperature - markersize: 5 - color: 'red' - label: 'JEDI' - do_linear_regression: False - - type: Scatter - x: - variable: experiment::MetaData::sensorChannelNumber - y: - variable: experiment::ObsValueMinusGsiHofXBcPassedQcCount::brightnessTemperature - markersize: 5 - color: 'green' - label: 'GSI' - do_linear_regression: False - - - figure: - layout: [1,1] - title: 'Number of Obs Passing QC - Used Channels Only | @NAME@ @CYCLE@' - output name: PassedQC_Counts_Used@CYCLE@_@NAME@.png - plots: - - add_xlabel: 'Channel' - add_ylabel: 'Counts' - add_grid: - add_legend: - loc: 'upper left' - layers: - - type: Scatter - x: - variable: experiment::MetaData::sensorChannelNumber_Used - y: - variable: experiment::PassedQc_Used::brightnessTemperature - markersize: 5 - color: 'red' - label: 'JEDI' - do_linear_regression: False - - type: Scatter - x: - variable: experiment::MetaData::sensorChannelNumber_Used - y: - variable: experiment::GsiPassedQc_Used::brightnessTemperature - markersize: 5 - color: 'green' - label: 'GSI' - do_linear_regression: False - - - figure: - layout: [1,1] - title: 'JEDI Obs Error - GSI Obs Error vs Channel | @NAME@ @CYCLE@' - output name: ObsErr_Difference_@CYCLE@_@NAME@.png - plots: - - add_xlabel: 'Channel' - add_ylabel: 'Observation Error Difference' - add_grid: - add_legend: - loc: 'upper left' - layers: - - type: Scatter - x: - variable: experiment::MetaData::sensorChannelNumber - y: - variable: experiment::ErrDiffMean::brightnessTemperature - markersize: 5 - color: 'red' - label: 'Mean' - do_linear_regression: False - - type: Scatter - x: - variable: experiment::MetaData::sensorChannelNumber - y: - variable: experiment::ErrDiffMax::brightnessTemperature - markersize: 5 - color: 'green' - label: 'Maximum' - do_linear_regression: False - - type: Scatter - x: - variable: experiment::MetaData::sensorChannelNumber - y: - variable: experiment::ErrDiffMin::brightnessTemperature - markersize: 5 - color: 'blue' - label: 'Minimum' - do_linear_regression: False - - # ---------- Scatter Plots ---------- - # JEDI h(x) vs Observations - # ------------------------- - - batch figure: - variables: *variables - @CHANNELSKEY@ - figure: - layout: [1,1] - title: 'Observations vs. JEDI h(x) | @NAME@ @CYCLE@ | ${variable_title}' - output name: observation_scatter_plots/jedi_hofx_vs_obs_@CYCLE@_@NAME@_${variable}@CHANNELVAR@.png - plots: - - add_xlabel: 'Observation Value' - add_ylabel: 'JEDI h(x)' - add_grid: - add_legend: - loc: 'upper left' - layers: - - type: Scatter - x: - variable: experiment::ObsValue::${variable} - y: - variable: experiment::hofx::${variable} - @CHANNELKEY@ - markersize: 5 - color: 'black' - label: 'JEDI h(x) versus obs (all obs)' - - type: Scatter - x: - variable: experiment::ObsValue::${variable} - y: - variable: experiment::hofxPassedQc::${variable} - @CHANNELKEY@ - markersize: 5 - color: 'red' - label: 'JEDI h(x) versus obs (passed QC in JEDI)' - - # GSI h(x) vs Observations - # ------------------------- - - batch figure: - variables: *variables - @CHANNELSKEY@ - figure: - layout: [1,1] - title: 'Observations vs. GSI h(x) | @NAME@ @CYCLE@ | ${variable_title}' - output name: observation_scatter_plots/gsi_hofx_vs_obs_@CYCLE@_@NAME@_${variable}@CHANNELVAR@.png - plots: - - add_xlabel: 'Observation Value' - add_ylabel: 'GSI h(x)' - add_grid: - add_legend: - loc: 'upper left' - layers: - - type: Scatter - x: - variable: experiment::ObsValue::${variable} - y: - variable: experiment::GsiHofXBc::${variable} - @CHANNELKEY@ - markersize: 5 - color: 'black' - label: 'GSI h(x) versus obs (all obs)' - - type: Scatter - x: - variable: experiment::ObsValue::${variable} - y: - variable: experiment::GsiHofXBcPassedQc::${variable} - @CHANNELKEY@ - markersize: 5 - color: 'red' - label: 'GSI h(x) versus obs (passed QC in JEDI)' - - # JEDI h(x) vs GSI h(x) - # --------------------- - - - batch figure: - variables: *variables - @CHANNELSKEY@ - figure: - layout: [1,1] - title: 'JEDI h(x) vs. GSI h(x) | @NAME@ @CYCLE@ | ${variable_title}' - output name: observation_scatter_plots/gsi_hofx_vs_jedi_hofx_@CYCLE@_@NAME@_${variable}@CHANNELVAR@.png - plots: - - add_xlabel: 'GSI h(x)' - add_ylabel: 'JEDI h(x)' - add_grid: - add_legend: - loc: 'lower left' - layers: - - type: Scatter - x: - variable: experiment::GsiHofXBc::${variable} - y: - variable: experiment::hofx::${variable} - @CHANNELKEY@ - markersize: 5 - color: 'black' - label: 'JEDI h(x) versus GSI h(x)' - - type: Scatter - x: - variable: experiment::GsiHofXBcPassedQc::${variable} - y: - variable: experiment::hofxPassedQc::${variable} - @CHANNELKEY@ - markersize: 5 - color: 'red' - label: 'JEDI h(x) versus GSI h(x) (passed QC in JEDI)' - - # JEDI omb vs GSI omb - # --------------------- - - - batch figure: - variables: *variables - @CHANNELSKEY@ - figure: - layout: [1,1] - title: 'JEDI omb vs. GSI omb | @NAME@ @CYCLE@ | ${variable_title}' - output name: observation_scatter_plots/gsi_omb_vs_jedi_omb_@CYCLE@_@NAME@_${variable}@CHANNELVAR@.png - plots: - - add_xlabel: 'GSI observation minus h(x)' - add_ylabel: 'JEDI observation minus h(x)' - add_grid: - add_legend: - loc: 'upper left' - layers: - - type: Scatter - x: - variable: experiment::ObsValueMinusGsiHofXBc::${variable} - y: - variable: experiment::ObsValueMinusHofx::${variable} - @CHANNELKEY@ - markersize: 5 - color: 'black' - label: 'GSI omb vs JEDI omb (all obs)' - - type: Scatter - x: - variable: experiment::ObsValueMinusGsiHofXBcPassedQc::${variable} - y: - variable: experiment::ObsValueMinushofxPassedQc::${variable} - @CHANNELKEY@ - markersize: 5 - color: 'red' - label: 'GSI omb vs JEDI omb (passed QC in JEDI)' - - # QC difference as a function of H(x) difference - - batch figure: - variables: *variables - @CHANNELSKEY@ - figure: - layout: [1,1] - title: 'H(x) JEDI-GSI vs QC JEDI-GSI. | @NAME@ @CYCLE@ | ${variable_title}' - output name: observation_scatter_plots/hofxdiff_vs_qcdiff_@CYCLE@_@NAME@_${variable}@CHANNELVAR@.png - plots: - - add_xlabel: 'h(x) JEDI - h(x) GSI' - add_ylabel: 'h(x) JEDI - h(x) GSI' - add_grid: - add_legend: - loc: 'lower left' - layers: - - type: Scatter - x: - variable: experiment::QCDiff::${variable} - y: - variable: experiment::HofXDiff::${variable} - @CHANNELKEY@ - markersize: 5 - color: 'red' - label: 'hofxdiff vs qcdiff' - statistics: - fields: - - field_name: experiment::QCDiff::${variable} - @CHANNELKEY@ - xloc: 0.5 - yloc: -0.10 - kwargs: - fontsize: 6 - statistics_variables: - - n - - min - - mean - - max - - std - - # ---------- Histograms ---------- - # Histogram of h(x) difference - - batch figure: - variables: *variables - @CHANNELSKEY@ - figure: - layout: [1,1] - title: 'Difference of h(x) JEDI-GSI | @NAME@ @CYCLE@ | ${variable_title}' - output name: histograms/hofxdiff_histogram_@CYCLE@_@NAME@_${variable}@CHANNELVAR@.png - plots: - - add_xlabel: 'h(x) JEDI - h(x) GSI' - add_ylabel: 'Count' - statistics: - fields: - - field_name: experiment::HofXDiff::${variable} - @CHANNELKEY@ - xloc: 0.5 - yloc: -0.10 - kwargs: - fontsize: 6 - statistics_variables: - - n - - min - - mean - - max - - std - layers: - - type: Histogram - data: - variable: experiment::HofXDiff::${variable} +datasets: + - name: experiment + type: IodaObsSpace + filenames: + - @FILENAME@ + @CHANNELSKEY@ + groups: + - name: ObsValue + variables: &variables @VARIABLES@ + - name: GsiHofXBc + - name: hofx + - name: EffectiveQC + - name: MetaData + - name: GsiEffectiveQC + - name: GsiFinalObsError + - name: EffectiveError +transforms: + + # difference of hofx + - transform: arithmetic + new name: experiment::HofXDiff::${variable} + equals: experiment::hofx::${variable}-experiment::GsiHofXBc::${variable} + for: + variable: *variables + + # difference of effective error + - transform: arithmetic + new name: experiment::ErrDiff::${variable} + equals: experiment::EffectiveError::${variable}-experiment::GsiFinalObsError::${variable} + for: + variable: *variables + + # difference of QC + - transform: arithmetic + new name: experiment::QCDiff::${variable} + equals: experiment::EffectiveQC::${variable}-experiment::GsiEffectiveQC::${variable} + for: + variable: *variables + + # Generate omb for GSI + - transform: arithmetic + new name: experiment::ObsValueMinusGsiHofXBc::${variable} + equals: experiment::ObsValue::${variable}-experiment::GsiHofXBc::${variable} + for: + variable: *variables + + # Generate omb for JEDI + - transform: arithmetic + new name: experiment::ObsValueMinusHofx::${variable} + equals: experiment::ObsValue::${variable}-experiment::hofx::${variable} + for: + variable: *variables + + # Generate hofx that passed QC for JEDI + - transform: accept where + new name: experiment::hofxPassedQc::${variable} + starting field: experiment::hofx::${variable} + where: + - experiment::EffectiveQC::${variable} == 0 + for: + variable: *variables + + # Generate GSI hofx that passed JEDI QC + - transform: accept where + new name: experiment::GsiHofXBcPassedQc::${variable} + starting field: experiment::GsiHofXBc::${variable} + where: + - experiment::EffectiveQC::${variable} == 0 + for: + variable: *variables + + # Generate omb that passed QC for JEDI + - transform: accept where + new name: experiment::ObsValueMinushofxPassedQc::${variable} + starting field: experiment::ObsValueMinusHofx::${variable} + where: + - experiment::EffectiveQC::${variable} == 0 + for: + variable: *variables + + # Generate omb that passed QC for GSI + - transform: accept where + new name: experiment::ObsValueMinusGsiHofXBcPassedQc::${variable} + starting field: experiment::ObsValueMinusGsiHofXBc::${variable} + where: + - experiment::GsiEffectiveQC::${variable} == 0 + for: + variable: *variables + + # hofxdiff that passed QC for JEDI + - transform: accept where + new name: experiment::HofXDiffPassedJediQc::${variable} + starting field: experiment::HofXDiff::${variable} + where: + - experiment::EffectiveQC::${variable} == 0 + for: + variable: *variables + + # hofxdiff that passed QC for GSI + - transform: accept where + new name: experiment::HofXDiffPassedGsiQc::${variable} + starting field: experiment::HofXDiff::${variable} + where: + - experiment::GsiEffectiveQC::${variable} == 0 + for: + variable: *variables + + # Stats for hofx + - transform: channel_stats + variable_name: experiment::HofXDiff::${variable} + for: + variable: *variables + + # Stats for omb that passed QC for GSI + - transform: channel_stats + variable_name: experiment::ObsValueMinusGsiHofXBcPassedQc::${variable} + for: + variable: *variables + + # Stats for omb that passed QC for JEDI + - transform: channel_stats + variable_name: experiment::ObsValueMinushofxPassedQc::${variable} + for: + variable: *variables + + # Stats for Obs Error Difference + - transform: channel_stats + variable_name: experiment::ErrDiff::${variable} + for: + variable: *variables + + # Generate QC Counts Difference + - transform: arithmetic + new name: experiment::QCCountsDiff::${variable} + equals: experiment::ObsValueMinushofxPassedQcCount::brightnessTemperature-experiment::ObsValueMinusGsiHofXBcPassedQcCount::brightnessTemperature + for: + variable: *variables + + # Because of some issues with passive IR channels, the following four transforms create QC counts just for channels with non-zero JEDI counts + - transform: accept where + new name: experiment::PassedQc_Used::${variable} + starting field: experiment::ObsValueMinushofxPassedQcCount::${variable} + where: + - experiment::ObsValueMinushofxPassedQcCount::${variable} > 0 + for: + variable: *variables + + - transform: accept where + new name: experiment::GsiPassedQc_Used::${variable} + starting field: experiment::ObsValueMinusGsiHofXBcPassedQcCount::${variable} + where: + - experiment::ObsValueMinushofxPassedQcCount::${variable} > 0 + for: + variable: *variables + + - transform: arithmetic + new name: experiment::QCCountsDiff_Used::${variable} + equals: experiment::PassedQc_Used::brightnessTemperature-experiment::GsiPassedQc_Used::brightnessTemperature + for: + variable: *variables + + - transform: accept where + new name: experiment::MetaData::sensorChannelNumber_Used + starting field: experiment::MetaData::sensorChannelNumber + where: + - experiment::ObsValueMinushofxPassedQcCount::${variable} > 0 + for: + variable: *variables + +graphics: + - figure: + layout: [1,1] + title: 'JEDI HofX - GSI HofX vs Channel | @NAME@ @CYCLE@' + output name: HofX_Difference_@CYCLE@_@NAME@.png + plots: + - add_xlabel: 'Channel' + add_ylabel: 'HofX Difference' + add_grid: + add_legend: + loc: 'upper left' + layers: + - type: Scatter + x: + variable: experiment::MetaData::sensorChannelNumber + y: + variable: experiment::HofXDiffMean::brightnessTemperature + markersize: 5 + color: 'red' + label: 'Mean' + do_linear_regression: False + - type: Scatter + x: + variable: experiment::MetaData::sensorChannelNumber + y: + variable: experiment::HofXDiffMax::brightnessTemperature + markersize: 5 + color: 'green' + label: 'Maximum' + do_linear_regression: False + - type: Scatter + x: + variable: experiment::MetaData::sensorChannelNumber + y: + variable: experiment::HofXDiffMin::brightnessTemperature + markersize: 5 + color: 'blue' + label: 'Minimum' + do_linear_regression: False + + - figure: + layout: [1,1] + title: 'Difference in Number of Obs Passing QC | @NAME@ @CYCLE@' + output name: QCCount_Diff_@CYCLE@_@NAME@.png + plots: + - add_xlabel: 'Channel' + add_ylabel: 'Counts' + add_grid: + add_legend: + loc: 'upper left' + layers: + - type: Scatter + x: + variable: experiment::MetaData::sensorChannelNumber + y: + variable: experiment::QCCountsDiff::brightnessTemperature + markersize: 5 + color: 'red' + label: 'JEDI-GSI Counts Difference' + do_linear_regression: False + + + - figure: + layout: [1,1] + title: 'Difference in Number of Obs Passing QC - Used Channels Only | @NAME@ @CYCLE@' + output name: QCCount_Diff_Used_@CYCLE@_@NAME@.png + plots: + - add_xlabel: 'Channel' + add_ylabel: 'Counts' + add_grid: + add_legend: + loc: 'upper left' + layers: + - type: Scatter + x: + variable: experiment::MetaData::sensorChannelNumber_Used + y: + variable: experiment::QCCountsDiff_Used::brightnessTemperature + markersize: 5 + color: 'red' + label: 'JEDI-GSI Counts Difference' + do_linear_regression: False + + - figure: + layout: [1,1] + title: 'Number of Obs Passing QC | @NAME@ @CYCLE@' + output name: PassedQC_Counts@CYCLE@_@NAME@.png + plots: + - add_xlabel: 'Channel' + add_ylabel: 'Counts' + add_grid: + add_legend: + loc: 'upper left' + layers: + - type: Scatter + x: + variable: experiment::MetaData::sensorChannelNumber + y: + variable: experiment::ObsValueMinushofxPassedQcCount::brightnessTemperature + markersize: 5 + color: 'red' + label: 'JEDI' + do_linear_regression: False + - type: Scatter + x: + variable: experiment::MetaData::sensorChannelNumber + y: + variable: experiment::ObsValueMinusGsiHofXBcPassedQcCount::brightnessTemperature + markersize: 5 + color: 'green' + label: 'GSI' + do_linear_regression: False + + - figure: + layout: [1,1] + title: 'Number of Obs Passing QC - Used Channels Only | @NAME@ @CYCLE@' + output name: PassedQC_Counts_Used@CYCLE@_@NAME@.png + plots: + - add_xlabel: 'Channel' + add_ylabel: 'Counts' + add_grid: + add_legend: + loc: 'upper left' + layers: + - type: Scatter + x: + variable: experiment::MetaData::sensorChannelNumber_Used + y: + variable: experiment::PassedQc_Used::brightnessTemperature + markersize: 5 + color: 'red' + label: 'JEDI' + do_linear_regression: False + - type: Scatter + x: + variable: experiment::MetaData::sensorChannelNumber_Used + y: + variable: experiment::GsiPassedQc_Used::brightnessTemperature + markersize: 5 + color: 'green' + label: 'GSI' + do_linear_regression: False + + - figure: + layout: [1,1] + title: 'JEDI Obs Error - GSI Obs Error vs Channel | @NAME@ @CYCLE@' + output name: ObsErr_Difference_@CYCLE@_@NAME@.png + plots: + - add_xlabel: 'Channel' + add_ylabel: 'Observation Error Difference' + add_grid: + add_legend: + loc: 'upper left' + layers: + - type: Scatter + x: + variable: experiment::MetaData::sensorChannelNumber + y: + variable: experiment::ErrDiffMean::brightnessTemperature + markersize: 5 + color: 'red' + label: 'Mean' + do_linear_regression: False + - type: Scatter + x: + variable: experiment::MetaData::sensorChannelNumber + y: + variable: experiment::ErrDiffMax::brightnessTemperature + markersize: 5 + color: 'green' + label: 'Maximum' + do_linear_regression: False + - type: Scatter + x: + variable: experiment::MetaData::sensorChannelNumber + y: + variable: experiment::ErrDiffMin::brightnessTemperature + markersize: 5 + color: 'blue' + label: 'Minimum' + do_linear_regression: False + + # ---------- Scatter Plots ---------- + # JEDI h(x) vs Observations + # ------------------------- + - batch figure: + variables: *variables + @CHANNELSKEY@ + figure: + layout: [1,1] + title: 'Observations vs. JEDI h(x) | @NAME@ @CYCLE@ | ${variable_title}' + output name: observation_scatter_plots/jedi_hofx_vs_obs_@CYCLE@_@NAME@_${variable}@CHANNELVAR@.png + plots: + - add_xlabel: 'Observation Value' + add_ylabel: 'JEDI h(x)' + add_grid: + add_legend: + loc: 'upper left' + layers: + - type: Scatter + x: + variable: experiment::ObsValue::${variable} + y: + variable: experiment::hofx::${variable} + @CHANNELKEY@ + markersize: 5 + color: 'black' + label: 'JEDI h(x) versus obs (all obs)' + - type: Scatter + x: + variable: experiment::ObsValue::${variable} + y: + variable: experiment::hofxPassedQc::${variable} + @CHANNELKEY@ + markersize: 5 + color: 'red' + label: 'JEDI h(x) versus obs (passed QC in JEDI)' + + # GSI h(x) vs Observations + # ------------------------- + - batch figure: + variables: *variables + @CHANNELSKEY@ + figure: + layout: [1,1] + title: 'Observations vs. GSI h(x) | @NAME@ @CYCLE@ | ${variable_title}' + output name: observation_scatter_plots/gsi_hofx_vs_obs_@CYCLE@_@NAME@_${variable}@CHANNELVAR@.png + plots: + - add_xlabel: 'Observation Value' + add_ylabel: 'GSI h(x)' + add_grid: + add_legend: + loc: 'upper left' + layers: + - type: Scatter + x: + variable: experiment::ObsValue::${variable} + y: + variable: experiment::GsiHofXBc::${variable} + @CHANNELKEY@ + markersize: 5 + color: 'black' + label: 'GSI h(x) versus obs (all obs)' + - type: Scatter + x: + variable: experiment::ObsValue::${variable} + y: + variable: experiment::GsiHofXBcPassedQc::${variable} + @CHANNELKEY@ + markersize: 5 + color: 'red' + label: 'GSI h(x) versus obs (passed QC in JEDI)' + + # JEDI h(x) vs GSI h(x) + # --------------------- + + - batch figure: + variables: *variables + @CHANNELSKEY@ + figure: + layout: [1,1] + title: 'JEDI h(x) vs. GSI h(x) | @NAME@ @CYCLE@ | ${variable_title}' + output name: observation_scatter_plots/gsi_hofx_vs_jedi_hofx_@CYCLE@_@NAME@_${variable}@CHANNELVAR@.png + plots: + - add_xlabel: 'GSI h(x)' + add_ylabel: 'JEDI h(x)' + add_grid: + add_legend: + loc: 'lower left' + layers: + - type: Scatter + x: + variable: experiment::GsiHofXBc::${variable} + y: + variable: experiment::hofx::${variable} + @CHANNELKEY@ + markersize: 5 + color: 'black' + label: 'JEDI h(x) versus GSI h(x)' + - type: Scatter + x: + variable: experiment::GsiHofXBcPassedQc::${variable} + y: + variable: experiment::hofxPassedQc::${variable} + @CHANNELKEY@ + markersize: 5 + color: 'red' + label: 'JEDI h(x) versus GSI h(x) (passed QC in JEDI)' + + # JEDI omb vs GSI omb + # --------------------- + + - batch figure: + variables: *variables + @CHANNELSKEY@ + figure: + layout: [1,1] + title: 'JEDI omb vs. GSI omb | @NAME@ @CYCLE@ | ${variable_title}' + output name: observation_scatter_plots/gsi_omb_vs_jedi_omb_@CYCLE@_@NAME@_${variable}@CHANNELVAR@.png + plots: + - add_xlabel: 'GSI observation minus h(x)' + add_ylabel: 'JEDI observation minus h(x)' + add_grid: + add_legend: + loc: 'upper left' + layers: + - type: Scatter + x: + variable: experiment::ObsValueMinusGsiHofXBc::${variable} + y: + variable: experiment::ObsValueMinusHofx::${variable} + @CHANNELKEY@ + markersize: 5 + color: 'black' + label: 'GSI omb vs JEDI omb (all obs)' + - type: Scatter + x: + variable: experiment::ObsValueMinusGsiHofXBcPassedQc::${variable} + y: + variable: experiment::ObsValueMinushofxPassedQc::${variable} + @CHANNELKEY@ + markersize: 5 + color: 'red' + label: 'GSI omb vs JEDI omb (passed QC in JEDI)' + + # QC difference as a function of H(x) difference + - batch figure: + variables: *variables + @CHANNELSKEY@ + figure: + layout: [1,1] + title: 'H(x) JEDI-GSI vs QC JEDI-GSI. | @NAME@ @CYCLE@ | ${variable_title}' + output name: observation_scatter_plots/hofxdiff_vs_qcdiff_@CYCLE@_@NAME@_${variable}@CHANNELVAR@.png + plots: + - add_xlabel: 'h(x) JEDI - h(x) GSI' + add_ylabel: 'h(x) JEDI - h(x) GSI' + add_grid: + add_legend: + loc: 'lower left' + layers: + - type: Scatter + x: + variable: experiment::QCDiff::${variable} + y: + variable: experiment::HofXDiff::${variable} + @CHANNELKEY@ + markersize: 5 + color: 'red' + label: 'hofxdiff vs qcdiff' + statistics: + fields: + - field_name: experiment::QCDiff::${variable} @CHANNELKEY@ - color: 'blue' - label: 'h(x) JEDI - h(x) GSI' - bins: 100 - - # Histogram of obs error difference - - batch figure: - variables: *variables - @CHANNELSKEY@ - figure: - layout: [1,1] - title: 'Difference of final obs error JEDI-GSI | @NAME@ @CYCLE@ | ${variable_title}' - output name: histograms/errordiff_histogram_@CYCLE@_@NAME@_${variable}@CHANNELVAR@.png - plots: - - add_xlabel: 'error JEDI - error GSI' - add_ylabel: 'Count' - statistics: - fields: - - field_name: experiment::ErrDiff::${variable} - @CHANNELKEY@ - xloc: 0.5 - yloc: -0.10 - kwargs: - fontsize: 6 - statistics_variables: - - n - - min - - mean - - max - - std - layers: - - type: Histogram - data: - variable: experiment::ErrDiff::${variable} + xloc: 0.5 + yloc: -0.10 + kwargs: + fontsize: 6 + statistics_variables: + - n + - min + - mean + - max + - std + + # ---------- Histograms ---------- + # Histogram of h(x) difference + - batch figure: + variables: *variables + @CHANNELSKEY@ + figure: + layout: [1,1] + title: 'Difference of h(x) JEDI-GSI | @NAME@ @CYCLE@ | ${variable_title}' + output name: histograms/hofxdiff_histogram_@CYCLE@_@NAME@_${variable}@CHANNELVAR@.png + plots: + - add_xlabel: 'h(x) JEDI - h(x) GSI' + add_ylabel: 'Count' + statistics: + fields: + - field_name: experiment::HofXDiff::${variable} @CHANNELKEY@ - color: 'blue' - label: 'error JEDI - error GSI' - bins: 100 - - # ---------- Map Plots ---------- - # Map plot of h(x) difference - # -------- - - - batch figure: - variables: *variables - @CHANNELSKEY@ - figure: - layout: [1,1] - figure size: [11,5] - title: 'Difference of h(x) JEDI-GSI | @NAME@ @CYCLE@ | ${variable_title}' - output name: map_plots/hofxdiff_map_@CYCLE@_@NAME@_${variable}@CHANNELVAR@.png - tight_layout: true - plots: - - mapping: - projection: plcarr - domain: global - add_map_features: ['coastline'] - add_grid: - add_colorbar: - layers: - - type: MapScatter - longitude: - variable: experiment::MetaData::longitude - latitude: - variable: experiment::MetaData::latitude - data: - variable: experiment::HofXDiff::${variable} + xloc: 0.5 + yloc: -0.10 + kwargs: + fontsize: 6 + statistics_variables: + - n + - min + - mean + - max + - std + layers: + - type: Histogram + data: + variable: experiment::HofXDiff::${variable} + @CHANNELKEY@ + color: 'blue' + label: 'h(x) JEDI - h(x) GSI' + bins: 100 + + # Histogram of obs error difference + - batch figure: + variables: *variables + @CHANNELSKEY@ + figure: + layout: [1,1] + title: 'Difference of final obs error JEDI-GSI | @NAME@ @CYCLE@ | ${variable_title}' + output name: histograms/errordiff_histogram_@CYCLE@_@NAME@_${variable}@CHANNELVAR@.png + plots: + - add_xlabel: 'error JEDI - error GSI' + add_ylabel: 'Count' + statistics: + fields: + - field_name: experiment::ErrDiff::${variable} @CHANNELKEY@ - markersize: 2 - label: '$(variable)' - colorbar: true - # below may need to be edited/removed - cmap: 'bwr' - vmin: -0.1 - vmax: 0.1 + xloc: 0.5 + yloc: -0.10 + kwargs: + fontsize: 6 + statistics_variables: + - n + - min + - mean + - max + - std + layers: + - type: Histogram + data: + variable: experiment::ErrDiff::${variable} + @CHANNELKEY@ + color: 'blue' + label: 'error JEDI - error GSI' + bins: 100 + + # ---------- Map Plots ---------- + # Map plot of h(x) difference + # -------- + + - batch figure: + variables: *variables + @CHANNELSKEY@ + figure: + layout: [1,1] + figure size: [11,5] + title: 'Difference of h(x) JEDI-GSI | @NAME@ @CYCLE@ | ${variable_title}' + output name: map_plots/hofxdiff_map_@CYCLE@_@NAME@_${variable}@CHANNELVAR@.png + tight_layout: true + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_grid: + add_colorbar: + layers: + - type: MapScatter + longitude: + variable: experiment::MetaData::longitude + latitude: + variable: experiment::MetaData::latitude + data: + variable: experiment::HofXDiff::${variable} + @CHANNELKEY@ + markersize: 2 + label: '$(variable)' + colorbar: true + # below may need to be edited/removed + cmap: 'bwr' + vmin: -0.1 + vmax: 0.1 diff --git a/ush/eva/jedi_gsi_compare_rad_summary.yaml b/ush/eva/jedi_gsi_compare_rad_summary.yaml index 8424c6935..76f2e59fd 100644 --- a/ush/eva/jedi_gsi_compare_rad_summary.yaml +++ b/ush/eva/jedi_gsi_compare_rad_summary.yaml @@ -1,354 +1,352 @@ # template YAML file to create EVA YAML files # based on obs spaces listed in JEDI YAML files -diagnostics: -- data: - datasets: - - name: experiment - type: IodaObsSpace - filenames: - - @FILENAME@ - @CHANNELSKEY@ - groups: - - name: ObsValue - variables: &variables @VARIABLES@ - - name: GsiHofXBc - - name: hofx - - name: EffectiveQC - - name: MetaData - - name: GsiEffectiveQC - - name: GsiFinalObsError - - name: EffectiveError - transforms: +datasets: + - name: experiment + type: IodaObsSpace + filenames: + - @FILENAME@ + @CHANNELSKEY@ + groups: + - name: ObsValue + variables: &variables @VARIABLES@ + - name: GsiHofXBc + - name: hofx + - name: EffectiveQC + - name: MetaData + - name: GsiEffectiveQC + - name: GsiFinalObsError + - name: EffectiveError +transforms: - # difference of hofx - - transform: arithmetic - new name: experiment::HofXDiff::${variable} - equals: experiment::hofx::${variable}-experiment::GsiHofXBc::${variable} - for: - variable: *variables + # difference of hofx + - transform: arithmetic + new name: experiment::HofXDiff::${variable} + equals: experiment::hofx::${variable}-experiment::GsiHofXBc::${variable} + for: + variable: *variables - # difference of effective error - - transform: arithmetic - new name: experiment::ErrDiff::${variable} - equals: experiment::EffectiveError::${variable}-experiment::GsiFinalObsError::${variable} - for: - variable: *variables + # difference of effective error + - transform: arithmetic + new name: experiment::ErrDiff::${variable} + equals: experiment::EffectiveError::${variable}-experiment::GsiFinalObsError::${variable} + for: + variable: *variables - # difference of QC - - transform: arithmetic - new name: experiment::QCDiff::${variable} - equals: experiment::EffectiveQC::${variable}-experiment::GsiEffectiveQC::${variable} - for: - variable: *variables + # difference of QC + - transform: arithmetic + new name: experiment::QCDiff::${variable} + equals: experiment::EffectiveQC::${variable}-experiment::GsiEffectiveQC::${variable} + for: + variable: *variables - # Generate omb for GSI - - transform: arithmetic - new name: experiment::ObsValueMinusGsiHofXBc::${variable} - equals: experiment::ObsValue::${variable}-experiment::GsiHofXBc::${variable} - for: - variable: *variables + # Generate omb for GSI + - transform: arithmetic + new name: experiment::ObsValueMinusGsiHofXBc::${variable} + equals: experiment::ObsValue::${variable}-experiment::GsiHofXBc::${variable} + for: + variable: *variables - # Generate omb for JEDI - - transform: arithmetic - new name: experiment::ObsValueMinusHofx::${variable} - equals: experiment::ObsValue::${variable}-experiment::hofx::${variable} - for: - variable: *variables + # Generate omb for JEDI + - transform: arithmetic + new name: experiment::ObsValueMinusHofx::${variable} + equals: experiment::ObsValue::${variable}-experiment::hofx::${variable} + for: + variable: *variables - # Generate hofx that passed QC for JEDI - - transform: accept where - new name: experiment::hofxPassedQc::${variable} - starting field: experiment::hofx::${variable} - where: - - experiment::EffectiveQC::${variable} == 0 - for: - variable: *variables + # Generate hofx that passed QC for JEDI + - transform: accept where + new name: experiment::hofxPassedQc::${variable} + starting field: experiment::hofx::${variable} + where: + - experiment::EffectiveQC::${variable} == 0 + for: + variable: *variables - # Generate GSI hofx that passed JEDI QC - - transform: accept where - new name: experiment::GsiHofXBcPassedQc::${variable} - starting field: experiment::GsiHofXBc::${variable} - where: - - experiment::EffectiveQC::${variable} == 0 - for: - variable: *variables + # Generate GSI hofx that passed JEDI QC + - transform: accept where + new name: experiment::GsiHofXBcPassedQc::${variable} + starting field: experiment::GsiHofXBc::${variable} + where: + - experiment::EffectiveQC::${variable} == 0 + for: + variable: *variables - # Generate omb that passed QC for JEDI - - transform: accept where - new name: experiment::ObsValueMinushofxPassedQc::${variable} - starting field: experiment::ObsValueMinusHofx::${variable} - where: - - experiment::EffectiveQC::${variable} == 0 - for: - variable: *variables + # Generate omb that passed QC for JEDI + - transform: accept where + new name: experiment::ObsValueMinushofxPassedQc::${variable} + starting field: experiment::ObsValueMinusHofx::${variable} + where: + - experiment::EffectiveQC::${variable} == 0 + for: + variable: *variables - # Generate omb that passed QC for GSI - - transform: accept where - new name: experiment::ObsValueMinusGsiHofXBcPassedQc::${variable} - starting field: experiment::ObsValueMinusGsiHofXBc::${variable} - where: - - experiment::GsiEffectiveQC::${variable} == 0 - for: - variable: *variables + # Generate omb that passed QC for GSI + - transform: accept where + new name: experiment::ObsValueMinusGsiHofXBcPassedQc::${variable} + starting field: experiment::ObsValueMinusGsiHofXBc::${variable} + where: + - experiment::GsiEffectiveQC::${variable} == 0 + for: + variable: *variables - # hofxdiff that passed QC for JEDI - - transform: accept where - new name: experiment::HofXDiffPassedJediQc::${variable} - starting field: experiment::HofXDiff::${variable} - where: - - experiment::EffectiveQC::${variable} == 0 - for: - variable: *variables + # hofxdiff that passed QC for JEDI + - transform: accept where + new name: experiment::HofXDiffPassedJediQc::${variable} + starting field: experiment::HofXDiff::${variable} + where: + - experiment::EffectiveQC::${variable} == 0 + for: + variable: *variables - # hofxdiff that passed QC for GSI - - transform: accept where - new name: experiment::HofXDiffPassedGsiQc::${variable} - starting field: experiment::HofXDiff::${variable} - where: - - experiment::GsiEffectiveQC::${variable} == 0 - for: - variable: *variables + # hofxdiff that passed QC for GSI + - transform: accept where + new name: experiment::HofXDiffPassedGsiQc::${variable} + starting field: experiment::HofXDiff::${variable} + where: + - experiment::GsiEffectiveQC::${variable} == 0 + for: + variable: *variables - # Stats for hofx - - transform: channel_stats - variable_name: experiment::HofXDiff::${variable} - for: - variable: *variables + # Stats for hofx + - transform: channel_stats + variable_name: experiment::HofXDiff::${variable} + for: + variable: *variables - # Stats for omb that passed QC for GSI - - transform: channel_stats - variable_name: experiment::ObsValueMinusGsiHofXBcPassedQc::${variable} - for: - variable: *variables + # Stats for omb that passed QC for GSI + - transform: channel_stats + variable_name: experiment::ObsValueMinusGsiHofXBcPassedQc::${variable} + for: + variable: *variables - # Stats for omb that passed QC for JEDI - - transform: channel_stats - variable_name: experiment::ObsValueMinushofxPassedQc::${variable} - for: - variable: *variables + # Stats for omb that passed QC for JEDI + - transform: channel_stats + variable_name: experiment::ObsValueMinushofxPassedQc::${variable} + for: + variable: *variables - # Stats for Obs Error Difference - - transform: channel_stats - variable_name: experiment::ErrDiff::${variable} - for: - variable: *variables + # Stats for Obs Error Difference + - transform: channel_stats + variable_name: experiment::ErrDiff::${variable} + for: + variable: *variables - # Generate QC Counts Difference - - transform: arithmetic - new name: experiment::QCCountsDiff::${variable} - equals: experiment::ObsValueMinushofxPassedQcCount::brightnessTemperature-experiment::ObsValueMinusGsiHofXBcPassedQcCount::brightnessTemperature - for: - variable: *variables + # Generate QC Counts Difference + - transform: arithmetic + new name: experiment::QCCountsDiff::${variable} + equals: experiment::ObsValueMinushofxPassedQcCount::brightnessTemperature-experiment::ObsValueMinusGsiHofXBcPassedQcCount::brightnessTemperature + for: + variable: *variables - # Because of some issues with passive IR channels, the following four transforms create QC counts just for channels with non-zero JEDI counts - - transform: accept where - new name: experiment::PassedQc_Used::${variable} - starting field: experiment::ObsValueMinushofxPassedQcCount::${variable} - where: - - experiment::ObsValueMinushofxPassedQcCount::${variable} > 0 - for: - variable: *variables + # Because of some issues with passive IR channels, the following four transforms create QC counts just for channels with non-zero JEDI counts + - transform: accept where + new name: experiment::PassedQc_Used::${variable} + starting field: experiment::ObsValueMinushofxPassedQcCount::${variable} + where: + - experiment::ObsValueMinushofxPassedQcCount::${variable} > 0 + for: + variable: *variables - - transform: accept where - new name: experiment::GsiPassedQc_Used::${variable} - starting field: experiment::ObsValueMinusGsiHofXBcPassedQcCount::${variable} - where: - - experiment::ObsValueMinushofxPassedQcCount::${variable} > 0 - for: - variable: *variables + - transform: accept where + new name: experiment::GsiPassedQc_Used::${variable} + starting field: experiment::ObsValueMinusGsiHofXBcPassedQcCount::${variable} + where: + - experiment::ObsValueMinushofxPassedQcCount::${variable} > 0 + for: + variable: *variables - - transform: arithmetic - new name: experiment::QCCountsDiff_Used::${variable} - equals: experiment::PassedQc_Used::brightnessTemperature-experiment::GsiPassedQc_Used::brightnessTemperature - for: - variable: *variables + - transform: arithmetic + new name: experiment::QCCountsDiff_Used::${variable} + equals: experiment::PassedQc_Used::brightnessTemperature-experiment::GsiPassedQc_Used::brightnessTemperature + for: + variable: *variables - - transform: accept where - new name: experiment::MetaData::sensorChannelNumber_Used - starting field: experiment::MetaData::sensorChannelNumber - where: - - experiment::ObsValueMinushofxPassedQcCount::${variable} > 0 - for: - variable: *variables + - transform: accept where + new name: experiment::MetaData::sensorChannelNumber_Used + starting field: experiment::MetaData::sensorChannelNumber + where: + - experiment::ObsValueMinushofxPassedQcCount::${variable} > 0 + for: + variable: *variables - graphics: - - figure: - layout: [1,1] - title: 'JEDI HofX - GSI HofX vs Channel | @NAME@ @CYCLE@' - output name: HofX_Difference_@CYCLE@_@NAME@.png - plots: - - add_xlabel: 'Channel' - add_ylabel: 'HofX Difference' - add_grid: - add_legend: - loc: 'upper left' - layers: - - type: Scatter - x: - variable: experiment::MetaData::sensorChannelNumber - y: - variable: experiment::HofXDiffMean::brightnessTemperature - markersize: 5 - color: 'red' - label: 'Mean' - do_linear_regression: False - - type: Scatter - x: - variable: experiment::MetaData::sensorChannelNumber - y: - variable: experiment::HofXDiffMax::brightnessTemperature - markersize: 5 - color: 'green' - label: 'Maximum' - do_linear_regression: False - - type: Scatter - x: - variable: experiment::MetaData::sensorChannelNumber - y: - variable: experiment::HofXDiffMin::brightnessTemperature - markersize: 5 - color: 'blue' - label: 'Minimum' - do_linear_regression: False +graphics: + - figure: + layout: [1,1] + title: 'JEDI HofX - GSI HofX vs Channel | @NAME@ @CYCLE@' + output name: HofX_Difference_@CYCLE@_@NAME@.png + plots: + - add_xlabel: 'Channel' + add_ylabel: 'HofX Difference' + add_grid: + add_legend: + loc: 'upper left' + layers: + - type: Scatter + x: + variable: experiment::MetaData::sensorChannelNumber + y: + variable: experiment::HofXDiffMean::brightnessTemperature + markersize: 5 + color: 'red' + label: 'Mean' + do_linear_regression: False + - type: Scatter + x: + variable: experiment::MetaData::sensorChannelNumber + y: + variable: experiment::HofXDiffMax::brightnessTemperature + markersize: 5 + color: 'green' + label: 'Maximum' + do_linear_regression: False + - type: Scatter + x: + variable: experiment::MetaData::sensorChannelNumber + y: + variable: experiment::HofXDiffMin::brightnessTemperature + markersize: 5 + color: 'blue' + label: 'Minimum' + do_linear_regression: False - - figure: - layout: [1,1] - title: 'Difference in Number of Obs Passing QC | @NAME@ @CYCLE@' - output name: QCCount_Diff_@CYCLE@_@NAME@.png - plots: - - add_xlabel: 'Channel' - add_ylabel: 'Counts' - add_grid: - add_legend: - loc: 'upper left' - layers: - - type: Scatter - x: - variable: experiment::MetaData::sensorChannelNumber - y: - variable: experiment::QCCountsDiff::brightnessTemperature - markersize: 5 - color: 'red' - label: 'JEDI-GSI Counts Difference' - do_linear_regression: False + - figure: + layout: [1,1] + title: 'Difference in Number of Obs Passing QC | @NAME@ @CYCLE@' + output name: QCCount_Diff_@CYCLE@_@NAME@.png + plots: + - add_xlabel: 'Channel' + add_ylabel: 'Counts' + add_grid: + add_legend: + loc: 'upper left' + layers: + - type: Scatter + x: + variable: experiment::MetaData::sensorChannelNumber + y: + variable: experiment::QCCountsDiff::brightnessTemperature + markersize: 5 + color: 'red' + label: 'JEDI-GSI Counts Difference' + do_linear_regression: False - - figure: - layout: [1,1] - title: 'Difference in Number of Obs Passing QC - Used Channels Only | @NAME@ @CYCLE@' - output name: QCCount_Diff_Used_@CYCLE@_@NAME@.png - plots: - - add_xlabel: 'Channel' - add_ylabel: 'Counts' - add_grid: - add_legend: - loc: 'upper left' - layers: - - type: Scatter - x: - variable: experiment::MetaData::sensorChannelNumber_Used - y: - variable: experiment::QCCountsDiff_Used::brightnessTemperature - markersize: 5 - color: 'red' - label: 'JEDI-GSI Counts Difference' - do_linear_regression: False + - figure: + layout: [1,1] + title: 'Difference in Number of Obs Passing QC - Used Channels Only | @NAME@ @CYCLE@' + output name: QCCount_Diff_Used_@CYCLE@_@NAME@.png + plots: + - add_xlabel: 'Channel' + add_ylabel: 'Counts' + add_grid: + add_legend: + loc: 'upper left' + layers: + - type: Scatter + x: + variable: experiment::MetaData::sensorChannelNumber_Used + y: + variable: experiment::QCCountsDiff_Used::brightnessTemperature + markersize: 5 + color: 'red' + label: 'JEDI-GSI Counts Difference' + do_linear_regression: False - - figure: - layout: [1,1] - title: 'Number of Obs Passing QC | @NAME@ @CYCLE@' - output name: PassedQC_Counts@CYCLE@_@NAME@.png - plots: - - add_xlabel: 'Channel' - add_ylabel: 'Counts' - add_grid: - add_legend: - loc: 'upper left' - layers: - - type: Scatter - x: - variable: experiment::MetaData::sensorChannelNumber - y: - variable: experiment::ObsValueMinushofxPassedQcCount::brightnessTemperature - markersize: 5 - color: 'red' - label: 'JEDI' - do_linear_regression: False - - type: Scatter - x: - variable: experiment::MetaData::sensorChannelNumber - y: - variable: experiment::ObsValueMinusGsiHofXBcPassedQcCount::brightnessTemperature - markersize: 5 - color: 'green' - label: 'GSI' - do_linear_regression: False + - figure: + layout: [1,1] + title: 'Number of Obs Passing QC | @NAME@ @CYCLE@' + output name: PassedQC_Counts@CYCLE@_@NAME@.png + plots: + - add_xlabel: 'Channel' + add_ylabel: 'Counts' + add_grid: + add_legend: + loc: 'upper left' + layers: + - type: Scatter + x: + variable: experiment::MetaData::sensorChannelNumber + y: + variable: experiment::ObsValueMinushofxPassedQcCount::brightnessTemperature + markersize: 5 + color: 'red' + label: 'JEDI' + do_linear_regression: False + - type: Scatter + x: + variable: experiment::MetaData::sensorChannelNumber + y: + variable: experiment::ObsValueMinusGsiHofXBcPassedQcCount::brightnessTemperature + markersize: 5 + color: 'green' + label: 'GSI' + do_linear_regression: False - - figure: - layout: [1,1] - title: 'Number of Obs Passing QC - Used Channels Only | @NAME@ @CYCLE@' - output name: PassedQC_Counts_Used@CYCLE@_@NAME@.png - plots: - - add_xlabel: 'Channel' - add_ylabel: 'Counts' - add_grid: - add_legend: - loc: 'upper left' - layers: - - type: Scatter - x: - variable: experiment::MetaData::sensorChannelNumber_Used - y: - variable: experiment::PassedQc_Used::brightnessTemperature - markersize: 5 - color: 'red' - label: 'JEDI' - do_linear_regression: False - - type: Scatter - x: - variable: experiment::MetaData::sensorChannelNumber_Used - y: - variable: experiment::GsiPassedQc_Used::brightnessTemperature - markersize: 5 - color: 'green' - label: 'GSI' - do_linear_regression: False + - figure: + layout: [1,1] + title: 'Number of Obs Passing QC - Used Channels Only | @NAME@ @CYCLE@' + output name: PassedQC_Counts_Used@CYCLE@_@NAME@.png + plots: + - add_xlabel: 'Channel' + add_ylabel: 'Counts' + add_grid: + add_legend: + loc: 'upper left' + layers: + - type: Scatter + x: + variable: experiment::MetaData::sensorChannelNumber_Used + y: + variable: experiment::PassedQc_Used::brightnessTemperature + markersize: 5 + color: 'red' + label: 'JEDI' + do_linear_regression: False + - type: Scatter + x: + variable: experiment::MetaData::sensorChannelNumber_Used + y: + variable: experiment::GsiPassedQc_Used::brightnessTemperature + markersize: 5 + color: 'green' + label: 'GSI' + do_linear_regression: False - - figure: - layout: [1,1] - title: 'JEDI Obs Error - GSI Obs Error vs Channel | @NAME@ @CYCLE@' - output name: ObsErr_Difference_@CYCLE@_@NAME@.png - plots: - - add_xlabel: 'Channel' - add_ylabel: 'Observation Error Difference' - add_grid: - add_legend: - loc: 'upper left' - layers: - - type: Scatter - x: - variable: experiment::MetaData::sensorChannelNumber - y: - variable: experiment::ErrDiffMean::brightnessTemperature - markersize: 5 - color: 'red' - label: 'Mean' - do_linear_regression: False - - type: Scatter - x: - variable: experiment::MetaData::sensorChannelNumber - y: - variable: experiment::ErrDiffMax::brightnessTemperature - markersize: 5 - color: 'green' - label: 'Maximum' - do_linear_regression: False - - type: Scatter - x: - variable: experiment::MetaData::sensorChannelNumber - y: - variable: experiment::ErrDiffMin::brightnessTemperature - markersize: 5 - color: 'blue' - label: 'Minimum' - do_linear_regression: False + - figure: + layout: [1,1] + title: 'JEDI Obs Error - GSI Obs Error vs Channel | @NAME@ @CYCLE@' + output name: ObsErr_Difference_@CYCLE@_@NAME@.png + plots: + - add_xlabel: 'Channel' + add_ylabel: 'Observation Error Difference' + add_grid: + add_legend: + loc: 'upper left' + layers: + - type: Scatter + x: + variable: experiment::MetaData::sensorChannelNumber + y: + variable: experiment::ErrDiffMean::brightnessTemperature + markersize: 5 + color: 'red' + label: 'Mean' + do_linear_regression: False + - type: Scatter + x: + variable: experiment::MetaData::sensorChannelNumber + y: + variable: experiment::ErrDiffMax::brightnessTemperature + markersize: 5 + color: 'green' + label: 'Maximum' + do_linear_regression: False + - type: Scatter + x: + variable: experiment::MetaData::sensorChannelNumber + y: + variable: experiment::ErrDiffMin::brightnessTemperature + markersize: 5 + color: 'blue' + label: 'Minimum' + do_linear_regression: False diff --git a/ush/eva/marine_eva_post.py b/ush/eva/marine_eva_post.py index bf5634f45..9cdf01c90 100755 --- a/ush/eva/marine_eva_post.py +++ b/ush/eva/marine_eva_post.py @@ -23,22 +23,21 @@ def marine_eva_post(inputyaml, outputdir, diagdir): logging.info(f'Loading input YAML from {inputyaml}') except Exception as e: logging.error(f'Error occurred when attempting to load: {inputyaml}, error: {e}') - for diagnostic in input_yaml_dict['diagnostics']: - for dataset in diagnostic['data']['datasets']: - newfilenames = [] - for filename in dataset['filenames']: - newfilename = os.path.join(diagdir, os.path.basename(filename)) - newfilenames.append(newfilename) - dataset['filenames'] = newfilenames - for graphic in diagnostic['graphics']: - # this assumes that there is only one variable, or that the - # variables are all the same - variable = graphic['batch figure']['variables'][0] - for plot in graphic['plots']: - for layer in plot['layers']: - if layer['type'] == 'MapScatter': - layer['vmin'] = vminmax[variable]['vmin'] - layer['vmax'] = vminmax[variable]['vmax'] + for dataset in input_yaml_dict['datasets']: + newfilenames = [] + for filename in dataset['filenames']: + newfilename = os.path.join(diagdir, os.path.basename(filename)) + newfilenames.append(newfilename) + dataset['filenames'] = newfilenames + for graphic in input_yaml_dict['graphics']: + # this assumes that there is only one variable, or that the + # variables are all the same + variable = graphic['batch figure']['variables'][0] + for plot in graphic['plots']: + for layer in plot['layers']: + if layer['type'] == 'MapScatter': + layer['vmin'] = vminmax[variable]['vmin'] + layer['vmax'] = vminmax[variable]['vmax'] # first, let us prepend some comments that tell someone this output YAML was generated now = datetime.datetime.now() diff --git a/ush/eva/marine_gdas_plots.yaml b/ush/eva/marine_gdas_plots.yaml index d42de87e2..78e9f6c17 100644 --- a/ush/eva/marine_gdas_plots.yaml +++ b/ush/eva/marine_gdas_plots.yaml @@ -1,221 +1,219 @@ # template YAML file to create EVA YAML files # based on obs spaces listed in JEDI YAML files -diagnostics: -- data: - datasets: - - name: experiment - type: IodaObsSpace - filenames: - - @FILENAME@ - @CHANNELSKEY@ - groups: - - name: ObsValue - variables: &variables @VARIABLES@ - - name: ObsError - - name: ombg - - name: oman - - name: hofx0 - - name: EffectiveQC0 - - name: MetaData - - name: PreQC - transforms: - - # bkg - - transform: arithmetic - new name: experiment::bkg::${variable} - equals: experiment::ObsValue::${variable}-experiment::ombg::${variable} - for: - variable: *variables - - # Generate omb that passed QC for JEDI - - transform: accept where - new name: experiment::OmBQC::${variable} - starting field: experiment::ombg::${variable} - where: - - experiment::EffectiveQC0::${variable} == 0 - for: - variable: *variables - - # Generate oma that passed QC for JEDI - - transform: accept where - new name: experiment::OmAQC::${variable} - starting field: experiment::oman::${variable} - where: - - experiment::EffectiveQC0::${variable} == 0 - for: - variable: *variables - - # Generate oma that passed QC for JEDI - - transform: accept where - new name: experiment::hofxQC::${variable} - starting field: experiment::hofx0::${variable} - where: - - experiment::EffectiveQC0::${variable} == 0 - for: - variable: *variables - - - - - - graphics: - - # ---------- Map Plots ---------- - # Map plot of OmBQC - # -------- - - - batch figure: - variables: *variables - @CHANNELSKEY@ - dynamic options: - - type: vminvmaxcmap - data variable: experiment::OmBQC::${variable} - figure: - layout: [1,1] - figure size: [11,5] - title: 'OmB post QC | @NAME@ @CYCLE@ | ${variable_title}' - output name: map_plots/@NAME@/${variable}/@CHANNELVAR@/@NAME@_${variable}@CHANNELVAR@OmBQC.png - tight_layout: true - plots: - - mapping: - projection: plcarr - domain: global - add_map_features: ['coastline'] - add_grid: - add_colorbar: - label: '${variable}' - layers: - - type: MapScatter - longitude: - variable: experiment::MetaData::longitude - latitude: - variable: experiment::MetaData::latitude - data: - variable: experiment::OmBQC::${variable} - @CHANNELKEY@ - markersize: 2 - label: '$(variable)' - colorbar: true - # below may need to be edited/removed - cmap: ${dynamic_cmap} - vmin: ${dynamic_vmin} - vmax: ${dynamic_vmax} - - - # Histogram plots - # --------------- - - # OmA pre and post QC - - batch figure: - variables: *variables - figure: - layout: [1,1] - title: 'OmA pre- and post QC | @NAME@ | ${variable_title}' - output name: histograms/@NAME@/${variable}/oma_pre_post_qc_${variable}.png - plots: - - add_xlabel: 'Observation minus anl pre- and post-QC' - add_ylabel: 'Count' - add_legend: - loc: 'upper left' - statistics: - fields: - - field_name: experiment::OmAQC::${variable} - xloc: 0.5 - yloc: -0.10 - kwargs: - fontsize: 6 - statistics_variables: - - n - - min - - mean - - max - - std - layers: - - type: Histogram - data: - variable: experiment::OmAQC::${variable} - color: 'blue' - label: 'OmA (post QC)' - bins: 100 - alpha: 0.5 - - type: Histogram - data: - variable: experiment::oman::${variable} - color: 'red' - label: 'OmA (pre QC)' - bins: 100 - alpha: 0.5 - - # diff between OmA and OmB - - batch figure: - variables: *variables - figure: - layout: [1,1] - title: 'OmA and OmB, post-QC | @NAME@ | ${variable_title}' - output name: histograms/@NAME@/${variable}/oma_omb_histogram_${variable}.png - plots: - - add_xlabel: 'OmA and OmB post-QC' - add_ylabel: 'Count' - add_legend: - loc: 'upper left' - statistics: - fields: - - field_name: experiment::OmAQC::${variable} - xloc: 0.5 - yloc: -0.10 - kwargs: - fontsize: 6 - statistics_variables: - - n - - min - - mean - - max - - std - layers: - - type: Histogram - data: - variable: experiment::OmAQC::${variable} - color: 'blue' - label: 'OmA (post QC)' - bins: 100 - alpha: 0.5 - - type: Histogram - data: - variable: experiment::OmBQC::${variable} - color: 'red' - label: 'OmB (post QC)' - bins: 100 - alpha: 0.5 - - - batch figure: - variables: *variables - @CHANNELSKEY@ - figure: - layout: [1,1] - title: 'Observations vs. JEDI h(x) | @NAME@ @CYCLE@ | ${variable_title}' - output name: observation_scatter_plots/jedi_hofx_vs_obs_@CYCLE@_@NAME@_${variable}@CHANNELVAR@.png - plots: - - add_xlabel: 'Observation Value' - add_ylabel: 'JEDI h(x)' - add_grid: - add_legend: - loc: 'upper left' - layers: - - type: Scatter - x: - variable: experiment::ObsValue::${variable} - y: - variable: experiment::hofx0::${variable} +datasets: + - name: experiment + type: IodaObsSpace + filenames: + - @FILENAME@ + @CHANNELSKEY@ + groups: + - name: ObsValue + variables: &variables @VARIABLES@ + - name: ObsError + - name: ombg + - name: oman + - name: hofx0 + - name: EffectiveQC0 + - name: MetaData + - name: PreQC +transforms: + + # bkg + - transform: arithmetic + new name: experiment::bkg::${variable} + equals: experiment::ObsValue::${variable}-experiment::ombg::${variable} + for: + variable: *variables + + # Generate omb that passed QC for JEDI + - transform: accept where + new name: experiment::OmBQC::${variable} + starting field: experiment::ombg::${variable} + where: + - experiment::EffectiveQC0::${variable} == 0 + for: + variable: *variables + + # Generate oma that passed QC for JEDI + - transform: accept where + new name: experiment::OmAQC::${variable} + starting field: experiment::oman::${variable} + where: + - experiment::EffectiveQC0::${variable} == 0 + for: + variable: *variables + + # Generate oma that passed QC for JEDI + - transform: accept where + new name: experiment::hofxQC::${variable} + starting field: experiment::hofx0::${variable} + where: + - experiment::EffectiveQC0::${variable} == 0 + for: + variable: *variables + + + + + +graphics: + + # ---------- Map Plots ---------- + # Map plot of OmBQC + # -------- + + - batch figure: + variables: *variables + @CHANNELSKEY@ + dynamic options: + - type: vminvmaxcmap + data variable: experiment::OmBQC::${variable} + figure: + layout: [1,1] + figure size: [11,5] + title: 'OmB post QC | @NAME@ @CYCLE@ | ${variable_title}' + output name: map_plots/@NAME@/${variable}/@CHANNELVAR@/@NAME@_${variable}@CHANNELVAR@OmBQC.png + tight_layout: true + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_grid: + add_colorbar: + label: '${variable}' + layers: + - type: MapScatter + longitude: + variable: experiment::MetaData::longitude + latitude: + variable: experiment::MetaData::latitude + data: + variable: experiment::OmBQC::${variable} @CHANNELKEY@ - markersize: 5 - color: 'black' - label: 'JEDI h(x) versus obs (all obs)' - - type: Scatter - x: - variable: experiment::ObsValue::${variable} - y: - variable: experiment::hofxQC::${variable} - @CHANNELKEY@ - markersize: 5 - color: 'red' - label: 'JEDI h(x) versus obs (passed QC in JEDI)' + markersize: 2 + label: '$(variable)' + colorbar: true + # below may need to be edited/removed + cmap: ${dynamic_cmap} + vmin: ${dynamic_vmin} + vmax: ${dynamic_vmax} + + + # Histogram plots + # --------------- + + # OmA pre and post QC + - batch figure: + variables: *variables + figure: + layout: [1,1] + title: 'OmA pre- and post QC | @NAME@ | ${variable_title}' + output name: histograms/@NAME@/${variable}/oma_pre_post_qc_${variable}.png + plots: + - add_xlabel: 'Observation minus anl pre- and post-QC' + add_ylabel: 'Count' + add_legend: + loc: 'upper left' + statistics: + fields: + - field_name: experiment::OmAQC::${variable} + xloc: 0.5 + yloc: -0.10 + kwargs: + fontsize: 6 + statistics_variables: + - n + - min + - mean + - max + - std + layers: + - type: Histogram + data: + variable: experiment::OmAQC::${variable} + color: 'blue' + label: 'OmA (post QC)' + bins: 100 + alpha: 0.5 + - type: Histogram + data: + variable: experiment::oman::${variable} + color: 'red' + label: 'OmA (pre QC)' + bins: 100 + alpha: 0.5 + + # diff between OmA and OmB + - batch figure: + variables: *variables + figure: + layout: [1,1] + title: 'OmA and OmB, post-QC | @NAME@ | ${variable_title}' + output name: histograms/@NAME@/${variable}/oma_omb_histogram_${variable}.png + plots: + - add_xlabel: 'OmA and OmB post-QC' + add_ylabel: 'Count' + add_legend: + loc: 'upper left' + statistics: + fields: + - field_name: experiment::OmAQC::${variable} + xloc: 0.5 + yloc: -0.10 + kwargs: + fontsize: 6 + statistics_variables: + - n + - min + - mean + - max + - std + layers: + - type: Histogram + data: + variable: experiment::OmAQC::${variable} + color: 'blue' + label: 'OmA (post QC)' + bins: 100 + alpha: 0.5 + - type: Histogram + data: + variable: experiment::OmBQC::${variable} + color: 'red' + label: 'OmB (post QC)' + bins: 100 + alpha: 0.5 + + - batch figure: + variables: *variables + @CHANNELSKEY@ + figure: + layout: [1,1] + title: 'Observations vs. JEDI h(x) | @NAME@ @CYCLE@ | ${variable_title}' + output name: observation_scatter_plots/jedi_hofx_vs_obs_@CYCLE@_@NAME@_${variable}@CHANNELVAR@.png + plots: + - add_xlabel: 'Observation Value' + add_ylabel: 'JEDI h(x)' + add_grid: + add_legend: + loc: 'upper left' + layers: + - type: Scatter + x: + variable: experiment::ObsValue::${variable} + y: + variable: experiment::hofx0::${variable} + @CHANNELKEY@ + markersize: 5 + color: 'black' + label: 'JEDI h(x) versus obs (all obs)' + - type: Scatter + x: + variable: experiment::ObsValue::${variable} + y: + variable: experiment::hofxQC::${variable} + @CHANNELKEY@ + markersize: 5 + color: 'red' + label: 'JEDI h(x) versus obs (passed QC in JEDI)' diff --git a/ush/land/genYAML_output_letkfoi.yaml b/ush/land/genYAML_output_letkfoi.yaml index 67ade909e..b55c3618a 100644 --- a/ush/land/genYAML_output_letkfoi.yaml +++ b/ush/land/genYAML_output_letkfoi.yaml @@ -10,7 +10,7 @@ background: filename_sfcd: 20210323.180000.sfc_data.nc filetype: fms restart state variables: - - snwdph + - snodl - vtype - slmsk zero padding: '3' @@ -119,6 +119,13 @@ observations: min_latitude: '-90' min_longitude: '-180' use_legacy_buddy_collector: 'false' + # Assign obsError. + - filter: Perform Action + filter variables: + - name: totalSnowDepth + action: + name: assign error + error parameter: 40.0 - assignments: - name: GrossErrorProbability/totalSnowDepth type: float diff --git a/ush/ufsda/soca_utils.py b/ush/ufsda/soca_utils.py index f46d497ee..0d3c3c0e1 100644 --- a/ush/ufsda/soca_utils.py +++ b/ush/ufsda/soca_utils.py @@ -18,8 +18,8 @@ pyiodaconv_lib = Path(os.path.join(jedilib, 'pyiodaconv')).resolve() sys.path.append(str(pyioda_lib)) sys.path.append(str(pyiodaconv_lib)) -import ioda_conv_engines as iconv -from orddicts import DefaultOrderedDict +import pyiodaconv.ioda_conv_engines as iconv +from pyiodaconv.orddicts import DefaultOrderedDict __all__ = ['concatenate_ioda']