From 18b170c4232645ce16a5480800ae1377af8246c0 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Fri, 24 Feb 2023 11:48:08 -0700 Subject: [PATCH] Update develop-ref after #2451 (#2466) Co-authored-by: Seth Linden Co-authored-by: John Halley Gotway Co-authored-by: johnhg Co-authored-by: Lisa Goodrich Co-authored-by: jprestop Co-authored-by: Howard Soh Co-authored-by: MET Tools Test Account Co-authored-by: Dave Albo Co-authored-by: j-opatz <59586397+j-opatz@users.noreply.github.com> Co-authored-by: George McCabe <23407799+georgemccabe@users.noreply.github.com> Co-authored-by: Julie Prestopnik Co-authored-by: Jonathan Vigh Co-authored-by: Julie Prestopnik Co-authored-by: Seth Linden Co-authored-by: hsoh-u Co-authored-by: lisagoodrich <33230218+lisagoodrich@users.noreply.github.com> Co-authored-by: davidalbo Co-authored-by: Daniel Adriaansen fix 2238 link error (#2255) fix #2271 develop nbrctc (#2272) fix #2309 develop tcmpr (#2310) fix #2306 ascii2nc airnow hourly (#2314) fix_spread_md (#2335) fix #2389 develop flowchart (#2392) Fix Python environment issue (#2407) fix definitions of G172 and G220 based on comments in NOAA-EMC/NCEPLIBS-w3emc#157. (#2406) fix #2380 develop override (#2382) fix #2408 develop empty config (#2410) fix #2390 develop compile zlib (#2404) fix #2412 develop climo (#2422) fix #2437 develop convert (#2439) fix for develop, for #2437, forgot one reference to the search_parent for a dictionary lookup. fix #2452 develop airnow (#2454) --- docs/Users_Guide/tc-pairs.rst | 3 +- .../scripts/installation/compile_MET_all.sh | 4 +- .../config/install_met_env.acorn_py3.10 | 36 ++++++++++++++ .../config/install_met_env.wcoss2_py3.10 | 47 +++++++++++++++++++ .../modulefiles/11.0.1.lua.wcoss2 | 27 +++++++++++ .../installation/modulefiles/11.0.1_acorn | 34 ++++++++++++++ .../test_unit/config/TCPairsConfig_CONSENSUS | 7 +++ src/basic/vx_config/config_constants.h | 1 + src/tools/tc_utils/tc_pairs/tc_pairs.cc | 17 ++++++- .../tc_utils/tc_pairs/tc_pairs_conf_info.cc | 17 ++++++- .../tc_utils/tc_pairs/tc_pairs_conf_info.h | 4 +- 11 files changed, 190 insertions(+), 7 deletions(-) create mode 100644 internal/scripts/installation/config/install_met_env.acorn_py3.10 create mode 100644 internal/scripts/installation/config/install_met_env.wcoss2_py3.10 create mode 100644 internal/scripts/installation/modulefiles/11.0.1.lua.wcoss2 create mode 100644 internal/scripts/installation/modulefiles/11.0.1_acorn diff --git a/docs/Users_Guide/tc-pairs.rst b/docs/Users_Guide/tc-pairs.rst index 40162dee4d..dbacd34839 100644 --- a/docs/Users_Guide/tc-pairs.rst +++ b/docs/Users_Guide/tc-pairs.rst @@ -199,10 +199,11 @@ ____________________ members = [ "MOD1", "MOD2", "MOD3" ]; required = [ true, false, false ]; min_req = 2; + write_members = TRUE; } ]; -The **consensus** field allows the user to generate a user-defined consensus forecasts from any number of models. All models used in the consensus forecast need to be included in the **model** field (first entry in **TCPairsConfig_default**). The name field is the desired consensus model name. The **members** field is a comma-separated list of model IDs that make up the members of the consensus. The **required** field is a comma-separated list of true/false values associated with each consensus member. If a member is designated as true, the member is required to be present in order for the consensus to be generated. If a member is false, the consensus will be generated regardless of whether the member is present. The length of the required array must be the same length as the members array. The **min_req** field is the number of members required in order for the consensus to be computed. The required and min_req field options are applied at each forecast lead time. If any member of the consensus has a non-valid position or intensity value, the consensus for that valid time will not be generated. If a consensus model is indicated in the configuration file there will be non-missing output for the consensus track variables in the output file (NUM_MEMBERS, TRACK_SPREAD, TRACK_STDEV, MSLP_STDEV, MAX_WIND_STDEV). See the TCMPR line type definitions below. +The **consensus** field allows the user to generate a user-defined consensus forecasts from any number of models. All models used in the consensus forecast need to be included in the **model** field (first entry in **TCPairsConfig_default**). The name field is the desired consensus model name. The **members** field is a comma-separated list of model IDs that make up the members of the consensus. The **required** field is a comma-separated list of true/false values associated with each consensus member. If a member is designated as true, the member is required to be present in order for the consensus to be generated. If a member is false, the consensus will be generated regardless of whether the member is present. The length of the required array must be the same length as the members array. The **min_req** field is the number of members required in order for the consensus to be computed. The required and min_req field options are applied at each forecast lead time. If any member of the consensus has a non-valid position or intensity value, the consensus for that valid time will not be generated. The **write_members** field is a boolean that indicates whether or not to write output for the individual consensus members. If set to true, standard output will show up for all members. If set to false, output for the consensus members is excluded from the output, even if they are used to define other consensus tracks in the configuration file. If a consensus model is defined in the configuration file, there will be non-missing output for the consensus track variables in the output file (NUM_MEMBERS, TRACK_SPREAD, TRACK_STDEV, MSLP_STDEV, MAX_WIND_STDEV). See the TCMPR line type definitions below. ____________________ diff --git a/internal/scripts/installation/compile_MET_all.sh b/internal/scripts/installation/compile_MET_all.sh index 9810a124f2..0ac540d147 100755 --- a/internal/scripts/installation/compile_MET_all.sh +++ b/internal/scripts/installation/compile_MET_all.sh @@ -628,9 +628,9 @@ export LDFLAGS="-Wl,--disable-new-dtags" # https://www.gnu.org/software/bash/manual/html_node/Shell-Parameter-Expansion.html # ${parameter:+word} # If parameter is null or unset, nothing is substituted, otherwise the expansion of word is substituted. -export LDFLAGS="${LDFLAGS} -Wl,-rpath,${LIB_DIR}/lib${MET_NETCDF:+:$MET_NETCDF/lib}${MET_HDF5:+:$MET_HDF5/lib}${MET_BUFRLIB:+:$MET_BUFRLIB}${MET_GRIB2CLIB:+:$MET_GRIB2CLIB}${MET_PYTHON:+:$MET_PYTHON/lib}${MET_GSL:+:$MET_GSL/lib}" +export LDFLAGS="${LDFLAGS} -Wl,-rpath,${LIB_DIR}/lib${ADDTL_DIR:+:$ADDTL_DIR}${LIB_DIR}/lib${MET_NETCDF:+:$MET_NETCDF/lib}${MET_HDF5:+:$MET_HDF5/lib}${MET_BUFRLIB:+:$MET_BUFRLIB}${MET_GRIB2CLIB:+:$MET_GRIB2CLIB}${MET_PYTHON:+:$MET_PYTHON/lib}${MET_GSL:+:$MET_GSL/lib}" export LDFLAGS="${LDFLAGS} -Wl,-rpath,${LIB_JASPER:+$LIB_JASPER}${LIB_LIBPNG:+:$LIB_PNG}${LIB_Z:+$LIB_Z}" -export LDFLAGS="${LDFLAGS} ${LIB_JASPER:+-L$LIB_JASPER} ${LIB_LIBPNG:+-L$LIB_LIBPNG} ${MET_HDF5:+-L$MET_HDF5/lib}" +export LDFLAGS="${LDFLAGS} ${LIB_JASPER:+-L$LIB_JASPER} ${LIB_LIBPNG:+-L$LIB_LIBPNG} ${MET_HDF5:+-L$MET_HDF5/lib} ${ADDTL_DIR:+-L$ADDTL_DIR}" export LIBS="${LIBS} -lhdf5_hl -lhdf5 -lz" export MET_FONT_DIR=${TEST_BASE}/fonts diff --git a/internal/scripts/installation/config/install_met_env.acorn_py3.10 b/internal/scripts/installation/config/install_met_env.acorn_py3.10 new file mode 100644 index 0000000000..b97699a187 --- /dev/null +++ b/internal/scripts/installation/config/install_met_env.acorn_py3.10 @@ -0,0 +1,36 @@ +module use /apps/ops/para/libs/modulefiles/compiler/intel/19.1.3.304/ +module load intel python/3.10.4 +module load ve/evs/1.0 +module load netcdf/4.7.4 +module load hdf5/1.12.2 +module load bufr/11.5.0 +module load zlib/1.2.11 +module load jasper/2.0.25 +module load libpng/1.6.37 +module load gsl/2.7 +module load g2c/1.6.4 + +export TEST_BASE=/apps/sw_review/emc/MET/11.0.1 +export LIB_DIR=${TEST_BASE}/external_libs +export BIN_DIR_PATH=${TEST_BASE}/exec +export COMPILER=intel_19.1.3.304 +export MET_SUBDIR=${TEST_BASE} +export MET_TARBALL=v11.0.1.tar.gz +export USE_MODULES=TRUE +export ADDTL_DIR=/apps/spack/gettext/0.21/intel/19.1.3.304/at2kdo4edvuhyzrt5g6zhwrdb7bdui4s/lib64 +export PYTHON_MODULE=python_3.10.4 +export MET_PYTHON=/apps/spack/python/3.10.4/intel/19.1.3.304/xqft4d45h4dp4xnbz2ue3nbxv65i6bgp +export MET_PYTHON_CC=-I/apps/spack/python/3.10.4/intel/19.1.3.304/xqft4d45h4dp4xnbz2ue3nbxv65i6bgp/include/python3.10 +export MET_PYTHON_LD=-L/apps/spack/python/3.10.4/intel/19.1.3.304/xqft4d45h4dp4xnbz2ue3nbxv65i6bgp/lib/python3.10/config-3.10-x86_64-linux-gnu/\ -L/apps/spack/python/3.10.4/intel/19.1.3.304/xqft4d45h4dp4xnbz2ue3nbxv65i6bgp/lib64\ -lpython3.10\ -lintl\ -lcrypt\ -ldl\ -lutil\ -lm\ -lm +export MET_NETCDF=/apps/prod/hpc-stack/intel-19.1.3.304/netcdf/4.7.4 +export MET_HDF5=/apps/prod/hpc-stack/intel-19.1.3.304/hdf5/1.12.2 +export MET_BUFRLIB=/apps/ops/prod/libs/intel/19.1.3.304/bufr/11.5.0/lib64 +export MET_GRIB2CLIB=/apps/ops/prod/libs/intel/19.1.3.304/g2c/1.6.4/lib64 +export MET_GRIB2CINC=/apps/ops/prod/libs/intel/19.1.3.304/g2c/1.6.4/include +export MET_GSL=/apps/spack/gsl/2.7/intel/19.1.3.304/xks7dxbowrdxhjck5zxc4rompopocevb +export BUFRLIB_NAME=-lbufr_4 +export GRIB2CLIB_NAME=-lg2c +export LIB_JASPER=/apps/spack/jasper/2.0.25/intel/19.1.3.304/sjib74krrorkyczqpqah4tvewmlnqdx4/lib64 +export LIB_LIBPNG=/apps/spack/libpng/1.6.37/intel/19.1.3.304/4ohkronuhlyherusoszzrmur5ewvlwzh/lib +export LIB_Z=/apps/spack/zlib/1.2.11/intel/19.1.3.304/hjotqkckeoyt6j6tibalwzrlfljcjtdh/lib +export SET_D64BIT=FALSE \ No newline at end of file diff --git a/internal/scripts/installation/config/install_met_env.wcoss2_py3.10 b/internal/scripts/installation/config/install_met_env.wcoss2_py3.10 new file mode 100644 index 0000000000..c6d550c3a3 --- /dev/null +++ b/internal/scripts/installation/config/install_met_env.wcoss2_py3.10 @@ -0,0 +1,47 @@ +# JY module use /apps/ops/para/libs/modulefiles/compiler/intel/19.1.3.304/ +module load intel +# JY add following two +module load craype/2.7.13 +module load cray-mpich/8.1.12 + +module load python/3.10.4 +module load netcdf/4.7.4 +module load hdf5/1.10.6 +module load bufr/11.6.0 +module load zlib/1.2.11 +module load jasper/2.0.25 +module load libpng/1.6.37 +module load gsl/2.7 +module load g2c/1.6.4 + +#export TEST_BASE=/apps/ops/para/libs/intel/19.1.3.304/met/11.0.1 +export TEST_BASE=$(pwd) +export LIB_DIR=${TEST_BASE}/external_libs +export BIN_DIR_PATH=${TEST_BASE}/bin +export COMPILER=intel_19.1.3.304 +export MET_SUBDIR=${TEST_BASE} +export MET_TARBALL=v11.0.1.tar.gz +export USE_MODULES=TRUE +export ADDTL_DIR=/apps/spack/gettext/0.21/intel/19.1.3.304/at2kdo4edvuhyzrt5g6zhwrdb7bdui4s/lib64 +export PYTHON_MODULE=python_3.10.4 +export MET_PYTHON=/apps/spack/python/3.10.4/intel/19.1.3.304/xqft4d45h4dp4xnbz2ue3nbxv65i6bgp +export MET_PYTHON_CC=-I/apps/spack/python/3.10.4/intel/19.1.3.304/xqft4d45h4dp4xnbz2ue3nbxv65i6bgp/include/python3.10 +export MET_PYTHON_LD=-L/apps/spack/python/3.10.4/intel/19.1.3.304/xqft4d45h4dp4xnbz2ue3nbxv65i6bgp/lib\ -lpython3.10\ -lpthread\ -ldl\ -lutil\ -lm\ -Xlinker\ -export-dynamic +export MET_NETCDF=/apps/prod/hpc-stack/intel-19.1.3.304/netcdf/4.7.4 +# JY export MET_HDF5=/apps/prod/hpc-stack/intel-19.1.3.304/hdf5/1.12.2 +export MET_HDF5=${HDF5_ROOT} +export MET_BUFRLIB=/apps/ops/prod/libs/intel/19.1.3.304/bufr/11.6.0/lib64 +# JY export MET_GRIB2CLIB=/apps/ops/prod/libs/intel/19.1.3.304/g2c/1.6.4/lib64 +# JY export MET_GRIB2CINC=/apps/ops/prod/libs/intel/19.1.3.304/g2c/1.6.4/include +export MET_GRIB2CLIB=${g2c_ROOT}/lib64 +export MET_GRIB2CINC=${G2C_INC} +export MET_GSL=/apps/spack/gsl/2.7/intel/19.1.3.304/xks7dxbowrdxhjck5zxc4rompopocevb +export BUFRLIB_NAME=-lbufr_4 +export GRIB2CLIB_NAME=-lg2c +# JY export LIB_JASPER=/apps/spack/jasper/2.0.25/intel/19.1.3.304/sjib74krrorkyczqpqah4tvewmlnqdx4/lib64 +export LIB_JASPER=${JASPER_LIBDIR} +# JY export LIB_LIBPNG=/apps/spack/libpng/1.6.37/intel/19.1.3.304/4ohkronuhlyherusoszzrmur5ewvlwzh/lib +export LIB_LIBPNG=${LIBPNG_LIBDIR} +# JY export LIB_Z=/apps/spack/zlib/1.2.11/intel/19.1.3.304/hjotqkckeoyt6j6tibalwzrlfljcjtdh/lib +export LIB_Z=${ZLIB_LIBDIR} +export SET_D64BIT=FALSE diff --git a/internal/scripts/installation/modulefiles/11.0.1.lua.wcoss2 b/internal/scripts/installation/modulefiles/11.0.1.lua.wcoss2 new file mode 100644 index 0000000000..c4afdd9fda --- /dev/null +++ b/internal/scripts/installation/modulefiles/11.0.1.lua.wcoss2 @@ -0,0 +1,27 @@ +help([[ +]]) + +local pkgName = myModuleName() +local pkgVersion = myModuleVersion() +local pkgNameVer = myModuleFullName() + +local hierA = hierarchyA(pkgNameVer,1) +local compNameVer = hierA[1] + + +conflict(pkgName) + +local opt = os.getenv("HPC_OPT") or os.getenv("OPT") or "/opt/modules" + +local base = pathJoin(opt,compNameVer,pkgName,pkgVersion) + +prepend_path("PATH", pathJoin(base,"bin")) + +setenv("MET_ROOT", base) +setenv("MET_BASE", pathJoin(base, "share", "met")) +setenv("MET_VERSION", pkgVersion) + +whatis("Name: ".. pkgName) +whatis("Version: " .. pkgVersion) +whatis("Category: applications") +whatis("Description: Model Evaluation Tools (MET)") diff --git a/internal/scripts/installation/modulefiles/11.0.1_acorn b/internal/scripts/installation/modulefiles/11.0.1_acorn new file mode 100644 index 0000000000..832194dc76 --- /dev/null +++ b/internal/scripts/installation/modulefiles/11.0.1_acorn @@ -0,0 +1,34 @@ +#%Module###################################################################### +## +## Model Evaluation Tools +## +proc ModulesHelp { } { + puts stderr "Sets up the paths and environment variables to use the Model Evaluation Tools v11.0.1 + *** For help see the official MET webpage at http://www.dtcenter.org/met/users ***" +} + +# The intel compiler is required to run MET + +module use /apps/ops/para/libs/modulefiles/compiler/intel/19.1.3.304/ +module load intel python/3.10.4 +module load ve/evs/1.0 +module load netcdf/4.7.4 +module load hdf5/1.12.2 +module load bufr/11.5.0 +module load zlib/1.2.11 +module load jasper/2.0.25 +module load libpng/1.6.37 +module load gsl/2.7 +module load g2c/1.6.4 + +set base /apps/sw_review/emc/MET/11.0.1 +set ver 11.0.1 +set share $base/share/met +set lib_base $base + +prepend-path PATH $base/exec + +setenv METversion V$ver +setenv MET_ROOT $base + + diff --git a/internal/test_unit/config/TCPairsConfig_CONSENSUS b/internal/test_unit/config/TCPairsConfig_CONSENSUS index b157c71264..f7382ae85b 100644 --- a/internal/test_unit/config/TCPairsConfig_CONSENSUS +++ b/internal/test_unit/config/TCPairsConfig_CONSENSUS @@ -107,42 +107,49 @@ consensus = [ "UE35" ]; required = []; min_req = 36; + write_members = FALSE; }, { name = "HCCA_CONS"; members = [ "AEMI", "GFSI", "CTCI", "DSHP", "EGRI", "EMN2", "EMXI", "HWFI", "LGEM" ]; required = []; min_req = 8; + write_members = TRUE; }, { name = "GFEX_CONS"; members = [ "GFSI", "EMXI" ]; required = []; min_req = 2; + write_members = TRUE; }, { name = "TVCA_CONS"; members = [ "GFSI", "EGRI", "HWFI", "EMHI", "CTCI", "EMNI" ]; required = []; min_req = 2; + write_members = TRUE; }, { name = "TVCX_CONS"; members = [ "GFSI", "EMXI", "EMXI", "HWFI", "CTCI", "EGRI" ]; required = [ TRUE, TRUE, FALSE, FALSE, FALSE, FALSE ]; min_req = 2; + write_members = TRUE; }, { name = "ICON_CONS"; members = [ "DSHP", "LGEM", "HWFI", "HMNI" ]; required = [ TRUE, TRUE, TRUE, TRUE ]; min_req = 4; + write_members = TRUE; }, { name = "IVCN_CONS"; members = [ "DSHP", "LGEM", "HWFI", "HMNI", "CTCI" ]; required = []; min_req = 2; + write_members = TRUE; } ]; diff --git a/src/basic/vx_config/config_constants.h b/src/basic/vx_config/config_constants.h index fb49e5cce7..d2e6a25f42 100644 --- a/src/basic/vx_config/config_constants.h +++ b/src/basic/vx_config/config_constants.h @@ -1086,6 +1086,7 @@ static const char conf_key_consensus[] = "consensus"; static const char conf_key_members[] = "members"; static const char conf_key_required[] = "required"; static const char conf_key_min_req[] = "min_req"; +static const char conf_key_write_members[] = "write_members"; static const char conf_key_lag_time[] = "lag_time"; static const char conf_key_best_technique[] = "best_technique"; static const char conf_key_best_baseline[] = "best_baseline"; diff --git a/src/tools/tc_utils/tc_pairs/tc_pairs.cc b/src/tools/tc_utils/tc_pairs/tc_pairs.cc index 734f83d256..618b818d51 100644 --- a/src/tools/tc_utils/tc_pairs/tc_pairs.cc +++ b/src/tools/tc_utils/tc_pairs/tc_pairs.cc @@ -36,6 +36,7 @@ // 012 07/06/22 Howard Soh METplus-Internal #19 Rename main to met_main // 013 09/28/22 Prestopnik MET #2227 Remove namespace std from header files // 014 10/06/22 Halley Gotway MET #392 Incorporate diagnostics +// 015 02/20/23 Seth Linden MET #2429 Added option to prevent output of consensus track members // //////////////////////////////////////////////////////////////////////// @@ -811,19 +812,30 @@ bool is_keeper(const ATCFLineBase * line) { void filter_tracks(TrackInfoArray &tracks) { int i, j; - int n_name, n_vld, n_mask_init, n_mask_vld, n_req_lead; + int n_name, n_vld, n_mask_init, n_mask_vld, n_req_lead, n_members; bool status; TrackInfoArray t = tracks; // Initialize tracks.clear(); - n_name = n_vld = n_mask_init = n_mask_vld = n_req_lead = 0; + n_name = n_vld = n_mask_init = n_mask_vld = n_req_lead = n_members = 0; // Loop through the tracks and determine which should be retained // The is_keeper() function has already filtered by model, storm id, // basin, cyclone, and timing information. for(i=0; i 0 && + conf_info.SkipConsensusMembers.has(t[i].technique())) { + mlog << Debug(4) + << "Discarding track " << i+1 << " since it is listed in SkipConsensusMembers: " + << t[i].technique() << "\n"; + n_members++; + continue; + } + // Check storm name if(conf_info.StormName.n() > 0 && !conf_info.StormName.has(t[i].storm_name())) { @@ -922,6 +934,7 @@ void filter_tracks(TrackInfoArray &tracks) { mlog << Debug(3) << "Total tracks read = " << t.n() << "\n" << "Total tracks kept = " << tracks.n() << "\n" + << "Rejected for skip members = " << n_members << "\n" << "Rejected for storm name = " << n_name << "\n" << "Rejected for valid time = " << n_vld << "\n" << "Rejected for required lead times = " << n_req_lead << "\n" diff --git a/src/tools/tc_utils/tc_pairs/tc_pairs_conf_info.cc b/src/tools/tc_utils/tc_pairs/tc_pairs_conf_info.cc index b4faab8aef..faafaab0e7 100644 --- a/src/tools/tc_utils/tc_pairs/tc_pairs_conf_info.cc +++ b/src/tools/tc_utils/tc_pairs/tc_pairs_conf_info.cc @@ -74,6 +74,7 @@ void TCPairsConfInfo::clear() { Basin.clear(); Cyclone.clear(); StormName.clear(); + SkipConsensusMembers.clear(); InitBeg = InitEnd = (unixtime) 0; InitInc.clear(); InitExc.clear(); @@ -240,12 +241,21 @@ void TCPairsConfInfo::process_config() { // Loop over the consensus entries for(i=0; idict_value()->lookup_string(conf_key_name); Consensus[i].Members = (*dict)[i]->dict_value()->lookup_string_array(conf_key_members); Consensus[i].Required = (*dict)[i]->dict_value()->lookup_num_array(conf_key_required); Consensus[i].MinReq = (*dict)[i]->dict_value()->lookup_int(conf_key_min_req); + // If write_members is missing, print warning message rather than error + Consensus[i].WriteMembers = (*dict)[i]->dict_value()->lookup_bool(conf_key_write_members, false); + if(!(*dict)[i]->dict_value()->last_lookup_status()) { + mlog << Warning + << "\nTCPairsConfInfo::process_config() -> " + << "\"consensus.write_members\" is missing. Using default value of true.\n\n"; + Consensus[i].WriteMembers = true; + } + // If required is empty, default to 0 if(Consensus[i].Required.n_elements() == 0) { for(j=0; j