diff --git a/src/libcode/vx_shapedata/mode_conf_info.cc b/src/libcode/vx_shapedata/mode_conf_info.cc index 03e806c663..e2db27326f 100644 --- a/src/libcode/vx_shapedata/mode_conf_info.cc +++ b/src/libcode/vx_shapedata/mode_conf_info.cc @@ -966,13 +966,14 @@ dict = conf.lookup_dictionary(conf_key_obs); if ( dict->lookup(conf_key_multivar_logic) ) obs_multivar_logic = dict->lookup_string(conf_key_multivar_logic); multivar_intensity.clear(); -BoolArray ba = lookup_bool_array("multivar_intensity_flag", dict); -ba.dump(cout); -for (int i=0; ilookup("multivar_intensity_flag")) { + BoolArray ba = lookup_bool_array("multivar_intensity_flag", dict); + //ba.dump(cout); + for (int i=0; ifile_type(); + // otype = obs_mtddf->file_type(); + + ftype = mvd.ftype; + otype = mvd.otype; + + // Process the configuration + engine.conf_info.process_config(ftype, otype); + + // const int shift = engine.conf_info.shift_right; + + // fcst_mtddf->set_shift_right(shift); + // obs_mtddf->set_shift_right(shift); + + // List the input files + // mlog << Debug(1) + // << "Forecast File: " << fcst_file << "\n" + // << "Observation File: " << obs_file << "\n"; + + engine.conf_info.nc_info.compress_level = engine.conf_info.get_compression_level(); + +return; + +} /////////////////////////////////////////////////////////////////////// @@ -1022,6 +1094,8 @@ MultiVarData *ModeExecutive::get_multivar_data() mvd->Fcst_sd = new ShapeData(Fcst_sd); mvd->Obs_sd = new ShapeData(Obs_sd); mvd->grid = new Grid(grid); + mvd->ftype = ftype; + mvd->otype = otype; return mvd; } @@ -1271,9 +1345,6 @@ if ( info.all_false() ) return; } - vector fcstValues, obsValues; - vector fcstCValues, obsCValues; - for(x=0; xdata(x, y)); - if (find(fcstValues.begin(), fcstValues.end(), fcst_obj_data[n]) == fcstValues.end()) - { - printf("FcstValue=%d\n", fcst_obj_data[n]); - fcstValues.push_back(fcst_obj_data[n]); - } } } else { @@ -1321,11 +1387,6 @@ if ( info.all_false() ) return; } if ( info.do_object_id && obs_obj_data != nullptr ) { obs_obj_data[n] = nint(engine.obs_split->data(x, y)); - if (find(obsValues.begin(), obsValues.end(), obs_obj_data[n]) == obsValues.end()) - { - printf("ObsValue=%d\n", obs_obj_data[n]); - obsValues.push_back(obs_obj_data[n]); - } } } else { diff --git a/src/tools/core/mode/mode_exec.h b/src/tools/core/mode/mode_exec.h index 73f889f200..75f574b5cd 100644 --- a/src/tools/core/mode/mode_exec.h +++ b/src/tools/core/mode/mode_exec.h @@ -77,6 +77,7 @@ class ModeExecutive { void clear(); void init(); + void init_final(const MultiVarData &mvd); int n_conv_radii () const; int n_conv_threshs () const; @@ -113,6 +114,8 @@ class ModeExecutive { ShapeData Fcst_sd, Obs_sd; + GrdFileType ftype, otype; + void clear_internal_r_index(); void setup_fcst_obs_data(); void setup_fcst_obs_data(const MultiVarData &mvd); diff --git a/src/tools/core/mode/mode_frontend.cc b/src/tools/core/mode/mode_frontend.cc index e8697d51c5..7ade93261b 100644 --- a/src/tools/core/mode/mode_frontend.cc +++ b/src/tools/core/mode/mode_frontend.cc @@ -64,7 +64,7 @@ if ( mode_exec ) { int ModeFrontEnd::run(const StringArray & Argv) { - Argv.dump(cout, 0); + // Argv.dump(cout, 0); // @@ -117,7 +117,7 @@ return (0); int ModeFrontEnd::run(const StringArray & Argv, const MultiVarData &mvd) { - Argv.dump(cout, 0); +//Argv.dump(cout, 0); // @@ -127,9 +127,9 @@ if ( mode_exec ) { delete mode_exec; mode_exec = 0; } mode_exec = new ModeExecutive; compress_level = -1; -field_index = -1; +//field_index = -1; -process_command_line(Argv); +process_command_line_final(Argv, mvd); // @@ -138,7 +138,7 @@ process_command_line(Argv); ModeConfInfo & conf = mode_exec->engine.conf_info; -if ( field_index >= 0 ) conf.set_field_index(field_index); +//if ( field_index >= 0 ) conf.set_field_index(field_index); mode_exec->setup_fcst_obs_data(mvd); @@ -317,6 +317,56 @@ void ModeFrontEnd::process_command_line(const StringArray & argv) } +void ModeFrontEnd::process_command_line_final(const StringArray & argv, + const MultiVarData &mvd) +{ + + CommandLine cline; + ConcatString s; + const int argc = argv.n(); + + + // Set the default output directory + mode_exec->out_dir = replace_path(default_out_dir); + + // Check for zero arguments + if(argc == 1) singlevar_usage(); // wrong, need something else + + // Parse the command line into tokens + cline.set(argv); + + // Set the usage function + cline.set_usage(singlevar_usage); // wrong, need something else + + // Add the options function calls + cline.add(set_config_merge_file, "-config_merge", 1); + cline.add(set_outdir, "-outdir", 1); + cline.add(set_logfile, "-log", 1); + cline.add(set_verbosity, "-v", 1); + cline.add(set_compress, "-compress", 1); + + // + // add for mode multivar ... undocumented + // + //cline.add(set_field_index, "-field_index", 1); + + // Parse the command line + cline.parse(); + + // Check for error. There should be 1 argument left: + // config filename + if(cline.n() != 1) singlevar_usage(); // wrong need something else + + // Store the input forecast and observation file names, placeholders + mode_exec->fcst_file = "not set"; + mode_exec->obs_file = "not set"; + mode_exec->match_config_file = cline[0]; + mode_exec->init_final(mvd); + + return; + +} + /////////////////////////////////////////////////////////////////////// void ModeFrontEnd::set_config_merge_file(const StringArray & a) diff --git a/src/tools/core/mode/mode_frontend.h b/src/tools/core/mode/mode_frontend.h index cfcff9c983..2440851416 100644 --- a/src/tools/core/mode/mode_frontend.h +++ b/src/tools/core/mode/mode_frontend.h @@ -45,6 +45,8 @@ class ModeFrontEnd { MultiVarData *get_multivar_data(); void process_command_line(const StringArray &); + void process_command_line_final(const StringArray & argv, + const MultiVarData &mvd); static void set_config_merge_file (const StringArray &); static void set_outdir (const StringArray &); static void set_logfile (const StringArray &); diff --git a/src/tools/core/mode/multivar_data.h b/src/tools/core/mode/multivar_data.h index 7eb0077925..401d97274d 100644 --- a/src/tools/core/mode/multivar_data.h +++ b/src/tools/core/mode/multivar_data.h @@ -36,7 +36,9 @@ class MultiVarData { obs_raw_data(0), nx(0), ny(0), Fcst_sd(0), Obs_sd(0), - grid(0) + grid(0), + ftype(FileType_None), + otype(FileType_None) {} inline ~MultiVarData() { @@ -57,6 +59,8 @@ class MultiVarData { ShapeData *Fcst_sd; ShapeData *Obs_sd; Grid *grid; + GrdFileType ftype; + GrdFileType otype; /* int get_fcst_obj_nx(); */ /* int get_fcst_obj_ny(); */ diff --git a/src/tools/core/mode/multivar_frontend.cc b/src/tools/core/mode/multivar_frontend.cc index 852060e9dd..6f3bfcc773 100644 --- a/src/tools/core/mode/multivar_frontend.cc +++ b/src/tools/core/mode/multivar_frontend.cc @@ -96,6 +96,19 @@ static void set_outdir (const StringArray &); static void set_logfile (const StringArray &); static void set_verbosity (const StringArray &); +static void multivar_consistency_checks(StringArray &fcst_filenames, StringArray &obs_filenames, + BoolCalc &f_calc, BoolCalc &o_calc, int &n_files); + +static ConcatString set_multivar_dir(int j); + +static MultiVarData *process_multivar_data(int j, const string &fcst_filename, + const string &obs_filename, + const ConcatString &dir); + +static void process_masked_multivar_data(int j, const BoolPlane &f_result, const BoolPlane &o_result, + int nx, int ny, const ConcatString &dir, + MultiVarData &mvd); + static void mask_data(int nx, int ny, const BoolPlane &mask, DataPlane &data); static void read_config(const string & filename); @@ -118,238 +131,56 @@ const int Argc = Argv.n(); if ( Argc < 4 ) multivar_usage(); -int j; +int j, n_files; StringArray fcst_filenames; StringArray obs_filenames; -ConcatString dir; - +BoolCalc f_calc, o_calc ; process_command_line(Argv); - read_config(config_file); -if ( config.fcst_multivar_logic.empty() ) { - - mlog << Error << "\n" << program_name - << ": fcst multivar logic not specified!\n\n"; - - exit ( 1 ); - -} - - -if ( config.obs_multivar_logic.empty() ) { - - mlog << Error << "\n" << program_name - << ": obs multivar logic not specified!\n\n"; - - exit ( 1 ); - -} - - // - // make sure the multivar logic programs are in the config file - // - -fcst_filenames = parse_ascii_file_list(fcst_fof.c_str()); - obs_filenames = parse_ascii_file_list(obs_fof.c_str()); - -if ( fcst_filenames.n() != obs_filenames.n() ) { - - mlog << Error << "\n" << program_name - << ": number of fcst and obs files should be the same!\n\n"; - - exit ( 1 ); - -} - -const int n_files = fcst_filenames.n(); -StringArray nc_files; - -//ConcatString mode_args; +multivar_consistency_checks(fcst_filenames, obs_filenames, f_calc, o_calc, n_files); mlog << Debug(2) << "\n" << sep << "\n"; - // - // check for no inputs - // - -if ( n_files == 0 ) { - - mlog << Error << "\n" << program_name - << ": no input forecast files to process!\n\n"; - - exit ( 1 ); - -} - - // - // create the BoolCalc objects now and check that the logic is in - // range and the right length before proceeding - - -BoolCalc f_calc, o_calc ; -f_calc.set(config.fcst_multivar_logic.text()); -o_calc.set(config.obs_multivar_logic.text()); -if (!f_calc.check_args(n_files)) { - exit ( 1 ); -} -if (!o_calc.check_args(n_files)) { - exit ( 1 ); -} - - // - // check that the multivar_intensity vector is the right length - // -if ((int)config.multivar_intensity.size() != n_files) { - mlog << Error << "\nmultivar_frontend() -> " - << "wrong size multivar_intensity array, wanted " - << n_files << " got " << config.multivar_intensity.size() << "\n\n"; - exit ( 1 ); - } - // // do the individual mode runs // - vector mvd; +vector mvd; +StringArray nc_files; for (j=0; j 0 ) dir << outdir << '/'; - - snprintf(junk, sizeof(junk), "%02d", j); - - dir << junk; - - if ( ! directory_exists(dir.c_str()) ) { - - mlog << Debug(2) - << program_name << ": creating output directory \"" - << dir << "\"\n\n"; - - status = mkdir(dir.c_str(), dir_creation_mode); - - if ( status < 0 ) { - - mlog << Error << "\n" << program_name - << ": unable to create output directory \"" - << dir << "\"\n\n"; - - exit ( 1 ); - - } - - } - - // - // build the command for running mode - // - - mode_argv.clear(); - - mode_argv.add(mode_path); - mode_argv.add(fcst_filenames[j]); - mode_argv.add(obs_filenames[j]); - mode_argv.add(config_file); - - command << cs_erase - << mode_path << ' ' - << fcst_filenames[j] << ' ' - << obs_filenames[j] << ' ' - << config_file; - - mode_argv.add("-v"); - snprintf(junk, sizeof(junk), "%d", mlog.verbosity_level()); - mode_argv.add(junk); - - mode_argv.add("-outdir"); - mode_argv.add(dir); - - command << " -v " << mlog.verbosity_level(); - - command << " -outdir " << dir; - - mode_argv.add("-field_index"); - snprintf(junk, sizeof(junk), "%d", j); - mode_argv.add(junk); - - command << " -field_index " << j; - - // - // run mode - // - - mlog << Debug(1) << "Running mode command: \"" << command << "\"\n\n"; - - - - //run_command(command); - // [TODO] MET #1238: run MODE in memory instead of via system calls. - // (void) mode_frontend(mode_argv); - - ModeFrontEnd *frontend = new ModeFrontEnd; - status = frontend->run(mode_argv); - MultiVarData *mvdi = frontend->get_multivar_data(); + ConcatString dir = set_multivar_dir(j); + MultiVarData *mvdi = process_multivar_data(j, fcst_filenames[j], obs_filenames[j], dir); mvd.push_back(mvdi); - - // int nx = frontend->get_fcst_obj_nx(); - // int ny = frontend->get_fcst_obj_ny(); - // int *fcst_obj_data = frontend->get_fcst_obj_data(); - // int *obs_obj_data = frontend->get_obs_obj_data(); - // float *fcst_raw_data = frontend->get_fcst_raw_data(); - // float *obs_raw_data = frontend->get_obs_raw_data(); - // mode_fcst_objects.push_back(fcst_obj_data); - // mode_fcst_raw_data.push_back(fcst_raw_data); - // mode_obs_objects.push_back(obs_obj_data); - // mode_obs_raw_data.push_back(obs_raw_data); - // mode_nx.push_back(nx); - // mode_ny.push_back(ny); - - delete frontend; mlog << Debug(2) << "\n finished mode run " << (j + 1) << " of " << n_files << "\n" << sep << "\n"; - - a = get_filenames_from_dir(dir.text(), "mode_", ".nc"); - + StringArray a = get_filenames_from_dir(dir.text(), "mode_", ".nc"); nc_files.add(a); } // for j -mlog << Debug(2) << "\n finished with individual mode runs " - << "\n" << sep << "\n"; +mlog << Debug(2) << "\n finished with individual mode runs " << "\n" << sep << "\n"; +nc_files.dump(cout, 0); BoolPlane * f_plane = new BoolPlane [n_files]; BoolPlane * o_plane = new BoolPlane [n_files]; BoolPlane f_result, o_result; -Pgm image; // - // load the objects from the mode output files + // set the BoolPlane objects using the mvd content // -nc_files.dump(cout, 0); - for (j=0; jfcst_obj_data, mvd[j]->obs_obj_data, mvd[j]->nx, mvd[j]->ny, f_plane[j], o_plane[j]); + objects_from_arrays(do_clusters, mvd[j]->fcst_obj_data, mvd[j]->obs_obj_data, mvd[j]->nx, mvd[j]->ny, f_plane[j], o_plane[j]); } @@ -367,111 +198,37 @@ o_result.set_size(nx, ny); combine_boolplanes(f_plane, n_files, f_calc, f_result); combine_boolplanes(o_plane, n_files, o_calc, o_result); - // Filter the data to within the superobjects only + // + // Filter the data to within the superobjects only and do statistics by invoking mode algorithm again + // + for (j=0; jFcst_sd->data); - mask_data(nx, ny, o_result, mvd[j]->Obs_sd->data); - ConcatString command; - StringArray a, mode_argv; - int status; - char junk [256]; + mlog << Debug(2) << "\n starting filtered data mode run " << (j + 1) << " of " << n_files << "\n" << sep << "\n"; - // - // test to see of the output directory for this - // mode runs exists, and if not, create it - // - - dir.clear(); - - if ( outdir.length() > 0 ) dir << outdir << '/'; - - snprintf(junk, sizeof(junk), "%02d", j); - - dir << junk; - - if ( ! directory_exists(dir.c_str()) ) { - - mlog << Debug(2) - << program_name << ": creating output directory \"" - << dir << "\"\n\n"; - - status = mkdir(dir.c_str(), dir_creation_mode); - - if ( status < 0 ) { - - mlog << Error << "\n" << program_name - << ": unable to create output directory \"" - << dir << "\"\n\n"; - - exit ( 1 ); - - } - } - // - // build the command for running mode again - // - mode_argv.clear(); - - mode_argv.add(mode_path); - mode_argv.add(fcst_filenames[j]); - mode_argv.add(obs_filenames[j]); - mode_argv.add(config_file); - - command << cs_erase - << mode_path << ' ' - << fcst_filenames[j] << ' ' - << obs_filenames[j] << ' ' - << config_file; - - mode_argv.add("-v"); - snprintf(junk, sizeof(junk), "%d", mlog.verbosity_level()); - mode_argv.add(junk); - - mode_argv.add("-outdir"); - mode_argv.add(dir); - - command << " -v " << mlog.verbosity_level(); - - command << " -outdir " << dir; - - mode_argv.add("-field_index"); - snprintf(junk, sizeof(junk), "%d", j); - mode_argv.add(junk); - - command << " -field_index " << j; + // same here as above, overwriting contents written to previously + ConcatString dir = set_multivar_dir(j); - // - // run mode - // - - mlog << Debug(1) << "Running filtered mode command: \"" << command << "\"\n\n"; - - - - //run_command(command); - // [TODO] MET #1238: run MODE in memory instead of via system calls. - // (void) mode_frontend(mode_argv); - - ModeFrontEnd *frontend = new ModeFrontEnd; - status = frontend->run(mode_argv, *mvd[j]); + process_masked_multivar_data(j, f_result, o_result, nx, ny, dir, *mvd[j]); + delete mvd[j]; + mvd[j] = 0; } +mlog << Debug(2) << "\n finished with individual masked mode runs " << "\n" << sep << "\n"; + // // write the superobject output files // MetNcFile met; // mostly to get grid ConcatString path; - - path = nc_files[0]; if ( ! met.open(path.text()) ) { @@ -494,10 +251,6 @@ write_output_nc_file( obs_file.text(), met, o_result); // // done // -for (size_t i=0; i " + << "empty multivar intensity array, setting to all FALSE \n\n"; + + for (int i=0; i 0 ) dir << outdir << '/'; +snprintf(junk, sizeof(junk), "%02d", j); +dir << junk; + +if ( ! directory_exists(dir.c_str()) ) { + + mlog << Debug(2) + << program_name << ": creating output directory \"" + << dir << "\"\n\n"; + + status = mkdir(dir.c_str(), dir_creation_mode); + + if ( status < 0 ) { + + mlog << Error << "\n" << program_name + << ": unable to create output directory \"" + << dir << "\"\n\n"; + + exit ( 1 ); + } +} +return dir; +} + +//////////////////////////////////////////////////////////////////////// + +MultiVarData *process_multivar_data(int j, const string &fcst_filename, const string &obs_filename, + const ConcatString &dir) +{ +ConcatString command; +StringArray a, mode_argv; + + // + // build the command for running mode + // + +mode_argv.clear(); +mode_argv.add(mode_path); +mode_argv.add(fcst_filename); +mode_argv.add(obs_filename); +mode_argv.add(config_file); + +command << cs_erase + << mode_path << ' ' + << fcst_filename << ' ' + << obs_filename << ' ' + << config_file; + +mode_argv.add("-v"); +char junk [256]; +snprintf(junk, sizeof(junk), "%d", mlog.verbosity_level()); +mode_argv.add(junk); + +mode_argv.add("-outdir"); +mode_argv.add(dir); + +command << " -v " << mlog.verbosity_level(); + +command << " -outdir " << dir; + +mode_argv.add("-field_index"); +snprintf(junk, sizeof(junk), "%d", j); +mode_argv.add(junk); + +command << " -field_index " << j; + + // + // run mode (sort of) + // + +mlog << Debug(1) << "Running mode command: \"" << command << "\"\n\n"; +ModeFrontEnd *frontend = new ModeFrontEnd; +// clean up this status thing, not used +int status = frontend->run(mode_argv); +MultiVarData *mvdi = frontend->get_multivar_data(); +delete frontend; +return mvdi; +} + +//////////////////////////////////////////////////////////////////////// + +void process_masked_multivar_data(int j, const BoolPlane &f_result, + const BoolPlane &o_result, int nx, int ny, + const ConcatString &dir, MultiVarData &mvd) +{ + + mask_data(nx, ny, f_result, mvd.Fcst_sd->data); + mask_data(nx, ny, o_result, mvd.Obs_sd->data); + + + StringArray mode_argv; + char junk [256]; + + // + // build the command for running mode again, sort of mode + // + mode_argv.clear(); + mode_argv.add(mode_path); + mode_argv.add(config_file); + mode_argv.add("-v"); + snprintf(junk, sizeof(junk), "%d", mlog.verbosity_level()); + mode_argv.add(junk); + mode_argv.add("-outdir"); + mode_argv.add(dir); + + mlog << Debug(1) << "Running filtered mode \n\n"; + + ModeFrontEnd *frontend = new ModeFrontEnd; + int status = frontend->run(mode_argv, mvd); + delete frontend; +} + //////////////////////////////////////////////////////////////////////// diff --git a/src/tools/core/mode/objects_from_netcdf.cc b/src/tools/core/mode/objects_from_netcdf.cc index ffcfa25f25..6b22bb2004 100644 --- a/src/tools/core/mode/objects_from_netcdf.cc +++ b/src/tools/core/mode/objects_from_netcdf.cc @@ -50,7 +50,7 @@ static const string obs_cluster_id_var_name = "obs_clus_id"; static void populate_bool_plane(const int * buf, const int nx, const int ny, BoolPlane & bp_out); -void objects_from_arrays(const char *netcf_filename, bool do_clusters, +void objects_from_arrays(bool do_clusters, int *fcst_objects, int *obs_objects, int nx, int ny, BoolPlane & fcst_out, BoolPlane & obs_out) diff --git a/src/tools/core/mode/objects_from_netcdf.h b/src/tools/core/mode/objects_from_netcdf.h index 4192e4033a..20f553791c 100644 --- a/src/tools/core/mode/objects_from_netcdf.h +++ b/src/tools/core/mode/objects_from_netcdf.h @@ -29,7 +29,7 @@ //////////////////////////////////////////////////////////////////////// -extern void objects_from_arrays(const char *netcf_filename, bool do_clusters, +extern void objects_from_arrays(bool do_clusters, int *fcst_objects, int *obs_objects, int nx, int ny, BoolPlane & fcst_out, BoolPlane & obs_out);