From e65313331d9e834134797867adaf4c404f4d9108 Mon Sep 17 00:00:00 2001 From: Cyrus Harrison Date: Mon, 4 Jan 2021 14:46:29 -0800 Subject: [PATCH 1/2] add generic io handle append and truncate options, add bp save_mesh to provide replay save (truncate) semantics --- CHANGELOG.md | 6 + src/libs/relay/conduit_relay_io_blueprint.cpp | 323 +++++++++++++++--- src/libs/relay/conduit_relay_io_blueprint.hpp | 45 ++- src/libs/relay/conduit_relay_io_handle.cpp | 260 ++++++++------ .../relay/conduit_relay_io_handle_api.hpp | 33 +- .../relay/conduit_relay_io_handle_sidre.cpp | 55 +-- .../relay/conduit_relay_mpi_io_blueprint.hpp | 49 ++- .../blueprint/t_blueprint_mesh_relay.cpp | 94 ++++- .../blueprint/t_blueprint_mpi_mesh_relay.cpp | 104 ++++++ src/tests/relay/t_relay_io_handle.cpp | 144 +++++++- 10 files changed, 883 insertions(+), 230 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 04e0c177b..5d4dda3ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,12 @@ and this project aspires to adhere to [Semantic Versioning](https://semver.org/s - Added a builtin sandboxed header-only version of fmt. The namespace and directory paths were changed to `conduit_fmt` to avoid potential symbol collisions with other codes using fmt. Downstream software can use by including `conduit_fmt/conduit_fmt.h`. - Added support for using C++11 initializer lists to set Node and DataArray values from numeric arrays. See C++ tutorial docs (https://llnl-conduit.readthedocs.io/en/latest/tutorial_cpp_numeric.html#c-11-initializer-lists) for more details. +#### Relay +- Added Relay IO Handle mode support for `a` (append) and `t` (truncate). Truncate allows you to overwrite files when the handle is opened. The default is append, which preserves prior IO Handle behavior. +- Added `conduit::relay::io::blueprint::save_mesh` variants, these overwrite existing files (providing relay save semantics) instead of adding mesh data to existing files. We recommend using `save_mesh` for most uses cases, b/c in many cases `write_mesh` to an existing HDF5 file set can fail due to conflicts with the current HDF5 tree. +- Added `truncate` option to `conduit::relay::io::blueprint::write_mesh`, this is used by `save_mesh`. +- Improve capture and reporting of I/O errors in `conduit::relay::[mpi::]io::blueprint::{save_mesh|write_mesh}`. Now in the MPI case, If any rank fails to open or write to a file all ranks will throw an exception. + ## [0.6.0] - Released 2020-11-02 diff --git a/src/libs/relay/conduit_relay_io_blueprint.cpp b/src/libs/relay/conduit_relay_io_blueprint.cpp index e7ae834a7..4f4046c7e 100644 --- a/src/libs/relay/conduit_relay_io_blueprint.cpp +++ b/src/libs/relay/conduit_relay_io_blueprint.cpp @@ -11,6 +11,7 @@ #include "conduit_relay_io.hpp" #include "conduit_relay_io_handle.hpp" +#include "conduit_fmt/conduit_fmt.h" #include "conduit_blueprint.hpp" #ifdef CONDUIT_RELAY_IO_MPI_ENABLED @@ -512,6 +513,101 @@ void gen_domain_to_file_map(int num_domains, }; +//----------------------------------------------------------------------------- +// Sig variants of save_mesh +//----------------------------------------------------------------------------- + +//----------------------------------------------------------------------------- +void save_mesh(const Node &mesh, + const std::string &path + CONDUIT_RELAY_COMMUNICATOR_ARG(MPI_Comm mpi_comm)) +{ + // empty opts + Node opts; +#ifdef CONDUIT_RELAY_IO_MPI_ENABLED + save_mesh(mesh, + path, + detail::identify_protocol(path), + opts, + mpi_comm); +#else + save_mesh(mesh, + path, + detail::identify_protocol(path), + opts); +#endif +} + +//----------------------------------------------------------------------------- +void save_mesh(const Node &mesh, + const std::string &path, + const std::string &protocol + CONDUIT_RELAY_COMMUNICATOR_ARG(MPI_Comm mpi_comm)) +{ + // empty opts + Node opts; +#ifdef CONDUIT_RELAY_IO_MPI_ENABLED + save_mesh(mesh, + path, + protocol, + opts, + mpi_comm); +#else + save_mesh(mesh, + path, + protocol, + opts); +#endif +} + +//----------------------------------------------------------------------------- +// Main Mesh Blueprint Save, taken from Ascent +//----------------------------------------------------------------------------- +//----------------------------------------------------------------------------- +/// The following options can be passed via the opts Node: +//----------------------------------------------------------------------------- +/// opts: +/// file_style: "default", "root_only", "multi_file" +/// when # of domains == 1, "default" ==> "root_only" +/// else, "default" ==> "multi_file" +/// +/// suffix: "default", "cycle", "none" +/// when # of domains == 1, "default" ==> "none" +/// else, "default" ==> "cycle" +/// +/// mesh_name: (used if present, default ==> "mesh") +/// +/// number_of_files: {# of files} +/// when "multi_file": +/// <= 0, use # of files == # of domains +/// > 0, # of files == number_of_files +/// +//----------------------------------------------------------------------------- +void save_mesh(const Node &mesh, + const std::string &path, + const std::string &protocol, + const Node &opts + CONDUIT_RELAY_COMMUNICATOR_ARG(MPI_Comm mpi_comm)) +{ + // we force overwrite to true, so we need a copy of the const opts passed. + Node save_opts; + save_opts.set(opts); + save_opts["truncate"] = "true"; + +#ifdef CONDUIT_RELAY_IO_MPI_ENABLED + write_mesh(mesh, + path, + protocol, + save_opts, + mpi_comm); +#else + write_mesh(mesh, + path, + protocol, + save_opts); +#endif +} + //----------------------------------------------------------------------------- // Sig variants of write_mesh //----------------------------------------------------------------------------- @@ -559,6 +655,7 @@ void write_mesh(const Node &mesh, #endif } + //----------------------------------------------------------------------------- // Main Mesh Blueprint Save, taken from Ascent //----------------------------------------------------------------------------- @@ -594,6 +691,7 @@ void write_mesh(const Node &mesh, std::string opts_suffix = "default"; std::string opts_mesh_name = "mesh"; int opts_num_files = -1; + bool opts_truncate = false; // check for + validate file_style option if(opts.has_child("file_style") && opts["file_style"].dtype().is_string()) @@ -640,6 +738,14 @@ void write_mesh(const Node &mesh, opts_num_files = (int) opts["number_of_files"].to_int(); } + // check for truncate (overwrite) + if(opts.has_child("truncate") && opts["truncate"].dtype().is_string()) + { + const std::string ow_string = opts["truncate"].as_string(); + if(ow_string == "true") + opts_truncate = true; + } + int num_files = opts_num_files; #ifdef CONDUIT_RELAY_IO_MPI_ENABLED @@ -753,9 +859,10 @@ void write_mesh(const Node &mesh, return; } + // TODO FMT - char fmt_buff[64] = {0}; - std::ostringstream oss; + // char fmt_buff[64] = {0}; + // std::ostringstream oss; std::string output_dir = ""; // resolve file_style == default @@ -779,17 +886,30 @@ void write_mesh(const Node &mesh, if(opts_file_style == "multi_file") { // setup the directory - oss << path; + output_dir = path; // at this point for suffix, we should only see // cycle or none -- default has been resolved if(opts_suffix == "cycle") { // TODO: FMT ? - snprintf(fmt_buff, sizeof(fmt_buff), "%06d",cycle); - oss << ".cycle_" << fmt_buff; + + // std::string res = conduit_fmt::format("The answer is {}.", 42); + // std::cout << res << std::endl; + // EXPECT_EQ(res, "The answer is 42."); + // + // res = conduit_fmt::format("The answer is {answer:0.2f}.", + // conduit_fmt::arg("answer",3.14)); + // std::cout << res << std::endl; + + + // snprintf(fmt_buff, sizeof(fmt_buff), "%06d",cycle); + + output_dir += conduit_fmt::format(".cycle_{:06d}",cycle); + // oss << + // oss << ".cycle_" << fmt_buff; } - output_dir = oss.str(); + // output_dir = oss.str(); bool dir_ok = false; // let rank zero handle dir creation @@ -828,18 +948,23 @@ void write_mesh(const Node &mesh, // setup root file name // ---------------------------------------------------- // TODO FMT - oss.str(""); - oss << path; + // oss.str(""); + // oss << path; + std::string root_filename = path; + // at this point for suffix, we should only see // cycle or none -- default has been resolved if(opts_suffix == "cycle") { - snprintf(fmt_buff, sizeof(fmt_buff), "%06d",cycle); - oss << ".cycle_" << fmt_buff; + root_filename += conduit_fmt::format(".cycle_{:06d}",cycle); + // snprintf(fmt_buff, sizeof(fmt_buff), "%06d",cycle); + // oss << ".cycle_" << fmt_buff; } - oss << ".root"; - std::string root_filename = oss.str(); + root_filename += ".root"; + // oss << ".root"; + + // std::string root_filename = oss.str(); // zero or negative (default cases), use one file per domain if(num_files <= 0) @@ -887,13 +1012,17 @@ void write_mesh(const Node &mesh, } else { + + // // TODO FMT + // snprintf(fmt_buff, sizeof(fmt_buff), "%06llu",domain); + // oss.str(""); + // oss << "domain_" << fmt_buff << "/" << opts_mesh_name; + // mesh_path = oss.str(); // multiple domains, we need to use a domain prefix uint64 domain = dom["state/domain_id"].to_uint64(); - // TODO FMT - snprintf(fmt_buff, sizeof(fmt_buff), "%06llu",domain); - oss.str(""); - oss << "domain_" << fmt_buff << "/" << opts_mesh_name; - mesh_path = oss.str(); + mesh_path = conduit_fmt::format("domain_{:06d}/{}", + domain, + opts_mesh_name); } hnd.write(dom,mesh_path); } @@ -914,11 +1043,22 @@ void write_mesh(const Node &mesh, const Node &dom = multi_dom.child(i); uint64 domain = dom["state/domain_id"].to_uint64(); - // TODO FMT - snprintf(fmt_buff, sizeof(fmt_buff), "%06llu",domain); - oss.str(""); - oss << "domain_" << fmt_buff << "." << file_protocol << ":" << opts_mesh_name; - std::string output_file = conduit::utils::join_file_path(output_dir,oss.str()); + // // TODO FMT + // snprintf(fmt_buff, sizeof(fmt_buff), "%06llu",domain); + // oss.str(""); + // oss << "domain_" << fmt_buff << "." << file_protocol << ":" << opts_mesh_name; + // + // std::string output_file = conduit::utils::join_file_path(output_dir,oss.str()); + // // TODO FMT + // snprintf(fmt_buff, sizeof(fmt_buff), "%06llu",domain); + // oss.str(""); + // oss << "domain_" << fmt_buff << "." << file_protocol << ":" << opts_mesh_name; + // + std::string output_file = conduit::utils::join_file_path(output_dir, + conduit_fmt::format("domain_{:06d}.{}:{}", + domain, + file_protocol, + opts_mesh_name)); relay::io::save(dom, output_file); } } @@ -936,6 +1076,10 @@ void write_mesh(const Node &mesh, books["local_file_batons"].set(DataType::int32(num_files)); books["global_file_batons"].set(DataType::int32(num_files)); + // used to track first touch + books["local_file_created"].set(DataType::int32(num_files)); + books["global_file_created"].set(DataType::int32(num_files)); + // local # of domains int32_array local_domain_to_file = books["local_domain_to_file"].value(); int32_array local_domain_status = books["local_domain_status"].value(); @@ -943,11 +1087,17 @@ void write_mesh(const Node &mesh, int32_array local_file_batons = books["local_file_batons"].value(); int32_array global_file_batons = books["global_file_batons"].value(); + int32_array local_file_created = books["local_file_created"].value(); + int32_array global_file_created = books["global_file_created"].value(); + + Node d2f_map; detail::gen_domain_to_file_map(global_num_domains, num_files, books); + // global first domain per file is important, b/c for save semantics + // we need to overwrite the file the first time we write int32_array global_d2f = books["global_domain_to_file"].value(); // init our local map and status array @@ -979,6 +1129,15 @@ void write_mesh(const Node &mesh, bool another_twirl = true; int twirls = 0; + + int local_all_is_good = 1; + int global_all_is_good = 1; + + books["local_all_is_good"].set_external(&local_all_is_good,1); + books["global_all_is_good"].set_external(&global_all_is_good,1); + + std::string local_io_exception_msg = ""; + while(another_twirl) { // update baton requests @@ -1002,14 +1161,24 @@ void write_mesh(const Node &mesh, global_file_batons.set(local_file_batons); #endif + // mpi max file created array + #ifdef CONDUIT_RELAY_IO_MPI_ENABLED + mpi::max_all_reduce(books["local_file_created"], + books["global_file_created"], + mpi_comm); + #else + global_file_created.set(local_file_created); + #endif + + // we now have valid batons (global_file_batons) - for(int f = 0; f < num_files; ++f) + for(int f = 0; f < num_files && local_all_is_good == 1 ; ++f) { // check if this rank has the global baton for this file if( global_file_batons[f] == par_rank ) { // check the domains this rank has pending - for(int d = 0; d < local_num_domains; ++d) + for(int d = 0; d < local_num_domains && local_all_is_good == 1; ++d) { // reuse this handle for all domains in the file relay::io::IOHandle hnd; @@ -1024,38 +1193,88 @@ void write_mesh(const Node &mesh, // construct file name // TODO FMT - snprintf(fmt_buff, sizeof(fmt_buff), "%06d",f); - oss.str(""); - oss << "file_" << fmt_buff << "." << file_protocol; - std::string file_name = oss.str(); + // snprintf(fmt_buff, sizeof(fmt_buff), "%06d",f); + // oss.str(""); + // oss << "file_" << fmt_buff << "." << file_protocol; + // std::string file_name = oss.str(); + + std::string file_name = conduit_fmt::format( + "file_{:06d}.{}", + f, + file_protocol); + std::string output_file = conduit::utils::join_file_path(output_dir, file_name); - // now the path in the file, - oss.str(""); - // and domain id - // TODO FMT - snprintf(fmt_buff, sizeof(fmt_buff), "%06llu",domain_id); - oss << "domain_" << fmt_buff << "/" << opts_mesh_name; - std::string curr_path = oss.str(); - + // // now the path in the file, + // oss.str(""); + // // and domain id + // // TODO FMT + // snprintf(fmt_buff, sizeof(fmt_buff), "%06llu",domain_id); + // oss << "domain_" << fmt_buff << "/" << opts_mesh_name; + // std::string curr_path = oss.str(); - if(!hnd.is_open()) + std::string curr_path = conduit_fmt::format( + "domain_{:06d}/{}", + domain_id, + opts_mesh_name); + + try { - hnd.open(output_file); + // if truncate == true check if this is the first time we are + // touching file, and use wt + Node open_opts; + if(opts_truncate && global_file_created[f] == 0) + { + open_opts["mode"] = "wt"; + + local_file_created[f] = 1; + global_file_created[f] = 1; + } + + if(!hnd.is_open()) + { + hnd.open(output_file, open_opts); + } + + hnd.write(dom, curr_path); + // CONDUIT_INFO("rank " << par_rank << " output_file" + // << output_file << " path " << path); + + // update status, we are done with this doman + local_domain_status[d] = 0; + } + catch(conduit::Error e) + { + local_all_is_good = 0; + local_io_exception_msg = e.message(); } - - hnd.write(dom, curr_path); - // CONDUIT_INFO("rank " << par_rank << " output_file" - // << output_file << " path " << path); - - // update status, we are done with this doman - local_domain_status[d] = 0; } } } } + // if any I/O errors happened stop and have all + // tasks bail out with an exception (to avoid hangs) + #ifdef CONDUIT_RELAY_IO_MPI_ENABLED + mpi::min_all_reduce(books["local_all_is_good"], + books["global_all_is_good"], + mpi_comm); + #else + global_all_is_good = local_all_is_good; + #endif + + if(global_all_is_good == 0) + { + std::string emsg = "Failed to write mesh data on one more more ranks."; + + if(!local_io_exception_msg.empty()) + { + emsg += conduit_fmt::format("Exception details from rank {}: {}.", + par_rank, local_io_exception_msg); + } + CONDUIT_ERROR(emsg); + } // If you need to debug the baton alog: // std::cout << "[" << par_rank << "] " // << " twirls: " << twirls @@ -1065,6 +1284,7 @@ void write_mesh(const Node &mesh, // check if we have another round // stop when all batons are -1 another_twirl = false; + for(int f = 0; f < num_files && !another_twirl; ++f) { // if any entry is not -1, we still have more work to do @@ -1372,10 +1592,10 @@ void read_mesh(const std::string &root_file_path, for(int i = domain_start ; i < domain_end; i++) { // TODO FMT - char domain_fmt_buff[64]; - snprintf(domain_fmt_buff, sizeof(domain_fmt_buff), "%06d",i); - oss.str(""); - oss << "domain_" << std::string(domain_fmt_buff); + // char domain_fmt_buff[64]; + // snprintf(domain_fmt_buff, sizeof(domain_fmt_buff), "%06d",i); + // oss.str(""); + // oss << "domain_" << std::string(domain_fmt_buff); std::string current, next; utils::rsplit_file_path (root_fname, current, next); @@ -1385,7 +1605,10 @@ void read_mesh(const std::string &root_file_path, // also need the tree path std::string tree_path = gen.GenerateTreePath(i); - Node &mesh_out = mesh[oss.str()]; + + std::string mesh_path = conduit_fmt::format("domain_{:06d}",i); + + Node &mesh_out = mesh[mesh_path]; // read components of the mesh according to the mesh index // for each child in the index diff --git a/src/libs/relay/conduit_relay_io_blueprint.hpp b/src/libs/relay/conduit_relay_io_blueprint.hpp index 4e15fc13d..019189118 100644 --- a/src/libs/relay/conduit_relay_io_blueprint.hpp +++ b/src/libs/relay/conduit_relay_io_blueprint.hpp @@ -50,10 +50,50 @@ namespace blueprint //----------------------------------------------------------------------------- // Save a blueprint mesh to root + file set //----------------------------------------------------------------------------- +/// Note: These methods use "save" semantics, they will overwrite existing +/// files. +/// +/// +//----------------------------------------------------------------------------- +void CONDUIT_RELAY_API save_mesh(const conduit::Node &mesh, + const std::string &path); + +//----------------------------------------------------------------------------- +void CONDUIT_RELAY_API save_mesh(const conduit::Node &mesh, + const std::string &path, + const std::string &protocol); + +//----------------------------------------------------------------------------- +/// The following options can be passed via the opts Node: +//----------------------------------------------------------------------------- +/// opts: +/// file_style: "default", "root_only", "multi_file" +/// when # of domains == 1, "default" ==> "root_only" +/// else, "default" ==> "multi_file" +/// +/// suffix: "default", "cycle", "none" +/// when # of domains == 1, "default" ==> "none" +/// else, "default" ==> "cycle" +/// +/// mesh_name: (used if present, default ==> "mesh") +/// +/// number_of_files: {# of files} +/// when "multi_file": +/// <= 0, use # of files == # of domains +/// > 0, # of files == number_of_files +/// +//----------------------------------------------------------------------------- +void CONDUIT_RELAY_API save_mesh(const conduit::Node &mesh, + const std::string &path, + const std::string &protocol, + const conduit::Node &opts); + +//----------------------------------------------------------------------------- +// Write a blueprint mesh to root + file set +//----------------------------------------------------------------------------- /// Note: These methods use "write" semantics, they will append to existing /// files. /// -/// TODO: Provide those with "save" sematics? /// //----------------------------------------------------------------------------- void CONDUIT_RELAY_API write_mesh(const conduit::Node &mesh, @@ -83,6 +123,9 @@ void CONDUIT_RELAY_API write_mesh(const conduit::Node &mesh, /// <= 0, use # of files == # of domains /// > 0, # of files == number_of_files /// +/// truncate: "false", "true" (used if present, default ==> "false") +/// when "true" overwrites existing files (relay 'save' semantics) +/// //----------------------------------------------------------------------------- void CONDUIT_RELAY_API write_mesh(const conduit::Node &mesh, const std::string &path, diff --git a/src/libs/relay/conduit_relay_io_handle.cpp b/src/libs/relay/conduit_relay_io_handle.cpp index e79afb121..afe45af73 100644 --- a/src/libs/relay/conduit_relay_io_handle.cpp +++ b/src/libs/relay/conduit_relay_io_handle.cpp @@ -182,12 +182,17 @@ IOHandle::HandleInterface::open() subpath); if( !subpath.empty() ) { - CONDUIT_ERROR("IOHandle does not (yet) support opening paths with " + CONDUIT_ERROR("IOHandle does not support opening paths with " "subpaths specified: \"" << path() << "\""); } - m_open_mode = "rw"; // default to rw - + m_open_mode = "rwa"; // default to rw, append + + m_open_mode_read = true; + m_open_mode_write = true; + m_open_mode_append = true; + m_open_mode_truncate = false; + // check if options includes open mode if(options().has_child("mode") && options()["mode"].dtype().is_string()) { @@ -195,20 +200,62 @@ IOHandle::HandleInterface::open() m_open_mode = ""; + m_open_mode_read = false; + m_open_mode_write = false; + m_open_mode_append = false; + m_open_mode_truncate = false; + if(opts_mode.find("r") != std::string::npos) { m_open_mode += "r"; + m_open_mode_read = true; } if(opts_mode.find("w") != std::string::npos) { m_open_mode += "w"; + m_open_mode_write = true; + } + + // we need at least read or write + if(! m_open_mode_read && ! m_open_mode_write) + { + CONDUIT_ERROR("IOHandle: invalid open mode:" + << "\"" << opts_mode << "\"." + << " 'mode' string must provide" + << " 'r' (read) and/or 'w' (write)." + << " Expected string: {rw}{a|t}"); + } + + // note append and truncate are mut-ex. + if(opts_mode.find("a") != std::string::npos) + { + if( opts_mode.find("t") != std::string::npos ) + { + CONDUIT_ERROR("IOHandle: invalid open mode:" + << "\"" << opts_mode << "\"." + << " In 'mode' string " + << " 'a' (append) and 't' (truncate)" + << " cannot be used together." + << " Expected string: {rw}{a|t}"); + } + m_open_mode += "a"; + m_open_mode_append = true; + } + + // we checked for both above, so it's safe just check for t here + if(opts_mode.find("t") != std::string::npos) + { + m_open_mode += "t"; + m_open_mode_truncate = true; } - if(m_open_mode == "") + if( !m_open_mode_append && !m_open_mode_truncate) { - CONDUIT_ERROR("IOHandle: invalid open mode: \"" - << open_mode() << "\""); + // if neither append or truncate were specified, + // default to append + m_open_mode += "a"; + m_open_mode_append = true; } } } @@ -346,21 +393,26 @@ BasicHandle::open() // we start out with a blank slate if( utils::is_file( path() ) ) { - // read if handle is not 'write' only - if( open_mode() != "w") + // read if handle is not 'write' only and we aren't truncating + if( open_mode_read() && !open_mode_truncate() ) { - // read from file + // read from file io::load(path(), protocol(), options(), m_node); } + else + { + m_node.reset(); + } } - else if( open_mode() == "r" ) // fail on read only if file doesn't exist + else if( open_mode_read_only() ) // fail on read only if file doesn't exist { CONDUIT_ERROR("path: \"" << path() - << "\" does not exist, cannot open read only (mode = 'r')"); + << "\" does not exist, cannot open read only " + << "(mode = '" << open_mode() << "')"); } else { @@ -384,12 +436,8 @@ BasicHandle::is_open() const void BasicHandle::read(Node &node) { - // throw an error if we opened in "w" mode - if( open_mode() == "w") - { - CONDUIT_ERROR("IOHandle: cannot read, handle is write only" - " (mode = 'w')"); - } + // note: wrong mode errors are handled before dispatch to interface + node.update(m_node); } @@ -398,12 +446,8 @@ void BasicHandle::read(const std::string &path, Node &node) { - // throw an error if we opened in "w" mode - if( open_mode() == "w") - { - CONDUIT_ERROR("IOHandle: cannot read, handle is write only" - " (mode = 'w')"); - } + // note: wrong mode errors are handled before dispatch to interface + if(m_node.has_path(path)) { node.update(m_node[path]); @@ -414,12 +458,8 @@ BasicHandle::read(const std::string &path, void BasicHandle::write(const Node &node) { - // throw an error if we opened in "r" mode - if( open_mode() == "r") - { - CONDUIT_ERROR("IOHandle: cannot write, handle is read only" - " (mode = 'r')"); - } + // note: wrong mode errors are handled before dispatch to interface + m_node.update(node); } @@ -429,12 +469,8 @@ void BasicHandle::write(const Node &node, const std::string &path) { - // throw an error if we opened in "r" mode - if( open_mode() == "r") - { - CONDUIT_ERROR("IOHandle: cannot write, handle is read only" - " (mode = 'r')"); - } + // note: wrong mode errors are handled before dispatch to interface + m_node[path].update(node); } @@ -442,11 +478,8 @@ BasicHandle::write(const Node &node, void BasicHandle::list_child_names(std::vector &res) { - if( open_mode() == "w") - { - CONDUIT_ERROR("IOHandle: cannot list_child_names, handle is write only" - " (mode = 'w')"); - } + // note: wrong mode errors are handled before dispatch to interface + res = m_node.child_names(); } @@ -455,11 +488,8 @@ void BasicHandle::list_child_names(const std::string &path, std::vector &res) { - if( open_mode() == "w") - { - CONDUIT_ERROR("IOHandle: cannot list_child_names, handle is write only" - " (mode = 'w')"); - } + // note: wrong mode errors are handled before dispatch to interface + res.clear(); if(m_node.has_path(path)) res = m_node[path].child_names(); @@ -469,12 +499,7 @@ BasicHandle::list_child_names(const std::string &path, void BasicHandle::remove(const std::string &path) { - // throw an error if we opened in "r" mode - if( open_mode() == "r") - { - CONDUIT_ERROR("IOHandle: cannot remove path, handle is read only" - " (mode = 'r')"); - } + // note: wrong mode errors are handled before dispatch to interface m_node.remove(path); } @@ -483,11 +508,8 @@ BasicHandle::remove(const std::string &path) bool BasicHandle::has_path(const std::string &path) { - if( open_mode() == "w") - { - CONDUIT_ERROR("IOHandle: cannot call has_path, handle is write only" - " (mode = 'w')"); - } + // note: wrong mode errors are handled before dispatch to interface + return m_node.has_path(path); } @@ -495,7 +517,7 @@ BasicHandle::has_path(const std::string &path) void BasicHandle::close() { - if(m_open && open_mode() != "r") + if(m_open && !open_mode_read_only() ) { // here is where it actually gets realized on disk io::save(m_node, @@ -544,16 +566,21 @@ HDF5Handle::open() if( utils::is_file( path() ) ) { // check open mode to select proper hdf5 call - if( open_mode() == "r" ) + + if( open_mode_read_only() ) { m_h5_id = hdf5_open_file_for_read( path() ); - } - else + } // support write with append + else if ( open_mode_append() ) { m_h5_id = hdf5_open_file_for_read_write( path() ); + } // support write with truncate + else if ( open_mode_truncate() ) + { + m_h5_id = hdf5_create_file( path() ); } } - else if( open_mode() == "r" ) + else if( open_mode_read_only() ) { CONDUIT_ERROR("path: \"" << path() @@ -561,7 +588,6 @@ HDF5Handle::open() } else { - m_h5_id = hdf5_create_file( path() ); } } @@ -578,11 +604,8 @@ HDF5Handle::is_open() const void HDF5Handle::read(Node &node) { - if( open_mode() == "w") - { - CONDUIT_ERROR("IOHandle: cannot read, handle is write only" - " (mode = 'w')"); - } + // note: wrong mode errors are handled before dispatch to interface + hdf5_read(m_h5_id,node); } @@ -591,11 +614,8 @@ void HDF5Handle::read(const std::string &path, Node &node) { - if( open_mode() == "w") - { - CONDUIT_ERROR("IOHandle: cannot read, handle is write only" - " (mode = 'w')"); - } + // note: wrong mode errors are handled before dispatch to interface + hdf5_read(m_h5_id,path,node); } @@ -603,13 +623,8 @@ HDF5Handle::read(const std::string &path, void HDF5Handle::write(const Node &node) { - // throw an error if we opened in "r" mode - if( open_mode() == "r") - { - CONDUIT_ERROR("IOHandle: cannot write, handle is read only" - " (mode = 'r')"); - } - + // note: wrong mode errors are handled before dispatch to interface + // Options Push / Pop (only needed for write, since hdf5 only supports // write options Node prev_options; @@ -633,12 +648,7 @@ void HDF5Handle::write(const Node &node, const std::string &path) { - // throw an error if we opened in "r" mode - if( open_mode() == "r") - { - CONDUIT_ERROR("IOHandle: cannot write, handle is read only" - " (mode = 'r')"); - } + // note: wrong mode errors are handled before dispatch to interface // Options Push / Pop (only needed for write, since hdf5 only supports // write options @@ -661,11 +671,8 @@ HDF5Handle::write(const Node &node, void HDF5Handle::list_child_names(std::vector &res) { - if( open_mode() == "w") - { - CONDUIT_ERROR("IOHandle: cannot list_child_names, handle is write only" - " (mode = 'w')"); - } + // note: wrong mode errors are handled before dispatch to interface + hdf5_group_list_child_names(m_h5_id, "/", res); } @@ -674,11 +681,8 @@ void HDF5Handle::list_child_names(const std::string &path, std::vector &res) { - if( open_mode() == "w") - { - CONDUIT_ERROR("IOHandle: cannot list_child_names, handle is write only" - " (mode = 'w')"); - } + // note: wrong mode errors are handled before dispatch to interface + hdf5_group_list_child_names(m_h5_id, path, res); } @@ -686,13 +690,8 @@ HDF5Handle::list_child_names(const std::string &path, void HDF5Handle::remove(const std::string &path) { - // throw an error if we opened in "r" mode - if( open_mode() == "r") - { - CONDUIT_ERROR("IOHandle: cannot remove path, handle is read only" - " (mode = 'r')"); - } - + // note: wrong mode errors are handled before dispatch to interface + hdf5_remove_path(m_h5_id,path); } @@ -700,11 +699,8 @@ HDF5Handle::remove(const std::string &path) bool HDF5Handle::has_path(const std::string &path) { - if( open_mode() == "w") - { - CONDUIT_ERROR("IOHandle: cannot call has_path, handle is write only" - " (mode = 'w')"); - } + // note: wrong mode errors are handled before dispatch to interface + return hdf5_has_path(m_h5_id,path); } @@ -816,6 +812,12 @@ IOHandle::read(Node &node) { if(m_handle != NULL) { + if( m_handle->open_mode_write_only() ) + { + CONDUIT_ERROR("IOHandle: cannot read, handle is write only" + " (mode = '" << m_handle->open_mode() << "')"); + } + m_handle->read(node); } else @@ -831,6 +833,12 @@ IOHandle::read(const std::string &path, { if(m_handle != NULL) { + if( m_handle->open_mode_write_only() ) + { + CONDUIT_ERROR("IOHandle: cannot read, handle is write only" + " (mode = '" << m_handle->open_mode() << "')"); + } + if(path.empty()) { m_handle->read(node); @@ -852,6 +860,12 @@ IOHandle::write(const Node &node) { if(m_handle != NULL) { + if( m_handle->open_mode_read_only() ) + { + CONDUIT_ERROR("IOHandle: cannot write, handle is read only" + " (mode = '" << m_handle->open_mode() << "')"); + } + m_handle->write(node); } else @@ -867,6 +881,12 @@ IOHandle::write(const Node &node, { if(m_handle != NULL) { + if( m_handle->open_mode_read_only() ) + { + CONDUIT_ERROR("IOHandle: cannot write, handle is read only" + " (mode = '" << m_handle->open_mode() << "')"); + } + m_handle->write(node, path); } else @@ -882,6 +902,12 @@ IOHandle::remove(const std::string &path) { if(m_handle != NULL) { + if( m_handle->open_mode_read_only() ) + { + CONDUIT_ERROR("IOHandle: cannot remove path, handle is read only" + " (mode = '" << m_handle->open_mode() << "')"); + } + m_handle->remove(path); } else @@ -897,6 +923,13 @@ IOHandle::list_child_names(std::vector &names) names.clear(); if(m_handle != NULL) { + if( m_handle->open_mode_write_only() ) + { + CONDUIT_ERROR("IOHandle: cannot list_child_names, handle is" + " write only" + " (mode = '" << m_handle->open_mode() << "')"); + } + return m_handle->list_child_names(names); } else @@ -914,6 +947,13 @@ IOHandle::list_child_names(const std::string &path, names.clear(); if(m_handle != NULL) { + if( m_handle->open_mode_write_only() ) + { + CONDUIT_ERROR("IOHandle: cannot list_child_names, handle is" + " write only" + " (mode = '" << m_handle->open_mode() << "')"); + } + return m_handle->list_child_names(path, names); } else @@ -928,6 +968,12 @@ IOHandle::has_path(const std::string &path) { if(m_handle != NULL) { + if( m_handle->open_mode_write_only() ) + { + CONDUIT_ERROR("IOHandle: cannot call has_path, handle is write" + " only" + " (mode = '" << m_handle->open_mode() << "')"); + } return m_handle->has_path(path); } else diff --git a/src/libs/relay/conduit_relay_io_handle_api.hpp b/src/libs/relay/conduit_relay_io_handle_api.hpp index 3f5c47555..7bb89a81c 100644 --- a/src/libs/relay/conduit_relay_io_handle_api.hpp +++ b/src/libs/relay/conduit_relay_io_handle_api.hpp @@ -110,6 +110,27 @@ class CONDUIT_RELAY_API IOHandle virtual bool has_path(const std::string &path) = 0; virtual void close() = 0; + // access to common state + const std::string &path() const; + const std::string &protocol() const; + const Node &options() const; + const std::string &open_mode() const; + + bool open_mode_append() const + { return m_open_mode_append;} + bool open_mode_truncate() const + { return m_open_mode_truncate;} + + bool open_mode_read() const + { return m_open_mode_read;} + bool open_mode_write() const + { return m_open_mode_write;} + + bool open_mode_read_only() const + { return m_open_mode_read && ! m_open_mode_write;} + bool open_mode_write_only() const + { return m_open_mode_write && ! m_open_mode_read;} + // factory helper methods used by interface class static HandleInterface *create(const std::string &path); @@ -122,19 +143,17 @@ class CONDUIT_RELAY_API IOHandle static HandleInterface *create(const std::string &path, const std::string &protocol, const Node &options); - protected: - // access to common state - const std::string &path() const; - const std::string &protocol() const; - const std::string &open_mode() const; - const Node &options() const; private: - std::string m_path; std::string m_protocol; std::string m_open_mode; Node m_options; + + bool m_open_mode_read; + bool m_open_mode_write; + bool m_open_mode_append; + bool m_open_mode_truncate; }; private: diff --git a/src/libs/relay/conduit_relay_io_handle_sidre.cpp b/src/libs/relay/conduit_relay_io_handle_sidre.cpp index 0dbc39726..716455d88 100644 --- a/src/libs/relay/conduit_relay_io_handle_sidre.cpp +++ b/src/libs/relay/conduit_relay_io_handle_sidre.cpp @@ -82,7 +82,7 @@ SidreIOHandle::open() // and processes standard options (mode = "rw", etc) HandleInterface::open(); - if( open_mode() == "w" ) + if( open_mode_write_only() ) CONDUIT_ERROR("SidreIOHandle does not support write mode " "(open_mode = 'w')"); @@ -173,11 +173,7 @@ SidreIOHandle::is_open() const void SidreIOHandle::read(Node &node) { - if( open_mode() == "w") - { - CONDUIT_ERROR("IOHandle: cannot read, handle is write only" - " (mode = 'w')"); - } + // note: wrong mode errors are handled before dispatch to interface std::vector child_names; list_child_names(child_names); @@ -193,11 +189,7 @@ void SidreIOHandle::read(const std::string &path, Node &node) { - if( open_mode() == "w") - { - CONDUIT_ERROR("IOHandle: cannot read, handle is write only" - " (mode = 'w')"); - } + // note: wrong mode errors are handled before dispatch to interface // if blank path or "/", use other method and early exist. if(path.empty() || path == "/") @@ -265,12 +257,7 @@ SidreIOHandle::read(const std::string &path, void SidreIOHandle::write(const Node & /*node*/) // node is unused { - // throw an error if we opened in "r" mode - if( open_mode() == "r") - { - CONDUIT_ERROR("IOHandle: cannot write, handle is read only" - " (mode = 'r')"); - } + // note: wrong mode errors are handled before dispatch to interface // not supported yet, so throw a fatal error CONDUIT_ERROR("IOHandle: sidre write support not implemented"); @@ -282,12 +269,7 @@ void SidreIOHandle::write(const Node & /*node*/, // node is unused const std::string & /*path*/ ) // path is unused { - // throw an error if we opened in "r" mode - if( open_mode() == "r") - { - CONDUIT_ERROR("IOHandle: cannot write, handle is read only" - " (mode = 'r')"); - } + // note: wrong mode errors are handled before dispatch to interface // not supported yet, so throw a fatal error CONDUIT_ERROR("IOHandle: sidre write support not implemented"); @@ -298,11 +280,7 @@ SidreIOHandle::write(const Node & /*node*/, // node is unused void SidreIOHandle::list_child_names(std::vector &res) { - if( open_mode() == "w") - { - CONDUIT_ERROR("IOHandle: cannot list_child_names, handle is write only" - " (mode = 'w')"); - } + // note: wrong mode errors are handled before dispatch to interface if(m_has_spio_index) { @@ -330,15 +308,11 @@ void SidreIOHandle::list_child_names(const std::string &path, std::vector &res) { + // note: wrong mode errors are handled before dispatch to interface + // note: if the path is bad, we return an empty list res.clear(); - if( open_mode() == "w") - { - CONDUIT_ERROR("IOHandle: cannot list_child_names, handle is write only" - " (mode = 'w')"); - } - if(m_has_spio_index) { std::string p_first; @@ -380,12 +354,7 @@ SidreIOHandle::list_child_names(const std::string &path, void SidreIOHandle::remove(const std::string &/*path*/) // path is unused { - // throw an error if we opened in "r" mode - if( open_mode() == "r") - { - CONDUIT_ERROR("IOHandle: cannot remove path, handle is read only" - " (mode = 'r')"); - } + // note: wrong mode errors are handled before dispatch to interface // not supported yet, so throw a fatal error CONDUIT_ERROR("IOHandle: sidre write support not implemented"); @@ -395,11 +364,7 @@ SidreIOHandle::remove(const std::string &/*path*/) // path is unused bool SidreIOHandle::has_path(const std::string &path) { - if( open_mode() == "w") - { - CONDUIT_ERROR("IOHandle: cannot call has_path, handle is write only" - " (mode = 'w')"); - } + // note: wrong mode errors are handled before dispatch to interface bool res = false; diff --git a/src/libs/relay/conduit_relay_mpi_io_blueprint.hpp b/src/libs/relay/conduit_relay_mpi_io_blueprint.hpp index 9314c4fc3..427097ef6 100644 --- a/src/libs/relay/conduit_relay_mpi_io_blueprint.hpp +++ b/src/libs/relay/conduit_relay_mpi_io_blueprint.hpp @@ -55,6 +55,50 @@ namespace io namespace blueprint { +//----------------------------------------------------------------------------- +// Save a blueprint mesh to root + file set +//----------------------------------------------------------------------------- +/// Note: These methods use "save" semantics, they will overwrite existing +/// files. +/// +/// +//----------------------------------------------------------------------------- +void CONDUIT_RELAY_API save_mesh(const conduit::Node &mesh, + const std::string &path, + MPI_Comm comm); + +//----------------------------------------------------------------------------- +void CONDUIT_RELAY_API save_mesh(const conduit::Node &mesh, + const std::string &path, + const std::string &protocol, + MPI_Comm comm); + +//----------------------------------------------------------------------------- +/// The following options can be passed via the opts Node: +//----------------------------------------------------------------------------- +/// opts: +/// file_style: "default", "root_only", "multi_file" +/// when # of domains == 1, "default" ==> "root_only" +/// else, "default" ==> "multi_file" +/// +/// suffix: "default", "cycle", "none" +/// when # of domains == 1, "default" ==> "none" +/// else, "default" ==> "cycle" +/// +/// mesh_name: (used if present, default ==> "mesh") +/// +/// number_of_files: {# of files} +/// when "multi_file": +/// <= 0, use # of files == # of domains +/// > 0, # of files == number_of_files +/// +//----------------------------------------------------------------------------- +void CONDUIT_RELAY_API save_mesh(const conduit::Node &mesh, + const std::string &path, + const std::string &protocol, + const conduit::Node &opts, + MPI_Comm comm); + //----------------------------------------------------------------------------- // Save a blueprint mesh to root + file set //----------------------------------------------------------------------------- @@ -62,8 +106,6 @@ namespace blueprint /// Note: These methods use "write" semantics, they will append to existing /// files. /// -/// TODO: Provide those with "save" sematics? -/// //----------------------------------------------------------------------------- //----------------------------------------------------------------------------- @@ -97,6 +139,9 @@ void CONDUIT_RELAY_API write_mesh(const conduit::Node &mesh, /// <= 0, use # of files == # of domains /// > 0, # of files == number_of_files /// +/// truncate: "false", "true" (used if present, default ==> "false") +/// when "true" overwrites existing files (relay 'save' semantics) +/// //----------------------------------------------------------------------------- void CONDUIT_RELAY_API write_mesh(const conduit::Node &mesh, const std::string &path, diff --git a/src/tests/blueprint/t_blueprint_mesh_relay.cpp b/src/tests/blueprint/t_blueprint_mesh_relay.cpp index 30b08bc73..6bc959b3e 100644 --- a/src/tests/blueprint/t_blueprint_mesh_relay.cpp +++ b/src/tests/blueprint/t_blueprint_mesh_relay.cpp @@ -105,7 +105,7 @@ TEST(conduit_blueprint_mesh_relay, save_read_mesh) bool hdf5_enabled = io_protos["io/protocols/hdf5"].as_string() == "enabled"; if(!hdf5_enabled) { - CONDUIT_INFO("HDF5 disabled, skipping spiral_multi_file test"); + CONDUIT_INFO("HDF5 disabled, skipping save_read_mesh test"); return; } @@ -123,7 +123,9 @@ TEST(conduit_blueprint_mesh_relay, save_read_mesh) opts["number_of_files"] = -1; remove_path_if_exists(output_base + ".cycle_000000.root"); - remove_path_if_exists(output_base + ".cycle_000000"); + remove_path_if_exists(output_base + ".cycle_000000/file_000000.hdf5"); + remove_path_if_exists(output_base + ".cycle_000000/file_000001.hdf5"); + remove_path_if_exists(output_base + ".cycle_000000/file_000002.hdf5"); relay::io::blueprint::write_mesh(data, output_base, "hdf5", opts); @@ -140,6 +142,94 @@ TEST(conduit_blueprint_mesh_relay, save_read_mesh) data.child(2).diff(n_read.child(2),info); } +//----------------------------------------------------------------------------- +TEST(conduit_blueprint_mesh_relay, save_read_mesh_truncate) +{ + Node io_protos; + relay::io::about(io_protos["io"]); + bool hdf5_enabled = io_protos["io/protocols/hdf5"].as_string() == "enabled"; + if(!hdf5_enabled) + { + CONDUIT_INFO("HDF5 disabled, skipping save_read_mesh_truncate test"); + return; + } + + std::string output_base = "tout_relay_mesh_save_load_truncate"; + + Node data; + + // 3 doms + blueprint::mesh::examples::braid("uniform", + 2, + 2, + 2, + data.append()); + + blueprint::mesh::examples::braid("uniform", + 2, + 2, + 2, + data.append()); + + blueprint::mesh::examples::braid("uniform", + 2, + 2, + 2, + data.append()); + + // set the domain ids + data.child(0)["state/domain_id"] = 0; + data.child(1)["state/domain_id"] = 1; + data.child(2)["state/domain_id"] = 2; + + // 3 doms 2 files + Node opts; + opts["number_of_files"] = 2; + + remove_path_if_exists(output_base + ".cycle_000100.root"); + remove_path_if_exists(output_base + ".cycle_000100/file_000000.hdf5"); + remove_path_if_exists(output_base + ".cycle_000100/file_000001.hdf5"); + + relay::io::blueprint::write_mesh(data, output_base, "hdf5", opts); + + // now save diff mesh to same file set, will fail with write_mesh + // b/c hdf5 paths won't be compat + + data.reset(); + // 3 doms + blueprint::mesh::examples::braid("uniform", + 5, + 5, + 0, + data.append()); + + blueprint::mesh::examples::braid("uniform", + 5, + 5, + 0, + data.append()); + + blueprint::mesh::examples::braid("uniform", + 5, + 5, + 0, + data.append()); + + // set the domain ids + data.child(0)["state/domain_id"] = 0; + data.child(1)["state/domain_id"] = 1; + data.child(2)["state/domain_id"] = 2; + + // non-trunk will error b/c leaf sizes aren't compat + EXPECT_THROW( relay::io::blueprint::write_mesh(data, output_base, "hdf5", opts),Error); + + // trunc will work + relay::io::blueprint::save_mesh(data, output_base, "hdf5", opts); + +} + + + //----------------------------------------------------------------------------- TEST(conduit_blueprint_mesh_relay, save_read_mesh_opts) diff --git a/src/tests/blueprint/t_blueprint_mpi_mesh_relay.cpp b/src/tests/blueprint/t_blueprint_mpi_mesh_relay.cpp index 674f042c5..58ba468f4 100644 --- a/src/tests/blueprint/t_blueprint_mpi_mesh_relay.cpp +++ b/src/tests/blueprint/t_blueprint_mpi_mesh_relay.cpp @@ -459,6 +459,110 @@ TEST(blueprint_mpi_relay, spiral_multi_file) // read this back using read_mesh } +//----------------------------------------------------------------------------- +TEST(blueprint_mpi_relay, test_write_error_hang) +{ + + Node io_protos; + relay::io::about(io_protos["io"]); + bool hdf5_enabled = io_protos["io/protocols/hdf5"].as_string() == "enabled"; + if(!hdf5_enabled) + { + CONDUIT_INFO("HDF5 disabled, skipping save_read_mesh_truncate test"); + return; + } + + MPI_Comm comm = MPI_COMM_WORLD; + int par_rank = mpi::rank(comm); + int par_size = mpi::size(comm); + + + std::string output_base = "tout_relay_mpi_mesh_save_load_truncate"; + + Node data; + + // 6 doms (2 per mpi task) + blueprint::mesh::examples::braid("uniform", + 2, + 2, + 2, + data.append()); + + blueprint::mesh::examples::braid("uniform", + 2, + 2, + 2, + data.append()); + + blueprint::mesh::examples::braid("uniform", + 2, + 2, + 2, + data.append()); + + // set the domain ids + data.child(0)["state/domain_id"] = 0; + data.child(1)["state/domain_id"] = 1; + data.child(2)["state/domain_id"] = 2; + + // use 2 files + Node opts; + opts["number_of_files"] = 2; + + remove_path_if_exists(output_base + ".cycle_000100.root"); + remove_path_if_exists(output_base + ".cycle_000100/file_000000.hdf5"); + remove_path_if_exists(output_base + ".cycle_000100/file_000001.hdf5"); + + conduit::relay::mpi::io::blueprint::write_mesh(data, + output_base, + "hdf5", + opts, + comm); + + // now save diff mesh to same file set, will fail with write_mesh + // b/c hdf5 paths won't be compat + + data.reset(); + + // 6 doms (2 per mpi task) + blueprint::mesh::examples::braid("uniform", + 5, + 5, + 0, + data.append()); + + blueprint::mesh::examples::braid("uniform", + 5, + 5, + 0, + data.append()); + + blueprint::mesh::examples::braid("uniform", + 5, + 5, + 0, + data.append()); + + // set the domain ids + data.child(0)["state/domain_id"] = 0; + data.child(1)["state/domain_id"] = 1; + data.child(2)["state/domain_id"] = 2; + + // non-trunk will error b/c leaf sizes aren't compat + EXPECT_THROW( conduit::relay::mpi::io::blueprint::write_mesh(data, + output_base, + "hdf5", + opts, + comm),Error); + + // trunc will work + conduit::relay::mpi::io::blueprint::save_mesh(data, + output_base, + "hdf5", + opts, + comm); +} + //----------------------------------------------------------------------------- int main(int argc, char* argv[]) diff --git a/src/tests/relay/t_relay_io_handle.cpp b/src/tests/relay/t_relay_io_handle.cpp index 89d7d5bee..c60aba63f 100644 --- a/src/tests/relay/t_relay_io_handle.cpp +++ b/src/tests/relay/t_relay_io_handle.cpp @@ -30,7 +30,7 @@ TEST(conduit_relay_io_handle, test_active_protos) Node n_about; io::about(n_about); - + if(n_about["protocols/hdf5"].as_string() == "enabled") protocols.push_back("hdf5"); @@ -38,7 +38,7 @@ TEST(conduit_relay_io_handle, test_active_protos) itr < protocols.end(); ++itr) { std::string protocol = *itr; - CONDUIT_INFO("Testing Relay IO Handle with protocol: " + CONDUIT_INFO("Testing Relay IO Handle with protocol: " << protocol ); std::string test_file_name = tfile_base + protocol; @@ -129,7 +129,7 @@ TEST(conduit_relay_io_handle, test_is_open) conduit::Error); EXPECT_FALSE(h.is_open()); - + h.open("tout_conduit_relay_io_handle_is_open.hdf5"); EXPECT_TRUE(h.is_open()); @@ -137,15 +137,15 @@ TEST(conduit_relay_io_handle, test_is_open) h.close(); EXPECT_FALSE(h.is_open()); - + } - + // test subpath for exceptions EXPECT_THROW(h.open("file.json:here/is/a/subpath/to/data"), conduit::Error); EXPECT_FALSE(h.is_open()); - + h.open("tout_conduit_relay_io_handle_is_open.conduit_json"); EXPECT_TRUE(h.is_open()); @@ -222,7 +222,7 @@ TEST(conduit_relay_io_handle, test_exceptions) // check throw on methods called on not open handle Node n; - + EXPECT_THROW(h.write(n), conduit::Error); EXPECT_THROW(h.read(n), conduit::Error); EXPECT_THROW(h.has_path("here"), conduit::Error); @@ -242,7 +242,7 @@ TEST(conduit_relay_io_handle, test_exceptions) EXPECT_THROW(h.open("here/is/a/garbage/file/path.hdf5"), conduit::Error); } - + // test subpath for exceptions EXPECT_THROW(h.open("file.json:here/is/a/subpath/to/data"), conduit::Error); @@ -276,14 +276,14 @@ TEST(conduit_relay_io_handle, test_mode) Node n_about; io::about(n_about); - + if(n_about["protocols/hdf5"].as_string() == "enabled") protocols.push_back("hdf5"); for (std::vector::const_iterator itr = protocols.begin(); itr < protocols.end(); ++itr) { - + std::string protocol = *itr; CONDUIT_INFO("Testing Relay IO Handle Open Mode 'r' with protocol: " << protocol ); @@ -313,13 +313,13 @@ TEST(conduit_relay_io_handle, test_mode) EXPECT_THROW(h_ro.write(n,"super"), conduit::Error); // read only, fail to remove: EXPECT_THROW(h_ro.remove("super"), conduit::Error); - + // make sure we can read something Node n_read; h_ro.read("d/here",n_read); EXPECT_EQ(n["d/here"].as_int64(),n_read.as_int64()); - CONDUIT_INFO("Testing Relay IO Handle Open Mode 'w' with protocol: " + CONDUIT_INFO("Testing Relay IO Handle Open Mode 'w' with protocol: " << protocol ); test_file_name = "tout_conduit_relay_io_handle_mode_wo." + protocol; @@ -327,7 +327,7 @@ TEST(conduit_relay_io_handle, test_mode) /// write only Node opts_wonly; opts_wonly["mode"] = "w"; - + io::IOHandle h_wo; h_wo.open(test_file_name,opts_wonly); @@ -343,14 +343,14 @@ TEST(conduit_relay_io_handle, test_mode) h_wo.write(n); h_wo.close(); - + // lets read what wrote to make sure the handled actually wrote // something n_read.reset(); relay::io::load(test_file_name,n_read); EXPECT_EQ(n["d/here"].as_int64(),n_read["d/here"].to_int64()); } - + // // io::IOHandle h(opts); // // if read only, it will fail to open a file that doesn't exist @@ -363,7 +363,7 @@ TEST(conduit_relay_io_handle, test_mode) // // read only, fail to write: // EXPECT_THROW(h.write(n), conduit::Error); - + } @@ -416,6 +416,118 @@ TEST(conduit_relay_io_handle, test_reuse_handle) } +//----------------------------------------------------------------------------- +TEST(conduit_relay_io_handle, test_handle_trunc) +{ + int64 a_val = 20; + int64 b_val = 8; + int64 c_val = 13; + int64 here_val = 10; + + Node n; + n["a"] = a_val; + n["b"] = b_val; + n["c"] = c_val; + n["d/here"] = here_val; + + // remove files if they already exist + utils::remove_path_if_exists("tout_conduit_relay_io_handle_trunc_1.conduit_bin"); + utils::remove_path_if_exists("tout_conduit_relay_io_handle_trunc_1.conduit_json"); + + io::IOHandle h; + h.open("tout_conduit_relay_io_handle_trunc_1.conduit_bin"); + h.write(n); + h.close(); + + // test truncate + n.reset(); + n["a"] = a_val; + + Node opts; + opts["mode"] = "wt"; + h.open("tout_conduit_relay_io_handle_trunc_1.conduit_bin",opts); + h.write(n); + h.close(); + + Node nread; + opts["mode"] = "r"; + h.open("tout_conduit_relay_io_handle_trunc_1.conduit_bin",opts); + h.read(nread); + + Node info; + EXPECT_FALSE(n.diff(nread, info, 0.0)); + + nread.print(); + info.print(); +} + +//----------------------------------------------------------------------------- +TEST(conduit_relay_io_handle, test_hdf5_trunc) +{ + std::string tfile_out = "tout_conduit_relay_io_handle_hdf5_trunc.hdf5"; + Node n_about; + io::about(n_about); + + // skip test if hdf5 isn't enabled + if(n_about["protocols/hdf5"].as_string() != "enabled") + return; + + int64 a_val = 20; + int64 b_val = 8; + int64 c_val = 13; + int64 here_val = 10; + + Node n; + n["a"] = a_val; + n["b"] = b_val; + n["c"] = c_val; + n["d/here"] = here_val; + + // remove files if they already exist + utils::remove_path_if_exists(tfile_out); + + io::IOHandle h; + h.open(tfile_out); + h.write(n); + h.close(); + + // test truncate + n.reset(); + n["a"] = a_val; + + Node opts; + opts["mode"] = "wt"; + h.open(tfile_out,opts); + h.write(n); + h.close(); + + Node nread; + opts["mode"] = "r"; + h.open(tfile_out,opts); + h.read(nread); + + Node info; + EXPECT_FALSE(n.diff(nread, info, 0.0)); + + nread.print(); + info.print(); + + h.open(tfile_out); + h.write(n,"b/c/d/e/f"); + h.close(); + + n["b/c/d/e/f/a"] = a_val; + + opts["mode"] = "r"; + h.open(tfile_out,opts); + h.read(nread); + + EXPECT_FALSE(n.diff(nread, info, 0.0)); + + nread.print(); + info.print(); +} + //----------------------------------------------------------------------------- TEST(conduit_relay_io_handle, test_empty_path_as_root) From 59977deca27ba474f87972f1e24be2bdd0dab70a Mon Sep 17 00:00:00 2001 From: Cyrus Harrison Date: Mon, 4 Jan 2021 14:59:45 -0800 Subject: [PATCH 2/2] use more fmt --- .../blueprint/t_blueprint_mesh_relay.cpp | 23 ++++++++++++++----- .../blueprint/t_blueprint_mpi_mesh_relay.cpp | 22 +++++++++++++----- 2 files changed, 33 insertions(+), 12 deletions(-) diff --git a/src/tests/blueprint/t_blueprint_mesh_relay.cpp b/src/tests/blueprint/t_blueprint_mesh_relay.cpp index 6bc959b3e..96acf0ee1 100644 --- a/src/tests/blueprint/t_blueprint_mesh_relay.cpp +++ b/src/tests/blueprint/t_blueprint_mesh_relay.cpp @@ -13,6 +13,8 @@ #include "conduit_relay.hpp" #include "conduit_log.hpp" +#include "conduit_fmt/conduit_fmt.h" + #include #include #include "gtest/gtest.h" @@ -83,12 +85,21 @@ TEST(conduit_blueprint_mesh_relay, spiral_multi_file) // domain_ fprefix = "domain_"; } - snprintf(fmt_buff, sizeof(fmt_buff), "%06d",i); - oss.str(""); - oss << join_file_path(output_base + ".cycle_000000", - fprefix) - << fmt_buff << ".hdf5"; - std::string fcheck = oss.str(); + + // std::string fcheck = conduit_fmt::format("{}.cycle_000000"); + + // snprintf(fmt_buff, sizeof(fmt_buff), "%06d",i); + // oss.str(""); + // oss << join_file_path(output_base + ".cycle_000000", + // fprefix) + // << fmt_buff << ".hdf5"; + // std::string fcheck = oss.str(); + + std::string fcheck = conduit_fmt::format("{}{:06d}.hdf5", + join_file_path(output_base + ".cycle_000000", + fprefix), + i); + std::cout << " checking: " << fcheck << std::endl; EXPECT_TRUE(is_file(fcheck)); } diff --git a/src/tests/blueprint/t_blueprint_mpi_mesh_relay.cpp b/src/tests/blueprint/t_blueprint_mpi_mesh_relay.cpp index 58ba468f4..ea98ad817 100644 --- a/src/tests/blueprint/t_blueprint_mpi_mesh_relay.cpp +++ b/src/tests/blueprint/t_blueprint_mpi_mesh_relay.cpp @@ -16,6 +16,7 @@ #include "conduit_relay_mpi_io.hpp" #include "conduit_relay_mpi_io_blueprint.hpp" #include "conduit_utils.hpp" +#include "conduit_fmt/conduit_fmt.h" #include #include @@ -445,12 +446,21 @@ TEST(blueprint_mpi_relay, spiral_multi_file) // domain_ fprefix = "domain_"; } - snprintf(fmt_buff, sizeof(fmt_buff), "%06d",i); - oss.str(""); - oss << conduit::utils::join_file_path(output_base + ".cycle_000000", - fprefix) - << fmt_buff << ".hdf5"; - std::string fcheck = oss.str(); + + // snprintf(fmt_buff, sizeof(fmt_buff), "%06d",i); + // oss.str(""); + // oss << conduit::utils::join_file_path(output_base + ".cycle_000000", + // fprefix) + // << fmt_buff << ".hdf5"; + // + // std::string fcheck = oss.str(); + + std::string fcheck = conduit_fmt::format("{}{:06d}.hdf5", + join_file_path(output_base + ".cycle_000000", + fprefix), + i); + + // std::string fcheck = oss.str(); std::cout << " checking: " << fcheck << std::endl; EXPECT_TRUE(conduit::utils::is_file(fcheck)); }