From f04d8fec65850193759553a4a1f51cce88a5df92 Mon Sep 17 00:00:00 2001 From: Cyrus Harrison Date: Wed, 13 Jan 2021 09:52:51 -0800 Subject: [PATCH] enhancements to matset to_silo (#677) * enhancements to matset to_silo * fix missing hdf5 guard, use diff to check to_silo results * fix from joe for unibuffer case --- CHANGELOG.md | 3 + src/libs/blueprint/CMakeLists.txt | 3 +- src/libs/blueprint/conduit_blueprint.hpp | 2 - src/libs/blueprint/conduit_blueprint_mesh.hpp | 11 +++ .../conduit_blueprint_mesh_examples_venn.cpp | 8 +- ... conduit_blueprint_mesh_matset_xforms.cpp} | 39 +++------ .../blueprint/conduit_blueprint_util_mesh.hpp | 86 ------------------- src/libs/relay/conduit_relay_io_blueprint.cpp | 5 +- src/tests/blueprint/CMakeLists.txt | 2 +- .../blueprint/t_blueprint_mesh_examples.cpp | 45 +++++++++- ...cpp => t_blueprint_mesh_matset_xforms.cpp} | 63 +++++++++++++- 11 files changed, 140 insertions(+), 127 deletions(-) rename src/libs/blueprint/{conduit_blueprint_util_mesh.cpp => conduit_blueprint_mesh_matset_xforms.cpp} (92%) delete mode 100644 src/libs/blueprint/conduit_blueprint_util_mesh.hpp rename src/tests/blueprint/{t_blueprint_util_mesh.cpp => t_blueprint_mesh_matset_xforms.cpp} (54%) diff --git a/CHANGELOG.md b/CHANGELOG.md index b4a08b9ce..18e3414c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,9 @@ and this project aspires to adhere to [Semantic Versioning](https://semver.org/s - Added `truncate` option to `conduit::relay::io::blueprint::write_mesh`, this is used by `save_mesh`. - Improve capture and reporting of I/O errors in `conduit::relay::[mpi::]io::blueprint::{save_mesh|write_mesh}`. Now in the MPI case, If any rank fails to open or write to a file all ranks will throw an exception. +#### Blueprint +- Added `conduit::blueprint::mesh::matset::to_silo()` which converts a valid blueprint matset to a node that contains arrays that follow Silo's sparse mix slot volume fraction representation. + ## [0.6.0] - Released 2020-11-02 diff --git a/src/libs/blueprint/CMakeLists.txt b/src/libs/blueprint/CMakeLists.txt index 0dc8bca60..534b87c09 100644 --- a/src/libs/blueprint/CMakeLists.txt +++ b/src/libs/blueprint/CMakeLists.txt @@ -22,7 +22,6 @@ set(blueprint_headers conduit_blueprint_mesh_examples.hpp conduit_blueprint_mesh_examples_julia.hpp conduit_blueprint_mesh_examples_venn.hpp - conduit_blueprint_util_mesh.hpp conduit_blueprint_mcarray.hpp conduit_blueprint_mcarray_examples.hpp conduit_blueprint_o2mrelation.hpp @@ -48,10 +47,10 @@ set(blueprint_c_headers set(blueprint_sources conduit_blueprint.cpp conduit_blueprint_mesh.cpp + conduit_blueprint_mesh_matset_xforms.cpp conduit_blueprint_mesh_examples.cpp conduit_blueprint_mesh_examples_julia.cpp conduit_blueprint_mesh_examples_venn.cpp - conduit_blueprint_util_mesh.cpp conduit_blueprint_mcarray.cpp conduit_blueprint_mcarray_examples.cpp conduit_blueprint_o2mrelation.cpp diff --git a/src/libs/blueprint/conduit_blueprint.hpp b/src/libs/blueprint/conduit_blueprint.hpp index 3e2a33755..91d284039 100644 --- a/src/libs/blueprint/conduit_blueprint.hpp +++ b/src/libs/blueprint/conduit_blueprint.hpp @@ -23,8 +23,6 @@ #include "conduit_blueprint_mesh_examples_julia.hpp" #include "conduit_blueprint_mesh_examples_venn.hpp" -#include "conduit_blueprint_util_mesh.hpp" - #include "conduit_blueprint_o2mrelation.hpp" #include "conduit_blueprint_o2mrelation_examples.hpp" #include "conduit_blueprint_o2mrelation_iterator.hpp" diff --git a/src/libs/blueprint/conduit_blueprint_mesh.hpp b/src/libs/blueprint/conduit_blueprint_mesh.hpp index 61de024fc..56cdad67f 100644 --- a/src/libs/blueprint/conduit_blueprint_mesh.hpp +++ b/src/libs/blueprint/conduit_blueprint_mesh.hpp @@ -407,6 +407,17 @@ namespace matset //------------------------------------------------------------------------- bool CONDUIT_BLUEPRINT_API is_material_dominant(const conduit::Node &n); + //------------------------------------------------------------------------- + // Converts a blueprint matset to the silo style sparse mixed slot + // representation. + // + // For details about the silo format, see documentation for + // 'DBPutMaterial' at: + // https://wci.llnl.gov/content/assets/docs/simulation/computer-codes/silo/LLNL-SM-654357.pdf + void CONDUIT_BLUEPRINT_API to_silo(const conduit::Node &n, + conduit::Node &dest, + const float64 epsilon = CONDUIT_EPSILON); + //------------------------------------------------------------------------- // blueprint::mesh::matset::index protocol interface //------------------------------------------------------------------------- diff --git a/src/libs/blueprint/conduit_blueprint_mesh_examples_venn.cpp b/src/libs/blueprint/conduit_blueprint_mesh_examples_venn.cpp index bf28ca8db..a446b6cfa 100644 --- a/src/libs/blueprint/conduit_blueprint_mesh_examples_venn.cpp +++ b/src/libs/blueprint/conduit_blueprint_mesh_examples_venn.cpp @@ -356,10 +356,10 @@ void venn_sparse_by_element_matset(Node &res) // Build the rest of the single-buffer matset res["matsets/matset/topology"] = "topo"; // This is the "key" that tells what material each volume fraction refers to - res["matsets/matset/material_map/cir_a"] = 1; - res["matsets/matset/material_map/cir_b"] = 2; - res["matsets/matset/material_map/cir_c"] = 3; - res["matsets/matset/material_map/bg"] = 0; + res["matsets/matset/material_map/circle_a"] = 1; + res["matsets/matset/material_map/circle_b"] = 2; + res["matsets/matset/material_map/circle_c"] = 3; + res["matsets/matset/material_map/background"] = 0; // All the volume fractions go here ("one big buffer") res["matsets/matset/volume_fractions"].set(DataType::float64(vfcount)); diff --git a/src/libs/blueprint/conduit_blueprint_util_mesh.cpp b/src/libs/blueprint/conduit_blueprint_mesh_matset_xforms.cpp similarity index 92% rename from src/libs/blueprint/conduit_blueprint_util_mesh.cpp rename to src/libs/blueprint/conduit_blueprint_mesh_matset_xforms.cpp index e24708621..f6a8eeeb8 100644 --- a/src/libs/blueprint/conduit_blueprint_util_mesh.cpp +++ b/src/libs/blueprint/conduit_blueprint_mesh_matset_xforms.cpp @@ -4,7 +4,7 @@ //----------------------------------------------------------------------------- /// -/// file: conduit_blueprint_util_mesh.cpp +/// file: conduit_blueprint_mesh_matset_xforms.cpp /// //----------------------------------------------------------------------------- @@ -22,7 +22,6 @@ #include "conduit_blueprint_mesh.hpp" #include "conduit_blueprint_o2mrelation.hpp" #include "conduit_blueprint_o2mrelation_iterator.hpp" -#include "conduit_blueprint_util_mesh.hpp" // FIXME(JRC): The helper functions below are hackily copied over from // 'conduit_blueprint_mesh.cpp'; these helpers should ultimately be abstracted @@ -118,19 +117,13 @@ namespace blueprint { //----------------------------------------------------------------------------- -// -- begin conduit::blueprint::util -- -//----------------------------------------------------------------------------- -namespace util -{ - -//----------------------------------------------------------------------------- -// -- begin conduit::blueprint::util::mesh -- +// -- begin conduit::blueprint::mesh -- //----------------------------------------------------------------------------- namespace mesh { //----------------------------------------------------------------------------- -// -- begin conduit::blueprint::util::mesh::matset -- +// -- begin conduit::blueprint::mesh::matset -- //----------------------------------------------------------------------------- namespace matset { @@ -154,7 +147,7 @@ to_silo(const conduit::Node &n, Node matset_mat_map; if(mset_is_unibuffer) { - matset_mat_map.set_external(n["volume_fractions/material_map"]); + matset_mat_map.set_external(n["material_map"]); } else // if(!mset_is_unibuffer) { @@ -187,12 +180,12 @@ to_silo(const conduit::Node &n, } else // if(!mset_is_matdom) { - // may need to do a bit of sculping here; embed the base array into + // may need to do a bit of sculpting here; embed the base array into // something w/ "values" child, as below Node mat_vfs; if(mset_is_unibuffer) { - mat_vfs["values"].set_external(n["volume_fractions/values"]); + mat_vfs.set_external(n); } else { @@ -253,8 +246,8 @@ to_silo(const conduit::Node &n, (void*)mat_mids.element_ptr(mat_ind_index)); const index_t mat_id = temp.to_index_t(); - // if this elem has a non-zero (or non-trival) volume fraction for this - // materal, add it do the map + // if this elem has a non-zero (or non-trivial) volume fraction for this + // material, add it do the map if(mat_vf > epsilon) { elem_mat_maps[elem_index][mat_id] = mat_vf; @@ -312,8 +305,8 @@ to_silo(const conduit::Node &n, } const index_t mat_elem = mset_is_matdom ? temp.to_index_t() : mat_index; - // if this elem has a non-zero (or non-trival) volume fraction for this - // materal, add it do the map + // if this elem has a non-zero (or non-trivial) volume fraction for this + // material, add it do the map if(mat_vf > epsilon) { elem_mat_maps[mat_elem][mat_id] = mat_vf; @@ -333,6 +326,9 @@ to_silo(const conduit::Node &n, dest.reset(); dest["topology"].set(n["topology"]); + // in some cases, this method will sort the material names + // so always include the material map + dest["material_map"].set(matset_mat_map); dest["matlist"].set(DataType(int_dtype.id(), mset_num_elems)); dest["mix_next"].set(DataType(int_dtype.id(), mset_num_slots)); dest["mix_mat"].set(DataType(int_dtype.id(), mset_num_slots)); @@ -388,7 +384,7 @@ to_silo(const conduit::Node &n, //----------------------------------------------------------------------------- } //----------------------------------------------------------------------------- -// -- end conduit::blueprint::util::mesh::matset -- +// -- end conduit::blueprint::mesh::matset -- //----------------------------------------------------------------------------- @@ -397,13 +393,6 @@ to_silo(const conduit::Node &n, // -- end conduit::blueprint::util::mesh -- //----------------------------------------------------------------------------- - -} -//----------------------------------------------------------------------------- -// -- end conduit::blueprint::util -- -//----------------------------------------------------------------------------- - - } //----------------------------------------------------------------------------- // -- end conduit::blueprint -- diff --git a/src/libs/blueprint/conduit_blueprint_util_mesh.hpp b/src/libs/blueprint/conduit_blueprint_util_mesh.hpp deleted file mode 100644 index 085646ab9..000000000 --- a/src/libs/blueprint/conduit_blueprint_util_mesh.hpp +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright (c) Lawrence Livermore National Security, LLC and other Conduit -// Project developers. See top-level LICENSE AND COPYRIGHT files for dates and -// other details. No copyright assignment is required to contribute to Conduit. - -//----------------------------------------------------------------------------- -/// -/// file: conduit_blueprint_util_mesh.hpp -/// -//----------------------------------------------------------------------------- - -#ifndef CONDUIT_BLUEPRINT_UTIL_MESH_HPP -#define CONDUIT_BLUEPRINT_UTIL_MESH_HPP - -//----------------------------------------------------------------------------- -// conduit lib includes -//----------------------------------------------------------------------------- -#include "conduit.hpp" -#include "conduit_blueprint_exports.h" - -//----------------------------------------------------------------------------- -// -- begin conduit -- -//----------------------------------------------------------------------------- -namespace conduit -{ - -//----------------------------------------------------------------------------- -// -- begin conduit::blueprint -- -//----------------------------------------------------------------------------- -namespace blueprint -{ - -//----------------------------------------------------------------------------- -// -- begin conduit::blueprint::util -- -//----------------------------------------------------------------------------- -namespace util -{ - -//----------------------------------------------------------------------------- -// -- begin conduit::blueprint::util::mesh -- -//----------------------------------------------------------------------------- -namespace mesh -{ - -//----------------------------------------------------------------------------- -// -- begin conduit::blueprint::util::mesh::matset -- -//----------------------------------------------------------------------------- -namespace matset -{ - //------------------------------------------------------------------------- - // See documentation for 'DBPutMaterial' at: - // https://wci.llnl.gov/content/assets/docs/simulation/computer-codes/silo/LLNL-SM-654357.pdf - void CONDUIT_BLUEPRINT_API to_silo(const conduit::Node &n, - conduit::Node &dest, - const float64 epsilon = CONDUIT_EPSILON); -} -//----------------------------------------------------------------------------- -// -- end conduit::blueprint::util::mesh::matset -- -//----------------------------------------------------------------------------- - -} -//----------------------------------------------------------------------------- -// -- end conduit::blueprint::util::mesh -- -//----------------------------------------------------------------------------- - -//----------------------------------------------------------------------------- -} -//----------------------------------------------------------------------------- -// -- end conduit::blueprint::util -- -//----------------------------------------------------------------------------- - - -} -//----------------------------------------------------------------------------- -// -- end conduit::blueprint -- -//----------------------------------------------------------------------------- - -} -//----------------------------------------------------------------------------- -// -- end conduit:: -- -//----------------------------------------------------------------------------- - - -#endif - - - diff --git a/src/libs/relay/conduit_relay_io_blueprint.cpp b/src/libs/relay/conduit_relay_io_blueprint.cpp index 2da9b695d..30e78cbc1 100644 --- a/src/libs/relay/conduit_relay_io_blueprint.cpp +++ b/src/libs/relay/conduit_relay_io_blueprint.cpp @@ -1452,7 +1452,10 @@ void read_mesh(const std::string &root_file_path, // assume hdf5, but check for json file std::string root_protocol = "hdf5"; - char buff[5] = {0,0,0,0,0}; + // we will read the first 5 bytes, but + // make sure our buff is null termed, unless you + // want a random chance at sadness. + char buff[6] = {0,0,0,0,0,0}; // heuristic, if json, we expect to see "{" in the first 5 chars of the file. std::ifstream ifs; diff --git a/src/tests/blueprint/CMakeLists.txt b/src/tests/blueprint/CMakeLists.txt index b55e26646..3aaad6d1b 100644 --- a/src/tests/blueprint/CMakeLists.txt +++ b/src/tests/blueprint/CMakeLists.txt @@ -19,7 +19,7 @@ set(BLUEPRINT_TESTS t_blueprint_smoke t_blueprint_mesh_relay t_blueprint_mesh_generate t_blueprint_mesh_examples - t_blueprint_util_mesh) + t_blueprint_mesh_matset_xforms) ################################ # Add our Main Unit Tests diff --git a/src/tests/blueprint/t_blueprint_mesh_examples.cpp b/src/tests/blueprint/t_blueprint_mesh_examples.cpp index ea7633714..8e47032e4 100644 --- a/src/tests/blueprint/t_blueprint_mesh_examples.cpp +++ b/src/tests/blueprint/t_blueprint_mesh_examples.cpp @@ -467,20 +467,40 @@ TEST(conduit_blueprint_mesh_examples, check_gen_index_state_prop) EXPECT_TRUE(idx.has_path("state/time")); } + + //----------------------------------------------------------------------------- void venn_test_small_yaml(const std::string &venn_type) { // provide small example save to yaml for folks to look at - const int nx = 25, ny = 25; + const int nx = 4, ny = 4; const double radius = 0.25; Node res, info, n_idx; blueprint::mesh::examples::venn(venn_type, nx, ny, radius, res); - blueprint::mesh::examples::venn(venn_type, nx, ny, radius, res); + EXPECT_TRUE(blueprint::mesh::verify(res, info)); blueprint::mesh::generate_index(res,"",1,n_idx); EXPECT_TRUE(blueprint::verify("mesh/index",n_idx,info)); - res.save("venn_small_example_" + venn_type + ".yaml"); + + std::string ofbase= "venn_small_example_" + venn_type; + + // save yaml and hdf5 versions + relay::io::blueprint::save_mesh(res, + ofbase + ".yaml", + "yaml"); + + Node io_protos; + relay::io::about(io_protos["io"]); + bool hdf5_enabled =io_protos["io/protocols/hdf5"].as_string() == "enabled"; + + if(hdf5_enabled) + { + relay::io::blueprint::save_mesh(res, + ofbase+ ".blueprint_root", + "hdf5"); + } + } //----------------------------------------------------------------------------- @@ -499,7 +519,24 @@ void venn_test(const std::string &venn_type) std::string ofbase = "venn_example_" + venn_type; std::cout << "[Saving " << ofbase << "]" << std::endl; - relay::io_blueprint::save(res, ofbase + ".blueprint_root"); + + std::string ofile_root= "venn_small_example_" + venn_type; + + // save yaml and hdf5 versions + relay::io::blueprint::save_mesh(res, + ofbase + ".yaml", + "yaml"); + + Node io_protos; + relay::io::about(io_protos["io"]); + bool hdf5_enabled =io_protos["io/protocols/hdf5"].as_string() == "enabled"; + + if(hdf5_enabled) + { + relay::io::blueprint::save_mesh(res, + ofbase+ ".blueprint_root", + "hdf5"); + } { std::cout << "[Verifying field area is correct]" << std::endl; diff --git a/src/tests/blueprint/t_blueprint_util_mesh.cpp b/src/tests/blueprint/t_blueprint_mesh_matset_xforms.cpp similarity index 54% rename from src/tests/blueprint/t_blueprint_util_mesh.cpp rename to src/tests/blueprint/t_blueprint_mesh_matset_xforms.cpp index 4c96f7448..fd01de290 100644 --- a/src/tests/blueprint/t_blueprint_util_mesh.cpp +++ b/src/tests/blueprint/t_blueprint_mesh_matset_xforms.cpp @@ -4,7 +4,7 @@ //----------------------------------------------------------------------------- /// -/// file: t_blueprint_util_mesh.cpp +/// file: t_blueprint_mesh_matset_xforms.cpp /// //----------------------------------------------------------------------------- @@ -39,7 +39,7 @@ TEST(conduit_blueprint_util_mesh, mesh_util_to_silo_basic) Node &mset = mesh["matsets/matset"]; Node silo, info; - blueprint::util::mesh::matset::to_silo(mset, silo); + blueprint::mesh::matset::to_silo(mset, silo); std::cout << silo.to_yaml() << std::endl; { // Check General Contents // @@ -66,3 +66,62 @@ TEST(conduit_blueprint_util_mesh, mesh_util_to_silo_basic) // EXPECT_FALSE(actual_matlist.diff(expected_matlist, info)); // } } + + +//----------------------------------------------------------------------------- +TEST(conduit_blueprint_util_mesh, mesh_util_venn_to_silo) +{ + const int nx = 4, ny = 4; + const double radius = 0.25; + + Node mset_silo_baseline; + + // all of these cases should create the same silo output + // we diff the 2 and 3 cases with the 1 to test this + + CONDUIT_INFO("venn full to silo"); + { + Node mesh; + blueprint::mesh::examples::venn("full", nx, ny, radius, mesh); + const Node &mset = mesh["matsets/matset"]; + + std::cout << mset.to_yaml() << std::endl; + + Node mset_silo; + blueprint::mesh::matset::to_silo(mset, mset_silo); + std::cout << mset_silo.to_yaml() << std::endl; + + mset_silo_baseline.set(mset_silo); + } + + CONDUIT_INFO("venn sparse_by_material to silo"); + { + Node mesh, info; + blueprint::mesh::examples::venn("sparse_by_material", nx, ny, radius, mesh); + const Node &mset = mesh["matsets/matset"]; + + std::cout << mset.to_yaml() << std::endl; + + Node mset_silo; + blueprint::mesh::matset::to_silo(mset, mset_silo); + std::cout << mset_silo.to_yaml() << std::endl; + + EXPECT_FALSE(mset_silo.diff(mset_silo_baseline,info)); + } + + CONDUIT_INFO("venn sparse_by_element to silo"); + { + Node mesh, info; + blueprint::mesh::examples::venn("sparse_by_element", nx, ny, radius, mesh); + const Node &mset = mesh["matsets/matset"]; + + std::cout << mset.to_yaml() << std::endl; + + Node mset_silo; + blueprint::mesh::matset::to_silo(mset, mset_silo); + std::cout << mset_silo.to_yaml() << std::endl; + + EXPECT_FALSE(mset_silo.diff(mset_silo_baseline,info)); + } + +}