From 9bf8591cbae0083a9acbe8c550c72482c7df0aea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 24 Apr 2024 13:02:48 +0200 Subject: [PATCH 01/16] Add define for ADIOS2_HAVE_BP5 This was removed from ADIOS2 v2.9 to v2.10. --- include/openPMD/IO/ADIOS/macros.hpp | 5 +++++ src/config.cpp | 1 + 2 files changed, 6 insertions(+) diff --git a/include/openPMD/IO/ADIOS/macros.hpp b/include/openPMD/IO/ADIOS/macros.hpp index 59b630a112..0c90e55043 100644 --- a/include/openPMD/IO/ADIOS/macros.hpp +++ b/include/openPMD/IO/ADIOS/macros.hpp @@ -19,6 +19,11 @@ #define openPMD_HAS_ADIOS_2_9 \ (ADIOS2_VERSION_MAJOR * 100 + ADIOS2_VERSION_MINOR >= 209) +#if !defined(ADIOS2_HAVE_BP5) && openPMD_HAS_ADIOS_2_9 +// ADIOS2 v2.10 no longer defines this +#define ADIOS2_HAVE_BP5 +#endif + #else #define openPMD_HAS_ADIOS_2_8 0 diff --git a/src/config.cpp b/src/config.cpp index 89a824500c..9245333026 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -22,6 +22,7 @@ #include "openPMD/version.hpp" #if openPMD_HAVE_ADIOS2 +#include "openPMD/IO/ADIOS/macros.hpp" #include #endif #include From c0681638643d61d107402fe79025f3135ecb4b04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 6 May 2024 14:56:11 +0200 Subject: [PATCH 02/16] Temporarily revert "Add define for ADIOS2_HAVE_BP5" This reverts commit 9bf8591cbae0083a9acbe8c550c72482c7df0aea. CI seems broken atm and need to fix that first. --- include/openPMD/IO/ADIOS/macros.hpp | 5 ----- src/config.cpp | 1 - 2 files changed, 6 deletions(-) diff --git a/include/openPMD/IO/ADIOS/macros.hpp b/include/openPMD/IO/ADIOS/macros.hpp index 0c90e55043..59b630a112 100644 --- a/include/openPMD/IO/ADIOS/macros.hpp +++ b/include/openPMD/IO/ADIOS/macros.hpp @@ -19,11 +19,6 @@ #define openPMD_HAS_ADIOS_2_9 \ (ADIOS2_VERSION_MAJOR * 100 + ADIOS2_VERSION_MINOR >= 209) -#if !defined(ADIOS2_HAVE_BP5) && openPMD_HAS_ADIOS_2_9 -// ADIOS2 v2.10 no longer defines this -#define ADIOS2_HAVE_BP5 -#endif - #else #define openPMD_HAS_ADIOS_2_8 0 diff --git a/src/config.cpp b/src/config.cpp index 9245333026..89a824500c 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -22,7 +22,6 @@ #include "openPMD/version.hpp" #if openPMD_HAVE_ADIOS2 -#include "openPMD/IO/ADIOS/macros.hpp" #include #endif #include From 4838c0cae6c7d81700da5cb21675484782296f97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 6 May 2024 15:50:05 +0200 Subject: [PATCH 03/16] Revert "Temporarily revert "Add define for ADIOS2_HAVE_BP5"" This reverts commit c0681638643d61d107402fe79025f3135ecb4b04. --- include/openPMD/IO/ADIOS/macros.hpp | 5 +++++ src/config.cpp | 1 + 2 files changed, 6 insertions(+) diff --git a/include/openPMD/IO/ADIOS/macros.hpp b/include/openPMD/IO/ADIOS/macros.hpp index 59b630a112..0c90e55043 100644 --- a/include/openPMD/IO/ADIOS/macros.hpp +++ b/include/openPMD/IO/ADIOS/macros.hpp @@ -19,6 +19,11 @@ #define openPMD_HAS_ADIOS_2_9 \ (ADIOS2_VERSION_MAJOR * 100 + ADIOS2_VERSION_MINOR >= 209) +#if !defined(ADIOS2_HAVE_BP5) && openPMD_HAS_ADIOS_2_9 +// ADIOS2 v2.10 no longer defines this +#define ADIOS2_HAVE_BP5 +#endif + #else #define openPMD_HAS_ADIOS_2_8 0 diff --git a/src/config.cpp b/src/config.cpp index 89a824500c..9245333026 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -22,6 +22,7 @@ #include "openPMD/version.hpp" #if openPMD_HAVE_ADIOS2 +#include "openPMD/IO/ADIOS/macros.hpp" #include #endif #include From dcba41693fb6fc6368c68ef3201db8221ef9f9e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 6 May 2024 15:50:16 +0200 Subject: [PATCH 04/16] More direct error output --- include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index c23fb81d49..efab738a20 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -409,10 +409,9 @@ class ADIOS2IOHandlerImpl { std::stringstream errorMessage; errorMessage << "Trying to access a dataset with wrong type " - "(trying to access dataset with type " - << determineDatatype() << ", but has type " - << detail::fromADIOS2Type(actualType, false) - << ")"; + "(trying to access dataset with type '" + << requiredType << "', but has type '" + << actualType << "')"; throw error::ReadError( error::AffectedObject::Dataset, error::Reason::UnexpectedContent, From 85e9018a6528a1bb5212410b760d5e276e70136b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 6 May 2024 16:26:18 +0200 Subject: [PATCH 05/16] Add normalizingVariableType --- include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp | 36 +-------- src/IO/ADIOS/ADIOS2Auxiliary.cpp | 78 ++++++++++---------- 2 files changed, 42 insertions(+), 72 deletions(-) diff --git a/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp b/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp index 0b279f38cb..bcf18610b5 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp @@ -95,39 +95,6 @@ namespace detail // we represent booleans as unsigned chars using bool_representation = unsigned char; - template - struct ToDatatypeHelper - { - static std::string type(); - }; - - template - struct ToDatatypeHelper> - { - static std::string type(); - }; - - template - struct ToDatatypeHelper> - { - static std::string type(); - }; - - template <> - struct ToDatatypeHelper - { - static std::string type(); - }; - - struct ToDatatype - { - template - std::string operator()(); - - template - std::string operator()(); - }; - /** * @brief Convert ADIOS2 datatype to openPMD type. * @param dt @@ -210,6 +177,9 @@ namespace detail } throw error::Internal("Control flow error: No ADIOS2 open mode."); } + + std::string + normalizingVariableType(adios2::IO const &, std::string const &name); } // namespace detail /** diff --git a/src/IO/ADIOS/ADIOS2Auxiliary.cpp b/src/IO/ADIOS/ADIOS2Auxiliary.cpp index 366be2b893..ec36ce7c1b 100644 --- a/src/IO/ADIOS/ADIOS2Auxiliary.cpp +++ b/src/IO/ADIOS/ADIOS2Auxiliary.cpp @@ -19,6 +19,7 @@ * If not, see . */ +#include "openPMD/auxiliary/TypeTraits.hpp" #include "openPMD/config.hpp" #if openPMD_HAVE_ADIOS2 #include "openPMD/Datatype.hpp" @@ -61,45 +62,6 @@ FlushTarget flushTargetFromString(std::string const &str) namespace openPMD::detail { -template -std::string ToDatatypeHelper::type() -{ - return adios2::GetType(); -} - -template -std::string ToDatatypeHelper>::type() -{ - return - - adios2::GetType(); -} - -template -std::string ToDatatypeHelper>::type() -{ - return - - adios2::GetType(); -} - -std::string ToDatatypeHelper::type() -{ - return ToDatatypeHelper::type(); -} - -template -std::string ToDatatype::operator()() -{ - return ToDatatypeHelper::type(); -} - -template -std::string ToDatatype::operator()() -{ - return ""; -} - Datatype fromADIOS2Type(std::string const &dt, bool verbose) { static std::map map{ @@ -272,5 +234,43 @@ Datatype attributeInfo( throw std::runtime_error("Unreachable!"); } } + +namespace +{ + struct ToDatatype + { + template + static std::string call() + { + if constexpr (std::is_same_v) + { + return ToDatatype::call(); + } + else if constexpr ( + auxiliary::IsVector_v || auxiliary::IsArray_v) + { + return ToDatatype::call(); + } + else + { + return adios2::GetType(); + } + } + + template + static std::string call() + { + return {}; + } + }; +} // namespace + +std::string +normalizingVariableType(const adios2::IO &IO, const std::string &name) +{ + auto const &unsanitized_result = IO.VariableType(name); + auto openpmd_type = fromADIOS2Type(unsanitized_result); + return switchAdios2VariableType(openpmd_type); +} } // namespace openPMD::detail #endif From 41b6b5776f64f6e19ad926892b55b196999833a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 6 May 2024 16:32:44 +0200 Subject: [PATCH 06/16] ... and use the new function --- include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp | 2 +- src/IO/ADIOS/ADIOS2Auxiliary.cpp | 2 +- src/IO/ADIOS/ADIOS2IOHandler.cpp | 10 ++++++---- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index efab738a20..610614ea18 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -403,7 +403,7 @@ class ADIOS2IOHandlerImpl { { auto requiredType = adios2::GetType(); - auto actualType = IO.VariableType(varName); + auto actualType = detail::normalizingVariableType(IO, varName); if (requiredType != actualType) { diff --git a/src/IO/ADIOS/ADIOS2Auxiliary.cpp b/src/IO/ADIOS/ADIOS2Auxiliary.cpp index ec36ce7c1b..b669a96bcc 100644 --- a/src/IO/ADIOS/ADIOS2Auxiliary.cpp +++ b/src/IO/ADIOS/ADIOS2Auxiliary.cpp @@ -166,7 +166,7 @@ Datatype attributeInfo( type = IO.AttributeType(attributeName); break; case VariableOrAttribute::Variable: - type = IO.VariableType(attributeName); + type = normalizingVariableType(IO, attributeName); break; } if (type.empty()) diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 2d93837980..690a0fcb71 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -865,7 +865,8 @@ void ADIOS2IOHandlerImpl::extendDataset( auto file = refreshFileFromParent(writable, /* preferParentFile = */ false); std::string name = nameOfVariable(writable); auto &filedata = getFileData(file, IfFileNotOpen::ThrowError); - Datatype dt = detail::fromADIOS2Type(filedata.m_IO.VariableType(name)); + Datatype dt = detail::fromADIOS2Type( + detail::normalizingVariableType(filedata.m_IO, name)); switchAdios2VariableType( dt, filedata.m_IO, name, parameters.extent); } @@ -962,8 +963,8 @@ void ADIOS2IOHandlerImpl::openDataset( auto file = refreshFileFromParent(writable, /* preferParentFile = */ true); auto varName = nameOfVariable(writable); auto &fileData = getFileData(file, IfFileNotOpen::ThrowError); - *parameters.dtype = - detail::fromADIOS2Type(fileData.m_IO.VariableType(varName)); + *parameters.dtype = detail::fromADIOS2Type( + detail::normalizingVariableType(fileData.m_IO, varName)); switchAdios2VariableType( *parameters.dtype, this, file, varName, parameters); writable->written = true; @@ -1464,7 +1465,8 @@ void ADIOS2IOHandlerImpl::availableChunks( detail::ADIOS2File &ba = getFileData(file, IfFileNotOpen::ThrowError); std::string varName = nameOfVariable(writable); auto engine = ba.getEngine(); // make sure that data are present - auto datatype = detail::fromADIOS2Type(ba.m_IO.VariableType(varName)); + auto datatype = detail::fromADIOS2Type( + detail::normalizingVariableType(ba.m_IO, varName)); bool allSteps = ba.m_mode != adios2::Mode::Read && ba.streamStatus == detail::ADIOS2File::StreamStatus::ReadWithoutStream; switchAdios2VariableType( From e18d608eafbdc139992b193fca43b845426da7a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 6 May 2024 17:00:50 +0200 Subject: [PATCH 07/16] Try verbose debugging --- src/IO/ADIOS/ADIOS2Auxiliary.cpp | 5 ++++- src/IO/ADIOS/ADIOS2IOHandler.cpp | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/IO/ADIOS/ADIOS2Auxiliary.cpp b/src/IO/ADIOS/ADIOS2Auxiliary.cpp index b669a96bcc..3fb76760e0 100644 --- a/src/IO/ADIOS/ADIOS2Auxiliary.cpp +++ b/src/IO/ADIOS/ADIOS2Auxiliary.cpp @@ -270,7 +270,10 @@ normalizingVariableType(const adios2::IO &IO, const std::string &name) { auto const &unsanitized_result = IO.VariableType(name); auto openpmd_type = fromADIOS2Type(unsanitized_result); - return switchAdios2VariableType(openpmd_type); + auto res = switchAdios2VariableType(openpmd_type); + std::cout << "[normalizingVariableType] for '" << name << "' " << res + << " (non-normalized: " << unsanitized_result << ")" << std::endl; + return res; } } // namespace openPMD::detail #endif diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 690a0fcb71..e4e3ab5dbf 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -965,6 +965,8 @@ void ADIOS2IOHandlerImpl::openDataset( auto &fileData = getFileData(file, IfFileNotOpen::ThrowError); *parameters.dtype = detail::fromADIOS2Type( detail::normalizingVariableType(fileData.m_IO, varName)); + std::cout << "[openDataset] opened '" << varName << "'as " + << *parameters.dtype << std::endl; switchAdios2VariableType( *parameters.dtype, this, file, varName, parameters); writable->written = true; From cdd2c28ab043f691907caa70e9869ff4c72edc69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 8 May 2024 10:59:43 +0200 Subject: [PATCH 08/16] Put the log into the err message so the CI actually shows it...... --- include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp | 4 ++++ include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp | 4 +++- src/IO/ADIOS/ADIOS2Auxiliary.cpp | 11 +++++++++-- src/IO/ADIOS/ADIOS2IOHandler.cpp | 4 ++-- 4 files changed, 18 insertions(+), 5 deletions(-) diff --git a/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp b/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp index bcf18610b5..a8fc6c0522 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp @@ -24,6 +24,7 @@ #include "openPMD/Error.hpp" #include "openPMD/IO/ADIOS/macros.hpp" #include "openPMD/config.hpp" +#include #if openPMD_HAVE_ADIOS2 #include "openPMD/Dataset.hpp" @@ -40,6 +41,9 @@ namespace openPMD { + +extern std::stringstream debugging_log; + enum class GroupOrDataset { GROUP, diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index 610614ea18..4d36d6e5e2 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -411,7 +411,9 @@ class ADIOS2IOHandlerImpl errorMessage << "Trying to access a dataset with wrong type " "(trying to access dataset with type '" << requiredType << "', but has type '" - << actualType << "')"; + << actualType << "')\nLOG IS:\n" + << debugging_log.str(); + debugging_log.clear(); throw error::ReadError( error::AffectedObject::Dataset, error::Reason::UnexpectedContent, diff --git a/src/IO/ADIOS/ADIOS2Auxiliary.cpp b/src/IO/ADIOS/ADIOS2Auxiliary.cpp index 3fb76760e0..a0ad2c1af3 100644 --- a/src/IO/ADIOS/ADIOS2Auxiliary.cpp +++ b/src/IO/ADIOS/ADIOS2Auxiliary.cpp @@ -21,6 +21,7 @@ #include "openPMD/auxiliary/TypeTraits.hpp" #include "openPMD/config.hpp" +#include #if openPMD_HAVE_ADIOS2 #include "openPMD/Datatype.hpp" #include "openPMD/DatatypeHelpers.hpp" @@ -271,9 +272,15 @@ normalizingVariableType(const adios2::IO &IO, const std::string &name) auto const &unsanitized_result = IO.VariableType(name); auto openpmd_type = fromADIOS2Type(unsanitized_result); auto res = switchAdios2VariableType(openpmd_type); - std::cout << "[normalizingVariableType] for '" << name << "' " << res - << " (non-normalized: " << unsanitized_result << ")" << std::endl; + debugging_log << "[normalizingVariableType] for '" << name << "' " << res + << " (non-normalized: " << unsanitized_result << ")" + << std::endl; return res; } + } // namespace openPMD::detail +namespace openPMD +{ +std::stringstream debugging_log; +} #endif diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index e4e3ab5dbf..6a1ed140a0 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -965,8 +965,8 @@ void ADIOS2IOHandlerImpl::openDataset( auto &fileData = getFileData(file, IfFileNotOpen::ThrowError); *parameters.dtype = detail::fromADIOS2Type( detail::normalizingVariableType(fileData.m_IO, varName)); - std::cout << "[openDataset] opened '" << varName << "'as " - << *parameters.dtype << std::endl; + debugging_log << "[openDataset] opened '" << varName << "'as " + << *parameters.dtype << std::endl; switchAdios2VariableType( *parameters.dtype, this, file, varName, parameters); writable->written = true; From ee14b9f716e4f199729d978137f04283c2b1411b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 8 May 2024 11:07:05 +0200 Subject: [PATCH 09/16] Use our own macro name --- include/openPMD/IO/ADIOS/macros.hpp | 6 ++++-- src/IO/ADIOS/ADIOS2IOHandler.cpp | 2 +- src/config.cpp | 2 +- test/CoreTest.cpp | 4 ++-- test/SerialIOTest.cpp | 14 +++++++------- 5 files changed, 15 insertions(+), 13 deletions(-) diff --git a/include/openPMD/IO/ADIOS/macros.hpp b/include/openPMD/IO/ADIOS/macros.hpp index 0c90e55043..d7e10385ee 100644 --- a/include/openPMD/IO/ADIOS/macros.hpp +++ b/include/openPMD/IO/ADIOS/macros.hpp @@ -19,9 +19,11 @@ #define openPMD_HAS_ADIOS_2_9 \ (ADIOS2_VERSION_MAJOR * 100 + ADIOS2_VERSION_MINOR >= 209) -#if !defined(ADIOS2_HAVE_BP5) && openPMD_HAS_ADIOS_2_9 +#if defined(ADIOS2_HAVE_BP5) || openPMD_HAS_ADIOS_2_9 // ADIOS2 v2.10 no longer defines this -#define ADIOS2_HAVE_BP5 +#define openPMD_HAVE_ADIOS2_BP5 1 +#else +#define openPMD_HAVE_ADIOS2_BP5 0 #endif #else diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 6a1ed140a0..c28a8280ef 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -354,7 +354,7 @@ std::string ADIOS2IOHandlerImpl::fileSuffix(bool verbose) const { // SST engine adds its suffix unconditionally // so we don't add it -#if defined(ADIOS2_HAVE_BP5) && openPMD_HAS_ADIOS_2_9 +#if openPMD_HAVE_ADIOS2_BP5 && openPMD_HAS_ADIOS_2_9 constexpr char const *const default_file_ending = ".bp5"; #else constexpr char const *const default_file_ending = ".bp4"; diff --git a/src/config.cpp b/src/config.cpp index 9245333026..d25c926fa3 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -61,7 +61,7 @@ std::vector openPMD::getFileExtensions() // BP4 is always available in ADIOS2 fext.emplace_back("bp4"); #endif -#ifdef ADIOS2_HAVE_BP5 +#if openPMD_HAVE_ADIOS2_BP5 fext.emplace_back("bp5"); #endif #ifdef ADIOS2_HAVE_SST diff --git a/test/CoreTest.cpp b/test/CoreTest.cpp index 0cdefd67cf..8a59233537 100644 --- a/test/CoreTest.cpp +++ b/test/CoreTest.cpp @@ -1184,7 +1184,7 @@ TEST_CASE("backend_via_json", "[core]") TEST_CASE("wildcard_extension", "[core]") { #if openPMD_HAVE_ADIOS2 -#if defined(ADIOS2_HAVE_BP5) && openPMD_HAS_ADIOS_2_9 +#if openPMD_HAVE_ADIOS2_BP5 && openPMD_HAS_ADIOS_2_9 constexpr char const *const default_file_ending = "bp5"; #else constexpr char const *const default_file_ending = "bp4"; @@ -1227,7 +1227,7 @@ TEST_CASE("wildcard_extension", "[core]") } }; #if openPMD_HAVE_ADIOS2 -#ifdef ADIOS2_HAVE_BP5 +#if openPMD_HAVE_ADIOS2_BP5 run_test( R"({"adios2": {"engine": {"type": "bp5"}}, "backend": "adios2"})", "bp5"); diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index f07a28d91a..770138bb04 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -4075,7 +4075,7 @@ TEST_CASE("git_adios2_early_chunk_query", "[serial][adios2]") /* * Require __unix__ since we need all that filestat stuff for this test. */ -#if defined(__unix__) && defined(ADIOS2_HAVE_BP5) +#if defined(__unix__) && openPMD_HAVE_ADIOS2_BP5 enum class FlushDuringStep { @@ -4406,7 +4406,7 @@ TEST_CASE("adios2_engines_and_file_endings") groupbased_test_explicit_backend( "", true, "bp4", "", "adios2.engine.type = \"bp4\""); -#ifdef ADIOS2_HAVE_BP5 +#if openPMD_HAVE_ADIOS2_BP5 // BP5 tests groupbased_test_explicit_backend(".bp5", true, "bp5", ""); groupbased_test_explicit_backend( @@ -4472,7 +4472,7 @@ TEST_CASE("adios2_engines_and_file_endings") REQUIRE_THROWS(groupbased_test_no_explicit_backend( "", true, "bp4", "", "adios2.engine.type = \"bp4\"")); -#ifdef ADIOS2_HAVE_BP5 +#if openPMD_HAVE_ADIOS2_BP5 // BP5 tests groupbased_test_no_explicit_backend(".bp5", true, "bp5", ""); groupbased_test_no_explicit_backend( @@ -4551,7 +4551,7 @@ TEST_CASE("adios2_engines_and_file_endings") filebased_test_explicit_backend( "", true, "bp4", "", "adios2.engine.type = \"bp4\""); -#ifdef ADIOS2_HAVE_BP5 +#if openPMD_HAVE_ADIOS2_BP5 // BP5 tests filebased_test_explicit_backend(".bp5", true, "bp5", ""); filebased_test_explicit_backend( @@ -4627,7 +4627,7 @@ TEST_CASE("adios2_engines_and_file_endings") REQUIRE_THROWS(filebased_test_no_explicit_backend( "", true, "bp4", "", "adios2.engine.type = \"bp4\"")); -#ifdef ADIOS2_HAVE_BP5 +#if openPMD_HAVE_ADIOS2_BP5 // BP5 tests filebased_test_no_explicit_backend(".bp5", true, "bp5", ""); filebased_test_no_explicit_backend( @@ -6118,7 +6118,7 @@ TEST_CASE("iterate_nonstreaming_series", "[serial][adios2]") backend.extension, false, backend.jsonBaseConfig()); -#if openPMD_HAVE_ADIOS2 && defined(ADIOS2_HAVE_BP5) +#if openPMD_HAVE_ADIOS2 && openPMD_HAVE_ADIOS2_BP5 if (backend.extension == "bp") { iterate_nonstreaming_series( @@ -6144,7 +6144,7 @@ TEST_CASE("iterate_nonstreaming_series", "[serial][adios2]") #endif } -#if openPMD_HAVE_ADIOS2 && defined(ADIOS2_HAVE_BP5) +#if openPMD_HAVE_ADIOS2 && openPMD_HAVE_ADIOS2_BP5 void adios2_bp5_no_steps(bool usesteps) { std::string const config = R"END( From a1581216a2da463d219e27ac283c4a7c63ca6458 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 8 May 2024 11:38:05 +0200 Subject: [PATCH 10/16] Fix a potential source for confused datatypes --- src/binding/python/RecordComponent.cpp | 244 ++++++++----------------- 1 file changed, 80 insertions(+), 164 deletions(-) diff --git a/src/binding/python/RecordComponent.cpp b/src/binding/python/RecordComponent.cpp index af58f0d2c8..4e2b3aac29 100644 --- a/src/binding/python/RecordComponent.cpp +++ b/src/binding/python/RecordComponent.cpp @@ -19,11 +19,14 @@ * If not, see . */ #include +#include #include #include #include #include +#include "openPMD/Dataset.hpp" +#include "openPMD/Datatype.hpp" #include "openPMD/DatatypeHelpers.hpp" #include "openPMD/Error.hpp" #include "openPMD/RecordComponent.hpp" @@ -309,6 +312,76 @@ inline void check_buffer_is_contiguous(py::array &a) // loop over the input data strides in store/load calls } +namespace +{ +struct StoreChunkFromPythonArray +{ + template + static void call( + RecordComponent &r, + py::array &a, + Offset const &offset, + Extent const &extent) + { + // here, we increase a reference on the user-passed data so that + // temporary and lost-scope variables stay alive until we flush + // note: this does not yet prevent the user, as in C++, to build + // a race condition by manipulating the data that was passed + a.inc_ref(); + void *data = a.mutable_data(); + std::shared_ptr shared((T *)data, [a](T *) { a.dec_ref(); }); + r.storeChunk(std::move(shared), offset, extent); + } + + static constexpr char const *errorMsg = "store_chunk()"; +}; +struct LoadChunkIntoPythonArray +{ + template + static void call( + RecordComponent &r, + py::array &a, + Offset const &offset, + Extent const &extent) + { + // here, we increase a reference on the user-passed data so that + // temporary and lost-scope variables stay alive until we flush + // note: this does not yet prevent the user, as in C++, to build + // a race condition by manipulating the data that was passed + a.inc_ref(); + void *data = a.mutable_data(); + std::shared_ptr shared((T *)data, [a](T *) { a.dec_ref(); }); + r.loadChunk(std::move(shared), offset, extent); + } + + static constexpr char const *errorMsg = "load_chunk()"; +}; +struct LoadChunkIntoPythonBuffer +{ + template + static void call( + RecordComponent &r, + py::buffer &buffer, + py::buffer_info const &buffer_info, + Offset const &offset, + Extent const &extent) + { + // here, we increase a reference on the user-passed data so that + // temporary and lost-scope variables stay alive until we flush + // note: this does not yet prevent the user, as in C++, to build + // a race condition by manipulating the data that was passed + buffer.inc_ref(); + // buffer_info.inc_ref(); + void *data = buffer_info.ptr; + std::shared_ptr shared( + (T *)data, [buffer](T *) { buffer.dec_ref(); }); + r.loadChunk(std::move(shared), offset, extent); + } + + static constexpr char const *errorMsg = "load_chunk()"; +}; +} // namespace + /** Store Chunk * * Called with offset and extent that are already in the record component's @@ -409,64 +482,9 @@ inline void store_chunk( check_buffer_is_contiguous(a); - // here, we increase a reference on the user-passed data so that - // temporary and lost-scope variables stay alive until we flush - // note: this does not yet prevent the user, as in C++, to build - // a race condition by manipulating the data they passed - auto store_data = [&r, &a, &offset, &extent](auto cxxtype) { - using CXXType = decltype(cxxtype); - a.inc_ref(); - void *data = a.mutable_data(); - std::shared_ptr shared( - (CXXType *)data, [a](CXXType *) { a.dec_ref(); }); - r.storeChunk(std::move(shared), offset, extent); - }; - - // store - auto const dtype = dtype_from_numpy(a.dtype()); - if (dtype == Datatype::CHAR) - store_data(char()); - else if (dtype == Datatype::UCHAR) - store_data((unsigned char)0); - else if (dtype == Datatype::SHORT) - store_data(short()); - else if (dtype == Datatype::INT) - store_data(int()); - else if (dtype == Datatype::LONG) - store_data(long()); - else if (dtype == Datatype::LONGLONG) - store_data((long long)0); - else if (dtype == Datatype::USHORT) - store_data((unsigned short)0); - else if (dtype == Datatype::UINT) - store_data((unsigned int)0); - else if (dtype == Datatype::ULONG) - store_data((unsigned long)0); - else if (dtype == Datatype::ULONGLONG) - store_data((unsigned long long)0); - else if (dtype == Datatype::LONG_DOUBLE) - store_data((long double)0); - else if (dtype == Datatype::DOUBLE) - store_data(double()); - else if (dtype == Datatype::FLOAT) - store_data(float()); - else if (dtype == Datatype::CLONG_DOUBLE) - store_data(std::complex()); - else if (dtype == Datatype::CDOUBLE) - store_data(std::complex()); - else if (dtype == Datatype::CFLOAT) - store_data(std::complex()); - /* @todo - .value("STRING", Datatype::STRING) - .value("VEC_STRING", Datatype::VEC_STRING) - .value("ARR_DBL_7", Datatype::ARR_DBL_7) - */ - else if (dtype == Datatype::BOOL) - store_data(bool()); - else - throw std::runtime_error( - std::string("Datatype '") + std::string(py::str(a.dtype())) + - std::string("' not known in 'storeChunk'!")); + // dtype_from_numpy(a.dtype()) + switchDatasetType( + r.getDatatype(), r, a, offset, extent); } /** Store Chunk @@ -682,60 +700,8 @@ void load_chunk( } } - // here, we increase a reference on the user-passed data so that - // temporary and lost-scope variables stay alive until we flush - // note: this does not yet prevent the user, as in C++, to build - // a race condition by manipulating the data they passed - auto load_data = - [&r, &buffer, &buffer_info, &offset, &extent](auto cxxtype) { - using CXXType = decltype(cxxtype); - buffer.inc_ref(); - // buffer_info.inc_ref(); - void *data = buffer_info.ptr; - std::shared_ptr shared( - (CXXType *)data, [buffer](CXXType *) { buffer.dec_ref(); }); - r.loadChunk(std::move(shared), offset, extent); - }; - - if (r.getDatatype() == Datatype::CHAR) - load_data((char)0); - else if (r.getDatatype() == Datatype::UCHAR) - load_data((unsigned char)0); - else if (r.getDatatype() == Datatype::SCHAR) - load_data((signed char)0); - else if (r.getDatatype() == Datatype::SHORT) - load_data((short)0); - else if (r.getDatatype() == Datatype::INT) - load_data((int)0); - else if (r.getDatatype() == Datatype::LONG) - load_data((long)0); - else if (r.getDatatype() == Datatype::LONGLONG) - load_data((long long)0); - else if (r.getDatatype() == Datatype::USHORT) - load_data((unsigned short)0); - else if (r.getDatatype() == Datatype::UINT) - load_data((unsigned int)0); - else if (r.getDatatype() == Datatype::ULONG) - load_data((unsigned long)0); - else if (r.getDatatype() == Datatype::ULONGLONG) - load_data((unsigned long long)0); - else if (r.getDatatype() == Datatype::LONG_DOUBLE) - load_data((long double)0); - else if (r.getDatatype() == Datatype::DOUBLE) - load_data((double)0); - else if (r.getDatatype() == Datatype::FLOAT) - load_data((float)0); - else if (r.getDatatype() == Datatype::CLONG_DOUBLE) - load_data((std::complex)0); - else if (r.getDatatype() == Datatype::CDOUBLE) - load_data((std::complex)0); - else if (r.getDatatype() == Datatype::CFLOAT) - load_data((std::complex)0); - else if (r.getDatatype() == Datatype::BOOL) - load_data((bool)0); - else - throw std::runtime_error( - std::string("Datatype not known in 'loadChunk'!")); + switchNonVectorType( + r.getDatatype(), r, buffer, buffer_info, offset, extent); } /** Load Chunk @@ -792,58 +758,8 @@ inline void load_chunk( check_buffer_is_contiguous(a); - // here, we increase a reference on the user-passed data so that - // temporary and lost-scope variables stay alive until we flush - // note: this does not yet prevent the user, as in C++, to build - // a race condition by manipulating the data they passed - auto load_data = [&r, &a, &offset, &extent](auto cxxtype) { - using CXXType = decltype(cxxtype); - a.inc_ref(); - void *data = a.mutable_data(); - std::shared_ptr shared( - (CXXType *)data, [a](CXXType *) { a.dec_ref(); }); - r.loadChunk(std::move(shared), offset, extent); - }; - - if (r.getDatatype() == Datatype::CHAR) - load_data(char()); - else if (r.getDatatype() == Datatype::UCHAR) - load_data((unsigned char)0); - else if (r.getDatatype() == Datatype::SCHAR) - load_data((signed char)0); - else if (r.getDatatype() == Datatype::SHORT) - load_data(short()); - else if (r.getDatatype() == Datatype::INT) - load_data(int()); - else if (r.getDatatype() == Datatype::LONG) - load_data(long()); - else if (r.getDatatype() == Datatype::LONGLONG) - load_data((long long)0); - else if (r.getDatatype() == Datatype::USHORT) - load_data((unsigned short)0); - else if (r.getDatatype() == Datatype::UINT) - load_data((unsigned int)0); - else if (r.getDatatype() == Datatype::ULONG) - load_data((unsigned long)0); - else if (r.getDatatype() == Datatype::ULONGLONG) - load_data((unsigned long long)0); - else if (r.getDatatype() == Datatype::LONG_DOUBLE) - load_data((long double)0); - else if (r.getDatatype() == Datatype::DOUBLE) - load_data(double()); - else if (r.getDatatype() == Datatype::FLOAT) - load_data(float()); - else if (r.getDatatype() == Datatype::CLONG_DOUBLE) - load_data(std::complex()); - else if (r.getDatatype() == Datatype::CDOUBLE) - load_data(std::complex()); - else if (r.getDatatype() == Datatype::CFLOAT) - load_data(std::complex()); - else if (r.getDatatype() == Datatype::BOOL) - load_data(bool()); - else - throw std::runtime_error( - std::string("Datatype not known in 'load_chunk'!")); + switchDatasetType( + r.getDatatype(), r, a, offset, extent); } /** Load Chunk From bbb285cee69cc37020a47caa53fd2be4d9e6e797 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 8 May 2024 11:45:19 +0200 Subject: [PATCH 11/16] Revert "Put the log into the err message so the CI actually shows it......" This reverts commit cdd2c28ab043f691907caa70e9869ff4c72edc69. --- include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp | 4 ---- include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp | 4 +--- src/IO/ADIOS/ADIOS2Auxiliary.cpp | 11 ++--------- src/IO/ADIOS/ADIOS2IOHandler.cpp | 4 ++-- 4 files changed, 5 insertions(+), 18 deletions(-) diff --git a/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp b/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp index a8fc6c0522..bcf18610b5 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp @@ -24,7 +24,6 @@ #include "openPMD/Error.hpp" #include "openPMD/IO/ADIOS/macros.hpp" #include "openPMD/config.hpp" -#include #if openPMD_HAVE_ADIOS2 #include "openPMD/Dataset.hpp" @@ -41,9 +40,6 @@ namespace openPMD { - -extern std::stringstream debugging_log; - enum class GroupOrDataset { GROUP, diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index 4d36d6e5e2..610614ea18 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -411,9 +411,7 @@ class ADIOS2IOHandlerImpl errorMessage << "Trying to access a dataset with wrong type " "(trying to access dataset with type '" << requiredType << "', but has type '" - << actualType << "')\nLOG IS:\n" - << debugging_log.str(); - debugging_log.clear(); + << actualType << "')"; throw error::ReadError( error::AffectedObject::Dataset, error::Reason::UnexpectedContent, diff --git a/src/IO/ADIOS/ADIOS2Auxiliary.cpp b/src/IO/ADIOS/ADIOS2Auxiliary.cpp index a0ad2c1af3..3fb76760e0 100644 --- a/src/IO/ADIOS/ADIOS2Auxiliary.cpp +++ b/src/IO/ADIOS/ADIOS2Auxiliary.cpp @@ -21,7 +21,6 @@ #include "openPMD/auxiliary/TypeTraits.hpp" #include "openPMD/config.hpp" -#include #if openPMD_HAVE_ADIOS2 #include "openPMD/Datatype.hpp" #include "openPMD/DatatypeHelpers.hpp" @@ -272,15 +271,9 @@ normalizingVariableType(const adios2::IO &IO, const std::string &name) auto const &unsanitized_result = IO.VariableType(name); auto openpmd_type = fromADIOS2Type(unsanitized_result); auto res = switchAdios2VariableType(openpmd_type); - debugging_log << "[normalizingVariableType] for '" << name << "' " << res - << " (non-normalized: " << unsanitized_result << ")" - << std::endl; + std::cout << "[normalizingVariableType] for '" << name << "' " << res + << " (non-normalized: " << unsanitized_result << ")" << std::endl; return res; } - } // namespace openPMD::detail -namespace openPMD -{ -std::stringstream debugging_log; -} #endif diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index c28a8280ef..b688c757a4 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -965,8 +965,8 @@ void ADIOS2IOHandlerImpl::openDataset( auto &fileData = getFileData(file, IfFileNotOpen::ThrowError); *parameters.dtype = detail::fromADIOS2Type( detail::normalizingVariableType(fileData.m_IO, varName)); - debugging_log << "[openDataset] opened '" << varName << "'as " - << *parameters.dtype << std::endl; + std::cout << "[openDataset] opened '" << varName << "'as " + << *parameters.dtype << std::endl; switchAdios2VariableType( *parameters.dtype, this, file, varName, parameters); writable->written = true; From d856da145992664f7795d71f3c9da52d2eadb2d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 8 May 2024 11:45:21 +0200 Subject: [PATCH 12/16] Revert "Try verbose debugging" This reverts commit e18d608eafbdc139992b193fca43b845426da7a8. --- src/IO/ADIOS/ADIOS2Auxiliary.cpp | 5 +---- src/IO/ADIOS/ADIOS2IOHandler.cpp | 2 -- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/src/IO/ADIOS/ADIOS2Auxiliary.cpp b/src/IO/ADIOS/ADIOS2Auxiliary.cpp index 3fb76760e0..b669a96bcc 100644 --- a/src/IO/ADIOS/ADIOS2Auxiliary.cpp +++ b/src/IO/ADIOS/ADIOS2Auxiliary.cpp @@ -270,10 +270,7 @@ normalizingVariableType(const adios2::IO &IO, const std::string &name) { auto const &unsanitized_result = IO.VariableType(name); auto openpmd_type = fromADIOS2Type(unsanitized_result); - auto res = switchAdios2VariableType(openpmd_type); - std::cout << "[normalizingVariableType] for '" << name << "' " << res - << " (non-normalized: " << unsanitized_result << ")" << std::endl; - return res; + return switchAdios2VariableType(openpmd_type); } } // namespace openPMD::detail #endif diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index b688c757a4..97b3a23c1d 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -965,8 +965,6 @@ void ADIOS2IOHandlerImpl::openDataset( auto &fileData = getFileData(file, IfFileNotOpen::ThrowError); *parameters.dtype = detail::fromADIOS2Type( detail::normalizingVariableType(fileData.m_IO, varName)); - std::cout << "[openDataset] opened '" << varName << "'as " - << *parameters.dtype << std::endl; switchAdios2VariableType( *parameters.dtype, this, file, varName, parameters); writable->written = true; From 42f6ace78a17a27a7124ddd25878135320c3788b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 8 May 2024 11:45:22 +0200 Subject: [PATCH 13/16] Revert "... and use the new function" This reverts commit 41b6b5776f64f6e19ad926892b55b196999833a9. --- include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp | 2 +- src/IO/ADIOS/ADIOS2Auxiliary.cpp | 2 +- src/IO/ADIOS/ADIOS2IOHandler.cpp | 10 ++++------ 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index 610614ea18..efab738a20 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -403,7 +403,7 @@ class ADIOS2IOHandlerImpl { { auto requiredType = adios2::GetType(); - auto actualType = detail::normalizingVariableType(IO, varName); + auto actualType = IO.VariableType(varName); if (requiredType != actualType) { diff --git a/src/IO/ADIOS/ADIOS2Auxiliary.cpp b/src/IO/ADIOS/ADIOS2Auxiliary.cpp index b669a96bcc..ec36ce7c1b 100644 --- a/src/IO/ADIOS/ADIOS2Auxiliary.cpp +++ b/src/IO/ADIOS/ADIOS2Auxiliary.cpp @@ -166,7 +166,7 @@ Datatype attributeInfo( type = IO.AttributeType(attributeName); break; case VariableOrAttribute::Variable: - type = normalizingVariableType(IO, attributeName); + type = IO.VariableType(attributeName); break; } if (type.empty()) diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 97b3a23c1d..52c2b8acfc 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -865,8 +865,7 @@ void ADIOS2IOHandlerImpl::extendDataset( auto file = refreshFileFromParent(writable, /* preferParentFile = */ false); std::string name = nameOfVariable(writable); auto &filedata = getFileData(file, IfFileNotOpen::ThrowError); - Datatype dt = detail::fromADIOS2Type( - detail::normalizingVariableType(filedata.m_IO, name)); + Datatype dt = detail::fromADIOS2Type(filedata.m_IO.VariableType(name)); switchAdios2VariableType( dt, filedata.m_IO, name, parameters.extent); } @@ -963,8 +962,8 @@ void ADIOS2IOHandlerImpl::openDataset( auto file = refreshFileFromParent(writable, /* preferParentFile = */ true); auto varName = nameOfVariable(writable); auto &fileData = getFileData(file, IfFileNotOpen::ThrowError); - *parameters.dtype = detail::fromADIOS2Type( - detail::normalizingVariableType(fileData.m_IO, varName)); + *parameters.dtype = + detail::fromADIOS2Type(fileData.m_IO.VariableType(varName)); switchAdios2VariableType( *parameters.dtype, this, file, varName, parameters); writable->written = true; @@ -1465,8 +1464,7 @@ void ADIOS2IOHandlerImpl::availableChunks( detail::ADIOS2File &ba = getFileData(file, IfFileNotOpen::ThrowError); std::string varName = nameOfVariable(writable); auto engine = ba.getEngine(); // make sure that data are present - auto datatype = detail::fromADIOS2Type( - detail::normalizingVariableType(ba.m_IO, varName)); + auto datatype = detail::fromADIOS2Type(ba.m_IO.VariableType(varName)); bool allSteps = ba.m_mode != adios2::Mode::Read && ba.streamStatus == detail::ADIOS2File::StreamStatus::ReadWithoutStream; switchAdios2VariableType( From 2aa66d8ce6b9d88a5292614e17f5f796976c80d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 8 May 2024 11:45:23 +0200 Subject: [PATCH 14/16] Revert "Add normalizingVariableType" This reverts commit 85e9018a6528a1bb5212410b760d5e276e70136b. --- include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp | 36 ++++++++- src/IO/ADIOS/ADIOS2Auxiliary.cpp | 78 ++++++++++---------- 2 files changed, 72 insertions(+), 42 deletions(-) diff --git a/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp b/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp index bcf18610b5..0b279f38cb 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp @@ -95,6 +95,39 @@ namespace detail // we represent booleans as unsigned chars using bool_representation = unsigned char; + template + struct ToDatatypeHelper + { + static std::string type(); + }; + + template + struct ToDatatypeHelper> + { + static std::string type(); + }; + + template + struct ToDatatypeHelper> + { + static std::string type(); + }; + + template <> + struct ToDatatypeHelper + { + static std::string type(); + }; + + struct ToDatatype + { + template + std::string operator()(); + + template + std::string operator()(); + }; + /** * @brief Convert ADIOS2 datatype to openPMD type. * @param dt @@ -177,9 +210,6 @@ namespace detail } throw error::Internal("Control flow error: No ADIOS2 open mode."); } - - std::string - normalizingVariableType(adios2::IO const &, std::string const &name); } // namespace detail /** diff --git a/src/IO/ADIOS/ADIOS2Auxiliary.cpp b/src/IO/ADIOS/ADIOS2Auxiliary.cpp index ec36ce7c1b..366be2b893 100644 --- a/src/IO/ADIOS/ADIOS2Auxiliary.cpp +++ b/src/IO/ADIOS/ADIOS2Auxiliary.cpp @@ -19,7 +19,6 @@ * If not, see . */ -#include "openPMD/auxiliary/TypeTraits.hpp" #include "openPMD/config.hpp" #if openPMD_HAVE_ADIOS2 #include "openPMD/Datatype.hpp" @@ -62,6 +61,45 @@ FlushTarget flushTargetFromString(std::string const &str) namespace openPMD::detail { +template +std::string ToDatatypeHelper::type() +{ + return adios2::GetType(); +} + +template +std::string ToDatatypeHelper>::type() +{ + return + + adios2::GetType(); +} + +template +std::string ToDatatypeHelper>::type() +{ + return + + adios2::GetType(); +} + +std::string ToDatatypeHelper::type() +{ + return ToDatatypeHelper::type(); +} + +template +std::string ToDatatype::operator()() +{ + return ToDatatypeHelper::type(); +} + +template +std::string ToDatatype::operator()() +{ + return ""; +} + Datatype fromADIOS2Type(std::string const &dt, bool verbose) { static std::map map{ @@ -234,43 +272,5 @@ Datatype attributeInfo( throw std::runtime_error("Unreachable!"); } } - -namespace -{ - struct ToDatatype - { - template - static std::string call() - { - if constexpr (std::is_same_v) - { - return ToDatatype::call(); - } - else if constexpr ( - auxiliary::IsVector_v || auxiliary::IsArray_v) - { - return ToDatatype::call(); - } - else - { - return adios2::GetType(); - } - } - - template - static std::string call() - { - return {}; - } - }; -} // namespace - -std::string -normalizingVariableType(const adios2::IO &IO, const std::string &name) -{ - auto const &unsanitized_result = IO.VariableType(name); - auto openpmd_type = fromADIOS2Type(unsanitized_result); - return switchAdios2VariableType(openpmd_type); -} } // namespace openPMD::detail #endif From a7bbf16890623e4037953dbaf6e06f32421f580d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 8 May 2024 13:06:59 +0200 Subject: [PATCH 15/16] CI fixes --- src/binding/python/RecordComponent.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/binding/python/RecordComponent.cpp b/src/binding/python/RecordComponent.cpp index 4e2b3aac29..42d2a6dc15 100644 --- a/src/binding/python/RecordComponent.cpp +++ b/src/binding/python/RecordComponent.cpp @@ -408,7 +408,7 @@ inline void store_chunk( size_t const numFlattenDims = std::count(flatten.begin(), flatten.end(), true); auto const r_extent = r.getExtent(); - auto const s_extent(extent); // selected extent in r + auto const &s_extent(extent); // selected extent in r std::vector r_shape(r_extent.size() - numFlattenDims); std::vector s_shape(s_extent.size() - numFlattenDims); auto maskIt = flatten.begin(); From 7fd7ddda58f1bd3e2efed45bebcebd12d5d156b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 8 May 2024 13:59:37 +0200 Subject: [PATCH 16/16] Minor cleanup --- src/binding/python/RecordComponent.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/binding/python/RecordComponent.cpp b/src/binding/python/RecordComponent.cpp index 42d2a6dc15..5bceeaee9c 100644 --- a/src/binding/python/RecordComponent.cpp +++ b/src/binding/python/RecordComponent.cpp @@ -19,7 +19,6 @@ * If not, see . */ #include -#include #include #include #include @@ -371,7 +370,6 @@ struct LoadChunkIntoPythonBuffer // note: this does not yet prevent the user, as in C++, to build // a race condition by manipulating the data that was passed buffer.inc_ref(); - // buffer_info.inc_ref(); void *data = buffer_info.ptr; std::shared_ptr shared( (T *)data, [buffer](T *) { buffer.dec_ref(); });