Skip to content

Commit

Permalink
Fix CI issues for ADIOS2 v2.10: Add openPMD_HAVE_ADIOS2_BP5, datatype…
Browse files Browse the repository at this point in the history 
… fixes in Python bindings (#1618)

* Add define for ADIOS2_HAVE_BP5

This was removed from ADIOS2 v2.9 to v2.10.

* Temporarily revert "Add define for ADIOS2_HAVE_BP5"

This reverts commit 9bf8591.
CI seems broken atm and need to fix that first.

* Revert "Temporarily revert "Add define for ADIOS2_HAVE_BP5""

This reverts commit c068163.

* More direct error output

* Add normalizingVariableType

* ... and use the new function

* Try verbose debugging

* Put the log into the err message so the CI actually shows it......

* Use our own macro name

* Fix a potential source for confused datatypes

* Revert "Put the log into the err message so the CI actually shows it......"

This reverts commit cdd2c28.

* Revert "Try verbose debugging"

This reverts commit e18d608.

* Revert "... and use the new function"

This reverts commit 41b6b57.

* Revert "Add normalizingVariableType"

This reverts commit 85e9018.

* CI fixes

* Minor cleanup
  • Loading branch information
@franzpoeschel
franzpoeschel committed May 8, 2024
1 parent 805c760 commit 95cbcb6
Show file tree
Hide file tree
Showing 7 changed files with 101 additions and 180 deletions.
7 changes: 3 additions & 4 deletions include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -409,10 +409,9 @@ class ADIOS2IOHandlerImpl
{
std::stringstream errorMessage;
errorMessage << "Trying to access a dataset with wrong type "
"(trying to access dataset with type "
<< determineDatatype<T>() << ", but has type "
<< detail::fromADIOS2Type(actualType, false)
<< ")";
"(trying to access dataset with type '"
<< requiredType << "', but has type '"
<< actualType << "')";
throw error::ReadError(
error::AffectedObject::Dataset,
error::Reason::UnexpectedContent,
Expand Down
7 changes: 7 additions & 0 deletions include/openPMD/IO/ADIOS/macros.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,13 @@
#define openPMD_HAS_ADIOS_2_9 \
(ADIOS2_VERSION_MAJOR * 100 + ADIOS2_VERSION_MINOR >= 209)

#if defined(ADIOS2_HAVE_BP5) || openPMD_HAS_ADIOS_2_9
// ADIOS2 v2.10 no longer defines this
#define openPMD_HAVE_ADIOS2_BP5 1
#else
#define openPMD_HAVE_ADIOS2_BP5 0
#endif

#else

#define openPMD_HAS_ADIOS_2_8 0
Expand Down
2 changes: 1 addition & 1 deletion src/IO/ADIOS/ADIOS2IOHandler.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -354,7 +354,7 @@ std::string ADIOS2IOHandlerImpl::fileSuffix(bool verbose) const
{
// SST engine adds its suffix unconditionally
// so we don't add it
#if defined(ADIOS2_HAVE_BP5) && openPMD_HAS_ADIOS_2_9
#if openPMD_HAVE_ADIOS2_BP5 && openPMD_HAS_ADIOS_2_9
constexpr char const *const default_file_ending = ".bp5";
#else
constexpr char const *const default_file_ending = ".bp4";
Expand Down
244 changes: 79 additions & 165 deletions src/binding/python/RecordComponent.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@
#include <pybind11/pybind11.h>
#include <pybind11/stl.h>

#include "openPMD/Dataset.hpp"
#include "openPMD/Datatype.hpp"
#include "openPMD/DatatypeHelpers.hpp"
#include "openPMD/Error.hpp"
#include "openPMD/RecordComponent.hpp"
Expand Down Expand Up @@ -309,6 +311,75 @@ inline void check_buffer_is_contiguous(py::array &a)
// loop over the input data strides in store/load calls
}

namespace
{
struct StoreChunkFromPythonArray
{
template <typename T>
static void call(
RecordComponent &r,
py::array &a,
Offset const &offset,
Extent const &extent)
{
// here, we increase a reference on the user-passed data so that
// temporary and lost-scope variables stay alive until we flush
// note: this does not yet prevent the user, as in C++, to build
// a race condition by manipulating the data that was passed
a.inc_ref();
void *data = a.mutable_data();
std::shared_ptr<T> shared((T *)data, [a](T *) { a.dec_ref(); });
r.storeChunk(std::move(shared), offset, extent);
}

static constexpr char const *errorMsg = "store_chunk()";
};
struct LoadChunkIntoPythonArray
{
template <typename T>
static void call(
RecordComponent &r,
py::array &a,
Offset const &offset,
Extent const &extent)
{
// here, we increase a reference on the user-passed data so that
// temporary and lost-scope variables stay alive until we flush
// note: this does not yet prevent the user, as in C++, to build
// a race condition by manipulating the data that was passed
a.inc_ref();
void *data = a.mutable_data();
std::shared_ptr<T> shared((T *)data, [a](T *) { a.dec_ref(); });
r.loadChunk(std::move(shared), offset, extent);
}

static constexpr char const *errorMsg = "load_chunk()";
};
struct LoadChunkIntoPythonBuffer
{
template <typename T>
static void call(
RecordComponent &r,
py::buffer &buffer,
py::buffer_info const &buffer_info,
Offset const &offset,
Extent const &extent)
{
// here, we increase a reference on the user-passed data so that
// temporary and lost-scope variables stay alive until we flush
// note: this does not yet prevent the user, as in C++, to build
// a race condition by manipulating the data that was passed
buffer.inc_ref();
void *data = buffer_info.ptr;
std::shared_ptr<T> shared(
(T *)data, [buffer](T *) { buffer.dec_ref(); });
r.loadChunk(std::move(shared), offset, extent);
}

static constexpr char const *errorMsg = "load_chunk()";
};
} // namespace

/** Store Chunk
*
* Called with offset and extent that are already in the record component's
Expand All @@ -335,7 +406,7 @@ inline void store_chunk(
size_t const numFlattenDims =
std::count(flatten.begin(), flatten.end(), true);
auto const r_extent = r.getExtent();
auto const s_extent(extent); // selected extent in r
auto const &s_extent(extent); // selected extent in r
std::vector<std::uint64_t> r_shape(r_extent.size() - numFlattenDims);
std::vector<std::uint64_t> s_shape(s_extent.size() - numFlattenDims);
auto maskIt = flatten.begin();
Expand Down Expand Up @@ -409,64 +480,9 @@ inline void store_chunk(

check_buffer_is_contiguous(a);

// here, we increase a reference on the user-passed data so that
// temporary and lost-scope variables stay alive until we flush
// note: this does not yet prevent the user, as in C++, to build
// a race condition by manipulating the data they passed
auto store_data = [&r, &a, &offset, &extent](auto cxxtype) {
using CXXType = decltype(cxxtype);
a.inc_ref();
void *data = a.mutable_data();
std::shared_ptr<CXXType> shared(
(CXXType *)data, [a](CXXType *) { a.dec_ref(); });
r.storeChunk(std::move(shared), offset, extent);
};

// store
auto const dtype = dtype_from_numpy(a.dtype());
if (dtype == Datatype::CHAR)
store_data(char());
else if (dtype == Datatype::UCHAR)
store_data((unsigned char)0);
else if (dtype == Datatype::SHORT)
store_data(short());
else if (dtype == Datatype::INT)
store_data(int());
else if (dtype == Datatype::LONG)
store_data(long());
else if (dtype == Datatype::LONGLONG)
store_data((long long)0);
else if (dtype == Datatype::USHORT)
store_data((unsigned short)0);
else if (dtype == Datatype::UINT)
store_data((unsigned int)0);
else if (dtype == Datatype::ULONG)
store_data((unsigned long)0);
else if (dtype == Datatype::ULONGLONG)
store_data((unsigned long long)0);
else if (dtype == Datatype::LONG_DOUBLE)
store_data((long double)0);
else if (dtype == Datatype::DOUBLE)
store_data(double());
else if (dtype == Datatype::FLOAT)
store_data(float());
else if (dtype == Datatype::CLONG_DOUBLE)
store_data(std::complex<long double>());
else if (dtype == Datatype::CDOUBLE)
store_data(std::complex<double>());
else if (dtype == Datatype::CFLOAT)
store_data(std::complex<float>());
/* @todo
.value("STRING", Datatype::STRING)
.value("VEC_STRING", Datatype::VEC_STRING)
.value("ARR_DBL_7", Datatype::ARR_DBL_7)
*/
else if (dtype == Datatype::BOOL)
store_data(bool());
else
throw std::runtime_error(
std::string("Datatype '") + std::string(py::str(a.dtype())) +
std::string("' not known in 'storeChunk'!"));
// dtype_from_numpy(a.dtype())
switchDatasetType<StoreChunkFromPythonArray>(
r.getDatatype(), r, a, offset, extent);
}

/** Store Chunk
Expand Down Expand Up @@ -682,60 +698,8 @@ void load_chunk(
}
}

// here, we increase a reference on the user-passed data so that
// temporary and lost-scope variables stay alive until we flush
// note: this does not yet prevent the user, as in C++, to build
// a race condition by manipulating the data they passed
auto load_data =
[&r, &buffer, &buffer_info, &offset, &extent](auto cxxtype) {
using CXXType = decltype(cxxtype);
buffer.inc_ref();
// buffer_info.inc_ref();
void *data = buffer_info.ptr;
std::shared_ptr<CXXType> shared(
(CXXType *)data, [buffer](CXXType *) { buffer.dec_ref(); });
r.loadChunk(std::move(shared), offset, extent);
};

if (r.getDatatype() == Datatype::CHAR)
load_data((char)0);
else if (r.getDatatype() == Datatype::UCHAR)
load_data((unsigned char)0);
else if (r.getDatatype() == Datatype::SCHAR)
load_data((signed char)0);
else if (r.getDatatype() == Datatype::SHORT)
load_data((short)0);
else if (r.getDatatype() == Datatype::INT)
load_data((int)0);
else if (r.getDatatype() == Datatype::LONG)
load_data((long)0);
else if (r.getDatatype() == Datatype::LONGLONG)
load_data((long long)0);
else if (r.getDatatype() == Datatype::USHORT)
load_data((unsigned short)0);
else if (r.getDatatype() == Datatype::UINT)
load_data((unsigned int)0);
else if (r.getDatatype() == Datatype::ULONG)
load_data((unsigned long)0);
else if (r.getDatatype() == Datatype::ULONGLONG)
load_data((unsigned long long)0);
else if (r.getDatatype() == Datatype::LONG_DOUBLE)
load_data((long double)0);
else if (r.getDatatype() == Datatype::DOUBLE)
load_data((double)0);
else if (r.getDatatype() == Datatype::FLOAT)
load_data((float)0);
else if (r.getDatatype() == Datatype::CLONG_DOUBLE)
load_data((std::complex<long double>)0);
else if (r.getDatatype() == Datatype::CDOUBLE)
load_data((std::complex<double>)0);
else if (r.getDatatype() == Datatype::CFLOAT)
load_data((std::complex<float>)0);
else if (r.getDatatype() == Datatype::BOOL)
load_data((bool)0);
else
throw std::runtime_error(
std::string("Datatype not known in 'loadChunk'!"));
switchNonVectorType<LoadChunkIntoPythonBuffer>(
r.getDatatype(), r, buffer, buffer_info, offset, extent);
}

/** Load Chunk
Expand Down Expand Up @@ -792,58 +756,8 @@ inline void load_chunk(

check_buffer_is_contiguous(a);

// here, we increase a reference on the user-passed data so that
// temporary and lost-scope variables stay alive until we flush
// note: this does not yet prevent the user, as in C++, to build
// a race condition by manipulating the data they passed
auto load_data = [&r, &a, &offset, &extent](auto cxxtype) {
using CXXType = decltype(cxxtype);
a.inc_ref();
void *data = a.mutable_data();
std::shared_ptr<CXXType> shared(
(CXXType *)data, [a](CXXType *) { a.dec_ref(); });
r.loadChunk(std::move(shared), offset, extent);
};

if (r.getDatatype() == Datatype::CHAR)
load_data(char());
else if (r.getDatatype() == Datatype::UCHAR)
load_data((unsigned char)0);
else if (r.getDatatype() == Datatype::SCHAR)
load_data((signed char)0);
else if (r.getDatatype() == Datatype::SHORT)
load_data(short());
else if (r.getDatatype() == Datatype::INT)
load_data(int());
else if (r.getDatatype() == Datatype::LONG)
load_data(long());
else if (r.getDatatype() == Datatype::LONGLONG)
load_data((long long)0);
else if (r.getDatatype() == Datatype::USHORT)
load_data((unsigned short)0);
else if (r.getDatatype() == Datatype::UINT)
load_data((unsigned int)0);
else if (r.getDatatype() == Datatype::ULONG)
load_data((unsigned long)0);
else if (r.getDatatype() == Datatype::ULONGLONG)
load_data((unsigned long long)0);
else if (r.getDatatype() == Datatype::LONG_DOUBLE)
load_data((long double)0);
else if (r.getDatatype() == Datatype::DOUBLE)
load_data(double());
else if (r.getDatatype() == Datatype::FLOAT)
load_data(float());
else if (r.getDatatype() == Datatype::CLONG_DOUBLE)
load_data(std::complex<long double>());
else if (r.getDatatype() == Datatype::CDOUBLE)
load_data(std::complex<double>());
else if (r.getDatatype() == Datatype::CFLOAT)
load_data(std::complex<float>());
else if (r.getDatatype() == Datatype::BOOL)
load_data(bool());
else
throw std::runtime_error(
std::string("Datatype not known in 'load_chunk'!"));
switchDatasetType<LoadChunkIntoPythonArray>(
r.getDatatype(), r, a, offset, extent);
}

/** Load Chunk
Expand Down
3 changes: 2 additions & 1 deletion src/config.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
#include "openPMD/version.hpp"

#if openPMD_HAVE_ADIOS2
#include "openPMD/IO/ADIOS/macros.hpp"
#include <adios2.h>
#endif
#include <map>
Expand Down Expand Up @@ -60,7 +61,7 @@ std::vector<std::string> openPMD::getFileExtensions()
// BP4 is always available in ADIOS2
fext.emplace_back("bp4");
#endif
#ifdef ADIOS2_HAVE_BP5
#if openPMD_HAVE_ADIOS2_BP5
fext.emplace_back("bp5");
#endif
#ifdef ADIOS2_HAVE_SST
Expand Down
4 changes: 2 additions & 2 deletions test/CoreTest.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1184,7 +1184,7 @@ TEST_CASE("backend_via_json", "[core]")
TEST_CASE("wildcard_extension", "[core]")
{
#if openPMD_HAVE_ADIOS2
#if defined(ADIOS2_HAVE_BP5) && openPMD_HAS_ADIOS_2_9
#if openPMD_HAVE_ADIOS2_BP5 && openPMD_HAS_ADIOS_2_9
constexpr char const *const default_file_ending = "bp5";
#else
constexpr char const *const default_file_ending = "bp4";
Expand Down Expand Up @@ -1227,7 +1227,7 @@ TEST_CASE("wildcard_extension", "[core]")
}
};
#if openPMD_HAVE_ADIOS2
#ifdef ADIOS2_HAVE_BP5
#if openPMD_HAVE_ADIOS2_BP5
run_test(
R"({"adios2": {"engine": {"type": "bp5"}}, "backend": "adios2"})",
"bp5");
Expand Down

0 comments on commit 95cbcb6

Please sign in to comment.