Skip to content

Commit

Permalink
Merge pull request #4074 from anagainaru/gpu-c-bindings
Browse files Browse the repository at this point in the history
C bindings for the memory space related functions
  • Loading branch information
anagainaru committed Mar 6, 2024
2 parents 90e51e3 + 6c45d67 commit 8c2ef6f
Show file tree
Hide file tree
Showing 8 changed files with 275 additions and 10 deletions.
9 changes: 9 additions & 0 deletions bindings/C/adios2/c/adios2_c_types.h
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,15 @@ union adios2_PrimitiveStdtypeUnion
char *str;
};

typedef enum
{
adios2_memory_space_detect = 0,
adios2_memory_space_host = 1,
#ifdef ADIOS2_HAVE_GPU_SUPPORT
adios2_memory_space_gpu = 2,
#endif
} adios2_memory_space;

typedef struct
{
int WriterID;
Expand Down
98 changes: 96 additions & 2 deletions bindings/C/adios2/c/adios2_c_variable.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,90 @@ adios2_error adios2_set_shape(adios2_variable *variable, const size_t ndims, con
}
}

adios2::MemorySpace adios2_ToMemorySpace(const adios2_memory_space Cmem)
{
#ifdef ADIOS2_HAVE_GPU_SUPPORT
adios2::MemorySpace mem = adios2::MemorySpace::Detect;
#else
adios2::MemorySpace mem = adios2::MemorySpace::Host;
#endif
switch (Cmem)
{

case adios2_memory_space_host:
mem = adios2::MemorySpace::Host;
break;

#ifdef ADIOS2_HAVE_GPU_SUPPORT
case adios2_memory_space_gpu:
mem = adios2::MemorySpace::GPU;
break;
#endif
default:
break;
}
return mem;
}

adios2_memory_space adios2_FromMemorySpace(const adios2::MemorySpace mem)
{
#ifdef ADIOS2_HAVE_GPU_SUPPORT
adios2_memory_space Cmem = adios2_memory_space_detect;
#else
adios2_memory_space Cmem = adios2_memory_space_host;
#endif
switch (mem)
{

case adios2::MemorySpace::Host:
Cmem = adios2_memory_space_host;
break;

#ifdef ADIOS2_HAVE_GPU_SUPPORT
case adios2::MemorySpace::GPU:
Cmem = adios2_memory_space_gpu;
break;
#endif
default:
break;
}
return Cmem;
}

adios2_error adios2_set_memory_space(adios2_variable *variable, const adios2_memory_space mem)
{
try
{
adios2::core::VariableBase *variableBase =
reinterpret_cast<adios2::core::VariableBase *>(variable);
variableBase->SetMemorySpace(adios2_ToMemorySpace(mem));

return adios2_error_none;
}
catch (...)
{
return static_cast<adios2_error>(
adios2::helper::ExceptionToError("adios2_set_memory_space"));
}
}

adios2_error adios2_get_memory_space(adios2_memory_space *mem, adios2_variable *variable)
{
try
{
adios2::core::VariableBase *variableBase =
reinterpret_cast<adios2::core::VariableBase *>(variable);
*mem = adios2_FromMemorySpace(variableBase->m_MemSpace);

return adios2_error_none;
}
catch (...)
{
return static_cast<adios2_error>(
adios2::helper::ExceptionToError("adios2_set_memory_space"));
}
}

adios2_error adios2_set_block_selection(adios2_variable *variable, const size_t block_id)
{
try
Expand Down Expand Up @@ -274,7 +358,8 @@ adios2_error adios2_variable_ndims(size_t *ndims, const adios2_variable *variabl
}
}

adios2_error adios2_variable_shape(size_t *shape, const adios2_variable *variable)
adios2_error adios2_variable_shape_with_memory_space(size_t *shape, const adios2_variable *variable,
adios2_memory_space mem)
{
try
{
Expand All @@ -296,7 +381,11 @@ adios2_error adios2_variable_shape(size_t *shape, const adios2_variable *variabl
{ \
const adios2::core::Variable<T> *variable = \
dynamic_cast<const adios2::core::Variable<T> *>(variableBase); \
const adios2::Dims shapeCpp = variable->Shape(adios2::EngineCurrentStep); \
adios2::Dims shapeCpp; \
if (mem == adios2_memory_space_host) \
shapeCpp = variable->Shape(adios2::EngineCurrentStep); \
else \
shapeCpp = variable->Shape(adios2::EngineCurrentStep, adios2_ToMemorySpace(mem)); \
std::copy(shapeCpp.begin(), shapeCpp.end(), shape); \
}
ADIOS2_FOREACH_STDTYPE_1ARG(declare_template_instantiation)
Expand All @@ -310,6 +399,11 @@ adios2_error adios2_variable_shape(size_t *shape, const adios2_variable *variabl
}
}

adios2_error adios2_variable_shape(size_t *shape, const adios2_variable *variable)
{
return adios2_variable_shape_with_memory_space(shape, variable, adios2_memory_space_host);
}

adios2_error adios2_variable_start(size_t *start, const adios2_variable *variable)
{
try
Expand Down
26 changes: 26 additions & 0 deletions bindings/C/adios2/c/adios2_c_variable.h
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,22 @@ extern "C" {
*/
adios2_error adios2_set_shape(adios2_variable *variable, const size_t ndims, const size_t *shape);

/**
* Sets the memory space for all following Puts/Gets
* to either host (default) or device
* @param mem memory space where Put/Get buffers are allocated
* @return adios2_error 0: success, see enum adios2_error for errors
*/
adios2_error adios2_set_memory_space(adios2_variable *variable, const adios2_memory_space mem);

/**
* Get the memory space that was set by the application for a given variable
* @param memory space output, the variable memory space
* @param variable handler
* @return adios2_error 0: success, see enum adios2_error for errors
*/
adios2_error adios2_get_memory_space(adios2_memory_space *mem, adios2_variable *variable);

/**
* Read mode only. Required for reading local variables. For Global Arrays it
* will Set
Expand Down Expand Up @@ -140,6 +156,16 @@ adios2_error adios2_variable_ndims(size_t *ndims, const adios2_variable *variabl
*/
adios2_error adios2_variable_shape(size_t *shape, const adios2_variable *variable);

/**
* Retrieve current variable shape for a given memory space
* @param shape output, must be pre-allocated with ndims
* @param variable handler
* @param memory space
* @return adios2_error 0: success, see enum adios2_error for errors
*/
adios2_error adios2_variable_shape_with_memory_space(size_t *shape, const adios2_variable *variable,
const adios2_memory_space mem);

/**
* Retrieve current variable start
* @param start output, single value
Expand Down
6 changes: 3 additions & 3 deletions bindings/CXX11/adios2/cxx11/Variable.h
Original file line number Diff line number Diff line change
Expand Up @@ -148,9 +148,9 @@ class Variable
explicit operator bool() const noexcept;

/**
* Sets the memory space for all following Puts
* to either host (default) or device (currently only CUDA supported)
* @param mem memory space where Put buffers are allocated
* Sets the memory space for all following Puts/Gets
* to either host (default) or device
* @param mem memory space where Put/Get buffers are allocated
*/
void SetMemorySpace(const MemorySpace mem);

Expand Down
4 changes: 1 addition & 3 deletions source/adios2/common/ADIOSTypes.h
Original file line number Diff line number Diff line change
Expand Up @@ -45,10 +45,8 @@ enum class DerivedVarType
/** Memory space for the user provided buffers */
enum class MemorySpace
{
#ifdef ADIOS2_HAVE_GPU_SUPPORT
Detect, ///< Detect the memory space automatically
#endif
Host, ///< Host memory space
Host, ///< Host memory space
#ifdef ADIOS2_HAVE_GPU_SUPPORT
GPU ///< GPU memory space
#endif
Expand Down
13 changes: 11 additions & 2 deletions source/adios2/core/VariableBase.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -77,19 +77,19 @@ MemorySpace VariableBase::GetMemorySpace(const void *ptr)
#if defined(ADIOS2_HAVE_KOKKOS) || defined(ADIOS2_HAVE_GPU_SUPPORT)
ArrayOrdering layout = m_BaseLayout;
#endif
#ifdef ADIOS2_HAVE_GPU_SUPPORT
// first time the memory space is set
if (m_MemSpace == MemorySpace::Detect)
{
#ifdef ADIOS2_HAVE_GPU_SUPPORT
if (helper::IsGPUbuffer(ptr))
{
m_MemSpace = MemorySpace::GPU;
layout = ArrayOrdering::ColumnMajor;
}
else
#endif
m_MemSpace = MemorySpace::Host;
}
#endif
#if defined(ADIOS2_HAVE_KOKKOS) || defined(ADIOS2_HAVE_GPU_SUPPORT)
// set the layout based on the buffer memory space
// skipping throwing an exception for a mismatch
Expand Down Expand Up @@ -647,6 +647,15 @@ Dims VariableBase::Shape(const size_t step, const MemorySpace memSpace,
const ArrayOrdering layout) const
{
auto dims = Shape(step);
#ifdef ADIOS2_HAVE_GPU_SUPPORT
if (memSpace == MemorySpace::Detect)
{
helper::Throw<std::invalid_argument>("Core", "Variable", "Shape",
"can't call Shape() for variable " + m_Name +
" using the Detect memory space."
"Host/GPU need to be selected");
}
#endif
#if defined(ADIOS2_HAVE_KOKKOS) || defined(ADIOS2_HAVE_GPU_SUPPORT)
bool mismatchMemSpace =
(memSpace != MemorySpace::Host && m_BaseLayout != ArrayOrdering::ColumnMajor);
Expand Down
1 change: 1 addition & 0 deletions testing/adios2/bindings/C/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -8,3 +8,4 @@ gtest_add_tests_helper(WriteReadMultiblock MPI_ALLOW BP Bindings.C. "")
gtest_add_tests_helper(NullWriteRead MPI_ALLOW "" Bindings.C. "")
gtest_add_tests_helper(WriteAggregateReadLocal MPI_ONLY BP Bindings.C. "")
gtest_add_tests_helper(AvailableVariablesAttribites MPI_ONLY BP Bindings.C. "")
gtest_add_tests_helper(MemorySpace MPI_NONE BP Bindings.C. "")
128 changes: 128 additions & 0 deletions testing/adios2/bindings/C/TestBPMemorySpace.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
/*
* Distributed under the OSI-approved Apache License, Version 2.0. See
* accompanying file Copyright.txt for details.
*
* TestBPMemorySpace.cpp : test the C bindings related to memory space
*/

#include <adios2_c.h>
#include <gtest/gtest.h>

class ADIOS2_C_API : public ::testing::Test
{
public:
ADIOS2_C_API() { adiosH = adios2_init_serial(); }

~ADIOS2_C_API() { adios2_finalize(adiosH); }

adios2_adios *adiosH;
};

#ifdef ADIOS2_HAVE_GPU_SUPPORT
TEST_F(ADIOS2_C_API, ADIOS2BPMemorySpaceGPU)
{
adios2_io *ioH = adios2_declare_io(adiosH, "CMemSpace");
adios2_set_engine(ioH, "BPFile");

size_t shape[1];
shape[0] = 10;

size_t start[1];
start[0] = 0;

size_t count[1];
count[0] = 10;

adios2_define_variable(ioH, "varI32", adios2_type_int32_t, 1, shape, start, count,
adios2_constant_dims_true);
adios2_variable *varI32 = adios2_inquire_variable(ioH, "varI32");

// test that the default memory space is Detect
adios2_memory_space mem;
adios2_error e = adios2_get_memory_space(&mem, varI32);
EXPECT_EQ(e, adios2_error_none);
EXPECT_EQ(mem, adios2_memory_space_detect);
// test that the set memory space is GPU
adios2_set_memory_space(varI32, adios2_memory_space_gpu);
e = adios2_get_memory_space(&mem, varI32);
EXPECT_EQ(e, adios2_error_none);
EXPECT_EQ(mem, adios2_memory_space_gpu);
}
#endif

TEST_F(ADIOS2_C_API, ADIOS2BPMemorySpaceShape)
{
const char fname[] = "ADIOS2_C_API.ADIOS2BPMemorySpace.bp";
// write
{
adios2_io *ioH = adios2_declare_io(adiosH, "CMemSpace");
adios2_set_engine(ioH, "BPFile");

size_t shape[2];
shape[0] = 5;
shape[1] = 2;

size_t start[2];
start[0] = 0;
start[1] = 0;

size_t count[2];
count[0] = 5;
count[1] = 2;

int32_t data_I32[10] = {131072, 131073, -131070, 131075, -131068,
131077, -131066, 131079, -131064, 131081};
adios2_define_variable(ioH, "varI32", adios2_type_int32_t, 2, shape, start, count,
adios2_constant_dims_true);
adios2_variable *varI32 = adios2_inquire_variable(ioH, "varI32");

// test that the set memory space is Host
adios2_memory_space mem;
adios2_set_memory_space(varI32, adios2_memory_space_host);
adios2_error e = adios2_get_memory_space(&mem, varI32);
EXPECT_EQ(e, adios2_error_none);
EXPECT_EQ(mem, adios2_memory_space_host);

adios2_engine *engineH = adios2_open(ioH, fname, adios2_mode_write);
adios2_put(engineH, varI32, data_I32, adios2_mode_deferred);
adios2_close(engineH);
}
// read shape
{
adios2_io *ioH = adios2_declare_io(adiosH, "Reader");
adios2_engine *engineH = adios2_open(ioH, fname, adios2_mode_readRandomAccess);
size_t steps;
adios2_steps(&steps, engineH);
EXPECT_EQ(steps, 1);

adios2_variable *varI32 = adios2_inquire_variable(ioH, "varI32");

// test that the shape function returns the correct dimensions
size_t cpu_shape[2];
adios2_error e = adios2_variable_shape(cpu_shape, varI32);
EXPECT_EQ(e, adios2_error_none);
EXPECT_EQ(cpu_shape[0], 5);
EXPECT_EQ(cpu_shape[1], 2);
e = adios2_variable_shape_with_memory_space(cpu_shape, varI32, adios2_memory_space_host);
EXPECT_EQ(e, adios2_error_none);
EXPECT_EQ(cpu_shape[0], 5);
EXPECT_EQ(cpu_shape[1], 2);
#ifdef ADIOS2_HAVE_GPU_SUPPORT
size_t gpu_shape[2];
e = adios2_variable_shape_with_memory_space(gpu_shape, varI32, adios2_memory_space_gpu);
EXPECT_EQ(e, adios2_error_none);
EXPECT_EQ(gpu_shape[0], 2);
EXPECT_EQ(gpu_shape[1], 5);
#endif
adios2_close(engineH);
}
}

int main(int argc, char **argv)
{
int result;
::testing::InitGoogleTest(&argc, argv);
result = RUN_ALL_TESTS();

return result;
}

0 comments on commit 8c2ef6f

Please sign in to comment.