From 216e7ead03916eaaadfecdbe779c1a947311e896 Mon Sep 17 00:00:00 2001 From: Vicente Bolea Date: Fri, 28 Apr 2023 11:04:45 -0400 Subject: [PATCH 1/3] Merge pull request #3593 from ax3l/fix-mpireal8-example Heat Transfer Example: MPI Datatype (cherry picked from commit 50aa9a3ce90a09222fb81b3dc5c3d0197433e5dc) --- examples/heatTransfer/write/HeatTransfer.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/heatTransfer/write/HeatTransfer.cpp b/examples/heatTransfer/write/HeatTransfer.cpp index 45510fbac3..56e4b90ec6 100644 --- a/examples/heatTransfer/write/HeatTransfer.cpp +++ b/examples/heatTransfer/write/HeatTransfer.cpp @@ -136,7 +136,7 @@ void HeatTransfer::exchange(MPI_Comm comm) { // Build a custom MPI type for the column vector to allow strided access MPI_Datatype tColumnVector; - MPI_Type_vector(m_s.ndx + 2, 1, m_s.ndy + 2, MPI_REAL8, &tColumnVector); + MPI_Type_vector(m_s.ndx + 2, 1, m_s.ndy + 2, MPI_DOUBLE, &tColumnVector); MPI_Type_commit(&tColumnVector); // Exchange ghost cells, in the order left-right-up-down @@ -184,14 +184,14 @@ void HeatTransfer::exchange(MPI_Comm comm) { // std::cout << "Rank " << m_s.rank << " send down to rank " // << m_s.rank_down << std::endl; - MPI_Send(m_TCurrent[m_s.ndx], m_s.ndy + 2, MPI_REAL8, m_s.rank_down, + MPI_Send(m_TCurrent[m_s.ndx], m_s.ndy + 2, MPI_DOUBLE, m_s.rank_down, tag, comm); } if (m_s.rank_up >= 0) { // std::cout << "Rank " << m_s.rank << " receive from above from rank " // << m_s.rank_up << std::endl; - MPI_Recv(m_TCurrent[0], m_s.ndy + 2, MPI_REAL8, m_s.rank_up, tag, comm, + MPI_Recv(m_TCurrent[0], m_s.ndy + 2, MPI_DOUBLE, m_s.rank_up, tag, comm, &status); } @@ -202,13 +202,13 @@ void HeatTransfer::exchange(MPI_Comm comm) // std::cout << "Rank " << m_s.rank << " send up to rank " << // m_s.rank_up // << std::endl; - MPI_Send(m_TCurrent[1], m_s.ndy + 2, MPI_REAL8, m_s.rank_up, tag, comm); + MPI_Send(m_TCurrent[1], m_s.ndy + 2, MPI_DOUBLE, m_s.rank_up, tag, comm); } if (m_s.rank_down >= 0) { // std::cout << "Rank " << m_s.rank << " receive from below from rank " // << m_s.rank_down << std::endl; - MPI_Recv(m_TCurrent[m_s.ndx + 1], m_s.ndy + 2, MPI_REAL8, m_s.rank_down, + MPI_Recv(m_TCurrent[m_s.ndx + 1], m_s.ndy + 2, MPI_DOUBLE, m_s.rank_down, tag, comm, &status); } } From b9a811c635478e1998fa83bd38a50fba5b6bf33e Mon Sep 17 00:00:00 2001 From: Ana Gainaru Date: Tue, 2 May 2023 16:28:55 -0400 Subject: [PATCH 2/3] Adding functionality to check where a buffer is allocated for HIP memory space --- source/adios2/helper/adiosKokkos.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/source/adios2/helper/adiosKokkos.cpp b/source/adios2/helper/adiosKokkos.cpp index 4916f079cb..763903a582 100644 --- a/source/adios2/helper/adiosKokkos.cpp +++ b/source/adios2/helper/adiosKokkos.cpp @@ -89,6 +89,15 @@ bool IsGPUbuffer(const void *ptr) { return true; } +#endif +#ifdef ADIOS2_HAVE_KOKKOS_HIP + hipError_t ret; + hipPointerAttribute_t attr; + ret = hipPointerGetAttributes(&attr, ptr); + if (ret == hipSuccess && attr.memoryType == hipMemoryTypeDevice) + { + return true; + } #endif return false; } From a17eda294ca5a3d30de65a4698ea29859dc8aba4 Mon Sep 17 00:00:00 2001 From: Ana Gainaru Date: Tue, 2 May 2023 16:33:35 -0400 Subject: [PATCH 3/3] Adding functionality to set the device id for HIP memory space --- source/adios2/helper/adiosKokkos.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/source/adios2/helper/adiosKokkos.cpp b/source/adios2/helper/adiosKokkos.cpp index 763903a582..7a9f53c66b 100644 --- a/source/adios2/helper/adiosKokkos.cpp +++ b/source/adios2/helper/adiosKokkos.cpp @@ -111,6 +111,15 @@ void KokkosInit() int device_id; cudaGetDevice(&device_id); settings.set_device_id(device_id); +#endif +#ifdef ADIOS2_HAVE_KOKKOS_HIP + int device_id; + hipError_t ret; + ret = hipGetDevice(&device_id); + if (ret == hipSuccess) + { + settings.set_device_id(device_id); + } #endif Kokkos::initialize(settings); }