Skip to content

Commit

Permalink
Add CMake test for GPU-aware MPI (#17)
Browse files Browse the repository at this point in the history
Try to compile and run send_recv_usm.cpp to determine
whether the MPI implementation is GPU-aware. Enable
building the MPI_with_SYCL examples only if at least
one backend is found to work with MPI over GPU.
  • Loading branch information
rafbiels committed May 15, 2024
1 parent 80cfe84 commit 561ee59
Show file tree
Hide file tree
Showing 3 changed files with 73 additions and 9 deletions.
2 changes: 1 addition & 1 deletion CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
# This work is licensed under the Apache License, Version 2.0.
# For a copy, see http://www.apache.org/licenses/LICENSE-2.0

cmake_minimum_required(VERSION 3.5)
cmake_minimum_required(VERSION 3.12)
project(SYCL-samples)

# Set build type to Release if unset
Expand Down
8 changes: 6 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -85,8 +85,12 @@ MPI installation. E.g.
```

Additionally, in order to run the examples, the MPI implementation needs
to be device-aware. This is only detectable at runtime, so the examples may build
fine but crash on execution if the linked MPI library isn't device-aware.
to be device-aware. The CMake configuration attempts to build and execute the
simplest example to evaluate whether the found MPI library supports any of the
enabled backends. This demo will be automatically skipped if this check does not
pass and a corresponding message will appear in the CMake configuration output.
The result of this check can be overwritten with the `-DMPI_DEVICE_AWARE=ON/OFF`
option.

### Parallel Inclusive Scan
Implementation of a parallel inclusive scan with a given associative binary
Expand Down
72 changes: 66 additions & 6 deletions src/MPI_with_SYCL/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,11 +1,71 @@
function(run_mpi_test)
cmake_parse_arguments(RUN_MPI_TEST "" "BINARY;BACKEND_SELECTOR" "" ${ARGN})
execute_process(
COMMAND bash -c "ONEAPI_DEVICE_SELECTOR=${RUN_MPI_TEST_BACKEND_SELECTOR}:gpu mpirun -n 2 ${RUN_MPI_TEST_BINARY} > mpi-test-output.txt && grep 'received status==0' mpi-test-output.txt"
RESULT_VARIABLE MPI_TEST_FAILED
OUTPUT_QUIET
ERROR_QUIET)
if (${MPI_TEST_FAILED})
set(MPI_TEST_SUCCEEDED FALSE PARENT_SCOPE)
else()
set(MPI_TEST_SUCCEEDED TRUE PARENT_SCOPE)
endif()
endfunction()

function(test_mpi_gpu_support)
set(MPI_TEST_BINARY ${CMAKE_CURRENT_BINARY_DIR}/test-mpi-gpu)
try_compile(MPI_TEST_COMPILE_SUCCEEDED
${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/send_recv_usm.cpp
COMPILE_DEFINITIONS "${SYCL_FLAGS} -I${MPI_CXX_INCLUDE_DIRS}"
LINK_OPTIONS ${SYCL_FLAGS}
LINK_LIBRARIES ${MPI_LIBRARIES}
COPY_FILE ${MPI_TEST_BINARY}
OUTPUT_VARIABLE MPI_TEST_COMPILE_OUTPUT)
set(MPI_DEVICE_AWARE 0)
if (${MPI_TEST_COMPILE_SUCCEEDED})
if (${ENABLE_CUDA})
run_mpi_test(BINARY ${MPI_TEST_BINARY} BACKEND_SELECTOR "cuda")
if (MPI_TEST_SUCCEEDED)
set(MPI_DEVICE_AWARE 1)
list(APPEND MPI_AVAILABLE_BACKENDS "CUDA")
endif()
endif()

if (${ENABLE_HIP})
run_mpi_test(BINARY ${MPI_TEST_BINARY} BACKEND_SELECTOR "hip")
if (MPI_TEST_SUCCEEDED)
set(MPI_DEVICE_AWARE 1)
list(APPEND MPI_AVAILABLE_BACKENDS "HIP")
endif()
endif()

if (${ENABLE_SPIR})
run_mpi_test(BINARY ${MPI_TEST_BINARY} BACKEND_SELECTOR "level_zero")
if (MPI_TEST_SUCCEEDED)
set(MPI_DEVICE_AWARE 1)
list(APPEND MPI_AVAILABLE_BACKENDS "Level Zero")
endif()
endif()
endif()

set(MPI_DEVICE_AWARE ${MPI_DEVICE_AWARE} CACHE BOOL "Offload device aware MPI implementation is available")
set(MPI_AVAILABLE_BACKENDS ${MPI_AVAILABLE_BACKENDS} PARENT_SCOPE)
endfunction()

find_package(MPI)
if(NOT MPI_FOUND)
message(STATUS "MPI not found, skipping the MPI_with_SYCL demo")
else()
message(STATUS "Found MPI, configuring the MPI_with_SYCL demo")
foreach(TARGET send_recv_usm send_recv_buff scatter_reduce_gather)
add_executable(${TARGET} ${TARGET}.cpp)
target_compile_options(${TARGET} PUBLIC ${SYCL_FLAGS} ${MPI_INCLUDE_DIRS})
target_link_options(${TARGET} PUBLIC ${SYCL_FLAGS} ${MPI_LIBRARIES})
endforeach()
test_mpi_gpu_support()
if (MPI_DEVICE_AWARE)
list(JOIN MPI_AVAILABLE_BACKENDS ", " MPI_AVAILABLE_BACKENDS)
message(STATUS "Found offload device aware MPI, configuring the MPI_with_SYCL demo. Available backends: ${MPI_AVAILABLE_BACKENDS}")
foreach(TARGET send_recv_usm send_recv_buff scatter_reduce_gather)
add_executable(${TARGET} ${TARGET}.cpp)
target_compile_options(${TARGET} PUBLIC ${SYCL_FLAGS} -I${MPI_CXX_INCLUDE_DIRS})
target_link_options(${TARGET} PUBLIC ${SYCL_FLAGS} ${MPI_LIBRARIES})
endforeach()
else()
message(STATUS "Found MPI which is not offload device aware - skipping the MPI_with_SYCL demo")
endif()
endif()

0 comments on commit 561ee59

Please sign in to comment.