Add ATOMICS_BYPASS configuration option to disable atomics (kokkos#…

…6692) * Add NAME_TBD_UNSAFE_ATOMICS configuration option to disable atomics * Rename configuration option and macro NAME_TBD_UNSAFE_ATOMICS -> ATOMICS_BYPASS * Improve option description and configuration time error message Co-Authored-By: Christian Trott <crtrott@sandia.gov> Co-Authored-By: Daniel Arndt <arndtd@ornl.gov> * Mention Kokkos_ENABLE_ATOMICS_BYPASS option in error msssage * Simplify sanity check for disabling atomics in <Kokkos_Macros.hpp> Co-Authored-By: Daniel Arndt <arndtd@ornl.gov> --------- Co-authored-by: Christian Trott <crtrott@sandia.gov> Co-authored-by: Daniel Arndt <arndtd@ornl.gov>
japlews · Jan 9, 2024 · 27286c3 · 27286c3
1 parent 23b02f0
commit 27286c3
Show file tree

Hide file tree

Showing 7 changed files with 22 additions and 8 deletions.
diff --git a/cmake/KokkosCore_config.h.in b/cmake/KokkosCore_config.h.in
@@ -52,6 +52,7 @@
 #cmakedefine KOKKOS_OPT_RANGE_AGGRESSIVE_VECTORIZATION  // deprecated
 #cmakedefine KOKKOS_ENABLE_AGGRESSIVE_VECTORIZATION
 #cmakedefine KOKKOS_ENABLE_IMPL_MDSPAN
+#cmakedefine KOKKOS_ENABLE_ATOMICS_BYPASS
 
 /* TPL Settings */
 #cmakedefine KOKKOS_ENABLE_HWLOC

diff --git a/cmake/kokkos_arch.cmake b/cmake/kokkos_arch.cmake
@@ -1139,3 +1139,14 @@ MESSAGE(STATUS "Architectures:")
 FOREACH(Arch ${KOKKOS_ENABLED_ARCH_LIST})
   MESSAGE(STATUS " ${Arch}")
 ENDFOREACH()
+
+
+IF(KOKKOS_ENABLE_ATOMICS_BYPASS)
+  IF(NOT _HOST_PARALLEL STREQUAL "NoTypeDefined" OR NOT _DEVICE_PARALLEL STREQUAL "NoTypeDefined")
+    MESSAGE(FATAL_ERROR "Not allowed to disable atomics (via -DKokkos_ENABLE_AROMICS_BYPASS=ON) if neither a host parallel nor a device backend is enabled!")
+  ENDIF()
+  IF(NOT KOKKOS_ENABLE_SERIAL)
+    MESSAGE(FATAL_ERROR "Implementation bug")  # safeguard
+  ENDIF()
+  MESSAGE(STATUS "Atomics: **DISABLED**")
+ENDIF()
diff --git a/cmake/kokkos_enable_options.cmake b/cmake/kokkos_enable_options.cmake
@@ -73,6 +73,7 @@ KOKKOS_ENABLE_OPTION(HIP_MULTIPLE_KERNEL_INSTANTIATIONS OFF "Whether multiple ke
 
 # This option will go away eventually, but allows fallback to old implementation when needed.
 KOKKOS_ENABLE_OPTION(DESUL_ATOMICS_EXTERNAL OFF "Whether to use an external desul installation")
+KOKKOS_ENABLE_OPTION(ATOMICS_BYPASS OFF "**NOT RECOMMENDED** Whether to make atomics non-atomic for non-threaded MPI-only use cases")
 
 KOKKOS_ENABLE_OPTION(IMPL_MDSPAN OFF "Whether to enable experimental mdspan support")
 KOKKOS_ENABLE_OPTION(MDSPAN_EXTERNAL OFF BOOL "Whether to use an external version of mdspan")

diff --git a/core/src/Kokkos_Atomics_Desul_Volatile_Wrapper.hpp b/core/src/Kokkos_Atomics_Desul_Volatile_Wrapper.hpp
@@ -25,7 +25,7 @@ static_assert(false,
 #include <Kokkos_Atomics_Desul_Config.hpp>
 #include <desul/atomics.hpp>
 
-#ifdef KOKKOS_INTERNAL_NOT_PARALLEL
+#ifdef KOKKOS_ENABLE_ATOMICS_BYPASS
 #define KOKKOS_DESUL_MEM_SCOPE desul::MemoryScopeCaller()
 #else
 #define KOKKOS_DESUL_MEM_SCOPE desul::MemoryScopeDevice()

diff --git a/core/src/Kokkos_Atomics_Desul_Wrapper.hpp b/core/src/Kokkos_Atomics_Desul_Wrapper.hpp
@@ -49,7 +49,7 @@ inline const char* atomic_query_version() { return "KOKKOS_DESUL_ATOMICS"; }
 #endif
 // ============================================================
 
-#ifdef KOKKOS_INTERNAL_NOT_PARALLEL
+#ifdef KOKKOS_ENABLE_ATOMICS_BYPASS
 #define KOKKOS_DESUL_MEM_SCOPE desul::MemoryScopeCaller()
 #else
 #define KOKKOS_DESUL_MEM_SCOPE desul::MemoryScopeDevice()

diff --git a/core/src/Kokkos_Macros.hpp b/core/src/Kokkos_Macros.hpp
@@ -84,11 +84,12 @@
 
 //----------------------------------------------------------------------------
 
-#if !defined(KOKKOS_ENABLE_THREADS) && !defined(KOKKOS_ENABLE_CUDA) &&     \
-    !defined(KOKKOS_ENABLE_OPENMP) && !defined(KOKKOS_ENABLE_HPX) &&       \
-    !defined(KOKKOS_ENABLE_OPENMPTARGET) && !defined(KOKKOS_ENABLE_HIP) && \
-    !defined(KOKKOS_ENABLE_SYCL) && !defined(KOKKOS_ENABLE_OPENACC)
-#define KOKKOS_INTERNAL_NOT_PARALLEL
+#if defined(KOKKOS_ENABLE_ATOMICS_BYPASS) &&                              \
+    (defined(KOKKOS_ENABLE_THREADS) || defined(KOKKOS_ENABLE_CUDA) ||     \
+     defined(KOKKOS_ENABLE_OPENMP) || defined(KOKKOS_ENABLE_HPX) ||       \
+     defined(KOKKOS_ENABLE_OPENMPTARGET) || defined(KOKKOS_ENABLE_HIP) || \
+     defined(KOKKOS_ENABLE_SYCL) || defined(KOKKOS_ENABLE_OPENACC))
+#error Atomics may only be disabled if neither a host parallel nor a device backend is enabled
 #endif
 
 #define KOKKOS_ENABLE_CXX11_DISPATCH_LAMBDA

diff --git a/core/src/Serial/Kokkos_Serial.cpp b/core/src/Serial/Kokkos_Serial.cpp
@@ -153,7 +153,7 @@ void Serial::print_configuration(std::ostream& os, bool /*verbose*/) const {
   os << "Host Serial Execution Space:\n";
   os << "  KOKKOS_ENABLE_SERIAL: yes\n";
 
-#ifdef KOKKOS_INTERNAL_NOT_PARALLEL
+#ifdef KOKKOS_ENABLE_ATOMICS_BYPASS
   os << "Kokkos atomics disabled\n";
 #endif