Remove CUDA code and replace it with Kokkos

adamantine-sim · Jan 24, 2024 · c259303 · c259303
1 parent 22b1499
commit c259303
Show file tree

Hide file tree

Showing 31 changed files with 454 additions and 780 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -15,10 +15,6 @@ project(adamantine LANGUAGES CXX VERSION 0.1.0)
 include(SetupTPLs)
 include(SetupAdamantine)
 
-if (ADAMANTINE_ENABLE_CUDA)
-  enable_language(CUDA)
-endif()
-
 add_subdirectory(application)
 add_subdirectory(source)
 

diff --git a/README.md b/README.md
@@ -33,7 +33,6 @@ The list of configuration options is:
 * ADAMANTINE\_ENABLE\_ADIAK=ON/OFF
 * ADAMANTINE\_ENABLE\_CALIPER=ON/OFF
 * ADAMANTINE\_ENABLE\_COVERAGE=ON/OFF
-* ADAMANTINE\_ENABLE\_CUDA=ON/OFF
 * ADAMANTINE\_ENABLE\_TESTS=ON/OFF
 * BOOST\_DIR=/path/to/boost
 * CMAKE\_BUILD\_TYPE=Debug/Release

diff --git a/application/CMakeLists.txt b/application/CMakeLists.txt
@@ -1,23 +1,10 @@
 # Create adamantine executable and link against the static library.
-if (ADAMANTINE_ENABLE_CUDA)
-  add_executable(adamantine adamantine.cu)
-  target_compile_definitions(adamantine PRIVATE ADAMANTINE_HAVE_CUDA)
-  set_target_properties(adamantine PROPERTIES
-      CXX_STANDARD 17
-      CXX_STANDARD_REQUIRED ON
-      CXX_EXTENSIONS OFF
-      CUDA_SEPARABLE_COMPILATION ON
-      CUDA_STANDARD 17
-      CUDA_STANDARD_REQUIRED ON
-  )
-else()
-  add_executable(adamantine adamantine.cc)
-  set_target_properties(adamantine PROPERTIES
-      CXX_STANDARD 17
-      CXX_STANDARD_REQUIRED ON
-      CXX_EXTENSIONS OFF
-  )
-endif()
+add_executable(adamantine adamantine.cc)
+set_target_properties(adamantine PROPERTIES
+    CXX_STANDARD 17
+    CXX_STANDARD_REQUIRED ON
+    CXX_EXTENSIONS OFF
+)
 
 DEAL_II_SETUP_TARGET(adamantine)
 target_link_libraries(adamantine Adamantine)

diff --git a/application/adamantine.cc b/application/adamantine.cc
@@ -7,7 +7,6 @@
 
 #include "adamantine.hh"
 
-#include "utils.hh"
 #include <validate_input_database.hh>
 
 #ifdef ADAMANTINE_WITH_ADIAK
@@ -39,7 +38,6 @@ int main(int argc, char *argv[])
   adiak::cmdline();
   adiak::clustername();
   adiak::jobsize();
-  adiak::value("MemorySpace", "Host");
 #endif
 
   std::vector<adamantine::Timer> timers;
@@ -123,20 +121,49 @@ int main(int argc, char *argv[])
 
     unsigned int rank = dealii::Utilities::MPI::this_mpi_process(communicator);
 
+    // PropertyTreeInput memory_space
+    std::string memory_space =
+        database.get<std::string>("memory_space", "host");
+
+#ifdef ADAMANTINE_WITH_ADIAK
+    if (memory_space == "device")
+      adiak::value("MemorySpace", "Device");
+    else
+      adiak::value("MemorySpace", "Host");
+#endif
+
     if (dim == 2)
     {
       if (ensemble_calc)
       {
         if (rank == 0)
           std::cout << "Starting ensemble simulation" << std::endl;
-        run_ensemble<2, dealii::MemorySpace::Host>(communicator, database,
-                                                   timers);
+        if (memory_space == "device")
+        {
+          // TODO: Add device version of run_ensemble and call it here
+          adamantine::ASSERT_THROW(
+              false, "Error: Device version of ensemble simulations not "
+                     "yet implemented.");
+        }
+        else
+        {
+          run_ensemble<2, dealii::MemorySpace::Host>(communicator, database,
+                                                     timers);
+        }
       }
       else
       {
         if (rank == 0)
           std::cout << "Starting non-ensemble simulation" << std::endl;
-        run<2, dealii::MemorySpace::Host>(communicator, database, timers);
+
+        if (memory_space == "device")
+        {
+          run<2, dealii::MemorySpace::Default>(communicator, database, timers);
+        }
+        else
+        {
+          run<2, dealii::MemorySpace::Host>(communicator, database, timers);
+        }
       }
     }
     else
@@ -145,14 +172,33 @@ int main(int argc, char *argv[])
       {
         if (rank == 0)
           std::cout << "Starting ensemble simulation" << std::endl;
-        run_ensemble<3, dealii::MemorySpace::Host>(communicator, database,
-                                                   timers);
+
+        if (memory_space == "device")
+        {
+          // TODO: Add device version of run_ensemble and call it here
+          adamantine::ASSERT_THROW(
+              false, "Error: Device version of ensemble simulations not "
+                     "yet implemented.");
+        }
+        else
+        {
+          run_ensemble<3, dealii::MemorySpace::Host>(communicator, database,
+                                                     timers);
+        }
       }
       else
       {
         if (rank == 0)
           std::cout << "Starting non-ensemble simulation" << std::endl;
-        run<3, dealii::MemorySpace::Host>(communicator, database, timers);
+
+        if (memory_space == "device")
+        {
+          run<3, dealii::MemorySpace::Default>(communicator, database, timers);
+        }
+        else
+        {
+          run<3, dealii::MemorySpace::Host>(communicator, database, timers);
+        }
       }
     }
 

diff --git a/application/adamantine.cu b/application/adamantine.cu