Merge pull request #119 from titoiride/fix_mpi

Change cmake and MPI
ALaDyn · Nov 27, 2023 · 612d8cd · 612d8cd
2 parents b73c4a2 + a5c7ef9
commit 612d8cd
Show file tree

Hide file tree

Showing 19 changed files with 331 additions and 340 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
@@ -0,0 +1,12 @@
+add_subdirectory(cpp_lib)
+add_subdirectory(diagnostics)
+add_subdirectory(dynamics)
+add_subdirectory(fft)
+add_subdirectory(fields)
+add_subdirectory(grid)
+add_subdirectory(ionization)
+add_subdirectory(IO)
+add_subdirectory(parallel)
+add_subdirectory(particles)
+add_subdirectory(start)
+add_subdirectory(work)
diff --git a/src/IO/CMakeLists.txt b/src/IO/CMakeLists.txt
@@ -0,0 +1,9 @@
+target_sources(
+    ALaDyn
+    PRIVATE init_beam_part_distrib.f90
+            init_laser_field.f90
+            init_part_distrib.f90
+            pic_in.f90
+            pic_out_util.f90
+            pic_out.f90
+            psolve.f90)
diff --git a/src/cpp_lib/CMakeLists.txt b/src/cpp_lib/CMakeLists.txt
@@ -0,0 +1,23 @@
+add_library(cpp_folder_tree STATIC cpp_folder_tree.cpp)
+add_library(find_last_addr STATIC find_last_addr.cpp)
+add_library(enable_gdb_attach STATIC enable_gdb_attach.cpp)
+
+target_link_libraries(ALaDyn PRIVATE cpp_folder_tree find_last_addr
+                                     enable_gdb_attach)
+
+option(USE_GDB_ATTACH "Enable GDB attach to debug while running" OFF)
+
+if(NOT WIN32 AND USE_GDB_ATTACH)
+    target_compile_definitions(enable_gdb_attach PUBLIC USE_GDB_ATTACH)
+endif()
+
+if(NOT WIN32)
+    find_package(Boost COMPONENTS system filesystem)
+    if(Boost_FOUND)
+        target_compile_definitions(cpp_folder_tree PUBLIC USE_BOOST)
+        target_include_directories(cpp_folder_tree PUBLIC ${Boost_INCLUDE_DIR})
+        target_link_libraries(cpp_folder_tree PUBLIC ${Boost_LIBRARIES})
+    else()
+        target_compile_definitions(cpp_folder_tree PUBLIC USE_FILESYSTEM)
+    endif()
+endif()
diff --git a/src/diagnostics/CMakeLists.txt b/src/diagnostics/CMakeLists.txt
@@ -0,0 +1 @@
+target_sources(ALaDyn PRIVATE diag_part_and_fields.f90 run_data_info.f90)
diff --git a/src/dynamics/CMakeLists.txt b/src/dynamics/CMakeLists.txt
@@ -0,0 +1,4 @@
+target_sources(
+    ALaDyn
+    PRIVATE boris_push.f90 curr_and_fields_util.f90 env_evolve_in_time.f90
+            fluid_density_momenta.f90 pic_evolve_in_time.f90 window.f90)
diff --git a/src/fft/CMakeLists.txt b/src/fft/CMakeLists.txt
@@ -0,0 +1 @@
+target_sources(ALaDyn PRIVATE legacy_fft_lib.F90 modern_fft_lib.F90 prl_fft.f90)
diff --git a/src/fft/prl_fft.f90 b/src/fft/prl_fft.f90
@@ -124,15 +124,15 @@ subroutine ft_overset_grid(w_s, w_r, nft1, nft2, nft3)
      do ip = 1, jl
       tag = 200 + ip
       pes = yp_prev(ip)
-      call mpi_send(w_s(1, 1, 1), lenw, mpi_sd, pes, tag, comm_col(dd), error)
+      call mpi_send(w_s(1, 1, 1), lenw, mpi_double_precision, pes, tag, comm_col(dd), error)
      end do
     endif
     j1 = loc_yft_ord(imody)
     do ip = 1, jr
      per = yp_next(ip)
      tag = 200 + ip
      call mpi_recv(fp1(1, 1, 1), lenw, &
-                   mpi_sd, per, tag, comm_col(dd), status, error)
+                   mpi_double_precision, per, tag, comm_col(dd), status, error)
      j1 = j1 + nft2
      j2 = j1 + nft2 - 1
      w_r(1:nft1, j1:j2, k1:k2) = fp1(1:nft1, 1:nft2, 1:nft3)
@@ -148,14 +148,14 @@ subroutine ft_overset_grid(w_s, w_r, nft1, nft2, nft3)
     do ip = 1, ybd
      tag = 200 - ip
      pes = yp_next(ip)
-     call mpi_send(w_s(1, 1, 1), lenw, mpi_sd, pes, tag, comm_col(dd), error)
+     call mpi_send(w_s(1, 1, 1), lenw, mpi_double_precision, pes, tag, comm_col(dd), error)
     end do
     j1 = loc_yft_ord(imody)
     do ip = 1, jjr
      tag = 200 - ip
      per = yp_prev(ip)
      call mpi_recv(fp1(1, 1, 1), lenw, &
-                   mpi_sd, per, tag, comm_col(dd), status, error)
+                   mpi_double_precision, per, tag, comm_col(dd), status, error)
      j1 = j1 - nft2
      j2 = j1 + nft2 - 1
      w_r(1:nft1, j1:j2, k1:k2) = fp1(1:nft1, 1:nft2, 1:nft3)
@@ -176,15 +176,15 @@ subroutine ft_overset_grid(w_s, w_r, nft1, nft2, nft3)
       do ip = 1, kl
        tag = 100 + ip
        pes = zp_prev(ip)
-       call mpi_send(w_s(1, 1, 1), lenw, mpi_sd, pes, tag, comm_col(dd), error)
+       call mpi_send(w_s(1, 1, 1), lenw, mpi_double_precision, pes, tag, comm_col(dd), error)
       end do
      endif
      k1 = loc_zft_ord(imodz)
      do ip = 1, kr
       tag = 100 + ip
       per = zp_next(ip)
       call mpi_recv(fp1(1, 1, 1), lenw, &
-                    mpi_sd, per, tag, comm_col(dd), status, error)
+                    mpi_double_precision, per, tag, comm_col(dd), status, error)
       k1 = k1 + nft3
       k2 = k1 + nft3 - 1
       w_r(1:nft1, j1:j2, k1:k2) = fp1(1:nft1, 1:nft2, 1:nft3)
@@ -197,14 +197,14 @@ subroutine ft_overset_grid(w_s, w_r, nft1, nft2, nft3)
      do ip = 1, zbd
       tag = 100 - ip
       pes = zp_next(ip)
-      call mpi_send(w_s(1, 1, 1), lenw, mpi_sd, pes, tag, comm_col(dd), error)
+      call mpi_send(w_s(1, 1, 1), lenw, mpi_double_precision, pes, tag, comm_col(dd), error)
      end do
      k1 = loc_zft_ord(imodz)
      do ip = 1, kkr
       per = zp_prev(ip)
       tag = 100 - ip
       call mpi_recv(fp1(1, 1, 1), lenw, &
-                    mpi_sd, per, tag, comm_col(dd), status, error)
+                    mpi_double_precision, per, tag, comm_col(dd), status, error)
       k1 = k1 - nft3
       k2 = k1 + nft3 - 1
       w_r(1:nft1, j1:j2, k1:k2) = fp1(1:nft1, 1:nft2, 1:nft3)
@@ -258,8 +258,8 @@ subroutine swap_yx_3data(waux, wdata, n1_loc, n2, n3)
       end do
      end do
     end do
-    call mpi_sendrecv(faux1(1), lenws, mpi_sd, pes, tag, faux2(1), &
-                      lenwr, mpi_sd, per, tag, comm_col(3), status, error)
+    call mpi_sendrecv(faux1(1), lenws, mpi_double_precision, pes, tag, faux2(1), &
+                      lenwr, mpi_double_precision, per, tag, comm_col(3), status, error)
     j1 = n2_xloc*per
     kk = 0
     do iz = 1, n3
@@ -318,8 +318,8 @@ subroutine swap_xy_3data(wp1, wp2, n1_loc, n2_loc, n3_loc)
       end do
      end do
     end do
-    call mpi_sendrecv(faux1(1), lenws, mpi_sd, pes, tag, faux2(1), &
-                      lenwr, mpi_sd, per, tag, comm_col(1), status, error)
+    call mpi_sendrecv(faux1(1), lenws, mpi_double_precision, pes, tag, faux2(1), &
+                      lenwr, mpi_double_precision, per, tag, comm_col(1), status, error)
     i1 = n1_loc*per
     kk = 0
     do iz = 1, n3_loc
@@ -376,8 +376,8 @@ subroutine swap_xz_3data(wp1, wp2, n1_loc, n2_loc, n3_loc)
       end do
      end do
     end do
-    call mpi_sendrecv(faux1(1), lenw, mpi_sd, pes, tag, faux2(1), lenw, &
-                      mpi_sd, per, tag, comm_col(2), status, error)
+    call mpi_sendrecv(faux1(1), lenw, mpi_double_precision, pes, tag, faux2(1), lenw, &
+                      mpi_double_precision, per, tag, comm_col(2), status, error)
     i1 = n1_loc*per
     kk = 0
     do iz = 1, n3_loc
@@ -443,8 +443,8 @@ subroutine swap_yx_3data_inv(wdata, waux, n1_loc, n2, n3)
     lenws = kk
     lenwr = lenws
 
-    call mpi_sendrecv(faux1(1), lenws, mpi_sd, pes, tag, faux2(1), &
-                      lenwr, mpi_sd, per, tag, comm_col(3), status, error)
+    call mpi_sendrecv(faux1(1), lenws, mpi_double_precision, pes, tag, faux2(1), &
+                      lenwr, mpi_double_precision, per, tag, comm_col(3), status, error)
     i1 = n1_loc*per
     kk = 0
     do iz = 1, n3
@@ -506,8 +506,8 @@ subroutine swap_xy_3data_inv(wp2, wp1, n1_loc, n2_loc, n3_loc)
     lenws = kk
     lenwr = lenws
 
-    call mpi_sendrecv(faux1(1), lenws, mpi_sd, pes, tag, faux2(1), &
-                      lenwr, mpi_sd, per, tag, comm_col(1), status, error)
+    call mpi_sendrecv(faux1(1), lenws, mpi_double_precision, pes, tag, faux2(1), &
+                      lenwr, mpi_double_precision, per, tag, comm_col(1), status, error)
     j1 = n2_loc*per
     kk = 0
     do iz = 1, n3_loc
@@ -564,8 +564,8 @@ subroutine swap_xz_3data_inv(wp2, wp1, n1_loc, n2_loc, n3_loc)
      end do
     end do
 
-    call mpi_sendrecv(faux1(1), lenw, mpi_sd, pes, tag, faux2(1), lenw, &
-                      mpi_sd, per, tag, comm_col(2), status, error)
+    call mpi_sendrecv(faux1(1), lenw, mpi_double_precision, pes, tag, faux2(1), lenw, &
+                      mpi_double_precision, per, tag, comm_col(2), status, error)
     k1 = n3_loc*per
     kk = 0
     do iz = 1, n3_loc

diff --git a/src/fields/CMakeLists.txt b/src/fields/CMakeLists.txt
@@ -0,0 +1,2 @@
+target_sources(ALaDyn PRIVATE grid_field_param.f90 grid_fields.f90
+                              init_grid_fields.f90)
diff --git a/src/grid/CMakeLists.txt b/src/grid/CMakeLists.txt
@@ -0,0 +1,2 @@
+target_sources(ALaDyn PRIVATE grid_param.f90 set_grid_param.f90
+                              stretched_grid.f90)
diff --git a/src/ionization/CMakeLists.txt b/src/ionization/CMakeLists.txt
@@ -0,0 +1 @@
+target_sources(ALaDyn PRIVATE ionize.f90 ionz_data.f90)
diff --git a/src/parallel/CMakeLists.txt b/src/parallel/CMakeLists.txt
@@ -0,0 +1,28 @@
+target_sources(ALaDyn PRIVATE mpi_var.F90)
+
+if(WIN32 AND CMAKE_CXX_COMPILER_ID STREQUAL "PGI")
+    find_package(MPI REQUIRED COMPONENTS C)
+    if(MPI_FOUND)
+        set(FORCE_OLD_MPI
+            ON
+            CACHE BOOL "Forcing old mpif.h linking" FORCE)
+        message(STATUS "ALaDyn: MPI libraries are linked against mpif.h")
+        target_compile_definitions(ALaDyn PRIVATE FORCE_OLD_MPI)
+        target_include_directories(ALaDyn PRIVATE ${MPI_INCLUDE_PATH})
+        target_link_libraries(ALaDyn PRIVATE ${MPI_LIBRARIES})
+    endif()
+else()
+    find_package(MPI REQUIRED COMPONENTS Fortran)
+    if(MPI_FOUND)
+        target_include_directories(ALaDyn PRIVATE ${MPI_INCLUDE_PATH}
+                                                  ${MPI_Fortran_INCLUDE_PATH})
+        target_link_libraries(ALaDyn PRIVATE ${MPI_LIBRARIES})
+    endif()
+endif()
+
+if(MPI_FOUND)
+    target_sources(
+        ALaDyn PRIVATE mpi_curr_interface.f90 mpi_field_interface.f90
+                       mpi_part_interface.f90 parallel.F90)
+    target_compile_definitions(ALaDyn PRIVATE ALaDyn_USE_MPI)
+endif()
diff --git a/src/parallel/mpi_var.f90 → src/parallel/mpi_var.F90 b/src/parallel/mpi_var.f90 → src/parallel/mpi_var.F90
@@ -22,7 +22,17 @@
  module mpi_var
 
   use precision_def
+
+#if defined (ALaDyn_USE_MPI) && ! defined (FORCE_OLD_MPI)
+  use mpi_f08, only: MPI_Comm, MPI_Datatype
   implicit none
+  type(MPI_Comm) :: comm, comm_col(3)
+  type(MPI_Datatype) :: partype
+#else
+  implicit none
+  integer :: comm, partype, comm_col(3)
+#endif
+
 
   integer, allocatable :: loc_npart(:, :, :, :), loc_nbpart(:, :, :, :)
   integer, allocatable :: loc_ne_ionz(:, :, :), loc_tpart(:)
@@ -34,13 +44,12 @@ module mpi_var
   integer :: mype, imodx, imody, imodz, npe, npe_yloc, npe_zloc, &
              npe_xloc
   integer :: npe_yz, mpi_size, mpi_rank
-  integer :: partype
   integer :: imodzx, imodyz, imodyx
   integer :: pe_min, pe_max
   integer :: pe_min_y, pe_max_y, pe_min_z, pe_max_z, pe_min_x, pe_max_x
   integer :: ndims, dims(3)
   logical :: pe0y, pe0z, pe1y, pe1z, pe0, pe1, prl, prlx, prly, prlz
   logical :: xl_bd, yl_bd, zl_bd, xr_bd, yr_bd, zr_bd
   logical :: pe0x, pe1x, pex0, pex1
-  integer :: comm, coor(3), comm_col(3), col_or(3)
+  integer :: coor(3), col_or(3)
  end module