Skip to content

Commit

Permalink
Bind MPI wrapper routines to the corresponding MPI data type
Browse files Browse the repository at this point in the history
  • Loading branch information
Frederick Stein authored and fstein93 committed Feb 6, 2023
1 parent 843d9fe commit 5c1b4f8
Show file tree
Hide file tree
Showing 329 changed files with 3,753 additions and 4,384 deletions.
5 changes: 2 additions & 3 deletions src/almo_scf.F
Original file line number Diff line number Diff line change
Expand Up @@ -81,8 +81,7 @@ MODULE almo_scf
USE kinds, ONLY: default_path_length,&
dp
USE mathlib, ONLY: binomial
USE message_passing, ONLY: mp_comm_type,&
mp_sum
USE message_passing, ONLY: mp_comm_type
USE molecule_types, ONLY: get_molecule_set_info,&
molecule_type
USE mscfg_types, ONLY: get_matrix_from_submatrices,&
Expand Down Expand Up @@ -2036,7 +2035,7 @@ SUBROUTINE nlmo_compactification(qs_env, almo_scf_env, matrix)
nfullrows_total=Nrows, &
nfullcols_total=Ncols)
CALL para_group%set_handle(para_group_handle)
CALL mp_sum(retained(ispin), para_group)
CALL para_group%sum(retained(ispin))
!devide by the total no. elements
occ(ispin) = retained(ispin)/Nrows/Ncols
Expand Down
11 changes: 5 additions & 6 deletions src/almo_scf_optimizer.F
Original file line number Diff line number Diff line change
Expand Up @@ -93,8 +93,7 @@ MODULE almo_scf_optimizer
USE kinds, ONLY: dp
USE machine, ONLY: m_flush,&
m_walltime
USE message_passing, ONLY: mp_comm_type,&
mp_sum
USE message_passing, ONLY: mp_comm_type
USE particle_methods, ONLY: get_particle_set
USE particle_types, ONLY: particle_type
USE qs_energy_types, ONLY: qs_energy_type
Expand Down Expand Up @@ -1342,7 +1341,7 @@ SUBROUTINE almo_scf_xalmo_pcg(qs_env, almo_scf_env, optimizer, quench_t, &

reim_diag = 0.0_dp
CALL dbcsr_get_diag(tempOccOcc(ispin), reim_diag)
CALL mp_sum(reim_diag, para_group)
CALL para_group%sum(reim_diag)
z2(:) = z2(:) + reim_diag(:)*reim_diag(:)

END DO
Expand Down Expand Up @@ -2433,7 +2432,7 @@ SUBROUTINE almo_scf_construct_nlmos(qs_env, optimizer, &
retain_sparsity=.TRUE.)
ALLOCATE (diagonal(nocc(ispin)))
CALL dbcsr_get_diag(m_sig_sqrti_ii(ispin), diagonal)
CALL mp_sum(diagonal, para_group)
CALL para_group%sum(diagonal)
! TODO: works for zero diagonal elements?
diagonal(:) = 1.0_dp/SQRT(diagonal(:))
CALL dbcsr_set(m_sig_sqrti_ii(ispin), 0.0_dp)
Expand Down Expand Up @@ -6527,7 +6526,7 @@ SUBROUTINE compute_obj_nlmos(localization_obj_function_ispin, penalty_func_ispin
reim_diag = 0.0_dp
CALL dbcsr_get_diag(tempOccOcc2, reim_diag)
CALL mp_sum(reim_diag, para_group)
CALL para_group%sum(reim_diag)
z2(:) = z2(:) + reim_diag(:)*reim_diag(:)
END DO
Expand Down Expand Up @@ -6660,7 +6659,7 @@ SUBROUTINE compute_gradient_nlmos(m_grad_out, m_B0, weights, &
CALL dbcsr_get_diag(m_temp_oo_4, tg_diagonal)
CALL dbcsr_set(m_temp_oo_4, 0.0_dp)
CALL dbcsr_set_diag(m_temp_oo_4, tg_diagonal)
!CALL mp_sum(tg_diagonal, para_group)
!CALL para_group%sum(tg_diagonal)
z2(:) = z2(:) + tg_diagonal(:)*tg_diagonal(:)
CALL dbcsr_multiply("N", "N", 1.0_dp, &
Expand Down
15 changes: 7 additions & 8 deletions src/almo_scf_qs.F
Original file line number Diff line number Diff line change
Expand Up @@ -50,8 +50,7 @@ MODULE almo_scf_qs
do_bondparm_covalent,&
do_bondparm_vdw
USE kinds, ONLY: dp
USE message_passing, ONLY: mp_allgather,&
mp_comm_type
USE message_passing, ONLY: mp_comm_type
USE molecule_types, ONLY: get_molecule_set_info,&
molecule_type
USE particle_types, ONLY: particle_type
Expand Down Expand Up @@ -980,7 +979,7 @@ SUBROUTINE almo_scf_construct_quencher(qs_env, almo_scf_env)

! third, communicate local length to the other nodes
ALLOCATE (list_length_cpu(nNodes), list_offset_cpu(nNodes))
CALL mp_allgather(2*local_list_length, list_length_cpu, group)
CALL group%allgather(2*local_list_length, list_length_cpu)

! fourth, create a global list
list_offset_cpu(1) = 0
Expand All @@ -992,8 +991,8 @@ SUBROUTINE almo_scf_construct_quencher(qs_env, almo_scf_env)

! fifth, communicate all list data
ALLOCATE (global_list(global_list_length))
CALL mp_allgather(local_list, global_list, &
list_length_cpu, list_offset_cpu, group)
CALL group%allgatherv(local_list, global_list, &
list_length_cpu, list_offset_cpu)
DEALLOCATE (list_length_cpu, list_offset_cpu)
DEALLOCATE (local_list)

Expand Down Expand Up @@ -1396,7 +1395,7 @@ SUBROUTINE almo_scf_construct_quencher(qs_env, almo_scf_env)

! first, communicate map sizes on the other nodes
ALLOCATE (domain_entries_cpu(nNodes), offset_for_cpu(nNodes))
CALL mp_allgather(2*domain_map_local_entries, domain_entries_cpu, group)
CALL group%allgather(2*domain_map_local_entries, domain_entries_cpu)

! second, create
offset_for_cpu(1) = 0
Expand All @@ -1413,8 +1412,8 @@ SUBROUTINE almo_scf_construct_quencher(qs_env, almo_scf_env)
domain_map_local(2*(ientry - 1) + 1) = almo_scf_env%domain_map(ispin)%pairs(ientry, 1)
domain_map_local(2*ientry) = almo_scf_env%domain_map(ispin)%pairs(ientry, 2)
END DO
CALL mp_allgather(domain_map_local, domain_map_global, &
domain_entries_cpu, offset_for_cpu, group)
CALL group%allgatherv(domain_map_local, domain_map_global, &
domain_entries_cpu, offset_for_cpu)
DEALLOCATE (domain_entries_cpu, offset_for_cpu)
DEALLOCATE (domain_map_local)

Expand Down
7 changes: 3 additions & 4 deletions src/arnoldi/arnoldi_api.F
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,7 @@ MODULE arnoldi_api
create_replicated_row_vec_from_matrix,&
dbcsr_matrix_colvec_multiply
USE kinds, ONLY: dp
USE message_passing, ONLY: mp_comm_type,&
mp_sum
USE message_passing, ONLY: mp_comm_type
#include "../base/base_uses.f90"

IMPLICIT NONE
Expand Down Expand Up @@ -369,7 +368,7 @@ SUBROUTINE arnoldi_conjugate_gradient(matrix_a, vec_x, matrix_p, converged, thre
END DO
CALL dbcsr_iterator_stop(dbcsr_iter)
control => get_control(my_arnoldi)
CALL mp_sum(vec_x, control%mp_group)
CALL control%mp_group%sum(vec_x)
! Deallocated the work vectors
CALL dbcsr_release(x)
CALL dbcsr_release(vectors%input_vec)
Expand Down Expand Up @@ -526,7 +525,7 @@ FUNCTION vec_dot_vec(avec, bvec, mpgrp) RESULT(adotb)
END IF
END DO
CALL dbcsr_iterator_stop(dbcsr_iter)
CALL mp_sum(adotb, mpgrp)
CALL mpgrp%sum(adotb)

END FUNCTION vec_dot_vec

Expand Down
33 changes: 16 additions & 17 deletions src/arnoldi/arnoldi_methods.F
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,7 @@ MODULE arnoldi_methods
USE dbcsr_vector, ONLY: dbcsr_matrix_colvec_multiply
USE kinds, ONLY: real_4, &
real_8
USE message_passing, ONLY: mp_bcast, &
mp_sum, mp_comm_type
USE message_passing, ONLY: mp_comm_type
#include "../base/base_uses.f90"

IMPLICIT NONE
Expand Down Expand Up @@ -417,7 +416,7 @@ SUBROUTINE arnoldi_init_${nametype1}$ (matrix, vectors, arnoldi_data)

! Put the projection into the Hessenberg matrix, and make the vectors orthonormal
ar_data%Hessenberg(1, 1) = DOT_PRODUCT(v_vec, w_vec)
CALL mp_sum(ar_data%Hessenberg(1, 1), pcol_group)
CALL pcol_group%sum(ar_data%Hessenberg(1, 1))
ar_data%f_vec = w_vec - v_vec*ar_data%Hessenberg(1, 1)

ar_data%local_history(:, 1) = v_vec(:)
Expand Down Expand Up @@ -563,7 +562,7 @@ SUBROUTINE build_subspace_${nametype1}$ (matrix, vectors, arnoldi_data)

! check convergence and inform everybody about it, a bit annoying to talk to everybody because of that
IF (control%myproc == 0) control%converged = rnorm .LT. REAL(control%threshold, ${type_prec}$)
CALL mp_bcast(control%converged, 0, control%mp_group)
CALL control%mp_group%bcast(control%converged, 0)
IF (control%converged) EXIT

! transfer normalized residdum to history and its norm to the Hessenberg matrix
Expand Down Expand Up @@ -603,7 +602,7 @@ SUBROUTINE build_subspace_${nametype1}$ (matrix, vectors, arnoldi_data)
ar_data%Hessenberg(control%current_step + 1, control%current_step) = norm
! broadcast the Hessenberg matrix so we don't need to care later on
CALL mp_bcast(ar_data%Hessenberg, 0, control%mp_group)
CALL control%mp_group%bcast(ar_data%Hessenberg, 0)

DEALLOCATE (v_vec, w_vec, h_vec, s_vec)

Expand Down Expand Up @@ -677,7 +676,7 @@ SUBROUTINE Gram_Schmidt_ortho_${nametype1}$ (h_vec, f_vec, s_vec, w_vec, nrow_lo
h_vec = ${nametype_zero}$; f_vec = ${nametype_zero}$; s_vec = ${nametype_zero}$
IF (local_comp) CALL ${nametype1}$gemv('T', nrow_local, j, ${nametype_one}$, local_history, &
nrow_local, w_vec, 1, ${nametype_zero}$, h_vec, 1)
CALL mp_sum(h_vec(1:j), pcol_group)
CALL pcol_group%sum(h_vec(1:j))
IF (local_comp) CALL ${nametype1}$gemv('N', nrow_local, j, ${nametype_one}$, reorth_mat, &
nrow_local, h_vec, 1, ${nametype_zero}$, f_vec, 1)
Expand Down Expand Up @@ -714,7 +713,7 @@ SUBROUTINE DGKS_ortho_${nametype1}$ (h_vec, f_vec, s_vec, nrow_local, j, &
IF (local_comp) CALL ${nametype1}$gemv('T', nrow_local, j, ${nametype_one}$, local_history, &
nrow_local, f_vec, 1, ${nametype_zero}$, s_vec, 1)
CALL mp_sum(s_vec(1:j), pcol_group)
CALL pcol_group%sum(s_vec(1:j))
IF (local_comp) CALL ${nametype1}$gemv('N', nrow_local, j, ${nametype_negone}$, reorth_mat, &
nrow_local, s_vec, 1, ${nametype_one}$, f_vec, 1)
h_vec(1:j) = h_vec(1:j) + s_vec(1:j)
Expand All @@ -739,7 +738,7 @@ SUBROUTINE compute_norms_${nametype1}$ (vec, norm, rnorm, pcol_group)
! the norm is computed along the processor column
norm = DOT_PRODUCT(vec, vec)
CALL mp_sum(norm, pcol_group)
CALL pcol_group%sum(norm)
rnorm = SQRT(REAL(norm, ${type_prec}$))
${rnorm_to_norm}$
Expand Down Expand Up @@ -816,7 +815,7 @@ SUBROUTINE gev_arnoldi_init_${nametype1}$ (matrix, matrix_arnoldi, vectors, arno
ar_data%rho_scale = ${nametype_zero}$
ar_data%rho_scale = DOT_PRODUCT(v_vec, w_vec)
CALL mp_sum(ar_data%rho_scale, pcol_group)
CALL pcol_group%sum(ar_data%rho_scale)
CALL dbcsr_matrix_colvec_multiply(matrix(2)%matrix, vectors%input_vec, vectors%result_vec, ${nametype_one}$, &
${nametype_zero}$, vectors%rep_row_vec, vectors%rep_col_vec)
Expand All @@ -825,9 +824,9 @@ SUBROUTINE gev_arnoldi_init_${nametype1}$ (matrix, matrix_arnoldi, vectors, arno
denom = ${nametype_zero}$
denom = DOT_PRODUCT(v_vec, w_vec)
CALL mp_sum(denom, pcol_group)
CALL pcol_group%sum(denom)
IF (control%myproc == 0) ar_data%rho_scale = ar_data%rho_scale/denom
CALL mp_bcast(ar_data%rho_scale, 0, control%mp_group)
CALL control%mp_group%bcast(ar_data%rho_scale, 0)
! if the maximum ev is requested we need to optimize with -A-rho*B
CALL dbcsr_copy(matrix_arnoldi(1)%matrix, matrix(1)%matrix)
Expand Down Expand Up @@ -879,7 +878,7 @@ SUBROUTINE gev_build_subspace_${nametype1}$ (matrix, vectors, arnoldi_data)
norm = ${nametype_zero}$
norm = DOT_PRODUCT(ar_data%x_vec, BZmat(:, 1))
CALL mp_sum(norm, pcol_group)
CALL pcol_group%sum(norm)
IF (control%local_comp) THEN
Zmat(:, 1) = ar_data%x_vec/SQRT(norm); BZmat(:, 1) = BZmat(:, 1)/SQRT(norm)
END IF
Expand Down Expand Up @@ -907,10 +906,10 @@ SUBROUTINE gev_build_subspace_${nametype1}$ (matrix, vectors, arnoldi_data)
CALL transfer_dbcsr_to_local_array_${nametype1}$ (vectors%result_vec, v_vec, nrow_local, control%local_comp)
norm = ${nametype_zero}$
norm = DOT_PRODUCT(ar_data%f_vec, v_vec)
CALL mp_sum(norm, pcol_group)
CALL pcol_group%sum(norm)
IF (control%myproc == 0) control%converged = REAL(norm, ${type_prec}$) .LT. EPSILON(REAL(1.0, ${type_prec}$))
CALL mp_bcast(control%converged, 0, control%mp_group)
CALL control%mp_group%bcast(control%converged, 0)
IF (control%converged) EXIT
IF (j == control%max_iter - 1) EXIT
Expand All @@ -927,7 +926,7 @@ SUBROUTINE gev_build_subspace_${nametype1}$ (matrix, vectors, arnoldi_data)
ar_data%Hessenberg(1:control%current_step, 1:control%current_step) = &
MATMUL(TRANSPOSE(CZmat(:, 1:control%current_step)), Zmat(:, 1:control%current_step))
END IF
CALL mp_sum(ar_data%Hessenberg, control%mp_group)
CALL control%mp_group%sum(ar_data%Hessenberg)
ar_data%local_history = Zmat
! broadcast the Hessenberg matrix so we don't need to care later on
Expand Down Expand Up @@ -1002,9 +1001,9 @@ SUBROUTINE gev_update_data_${nametype1}$ (matrix, matrix_arnoldi, vectors, arnol
DEALLOCATE (v_vec)
END IF
! and broadcast the real eigenvalue
CALL mp_bcast(control%converged, 0, control%mp_group)
CALL control%mp_group%bcast(control%converged, 0)
ind = control%selected_ind(1)
CALL mp_bcast(ar_data%rho_scale, 0, control%mp_group)
CALL control%mp_group%bcast(ar_data%rho_scale, 0)

! Again the maximum value request is done on -A therefore the eigenvalue needs the opposite sign
ar_data%evals(ind) = ar_data%rho_scale
Expand Down
14 changes: 6 additions & 8 deletions src/bse.F
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,7 @@ MODULE bse
USE group_dist_types, ONLY: get_group_dist,&
group_dist_d1_type
USE kinds, ONLY: dp
USE message_passing, ONLY: mp_alltoall,&
mp_request_type,&
mp_sum
USE message_passing, ONLY: mp_request_type
USE mp2_types, ONLY: integ_mat_buffer_type
USE parallel_gemm_api, ONLY: parallel_gemm
USE rpa_communication, ONLY: communicate_buffer
Expand Down Expand Up @@ -164,8 +162,8 @@ SUBROUTINE compute_BZ(BZ, Z_vectors, B_iaQ_bse_local, B_bar_iaQ_bse_local, &

END DO

! we make the mp_sum to sum over all RI basis functions
CALL mp_sum(BZ, para_env%group)
! we make the sum over all RI basis functions
CALL para_env%group%sum(BZ)

END SUBROUTINE

Expand Down Expand Up @@ -225,8 +223,8 @@ SUBROUTINE compute_AZ(AZ, Z_vectors, B_iaQ_bse_local, B_bar_ijQ_bse_local, B_abQ

END DO

! we make the mp_sum to sum over all RI basis functions
CALL mp_sum(AZ, para_env%group)
! we make the sum over all RI basis functions
CALL para_env%group%sum(AZ)

! add (e_a-e_i)*Z_ia
DO i_occ = 1, homo
Expand Down Expand Up @@ -550,7 +548,7 @@ SUBROUTINE allocate_and_fill_local_array(B_local, fm_mat_S, gd_array, &

END DO

CALL mp_alltoall(num_entries_send, num_entries_rec, 1, para_env%group)
CALL para_env%group%alltoall(num_entries_send, num_entries_rec, 1)

ALLOCATE (buffer_rec(0:para_env%num_pe - 1))
ALLOCATE (buffer_send(0:para_env%num_pe - 1))
Expand Down
8 changes: 3 additions & 5 deletions src/colvar_methods.F
Original file line number Diff line number Diff line change
Expand Up @@ -81,8 +81,6 @@ MODULE colvar_methods
twopi
USE mathlib, ONLY: vector_product
USE memory_utilities, ONLY: reallocate
USE message_passing, ONLY: mp_sum,&
mp_sync
USE mixed_energy_types, ONLY: mixed_force_type
USE mixed_environment_utils, ONLY: get_subsys_map_index
USE molecule_kind_types, ONLY: fixd_constraint_type
Expand Down Expand Up @@ -2212,8 +2210,8 @@ SUBROUTINE u_colvar(colvar, force_env)
END DO
! Handling Parallel execution
CALL mp_sync(force_env%para_env%group)
CALL mp_sum(glob_natoms, force_env%para_env%group)
CALL force_env%para_env%group%sync()
CALL force_env%para_env%group%sum(glob_natoms)
! Transfer forces
DO iforce_eval = 1, nforce_eval
Expand All @@ -2228,7 +2226,7 @@ SUBROUTINE u_colvar(colvar, force_env)
END DO
END IF
END IF
CALL mp_sum(global_forces(iforce_eval)%forces, force_env%para_env%group)
CALL force_env%para_env%group%sum(global_forces(iforce_eval)%forces)
END DO
wrk_section => colvar%u_param%mixed_energy_section
Expand Down
7 changes: 3 additions & 4 deletions src/colvar_utils.F
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@ MODULE colvar_utils
USE kinds, ONLY: dp
USE mathlib, ONLY: invert_matrix
USE memory_utilities, ONLY: reallocate
USE message_passing, ONLY: mp_sum
USE molecule_kind_list_types, ONLY: molecule_kind_list_type
USE molecule_kind_types, ONLY: colvar_constraint_type,&
fixd_constraint_type,&
Expand Down Expand Up @@ -262,9 +261,9 @@ SUBROUTINE eval_colvar(force_env, coords, cvalues, Bmatrix, MassI, Amatrix)
END IF
END DO
END DO MOL
CALL mp_sum(n_tot, force_env%para_env%group)
CALL mp_sum(cvalues, force_env%para_env%group)
IF (PRESENT(Bmatrix)) CALL mp_sum(Bmatrix, force_env%para_env%group)
CALL force_env%para_env%group%sum(n_tot)
CALL force_env%para_env%group%sum(cvalues)
IF (PRESENT(Bmatrix)) CALL force_env%para_env%group%sum(Bmatrix)
END IF
offset = n_tot
! Intermolecular Colvars
Expand Down
3 changes: 1 addition & 2 deletions src/common/cp_log_handling.F
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,6 @@ MODULE cp_log_handling
USE machine, ONLY: default_output_unit,&
m_getpid,&
m_hostnm
USE message_passing, ONLY: mp_comm_free
USE string_utilities, ONLY: compress
USE timings, ONLY: print_stack
#include "../base/base_uses.f90"
Expand Down Expand Up @@ -689,7 +688,7 @@ SUBROUTINE my_cp_para_env_release(para_env)
para_env%ref_count = para_env%ref_count - 1
IF (para_env%ref_count < 1) THEN
IF (para_env%owns_group) THEN
CALL mp_comm_free(para_env%group)
CALL para_env%group%free()
END IF
DEALLOCATE (para_env)
END IF
Expand Down

0 comments on commit 5c1b4f8

Please sign in to comment.