Fix Flops rate for output and enable output flush (#1931)

cp2k · Feb 7, 2022 · 7d5a36e · 7d5a36e
1 parent e974820
commit 7d5a36e
Show file tree

Hide file tree

Showing 4 changed files with 29 additions and 14 deletions.
diff --git a/src/mp2_ri_gpw.F b/src/mp2_ri_gpw.F
@@ -139,7 +139,7 @@ SUBROUTINE mp2_ri_gpw_compute_en(Emp2_Cou, Emp2_EX, Emp2_S, Emp2_T, BIb_C, mp2_e
 
       CALL get_group_dist(gd_array, color_sub, my_group_L_start, my_group_L_end, my_group_L_size)
 
-      CALL dgemm_counter_init(dgemm_counter)
+      CALL dgemm_counter_init(dgemm_counter, unit_nr)
 
       ! We cannot fix the tag because of the recv routine
       tag = 42
@@ -742,7 +742,7 @@ SUBROUTINE mp2_ri_gpw_compute_en(Emp2_Cou, Emp2_EX, Emp2_S, Emp2_T, BIb_C, mp2_e
       IF (calc_forces .AND. .NOT. compare_potential_types(mp2_env%ri_metric, mp2_env%potential_parameter)) &
          DEALLOCATE (mp2_env%ri_grad%operator_half)
 
-      CALL dgemm_counter_write(dgemm_counter, para_env, unit_nr)
+      CALL dgemm_counter_write(dgemm_counter, para_env)
 
       CALL timestop(handle)
 

diff --git a/src/mp2_types.F b/src/mp2_types.F
@@ -272,6 +272,7 @@ MODULE mp2_types
       PRIVATE
       REAL(KIND=dp) :: flop_rate, t_start
       INTEGER :: num_dgemm_call
+      INTEGER :: unit_nr
    END TYPE
 
 CONTAINS
@@ -331,12 +332,15 @@ END SUBROUTINE mp2_env_create
 ! **************************************************************************************************
 !> \brief Initialize a dgemm_counter
 !> \param dgemm_counter ...
+!> \param unit_nr ...
 ! **************************************************************************************************
-   ELEMENTAL SUBROUTINE dgemm_counter_init(dgemm_counter)
+   ELEMENTAL SUBROUTINE dgemm_counter_init(dgemm_counter, unit_nr)
       TYPE(dgemm_counter_type), INTENT(OUT)              :: dgemm_counter
+      INTEGER, INTENT(IN)                                :: unit_nr
 
       dgemm_counter%num_dgemm_call = 0
       dgemm_counter%flop_rate = 0.0
+      dgemm_counter%unit_nr = unit_nr
 
    END SUBROUTINE dgemm_counter_init
 
@@ -366,27 +370,34 @@ SUBROUTINE dgemm_counter_stop(dgemm_counter, size1, size2, size3)
 
       t_end = m_walltime()
       flop_rate = 2.0_dp*REAL(size1, dp)*REAL(size2, dp)*REAL(size3, dp)/MAX(0.001_dp, t_end - dgemm_counter%t_start)
-      dgemm_counter%flop_rate = dgemm_counter%flop_rate + flop_rate
       dgemm_counter%num_dgemm_call = dgemm_counter%num_dgemm_call + 1
+      IF (dgemm_counter%unit_nr > 0) THEN
+         WRITE (UNIT=dgemm_counter%unit_nr, FMT="(T3,A,I10)") &
+            "PERFORMANCE| DGEMM call #", dgemm_counter%num_dgemm_call
+         WRITE (UNIT=dgemm_counter%unit_nr, FMT="(T3,A,I12,A,I12,A,I12)") &
+            "PERFORMANCE| DGEMM size (M x N x K) = ", size1, " x ", size2, " x ", size3
+         WRITE (UNIT=dgemm_counter%unit_nr, FMT="(T3,A,F15.5)") &
+            "PERFORMANCE| DGEMM time (s) = ", t_end - dgemm_counter%t_start
+      END IF
+      dgemm_counter%flop_rate = dgemm_counter%flop_rate + flop_rate
 
    END SUBROUTINE dgemm_counter_stop
 
 ! **************************************************************************************************
 !> \brief calculate and print flop rates
 !> \param dgemm_counter ...
 !> \param para_env ...
-!> \param unit_nr ...
 ! **************************************************************************************************
-   SUBROUTINE dgemm_counter_write(dgemm_counter, para_env, unit_nr)
+   SUBROUTINE dgemm_counter_write(dgemm_counter, para_env)
       TYPE(dgemm_counter_type), INTENT(INOUT)            :: dgemm_counter
       TYPE(cp_para_env_type), INTENT(IN), POINTER        :: para_env
-      INTEGER, INTENT(IN)                                :: unit_nr
 
       dgemm_counter%flop_rate = dgemm_counter%flop_rate/REAL(MAX(dgemm_counter%num_dgemm_call, 1), dp)/1.0E9_dp
+      ! Average over all ranks
       CALL mp_sum(dgemm_counter%flop_rate, para_env%group)
-      dgemm_counter%flop_rate = dgemm_counter%flop_rate/REAL(para_env%num_pe, dp)
-      IF (unit_nr > 0) WRITE (UNIT=unit_nr, FMT="(T3,A,T66,F15.2)") &
-         "PERFORMANCE| DGEMM flop rate (Gflops / MPI rank):", dgemm_counter%flop_rate
+      dgemm_counter%flop_rate = dgemm_counter%flop_rate/(REAL(para_env%num_pe, dp)*REAL(para_env%num_pe, dp))
+      IF (dgemm_counter%unit_nr > 0) WRITE (UNIT=dgemm_counter%unit_nr, FMT="(T3,A,T66,F15.2)") &
+         "PERFORMANCE| Average DGEMM flop rate (Gflops / MPI rank):", dgemm_counter%flop_rate
 
    END SUBROUTINE dgemm_counter_write
 

diff --git a/src/rpa_main.F b/src/rpa_main.F
@@ -61,7 +61,8 @@ MODULE rpa_main
                                               z_zero
    USE message_passing,                 ONLY: mp_min,&
                                               mp_sendrecv,&
-                                              mp_sum
+                                              mp_sum,&
+                                              mp_sync
    USE minimax_exp,                     ONLY: check_exp_minimax_range
    USE mp2_grids,                       ONLY: get_clenshaw_grid,&
                                               get_minimax_grid
@@ -1579,7 +1580,7 @@ SUBROUTINE rpa_num_int(qs_env, Erpa, mp2_env, para_env, para_env_RPA, para_env_s
       IF (mp2_env%ri_rpa%do_ri_axk) e_axk = 0.0_dp
       first_cycle = .TRUE.
       omega_old = 0.0_dp
-      CALL dgemm_counter_init(dgemm_counter)
+      CALL dgemm_counter_init(dgemm_counter, unit_nr)
 
       DO count_ev_sc_GW = 1, iter_evGW
 
@@ -1787,6 +1788,9 @@ SUBROUTINE rpa_num_int(qs_env, Erpa, mp2_env, para_env, para_env_RPA, para_env_s
                                          kpoints, qs_env, mp2_env, do_kpoints_cubic_RPA, do_kpoints_from_Gamma)
             END IF
 
+            IF (unit_nr > 0) CALL m_flush(unit_nr)
+            CALL mp_sync(para_env%group) ! sync to see output
+
          END DO ! jquad
 
          CALL mp_sum(Erpa, para_env%group)
@@ -1815,7 +1819,7 @@ SUBROUTINE rpa_num_int(qs_env, Erpa, mp2_env, para_env, para_env_RPA, para_env_s
 
          ELSE
 
-            CALL dgemm_counter_write(dgemm_counter, para_env, unit_nr)
+            CALL dgemm_counter_write(dgemm_counter, para_env)
 
          END IF
 

diff --git a/src/rpa_util.F b/src/rpa_util.F
@@ -773,7 +773,7 @@ SUBROUTINE contract_S_to_Q(mm_style, dimen_RI, dimen_ia, alpha, fm_mat_S, fm_mat
       CASE DEFAULT
          CPABORT("")
       END SELECT
-      CALL dgemm_counter_stop(dgemm_counter, dimen_ia, dimen_ia, dimen_RI)
+      CALL dgemm_counter_stop(dgemm_counter, dimen_RI, dimen_RI, dimen_ia)
 
       ! copy/redistribute fm_mat_Q_gemm to fm_mat_Q
       CALL cp_fm_set_all(matrix=fm_mat_Q, alpha=0.0_dp)