Skip to content

Commit

Permalink
Improve memory efficiency for the SCPH + FE_BUBBLE
Browse files Browse the repository at this point in the history 
The code still requires lots of RAM. Therefore, increasing the number of
temperature points and the number of MPI threads simultaneously can lead
to an error.
  • Loading branch information
@ttadano
ttadano committed Apr 3, 2018
1 parent fe57e8e commit e56745d
Show file tree
Hide file tree
Showing 2 changed files with 60 additions and 41 deletions.
97 changes: 58 additions & 39 deletions anphon/scph.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -187,20 +187,19 @@ void Scph::exec_scph()
double Tmax = system->Tmax;
double dT = system->dT;

double ***eval_anharm;
std::complex<double> ****evec_anharm;
std::complex<double> ****delta_dymat_scph;
double ***eval_anharm = nullptr;
std::complex<double> ****evec_anharm = nullptr;
std::complex<double> ****delta_dymat_scph = nullptr;


unsigned int NT = static_cast<unsigned int>((Tmax - Tmin) / dT) + 1;
auto NT = static_cast<unsigned int>((Tmax - Tmin) / dT) + 1;

MPI_Bcast(&restart_scph, 1, MPI_LOGICAL, 0, MPI_COMM_WORLD);
MPI_Bcast(&selfenergy_offdiagonal, 1, MPI_LOGICAL, 0, MPI_COMM_WORLD);
MPI_Bcast(&ialgo, 1, MPI_UNSIGNED, 0, MPI_COMM_WORLD);

memory->allocate(delta_dymat_scph, NT, ns, ns, nk_interpolate);
memory->allocate(eval_anharm, NT, nk_ref, ns);
memory->allocate(evec_anharm, NT, nk_ref, ns, ns);
if (mympi->my_rank == 0) {
memory->allocate(delta_dymat_scph, NT, ns, ns, nk_interpolate);
}

if (restart_scph) {

Expand All @@ -218,18 +217,35 @@ void Scph::exec_scph()
}
}

for (unsigned int iT = 0; iT < NT; ++iT) {
exec_interpolation(delta_dymat_scph[iT],
eval_anharm[iT],
evec_anharm[iT]);
if (mympi->my_rank == 0) {
memory->allocate(eval_anharm, NT, nk_ref, ns);
memory->allocate(evec_anharm, NT, nk_ref, ns, ns); // This requires lots of RAM

for (auto iT = 0; iT < NT; ++iT) {
exec_interpolation(delta_dymat_scph[iT],
eval_anharm[iT],
evec_anharm[iT]);
}
}

if (delta_dymat_scph) {
memory->deallocate(delta_dymat_scph);
}

if (kpoint->kpoint_mode == 2) {
if (thermodynamics->calc_FE_bubble) {

if (mympi->my_rank > 0) {
memory->allocate(eval_anharm, NT, nk_ref, ns);
memory->allocate(evec_anharm, NT, nk_ref, ns, ns); // Memory intensive
}

MPI_Bcast(&eval_anharm[0][0][0], NT * nk_ref * ns, MPI_DOUBLE, 0, MPI_COMM_WORLD);
mpi_bcast_complex(evec_anharm, NT, nk_ref, ns);

thermodynamics->compute_free_energy_bubble_SCPH(eval_anharm,
evec_anharm);
}

}

if (mympi->my_rank == 0) {
Expand All @@ -248,9 +264,12 @@ void Scph::exec_scph()
}
}

memory->deallocate(eval_anharm);
memory->deallocate(evec_anharm);
memory->deallocate(delta_dymat_scph);
if (eval_anharm) {
memory->deallocate(eval_anharm);
}
if (evec_anharm) {
memory->deallocate(evec_anharm);
}
}


Expand Down Expand Up @@ -364,7 +383,7 @@ void Scph::load_scph_dymat_from_file(std::complex<double> ****dymat_out)
std::cout << " done." << std::endl;
}
// Broadcast to all MPI threads
mpi_bcast_complex(dymat_out, NT, nk_interpolate, ns);
// mpi_bcast_complex(dymat_out, NT, nk_interpolate, ns);
}

void Scph::store_scph_dymat_to_file(std::complex<double> ****dymat_in)
Expand Down Expand Up @@ -532,7 +551,7 @@ void Scph::exec_scph_main(std::complex<double> ****dymat_anharm)

}

mpi_bcast_complex(dymat_anharm, NT, nk_interpolate, ns);
// mpi_bcast_complex(dymat_anharm, NT, nk_interpolate, ns);

memory->deallocate(omega2_anharm);
memory->deallocate(v4_array_all);
Expand Down Expand Up @@ -3531,45 +3550,45 @@ void Scph::mpi_bcast_complex(std::complex<double> ****data,
const int nk,
const int ns)
{
#ifdef MPI_COMPLEX16
#ifdef MPI_COMPLEX16_M
MPI_Bcast(&data[0][0][0][0], NT * nk * ns * ns, MPI_COMPLEX16, 0, MPI_COMM_WORLD);
#else
unsigned int i, iT, ik, is, js;
double ****data_real, ****data_imag;
unsigned int iT, ik, is, js;
double ***data_real, ***data_imag;

memory->allocate(data_real, NT, ns, ns, nk);
memory->allocate(data_imag, NT, ns, ns, nk);
memory->allocate(data_real, ns, ns, nk);
memory->allocate(data_imag, ns, ns, nk);

if (mympi->my_rank == 0) {
for (iT = 0; iT < NT; ++iT) {
for (iT = 0; iT < NT; ++iT) {

if (mympi->my_rank == 0) {
for (is = 0; is < ns; ++is) {
for (js = 0; js < ns; ++js) {
for (ik = 0; ik < nk; ++ik) {

data_real[iT][is][js][ik] = data[iT][is][js][ik].real();
data_imag[iT][is][js][ik] = data[iT][is][js][ik].imag();
data_real[is][js][ik] = data[iT][is][js][ik].real();
data_imag[is][js][ik] = data[iT][is][js][ik].imag();
}
}
}
}
}

MPI_Bcast(&data_real[0][0][0][0], NT * nk * ns * ns, MPI_DOUBLE, 0, MPI_COMM_WORLD);
MPI_Bcast(&data_imag[0][0][0][0], NT * nk * ns * ns, MPI_DOUBLE, 0, MPI_COMM_WORLD);
MPI_Bcast(&data_real[0][0][0], nk * ns * ns, MPI_DOUBLE, 0, MPI_COMM_WORLD);
MPI_Bcast(&data_imag[0][0][0], nk * ns * ns, MPI_DOUBLE, 0, MPI_COMM_WORLD);

if (mympi->my_rank > 0) {
for (iT = 0; iT < NT; ++iT) {
for (is = 0; is < ns; ++is) {
for (js = 0; js < ns; ++js) {
for (ik = 0; ik < nk; ++ik) {
data[iT][is][js][ik]
= std::complex<double>(data_real[iT][is][js][ik],
data_imag[iT][is][js][ik]);
if (mympi->my_rank > 0) {
for (iT = 0; iT < NT; ++iT) {
for (is = 0; is < ns; ++is) {
for (js = 0; js < ns; ++js) {
for (ik = 0; ik < nk; ++ik) {
data[iT][is][js][ik]
= std::complex<double>(data_real[is][js][ik],
data_imag[is][js][ik]);
}
}
}
}
}

}

memory->deallocate(data_real);
Expand Down
4 changes: 2 additions & 2 deletions anphon/thermodynamics.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -646,8 +646,8 @@ void Thermodynamics::compute_FE_bubble_SCPH(double ***eval,
omega_sum[0] = 1.0 / (omega0 + omega1 + omega2);
omega_sum[1] = 1.0 / (-omega0 + omega1 + omega2);

v3_tmp = std::norm(anharmonic_core->V3(arr_cubic, eval[iT], evec[iT])) * static_cast<double>
(multi);
v3_tmp = std::norm(anharmonic_core->V3(arr_cubic, eval[iT], evec[iT]))
* static_cast<double>(multi);

temp = system->Tmin + static_cast<double>(iT) * system->dT;
n0 = thermodynamics->fB(omega0, temp);
Expand Down

0 comments on commit e56745d

Please sign in to comment.