PID 44025 #0 0x00002aaac1cf65c6 in MPIDI_CH3I_Progress () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #1 0x00002aaac1de5657 in MPIC_Wait () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #2 0x00002aaac1de5c9c in MPIC_Recv () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #3 0x00002aaac1dde963 in MPIC_Recv_ft () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #4 0x00002aaac1ccfff2 in MPIR_Allreduce_intra () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #5 0x00002aaac1cd2acd in MPIR_Allreduce_impl () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #6 0x00002aaac1cd23c2 in PMPI_Allreduce () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #7 0x00002aaabc84693e in Epetra_MpiComm::MinAll(int*, int*, int) const () from /home/s/spc29/trilinos/12.0.1/lib/libepetra.so.12 #8 0x00002aaabc785f52 in Epetra_BlockMap::SameAs(Epetra_BlockMap const&) const () from /home/s/spc29/trilinos/12.0.1/lib/libepetra.so.12 #9 0x00000000009fda46 in dealii::TrilinosWrappers::PreconditionBase::vmult(dealii::TrilinosWrappers::VectorBase&, dealii::TrilinosWrappers::VectorBase const&) const () at /home/s/spc29/dealii-git/include/deal.II/lac/trilinos_precondition.h:1861 #10 0x0000000000aea2c6 in void dealii::SolverGMRES::solve(dealii::TrilinosWrappers::SparseMatrix const&, dealii::TrilinosWrappers::MPI::Vector&, dealii::TrilinosWrappers::MPI::Vector const&, dealii::TrilinosWrappers::PreconditionILU const&) () at /home/s/spc29/dealii-git/include/deal.II/lac/solver_gmres.h:911 #11 0x0000000000aebb57 in aspect::Simulator<2>::solve_advection(aspect::Simulator<2>::AdvectionField const&) () at /home/s/spc29/git/aspect/source/simulator/solver.cc:427 #12 0x0000000000a6f767 in aspect::Simulator<2>::solve_timestep() () at /home/s/spc29/git/aspect/source/simulator/core.cc:1681 #13 0x0000000000a72935 in aspect::Simulator<2>::run() () at /home/s/spc29/git/aspect/source/simulator/core.cc:2065 #14 0x0000000000828b17 in main () at /home/s/spc29/git/aspect/source/main.cc:506 PID 44026 #0 0x00002aaac1cfc3fd in ?? () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #1 0x00002aaac1cf6641 in MPIDI_CH3I_Progress () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #2 0x00002aaac1de5657 in MPIC_Wait () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #3 0x00002aaac1de53b3 in MPIC_Sendrecv () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #4 0x00002aaac1ddecf3 in MPIC_Sendrecv_ft () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #5 0x00002aaac1ccf28b in MPIR_Allreduce_intra () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #6 0x00002aaac1cd2acd in MPIR_Allreduce_impl () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #7 0x00002aaac1cd23c2 in PMPI_Allreduce () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #8 0x00002aaabc84693e in Epetra_MpiComm::MinAll(int*, int*, int) const () from /home/s/spc29/trilinos/12.0.1/lib/libepetra.so.12 #9 0x00002aaabc785f52 in Epetra_BlockMap::SameAs(Epetra_BlockMap const&) const () from /home/s/spc29/trilinos/12.0.1/lib/libepetra.so.12 #10 0x00000000009fda46 in dealii::TrilinosWrappers::PreconditionBase::vmult(dealii::TrilinosWrappers::VectorBase&, dealii::TrilinosWrappers::VectorBase const&) const () at /home/s/spc29/dealii-git/include/deal.II/lac/trilinos_precondition.h:1861 #11 0x0000000000aea2c6 in void dealii::SolverGMRES::solve(dealii::TrilinosWrappers::SparseMatrix const&, dealii::TrilinosWrappers::MPI::Vector&, dealii::TrilinosWrappers::MPI::Vector const&, dealii::TrilinosWrappers::PreconditionILU const&) () at /home/s/spc29/dealii-git/include/deal.II/lac/solver_gmres.h:911 #12 0x0000000000aebb57 in aspect::Simulator<2>::solve_advection(aspect::Simulator<2>::AdvectionField const&) () at /home/s/spc29/git/aspect/source/simulator/solver.cc:427 #13 0x0000000000a6f767 in aspect::Simulator<2>::solve_timestep() () at /home/s/spc29/git/aspect/source/simulator/core.cc:1681 #14 0x0000000000a72935 in aspect::Simulator<2>::run() () at /home/s/spc29/git/aspect/source/simulator/core.cc:2065 #15 0x0000000000828b17 in main () at /home/s/spc29/git/aspect/source/main.cc:506 PID 44027 #0 0x00002aaac1cf65e4 in MPIDI_CH3I_Progress () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #1 0x00002aaac1de5657 in MPIC_Wait () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #2 0x00002aaac1de53b3 in MPIC_Sendrecv () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #3 0x00002aaac1ddecf3 in MPIC_Sendrecv_ft () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #4 0x00002aaac1ccf28b in MPIR_Allreduce_intra () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #5 0x00002aaac1cd2acd in MPIR_Allreduce_impl () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #6 0x00002aaac1cd23c2 in PMPI_Allreduce () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #7 0x00002aaabc84693e in Epetra_MpiComm::MinAll(int*, int*, int) const () from /home/s/spc29/trilinos/12.0.1/lib/libepetra.so.12 #8 0x00002aaabc785f52 in Epetra_BlockMap::SameAs(Epetra_BlockMap const&) const () from /home/s/spc29/trilinos/12.0.1/lib/libepetra.so.12 #9 0x00000000009fda46 in dealii::TrilinosWrappers::PreconditionBase::vmult(dealii::TrilinosWrappers::VectorBase&, dealii::TrilinosWrappers::VectorBase const&) const () at /home/s/spc29/dealii-git/include/deal.II/lac/trilinos_precondition.h:1861 #10 0x0000000000aea2c6 in void dealii::SolverGMRES::solve(dealii::TrilinosWrappers::SparseMatrix const&, dealii::TrilinosWrappers::MPI::Vector&, dealii::TrilinosWrappers::MPI::Vector const&, dealii::TrilinosWrappers::PreconditionILU const&) () at /home/s/spc29/dealii-git/include/deal.II/lac/solver_gmres.h:911 #11 0x0000000000aebb57 in aspect::Simulator<2>::solve_advection(aspect::Simulator<2>::AdvectionField const&) () at /home/s/spc29/git/aspect/source/simulator/solver.cc:427 #12 0x0000000000a6f767 in aspect::Simulator<2>::solve_timestep() () at /home/s/spc29/git/aspect/source/simulator/core.cc:1681 #13 0x0000000000a72935 in aspect::Simulator<2>::run() () at /home/s/spc29/git/aspect/source/simulator/core.cc:2065 #14 0x0000000000828b17 in main () at /home/s/spc29/git/aspect/source/main.cc:506 PID 44028 #0 0x00002aaac1cfc11c in ?? () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #1 0x00002aaac1cf6641 in MPIDI_CH3I_Progress () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #2 0x00002aaac1de5657 in MPIC_Wait () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #3 0x00002aaac1de53b3 in MPIC_Sendrecv () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #4 0x00002aaac1ddecf3 in MPIC_Sendrecv_ft () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #5 0x00002aaac1ccf28b in MPIR_Allreduce_intra () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #6 0x00002aaac1cd2acd in MPIR_Allreduce_impl () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #7 0x00002aaac1cd23c2 in PMPI_Allreduce () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #8 0x00002aaabc84693e in Epetra_MpiComm::MinAll(int*, int*, int) const () from /home/s/spc29/trilinos/12.0.1/lib/libepetra.so.12 #9 0x00002aaabc785f52 in Epetra_BlockMap::SameAs(Epetra_BlockMap const&) const () from /home/s/spc29/trilinos/12.0.1/lib/libepetra.so.12 #10 0x00000000009fda46 in dealii::TrilinosWrappers::PreconditionBase::vmult(dealii::TrilinosWrappers::VectorBase&, dealii::TrilinosWrappers::VectorBase const&) const () at /home/s/spc29/dealii-git/include/deal.II/lac/trilinos_precondition.h:1861 #11 0x0000000000aea2c6 in void dealii::SolverGMRES::solve(dealii::TrilinosWrappers::SparseMatrix const&, dealii::TrilinosWrappers::MPI::Vector&, dealii::TrilinosWrappers::MPI::Vector const&, dealii::TrilinosWrappers::PreconditionILU const&) () at /home/s/spc29/dealii-git/include/deal.II/lac/solver_gmres.h:911 #12 0x0000000000aebb57 in aspect::Simulator<2>::solve_advection(aspect::Simulator<2>::AdvectionField const&) () at /home/s/spc29/git/aspect/source/simulator/solver.cc:427 #13 0x0000000000a6f767 in aspect::Simulator<2>::solve_timestep() () at /home/s/spc29/git/aspect/source/simulator/core.cc:1681 #14 0x0000000000a72935 in aspect::Simulator<2>::run() () at /home/s/spc29/git/aspect/source/simulator/core.cc:2065 #15 0x0000000000828b17 in main () at /home/s/spc29/git/aspect/source/main.cc:506 PID 44029 #0 0x00002aaac1cfc3f1 in ?? () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #1 0x00002aaac1cf6641 in MPIDI_CH3I_Progress () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #2 0x00002aaac1de5657 in MPIC_Wait () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #3 0x00002aaac1de53b3 in MPIC_Sendrecv () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #4 0x00002aaac1ddecf3 in MPIC_Sendrecv_ft () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #5 0x00002aaac1ccf28b in MPIR_Allreduce_intra () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #6 0x00002aaac1cd2acd in MPIR_Allreduce_impl () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #7 0x00002aaac1cd23c2 in PMPI_Allreduce () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #8 0x00002aaabc84693e in Epetra_MpiComm::MinAll(int*, int*, int) const () from /home/s/spc29/trilinos/12.0.1/lib/libepetra.so.12 #9 0x00002aaabc785f52 in Epetra_BlockMap::SameAs(Epetra_BlockMap const&) const () from /home/s/spc29/trilinos/12.0.1/lib/libepetra.so.12 #10 0x00000000009fda46 in dealii::TrilinosWrappers::PreconditionBase::vmult(dealii::TrilinosWrappers::VectorBase&, dealii::TrilinosWrappers::VectorBase const&) const () at /home/s/spc29/dealii-git/include/deal.II/lac/trilinos_precondition.h:1861 #11 0x0000000000aea2c6 in void dealii::SolverGMRES::solve(dealii::TrilinosWrappers::SparseMatrix const&, dealii::TrilinosWrappers::MPI::Vector&, dealii::TrilinosWrappers::MPI::Vector const&, dealii::TrilinosWrappers::PreconditionILU const&) () at /home/s/spc29/dealii-git/include/deal.II/lac/solver_gmres.h:911 #12 0x0000000000aebb57 in aspect::Simulator<2>::solve_advection(aspect::Simulator<2>::AdvectionField const&) () at /home/s/spc29/git/aspect/source/simulator/solver.cc:427 #13 0x0000000000a6f767 in aspect::Simulator<2>::solve_timestep() () at /home/s/spc29/git/aspect/source/simulator/core.cc:1681 #14 0x0000000000a72935 in aspect::Simulator<2>::run() () at /home/s/spc29/git/aspect/source/simulator/core.cc:2065 #15 0x0000000000828b17 in main () at /home/s/spc29/git/aspect/source/main.cc:506 PID 44030 #0 0x00002aaac1cf69d6 in MPIDI_CH3I_Progress () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #1 0x00002aaac1de5657 in MPIC_Wait () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #2 0x00002aaac1de53b3 in MPIC_Sendrecv () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #3 0x00002aaac1ddecf3 in MPIC_Sendrecv_ft () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #4 0x00002aaac1ccf28b in MPIR_Allreduce_intra () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #5 0x00002aaac1cd2acd in MPIR_Allreduce_impl () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #6 0x00002aaac1cd23c2 in PMPI_Allreduce () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #7 0x00002aaabc84693e in Epetra_MpiComm::MinAll(int*, int*, int) const () from /home/s/spc29/trilinos/12.0.1/lib/libepetra.so.12 #8 0x00002aaabc785f52 in Epetra_BlockMap::SameAs(Epetra_BlockMap const&) const () from /home/s/spc29/trilinos/12.0.1/lib/libepetra.so.12 #9 0x00000000009fda46 in dealii::TrilinosWrappers::PreconditionBase::vmult(dealii::TrilinosWrappers::VectorBase&, dealii::TrilinosWrappers::VectorBase const&) const () at /home/s/spc29/dealii-git/include/deal.II/lac/trilinos_precondition.h:1861 #10 0x0000000000aea2c6 in void dealii::SolverGMRES::solve(dealii::TrilinosWrappers::SparseMatrix const&, dealii::TrilinosWrappers::MPI::Vector&, dealii::TrilinosWrappers::MPI::Vector const&, dealii::TrilinosWrappers::PreconditionILU const&) () at /home/s/spc29/dealii-git/include/deal.II/lac/solver_gmres.h:911 #11 0x0000000000aebb57 in aspect::Simulator<2>::solve_advection(aspect::Simulator<2>::AdvectionField const&) () at /home/s/spc29/git/aspect/source/simulator/solver.cc:427 #12 0x0000000000a6f767 in aspect::Simulator<2>::solve_timestep() () at /home/s/spc29/git/aspect/source/simulator/core.cc:1681 #13 0x0000000000a72935 in aspect::Simulator<2>::run() () at /home/s/spc29/git/aspect/source/simulator/core.cc:2065 #14 0x0000000000828b17 in main () at /home/s/spc29/git/aspect/source/main.cc:506 PID 44031 #0 0x00002aaac3ab62f7 in sched_yield () from /lib64/libc.so.6 #1 0x00002aaac1cf6ac0 in MPIDI_CH3I_Progress () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #2 0x00002aaac1de5657 in MPIC_Wait () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #3 0x00002aaac1de53b3 in MPIC_Sendrecv () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #4 0x00002aaac1ddecf3 in MPIC_Sendrecv_ft () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #5 0x00002aaac1ccf28b in MPIR_Allreduce_intra () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #6 0x00002aaac1cd2acd in MPIR_Allreduce_impl () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #7 0x00002aaac1cd23c2 in PMPI_Allreduce () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #8 0x00002aaabc84693e in Epetra_MpiComm::MinAll(int*, int*, int) const () from /home/s/spc29/trilinos/12.0.1/lib/libepetra.so.12 #9 0x00002aaabc785f52 in Epetra_BlockMap::SameAs(Epetra_BlockMap const&) const () from /home/s/spc29/trilinos/12.0.1/lib/libepetra.so.12 #10 0x00000000009fda46 in dealii::TrilinosWrappers::PreconditionBase::vmult(dealii::TrilinosWrappers::VectorBase&, dealii::TrilinosWrappers::VectorBase const&) const () at /home/s/spc29/dealii-git/include/deal.II/lac/trilinos_precondition.h:1861 #11 0x0000000000aea2c6 in void dealii::SolverGMRES::solve(dealii::TrilinosWrappers::SparseMatrix const&, dealii::TrilinosWrappers::MPI::Vector&, dealii::TrilinosWrappers::MPI::Vector const&, dealii::TrilinosWrappers::PreconditionILU const&) () at /home/s/spc29/dealii-git/include/deal.II/lac/solver_gmres.h:911 #12 0x0000000000aebb57 in aspect::Simulator<2>::solve_advection(aspect::Simulator<2>::AdvectionField const&) () at /home/s/spc29/git/aspect/source/simulator/solver.cc:427 #13 0x0000000000a6f767 in aspect::Simulator<2>::solve_timestep() () at /home/s/spc29/git/aspect/source/simulator/core.cc:1681 #14 0x0000000000a72935 in aspect::Simulator<2>::run() () at /home/s/spc29/git/aspect/source/simulator/core.cc:2065 #15 0x0000000000828b17 in main () at /home/s/spc29/git/aspect/source/main.cc:506 PID 44032 #0 0x00002aaace5784a0 in ?? () from /usr/lib64/libmlx4-rdmav2.so #1 0x00002aaacdeed687 in ?? () from /usr/lib64/libdaploucm.so.2 #2 0x00002aaacdee7201 in ?? () from /usr/lib64/libdaploucm.so.2 #3 0x00002aaacdee5f23 in ?? () from /usr/lib64/libdaploucm.so.2 #4 0x00002aaac1d6b91d in ?? () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #5 0x00002aaac1d6b4b5 in MPID_nem_dapl_ud_poll () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #6 0x00002aaac1e44c40 in MPID_nem_network_poll () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #7 0x00002aaac1cf6a4c in MPIDI_CH3I_Progress () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #8 0x00002aaac1de5657 in MPIC_Wait () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #9 0x00002aaac1de53b3 in MPIC_Sendrecv () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #10 0x00002aaac1ddecf3 in MPIC_Sendrecv_ft () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #11 0x00002aaac1ccf28b in MPIR_Allreduce_intra () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #12 0x00002aaac1cd2acd in MPIR_Allreduce_impl () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #13 0x00002aaac1cd23c2 in PMPI_Allreduce () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #14 0x00002aaabc84693e in Epetra_MpiComm::MinAll(int*, int*, int) const () from /home/s/spc29/trilinos/12.0.1/lib/libepetra.so.12 #15 0x00002aaabc785f52 in Epetra_BlockMap::SameAs(Epetra_BlockMap const&) const () from /home/s/spc29/trilinos/12.0.1/lib/libepetra.so.12 #16 0x00000000009fd93d in dealii::TrilinosWrappers::PreconditionBase::vmult(dealii::TrilinosWrappers::VectorBase&, dealii::TrilinosWrappers::VectorBase const&) const () at /home/s/spc29/dealii-git/include/deal.II/lac/trilinos_precondition.h:1859 #17 0x0000000000aea2c6 in void dealii::SolverGMRES::solve(dealii::TrilinosWrappers::SparseMatrix const&, dealii::TrilinosWrappers::MPI::Vector&, dealii::TrilinosWrappers::MPI::Vector const&, dealii::TrilinosWrappers::PreconditionILU const&) () at /home/s/spc29/dealii-git/include/deal.II/lac/solver_gmres.h:911 #18 0x0000000000aebb57 in aspect::Simulator<2>::solve_advection(aspect::Simulator<2>::AdvectionField const&) () at /home/s/spc29/git/aspect/source/simulator/solver.cc:427 #19 0x0000000000a6f767 in aspect::Simulator<2>::solve_timestep() () at /home/s/spc29/git/aspect/source/simulator/core.cc:1681 #20 0x0000000000a72935 in aspect::Simulator<2>::run() () at /home/s/spc29/git/aspect/source/simulator/core.cc:2065 #21 0x0000000000828b17 in main () at /home/s/spc29/git/aspect/source/main.cc:506 PID 44033 #0 0x00002aaac1cfc0d4 in ?? () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #1 0x00002aaac1cf6641 in MPIDI_CH3I_Progress () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #2 0x00002aaac1de5657 in MPIC_Wait () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #3 0x00002aaac1de53b3 in MPIC_Sendrecv () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #4 0x00002aaac1ddecf3 in MPIC_Sendrecv_ft () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #5 0x00002aaac1ccf28b in MPIR_Allreduce_intra () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #6 0x00002aaac1cd2acd in MPIR_Allreduce_impl () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #7 0x00002aaac1cd23c2 in PMPI_Allreduce () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #8 0x00002aaabc84693e in Epetra_MpiComm::MinAll(int*, int*, int) const () from /home/s/spc29/trilinos/12.0.1/lib/libepetra.so.12 #9 0x00002aaabc785f52 in Epetra_BlockMap::SameAs(Epetra_BlockMap const&) const () from /home/s/spc29/trilinos/12.0.1/lib/libepetra.so.12 #10 0x00000000009fda46 in dealii::TrilinosWrappers::PreconditionBase::vmult(dealii::TrilinosWrappers::VectorBase&, dealii::TrilinosWrappers::VectorBase const&) const () at /home/s/spc29/dealii-git/include/deal.II/lac/trilinos_precondition.h:1861 #11 0x0000000000aea2c6 in void dealii::SolverGMRES::solve(dealii::TrilinosWrappers::SparseMatrix const&, dealii::TrilinosWrappers::MPI::Vector&, dealii::TrilinosWrappers::MPI::Vector const&, dealii::TrilinosWrappers::PreconditionILU const&) () at /home/s/spc29/dealii-git/include/deal.II/lac/solver_gmres.h:911 #12 0x0000000000aebb57 in aspect::Simulator<2>::solve_advection(aspect::Simulator<2>::AdvectionField const&) () at /home/s/spc29/git/aspect/source/simulator/solver.cc:427 #13 0x0000000000a6f767 in aspect::Simulator<2>::solve_timestep() () at /home/s/spc29/git/aspect/source/simulator/core.cc:1681 #14 0x0000000000a72935 in aspect::Simulator<2>::run() () at /home/s/spc29/git/aspect/source/simulator/core.cc:2065 #15 0x0000000000828b17 in main () at /home/s/spc29/git/aspect/source/main.cc:506 PID 44034 #1 0x00002aaac1de5657 in MPIC_Wait () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #2 0x00002aaac1de53b3 in MPIC_Sendrecv () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #3 0x00002aaac1ddecf3 in MPIC_Sendrecv_ft () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #4 0x00002aaac1ccf28b in MPIR_Allreduce_intra () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #5 0x00002aaac1cd2acd in MPIR_Allreduce_impl () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #6 0x00002aaac1cd23c2 in PMPI_Allreduce () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #7 0x00002aaabc84693e in Epetra_MpiComm::MinAll(int*, int*, int) const () from /home/s/spc29/trilinos/12.0.1/lib/libepetra.so.12 #8 0x00002aaabc785f52 in Epetra_BlockMap::SameAs(Epetra_BlockMap const&) const () from /home/s/spc29/trilinos/12.0.1/lib/libepetra.so.12 #9 0x00000000009fda46 in dealii::TrilinosWrappers::PreconditionBase::vmult(dealii::TrilinosWrappers::VectorBase&, dealii::TrilinosWrappers::VectorBase const&) const () at /home/s/spc29/dealii-git/include/deal.II/lac/trilinos_precondition.h:1861 #10 0x0000000000aea2c6 in void dealii::SolverGMRES::solve(dealii::TrilinosWrappers::SparseMatrix const&, dealii::TrilinosWrappers::MPI::Vector&, dealii::TrilinosWrappers::MPI::Vector const&, dealii::TrilinosWrappers::PreconditionILU const&) () at /home/s/spc29/dealii-git/include/deal.II/lac/solver_gmres.h:911 #11 0x0000000000aebb57 in aspect::Simulator<2>::solve_advection(aspect::Simulator<2>::AdvectionField const&) () at /home/s/spc29/git/aspect/source/simulator/solver.cc:427 #12 0x0000000000a6f767 in aspect::Simulator<2>::solve_timestep() () at /home/s/spc29/git/aspect/source/simulator/core.cc:1681 #13 0x0000000000a72935 in aspect::Simulator<2>::run() () at /home/s/spc29/git/aspect/source/simulator/core.cc:2065 #14 0x0000000000828b17 in main () at /home/s/spc29/git/aspect/source/main.cc:506 PID 44035 #0 0x00002aaac1d6b612 in MPID_nem_dapl_ud_poll () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #1 0x0000000000000001 in ?? () #2 0x00002aaac1e44c40 in MPID_nem_network_poll () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #3 0x00002aaac1cf6a4c in MPIDI_CH3I_Progress () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #4 0x00002aaac1de5657 in MPIC_Wait () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #5 0x00002aaac1de53b3 in MPIC_Sendrecv () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #6 0x00002aaac1ddecf3 in MPIC_Sendrecv_ft () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #7 0x00002aaac1ccf28b in MPIR_Allreduce_intra () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #8 0x00002aaac1cd2acd in MPIR_Allreduce_impl () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #9 0x00002aaac1cd23c2 in PMPI_Allreduce () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #10 0x00002aaabc84693e in Epetra_MpiComm::MinAll(int*, int*, int) const () from /home/s/spc29/trilinos/12.0.1/lib/libepetra.so.12 #11 0x00002aaabc785f52 in Epetra_BlockMap::SameAs(Epetra_BlockMap const&) const () from /home/s/spc29/trilinos/12.0.1/lib/libepetra.so.12 #12 0x00000000009fda46 in dealii::TrilinosWrappers::PreconditionBase::vmult(dealii::TrilinosWrappers::VectorBase&, dealii::TrilinosWrappers::VectorBase const&) const () at /home/s/spc29/dealii-git/include/deal.II/lac/trilinos_precondition.h:1861 #13 0x0000000000aea2c6 in void dealii::SolverGMRES::solve(dealii::TrilinosWrappers::SparseMatrix const&, dealii::TrilinosWrappers::MPI::Vector&, dealii::TrilinosWrappers::MPI::Vector const&, dealii::TrilinosWrappers::PreconditionILU const&) () at /home/s/spc29/dealii-git/include/deal.II/lac/solver_gmres.h:911 #14 0x0000000000aebb57 in aspect::Simulator<2>::solve_advection(aspect::Simulator<2>::AdvectionField const&) () at /home/s/spc29/git/aspect/source/simulator/solver.cc:427 #15 0x0000000000a6f767 in aspect::Simulator<2>::solve_timestep() () at /home/s/spc29/git/aspect/source/simulator/core.cc:1681 #16 0x0000000000a72935 in aspect::Simulator<2>::run() () at /home/s/spc29/git/aspect/source/simulator/core.cc:2065 #17 0x0000000000828b17 in main () at /home/s/spc29/git/aspect/source/main.cc:506 PID 44036 #0 0x00002aaac1d6b612 in MPID_nem_dapl_ud_poll () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #1 0x0000000000000001 in ?? () #2 0x00002aaac1e44c40 in MPID_nem_network_poll () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #3 0x00002aaac1cf6a4c in MPIDI_CH3I_Progress () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #4 0x00002aaac1de5657 in MPIC_Wait () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #5 0x00002aaac1de53b3 in MPIC_Sendrecv () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #6 0x00002aaac1ddecf3 in MPIC_Sendrecv_ft () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #7 0x00002aaac1ccf28b in MPIR_Allreduce_intra () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #8 0x00002aaac1cd2acd in MPIR_Allreduce_impl () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #9 0x00002aaac1cd23c2 in PMPI_Allreduce () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #10 0x00002aaabc84693e in Epetra_MpiComm::MinAll(int*, int*, int) const () from /home/s/spc29/trilinos/12.0.1/lib/libepetra.so.12 #11 0x00002aaabc785f52 in Epetra_BlockMap::SameAs(Epetra_BlockMap const&) const () from /home/s/spc29/trilinos/12.0.1/lib/libepetra.so.12 #12 0x00000000009fda46 in dealii::TrilinosWrappers::PreconditionBase::vmult(dealii::TrilinosWrappers::VectorBase&, dealii::TrilinosWrappers::VectorBase const&) const () at /home/s/spc29/dealii-git/include/deal.II/lac/trilinos_precondition.h:1861 #13 0x0000000000aea2c6 in void dealii::SolverGMRES::solve(dealii::TrilinosWrappers::SparseMatrix const&, dealii::TrilinosWrappers::MPI::Vector&, dealii::TrilinosWrappers::MPI::Vector const&, dealii::TrilinosWrappers::PreconditionILU const&) () at /home/s/spc29/dealii-git/include/deal.II/lac/solver_gmres.h:911 #14 0x0000000000aebb57 in aspect::Simulator<2>::solve_advection(aspect::Simulator<2>::AdvectionField const&) () at /home/s/spc29/git/aspect/source/simulator/solver.cc:427 #15 0x0000000000a6f767 in aspect::Simulator<2>::solve_timestep() () at /home/s/spc29/git/aspect/source/simulator/core.cc:1681 #16 0x0000000000a72935 in aspect::Simulator<2>::run() () at /home/s/spc29/git/aspect/source/simulator/core.cc:2065 #17 0x0000000000828b17 in main () at /home/s/spc29/git/aspect/source/main.cc:506 PID 44037 #0 0x00002aaac1d6b612 in MPID_nem_dapl_ud_poll () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #1 0x0000000000000001 in ?? () #2 0x00002aaac1e44c40 in MPID_nem_network_poll () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #3 0x00002aaac1cf6a4c in MPIDI_CH3I_Progress () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #4 0x00002aaac1de5657 in MPIC_Wait () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #5 0x00002aaac1de53b3 in MPIC_Sendrecv () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #6 0x00002aaac1ddecf3 in MPIC_Sendrecv_ft () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #7 0x00002aaac1ccf28b in MPIR_Allreduce_intra () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #8 0x00002aaac1cd2acd in MPIR_Allreduce_impl () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #9 0x00002aaac1cd23c2 in PMPI_Allreduce () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #10 0x00002aaabc84693e in Epetra_MpiComm::MinAll(int*, int*, int) const () from /home/s/spc29/trilinos/12.0.1/lib/libepetra.so.12 #11 0x00002aaabc785f52 in Epetra_BlockMap::SameAs(Epetra_BlockMap const&) const () from /home/s/spc29/trilinos/12.0.1/lib/libepetra.so.12 #12 0x00000000009fda46 in dealii::TrilinosWrappers::PreconditionBase::vmult(dealii::TrilinosWrappers::VectorBase&, dealii::TrilinosWrappers::VectorBase const&) const () at /home/s/spc29/dealii-git/include/deal.II/lac/trilinos_precondition.h:1861 #13 0x0000000000aea2c6 in void dealii::SolverGMRES::solve(dealii::TrilinosWrappers::SparseMatrix const&, dealii::TrilinosWrappers::MPI::Vector&, dealii::TrilinosWrappers::MPI::Vector const&, dealii::TrilinosWrappers::PreconditionILU const&) () at /home/s/spc29/dealii-git/include/deal.II/lac/solver_gmres.h:911 #14 0x0000000000aebb57 in aspect::Simulator<2>::solve_advection(aspect::Simulator<2>::AdvectionField const&) () at /home/s/spc29/git/aspect/source/simulator/solver.cc:427 #15 0x0000000000a6f767 in aspect::Simulator<2>::solve_timestep() () at /home/s/spc29/git/aspect/source/simulator/core.cc:1681 #16 0x0000000000a72935 in aspect::Simulator<2>::run() () at /home/s/spc29/git/aspect/source/simulator/core.cc:2065 #17 0x0000000000828b17 in main () at /home/s/spc29/git/aspect/source/main.cc:506 PID 44038 #0 0x00002aaac1d6b614 in MPID_nem_dapl_ud_poll () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #1 0x0000000000000001 in ?? () #2 0x00002aaac1e44c40 in MPID_nem_network_poll () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #3 0x00002aaac1cf6a4c in MPIDI_CH3I_Progress () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #4 0x00002aaac1de5657 in MPIC_Wait () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #5 0x00002aaac1de53b3 in MPIC_Sendrecv () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #6 0x00002aaac1ddecf3 in MPIC_Sendrecv_ft () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #7 0x00002aaac1ccf28b in MPIR_Allreduce_intra () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #8 0x00002aaac1cd2acd in MPIR_Allreduce_impl () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #9 0x00002aaac1cd23c2 in PMPI_Allreduce () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #10 0x00002aaabc84693e in Epetra_MpiComm::MinAll(int*, int*, int) const () from /home/s/spc29/trilinos/12.0.1/lib/libepetra.so.12 #11 0x00002aaabc785f52 in Epetra_BlockMap::SameAs(Epetra_BlockMap const&) const () from /home/s/spc29/trilinos/12.0.1/lib/libepetra.so.12 #12 0x00000000009fda46 in dealii::TrilinosWrappers::PreconditionBase::vmult(dealii::TrilinosWrappers::VectorBase&, dealii::TrilinosWrappers::VectorBase const&) const () at /home/s/spc29/dealii-git/include/deal.II/lac/trilinos_precondition.h:1861 #13 0x0000000000aea2c6 in void dealii::SolverGMRES::solve(dealii::TrilinosWrappers::SparseMatrix const&, dealii::TrilinosWrappers::MPI::Vector&, dealii::TrilinosWrappers::MPI::Vector const&, dealii::TrilinosWrappers::PreconditionILU const&) () at /home/s/spc29/dealii-git/include/deal.II/lac/solver_gmres.h:911 #14 0x0000000000aebb57 in aspect::Simulator<2>::solve_advection(aspect::Simulator<2>::AdvectionField const&) () at /home/s/spc29/git/aspect/source/simulator/solver.cc:427 #15 0x0000000000a6f767 in aspect::Simulator<2>::solve_timestep() () at /home/s/spc29/git/aspect/source/simulator/core.cc:1681 #16 0x0000000000a72935 in aspect::Simulator<2>::run() () at /home/s/spc29/git/aspect/source/simulator/core.cc:2065 #17 0x0000000000828b17 in main () at /home/s/spc29/git/aspect/source/main.cc:506 PID 19294 #0 0x00002aaac1cf6641 in MPIDI_CH3I_Progress () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #1 0x00002aaac1de5657 in MPIC_Wait () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #2 0x00002aaac1de53b3 in MPIC_Sendrecv () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #3 0x00002aaac1ddecf3 in MPIC_Sendrecv_ft () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #4 0x00002aaac1ccf28b in MPIR_Allreduce_intra () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #5 0x00002aaac1cd2acd in MPIR_Allreduce_impl () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #6 0x00002aaac1cd23c2 in PMPI_Allreduce () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #7 0x00002aaabc84693e in Epetra_MpiComm::MinAll(int*, int*, int) const () from /home/s/spc29/trilinos/12.0.1/lib/libepetra.so.12 #8 0x00002aaabc785f52 in Epetra_BlockMap::SameAs(Epetra_BlockMap const&) const () from /home/s/spc29/trilinos/12.0.1/lib/libepetra.so.12 #9 0x00000000009fda46 in dealii::TrilinosWrappers::PreconditionBase::vmult(dealii::TrilinosWrappers::VectorBase&, dealii::TrilinosWrappers::VectorBase const&) const () at /home/s/spc29/dealii-git/include/deal.II/lac/trilinos_precondition.h:1861 #10 0x0000000000aea2c6 in void dealii::SolverGMRES::solve(dealii::TrilinosWrappers::SparseMatrix const&, dealii::TrilinosWrappers::MPI::Vector&, dealii::TrilinosWrappers::MPI::Vector const&, dealii::TrilinosWrappers::PreconditionILU const&) () at /home/s/spc29/dealii-git/include/deal.II/lac/solver_gmres.h:911 #11 0x0000000000aebb57 in aspect::Simulator<2>::solve_advection(aspect::Simulator<2>::AdvectionField const&) () at /home/s/spc29/git/aspect/source/simulator/solver.cc:427 #12 0x0000000000a6f767 in aspect::Simulator<2>::solve_timestep() () at /home/s/spc29/git/aspect/source/simulator/core.cc:1681 #13 0x0000000000a72935 in aspect::Simulator<2>::run() () at /home/s/spc29/git/aspect/source/simulator/core.cc:2065 #14 0x0000000000828b17 in main () at /home/s/spc29/git/aspect/source/main.cc:506 PID 19295 #0 0x00002aaac24a4859 in pthread_mutex_unlock () from /lib64/libpthread.so.0 #1 0x00002aaac7be1f09 in ?? () from /usr/lib64/libdaploucm.so.2 #2 0x00002aaac1d6b91d in ?? () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #3 0x00002aaac1d6b4b5 in MPID_nem_dapl_ud_poll () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #4 0x00002aaac1e44c40 in MPID_nem_network_poll () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #5 0x00002aaac1cf6a4c in MPIDI_CH3I_Progress () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #6 0x00002aaac1de5657 in MPIC_Wait () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #7 0x00002aaac1de53b3 in MPIC_Sendrecv () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #8 0x00002aaac1ddecf3 in MPIC_Sendrecv_ft () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #9 0x00002aaac1ccf28b in MPIR_Allreduce_intra () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #10 0x00002aaac1cd2acd in MPIR_Allreduce_impl () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #11 0x00002aaac1cd23c2 in PMPI_Allreduce () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #12 0x00002aaabc84693e in Epetra_MpiComm::MinAll(int*, int*, int) const () from /home/s/spc29/trilinos/12.0.1/lib/libepetra.so.12 #13 0x00002aaabc785f52 in Epetra_BlockMap::SameAs(Epetra_BlockMap const&) const () from /home/s/spc29/trilinos/12.0.1/lib/libepetra.so.12 #14 0x00000000009fd93d in dealii::TrilinosWrappers::PreconditionBase::vmult(dealii::TrilinosWrappers::VectorBase&, dealii::TrilinosWrappers::VectorBase const&) const () at /home/s/spc29/dealii-git/include/deal.II/lac/trilinos_precondition.h:1859 #15 0x0000000000aea2c6 in void dealii::SolverGMRES::solve(dealii::TrilinosWrappers::SparseMatrix const&, dealii::TrilinosWrappers::MPI::Vector&, dealii::TrilinosWrappers::MPI::Vector const&, dealii::TrilinosWrappers::PreconditionILU const&) () at /home/s/spc29/dealii-git/include/deal.II/lac/solver_gmres.h:911 #16 0x0000000000aebb57 in aspect::Simulator<2>::solve_advection(aspect::Simulator<2>::AdvectionField const&) () at /home/s/spc29/git/aspect/source/simulator/solver.cc:427 #17 0x0000000000a6f767 in aspect::Simulator<2>::solve_timestep() () at /home/s/spc29/git/aspect/source/simulator/core.cc:1681 #18 0x0000000000a72935 in aspect::Simulator<2>::run() () at /home/s/spc29/git/aspect/source/simulator/core.cc:2065 #19 0x0000000000828b17 in main () at /home/s/spc29/git/aspect/source/main.cc:506 PID 19296 #0 0x00002aaac1cf65cd in MPIDI_CH3I_Progress () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #1 0x00002aaac1de5657 in MPIC_Wait () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #2 0x00002aaac1de53b3 in MPIC_Sendrecv () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #3 0x00002aaac1ddecf3 in MPIC_Sendrecv_ft () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #4 0x00002aaac1ccf28b in MPIR_Allreduce_intra () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #5 0x00002aaac1cd2acd in MPIR_Allreduce_impl () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #6 0x00002aaac1cd23c2 in PMPI_Allreduce () from /cm/shared/apps/ics/impi/4.1.3.048/intel64/lib/libmpi.so.4 #7 0x00002aaabc84693e in Epetra_MpiComm::MinAll(int*, int*, int) const () from /home/s/spc29/trilinos/12.0.1/lib/libepetra.so.12 #8 0x00002aaabc785f52 in Epetra_BlockMap::SameAs(Epetra_BlockMap const&) const () from /home/s/spc29/trilinos/12.0.1/lib/libepetra.so.12 #9 0x00000000009fda46 in dealii::TrilinosWrappers::PreconditionBase::vmult(dealii::TrilinosWrappers::VectorBase&, dealii::TrilinosWrappers::VectorBase const&) const () at /home/s/spc29/dealii-git/include/deal.II/lac/trilinos_precondition.h:1861 #10 0x0000000000aea2c6 in void dealii::SolverGMRES::solve(dealii::TrilinosWrappers::SparseMatrix const&, dealii::TrilinosWrappers::MPI::Vector&, dealii::TrilinosWrappers::MPI::Vector const&, dealii::TrilinosWrappers::PreconditionILU const&) () at /home/s/spc29/dealii-git/include/deal.II/lac/solver_gmres.h:911 #11 0x0000000000aebb57 in aspect::Simulator<2>::solve_advection(aspect::Simulator<2>::AdvectionField const&) () at /home/s/spc29/git/aspect/source/simulator/solver.cc:427 #12 0x0000000000a6f767 in aspect::Simulator<2>::solve_timestep() () at /home/s/spc29/git/aspect/source/simulator/core.cc:1681 #13 0x0000000000a72935 in aspect::Simulator<2>::run() () at /home/s/spc29/git/aspect/source/simulator/core.cc:2065 #14 0x0000000000828b17 in main () at /home/s/spc29/git/aspect/source/main.cc:506