Skip to content

Commit

Permalink
Merge pull request #2130 from gassmoeller/threads
Browse files Browse the repository at this point in the history 
Threads
  • Loading branch information
@gassmoeller
gassmoeller committed Mar 20, 2018
2 parents ebe7b81 + ee5780f commit e006f57
Show file tree
Hide file tree
Showing 2 changed files with 127 additions and 136 deletions.
24 changes: 24 additions & 0 deletions doc/manual/manual.tex
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3873,6 +3873,30 @@ \subsubsection{Regularizing models with large coefficient variation}
\index[prmindexfull]{Material model!Material averaging}


\subsubsection{Using multithreading}
In most cases using as many MPI processes as possible is the optimal
parallelization strategy for \aspect{} models, but if you are limited by the
amount of MPI communication it can be beneficial to use multiple threads per
MPI process. While not utilized by our linear solvers, this parallelization can
speed up the assembly of the system matrices, e.g. by around 10-15\% if you
utilize unused logical cores, or nearly linearly if you use otherwise
unused physical cores. This can also reduce the performance cost if you are
memory limited and need to run your model on less than the available number of
cores per node on a cluster to increase the available memory per core. Running
with for example two threads per process will offset some of the performance
loss you will see in these situations.

Multithreading is controlled by setting the command line parameter \texttt{-j}
or \texttt{-{}-threads}. If the parameter is not set, \aspect{} will create
exactly one thread per MPI process, i.e. multithreading is disabled. Appending
the parameter allows \aspect{} to spawn several threads per MPI process. Note
that the internally used TBB library will determine the number of threads based
on the number of available cores, i.e., if you start 2~MPI processes on a
quadcore machine with hyperthreading (8 logical cores), \aspect{} will spawn 4
threads on each MPI process. Also note that there is no guarantee that the
final number of threads will exactly match the number of available logical
cores if you start with a number of processes that is not a divisor of your
logical cores (e.g. 3 MPI processes for 8 logical cores).

\subsection{Input parameter files}
\label{sec:parameters-overview}
Expand Down
239 changes: 103 additions & 136 deletions source/main.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -368,9 +368,11 @@ void print_help()
<< std::endl;
std::cout << " optional arguments [args]:"
<< std::endl
<< " --version (for information about library versions)"
<< " -h, --help (for this usage help)"
<< std::endl
<< " --help (for this usage help)"
<< " -v, --version (for information about library versions)"
<< std::endl
<< " -j, --threads (to use multi-threading)"
<< std::endl
<< " --output-xml (print parameters in xml format to standard output and exit)"
<< std::endl
Expand All @@ -380,35 +382,6 @@ void print_help()
}


/**
* Print information about the versions of underlying libraries.
*/
template <class Stream>
void print_version_information(Stream &stream)
{
stream << "Version information of underlying libraries:\n"
<< " . deal.II: "
<< DEAL_II_PACKAGE_VERSION << '\t'
<< " (git revision "
<< DEAL_II_GIT_SHORTREV << ")\n"
#ifndef ASPECT_USE_PETSC
<< " . Trilinos: "
<< DEAL_II_TRILINOS_VERSION_MAJOR << '.'
<< DEAL_II_TRILINOS_VERSION_MINOR << '.'
<< DEAL_II_TRILINOS_VERSION_SUBMINOR << '\n'
#else
<< " . PETSc: "
<< PETSC_VERSION_MAJOR << '.'
<< PETSC_VERSION_MINOR << '.'
<< PETSC_VERSION_SUBMINOR << '\n'
#endif
<< " . p4est: "
<< DEAL_II_P4EST_VERSION_MAJOR << '.'
<< DEAL_II_P4EST_VERSION_MINOR << '.'
<< DEAL_II_P4EST_VERSION_SUBMINOR << '\n'
<< std::endl;
}


// hook into SIGABRT/SIGFPE and kill off the program
void signal_handler(int signal)
Expand Down Expand Up @@ -437,6 +410,41 @@ void signal_handler(int signal)
#endif
}



template<int dim>
void
run_simulator(const std::string &input_as_string,
const bool output_xml,
const bool output_plugin_graph)
{
using namespace dealii;

ParameterHandler prm;
const bool i_am_proc_0 = (Utilities::MPI::this_mpi_process(MPI_COMM_WORLD) == 0);
aspect::Simulator<dim>::declare_parameters(prm);
parse_parameters (input_as_string, prm);

if (output_xml)
{
if (i_am_proc_0)
prm.print_parameters(std::cout, ParameterHandler::XML);
}
else if (output_plugin_graph)
{
aspect::Simulator<dim> flow_problem(MPI_COMM_WORLD, prm);
if (i_am_proc_0)
flow_problem.write_plugin_graph (std::cout);
}
else
{
aspect::Simulator<dim> flow_problem(MPI_COMM_WORLD, prm);
flow_problem.run();
}
}



int main (int argc, char *argv[])
{
using namespace dealii;
Expand All @@ -452,26 +460,77 @@ int main (int argc, char *argv[])
#endif
#endif

std::string prm_name = "";
bool output_xml = false;
bool output_plugin_graph = false;
bool output_version = false;
bool output_help = false;
bool use_threads = false;
int current_argument = 1;

// Loop over all command line arguments. Handle a number of special ones
// starting with a dash, and then take the first non-special one as the
// name of the input file. We will later check that there are no further
// arguments left after that (though there may be with PETSc, see
// below).
while (current_argument<argc)
{
const std::string arg = argv[current_argument];
++current_argument;
if (arg == "--output-xml")
{
output_xml = true;
}
else if (arg == "--output-plugin-graph")
{
output_plugin_graph = true;
}
else if (arg=="-h" || arg =="--help")
{
output_help = true;
}
else if (arg=="-v" || arg =="--version")
{
output_version = true;
}
else if (arg=="-j" || arg =="--threads")
{
use_threads = true;
}
else
{
// Not a special argument, so we assume that this is the .prm
// filename (or "--"). We can now break out of this loop because
// we are not going to pass arguments passed after the filename
prm_name = arg;
break;
}
}

try
{
// Disable the use of threads. If that is not what you want,
// use numbers::invalid_unsigned_int instead of 1 to use as many threads
// as deemed useful by TBB.
//
// Note: we initialize this class inside the try/catch block and not
// before, so that the destructor of this instance can react if we are
// currently unwinding the stack if an unhandled exception is being
// thrown to avoid MPI deadlocks.
Utilities::MPI::MPI_InitFinalize mpi_initialization(argc, argv, /*n_threads =*/ 1);
Utilities::MPI::MPI_InitFinalize mpi_initialization(argc, argv, use_threads ? numbers::invalid_unsigned_int : 1);

deallog.depth_console(0);

int current_idx = 1;
const bool i_am_proc_0 = (Utilities::MPI::this_mpi_process(MPI_COMM_WORLD) == 0);

std::string prm_name = "";
bool output_xml = false;
bool output_plugin_graph = false;
if (output_help && i_am_proc_0)
{
print_aspect_header(std::cout);
print_help();
return 0;
}

if (output_version && i_am_proc_0)
{
print_aspect_header(std::cout);
return 0;
}

// We hook into the abort handler on ranks != 0 to avoid an MPI
// deadlock. The deal.II library will call std::abort() when an
Expand All @@ -491,52 +550,6 @@ int main (int argc, char *argv[])
std::signal(SIGFPE, signal_handler);
}

// Loop over all command line arguments. Handle a number of special ones
// starting with a dash, and then take the first non-special one as the
// name of the input file. We will later check that there are no further
// arguments left after that (though there may be with PETSc, see
// below).
while (current_idx<argc)
{
const std::string arg = argv[current_idx];
++current_idx;
if (arg == "--output-xml")
{
output_xml = true;
}
else if (arg == "--output-plugin-graph")
{
output_plugin_graph = true;
}
else if (arg=="-h" || arg =="--help")
{
if (i_am_proc_0)
{
print_aspect_header(std::cout);
print_help();
}
return 0;
}
else if (arg=="-v" || arg =="--version")
{
if (i_am_proc_0)
{
print_aspect_header(std::cout);
print_version_information(std::cout);
}
return 0;
}
else
{
// Not a special argument, so we assume that this is the .prm
// filename (or "--"). We can now break out of this loop because
// we are not going to pass arguments passed after the filename
prm_name = arg;
break;
}
}


// if no parameter given or somebody gave additional parameters,
// show help and exit.
// However, this does not work with PETSc because for PETSc, one
Expand All @@ -545,7 +558,7 @@ int main (int argc, char *argv[])
// MPI_InitFinalize above) does not filter these out.
if ((prm_name == "")
#ifndef ASPECT_USE_PETSC
|| (current_idx < argc)
|| (current_argument < argc)
#endif
)
{
Expand All @@ -563,7 +576,6 @@ int main (int argc, char *argv[])
print_aspect_header(std::cout);
}


// See where to read input from, then do the reading and
// put the contents of the input into a string.
//
Expand Down Expand Up @@ -627,7 +639,6 @@ int main (int argc, char *argv[])
// like "include $ASPECT_SOURCE_DIR/tests/bla.prm" work.
input_as_string = aspect::Utilities::expand_ASPECT_SOURCE_DIR(input_as_string);


// try to determine the dimension we want to work in. the default
// is 2, but if we find a line of the kind "set Dimension = ..."
// then the last such line wins
Expand All @@ -641,63 +652,19 @@ int main (int argc, char *argv[])
// the parameter file
possibly_load_shared_libs (input_as_string);

// now switch between the templates that code for 2d or 3d. it
// would be nicer if we didn't have to duplicate code, but the
// following needs to be known at compile time whereas the dimensionality
// is only read at run-time
ParameterHandler prm;

// Now switch between the templates that start the model for 2d or 3d.
switch (dim)
{
case 2:
{
aspect::Simulator<2>::declare_parameters(prm);
parse_parameters (input_as_string, prm);

if (output_xml)
{
if (i_am_proc_0)
prm.print_parameters(std::cout, ParameterHandler::XML);
}
else if (output_plugin_graph)
{
aspect::Simulator<2> flow_problem(MPI_COMM_WORLD, prm);
if (i_am_proc_0)
flow_problem.write_plugin_graph (std::cout);
}
else
{
aspect::Simulator<2> flow_problem(MPI_COMM_WORLD, prm);
flow_problem.run();
}
run_simulator<2>(input_as_string,output_xml,output_plugin_graph);
break;
}

case 3:
{
aspect::Simulator<3>::declare_parameters(prm);
parse_parameters (input_as_string, prm);

if (output_xml)
{
if (i_am_proc_0)
prm.print_parameters(std::cout, ParameterHandler::XML);
}
else if (output_plugin_graph)
{
aspect::Simulator<3> flow_problem(MPI_COMM_WORLD, prm);
if (i_am_proc_0)
flow_problem.write_plugin_graph (std::cout);
}
else
{
aspect::Simulator<3> flow_problem(MPI_COMM_WORLD, prm);
flow_problem.run();
}

run_simulator<3>(input_as_string,output_xml,output_plugin_graph);
break;
}

default:
AssertThrow((dim >= 2) && (dim <= 3),
ExcMessage ("ASPECT can only be run in 2d and 3d but a "
Expand Down

0 comments on commit e006f57

Please sign in to comment.