diff --git a/framework/include/vectorpostprocessors/WorkBalance.h b/framework/include/vectorpostprocessors/WorkBalance.h new file mode 100644 index 000000000000..a8a55aa92fce --- /dev/null +++ b/framework/include/vectorpostprocessors/WorkBalance.h @@ -0,0 +1,49 @@ +//* This file is part of the MOOSE framework +//* https://www.mooseframework.org +//* +//* All rights reserved, see COPYRIGHT for full restrictions +//* https://github.com/idaholab/moose/blob/master/COPYRIGHT +//* +//* Licensed under LGPL 2.1, please see LICENSE for details +//* https://www.gnu.org/licenses/lgpl-2.1.html + +#ifndef WORKBALANCE_H +#define WORKBALANCE_H + +#include "ElementVectorPostprocessor.h" + +class WorkBalance; + +template <> +InputParameters validParams(); + +/** + * Compute several metrics for each MPI process. + * + * Note: this is somewhat expensive. It does loops over elements, sides and nodes + */ +class WorkBalance : public GeneralVectorPostprocessor +{ +public: + WorkBalance(const InputParameters & parameters); + + virtual void initialize() override; + virtual void execute() override; + virtual void finalize() override; + +protected: + dof_id_type _local_num_elems; + dof_id_type _local_num_nodes; + dof_id_type _local_num_dofs; + dof_id_type _local_num_partition_sides; + Real _local_partition_surface_area; + + VectorPostprocessorValue & _pid; + VectorPostprocessorValue & _num_elems; + VectorPostprocessorValue & _num_nodes; + VectorPostprocessorValue & _num_dofs; + VectorPostprocessorValue & _num_partition_sides; + VectorPostprocessorValue & _partition_surface_area; +}; + +#endif // WORKBALANCE_H diff --git a/framework/src/vectorpostprocessors/WorkBalance.C b/framework/src/vectorpostprocessors/WorkBalance.C new file mode 100644 index 000000000000..56d86d00e358 --- /dev/null +++ b/framework/src/vectorpostprocessors/WorkBalance.C @@ -0,0 +1,224 @@ +//* This file is part of the MOOSE framework +//* https://www.mooseframework.org +//* +//* All rights reserved, see COPYRIGHT for full restrictions +//* https://github.com/idaholab/moose/blob/master/COPYRIGHT +//* +//* Licensed under LGPL 2.1, please see LICENSE for details +//* https://www.gnu.org/licenses/lgpl-2.1.html + +#include "WorkBalance.h" + +// MOOSE includes +#include "MooseVariableField.h" +#include "ThreadedElementLoopBase.h" +#include "ThreadedNodeLoop.h" + +#include "libmesh/quadrature.h" + +registerMooseObject("MooseApp", WorkBalance); + +template <> +InputParameters +validParams() +{ + InputParameters params = validParams(); + return params; +} + +WorkBalance::WorkBalance(const InputParameters & parameters) + : GeneralVectorPostprocessor(parameters), + _local_num_elems(0), + _local_num_nodes(0), + _local_num_dofs(0), + _local_num_partition_sides(0), + _local_partition_surface_area(0), + _pid(declareVector("pid")), + _num_elems(declareVector("num_elems")), + _num_nodes(declareVector("num_nodes")), + _num_dofs(declareVector("num_dofs")), + _num_partition_sides(declareVector("num_partition_sides")), + _partition_surface_area(declareVector("partition_surface_area")) +{ +} + +void +WorkBalance::initialize() +{ + _local_num_elems = 0; + _local_num_nodes = 0; + _local_num_dofs = 0; + _local_num_partition_sides = 0; + _local_partition_surface_area = 0; +} + +namespace +{ + +// Helper Threaded Loop for Elements +class WBElementLoop : public ThreadedElementLoopBase +{ +public: + WBElementLoop(MooseMesh & mesh) + : ThreadedElementLoopBase(mesh), + _local_num_elems(0), + _local_num_dofs(0), + _local_num_partition_sides(0), + _local_partition_surface_area(0), + _this_pid(_mesh.processor_id()) // Get this once because it is expensive + { + } + + WBElementLoop(WBElementLoop & x, Threads::split split) + : ThreadedElementLoopBase(x, split), + _local_num_elems(0), + _local_num_dofs(0), + _local_num_partition_sides(0), + _local_partition_surface_area(0), + _this_pid(x._this_pid) + { + } + + virtual ~WBElementLoop() {} + + virtual void pre() override + { + _local_num_elems = 0; + _local_num_dofs = 0; + _local_num_partition_sides = 0; + _local_partition_surface_area = 0; + } + + virtual void onElement(const Elem * elem) override + { + _local_num_elems++; + + /* + // Find out how many dofs there are on this element + auto n_sys = elem->n_systems(); + for (decltype(n_sys) sys = 0; sys < n_sys; sys++) + { + */ + + // For MOOSE, system 0 is the nonnlinear system - that's what we care about here + // I've left the other code commented out here because I might change my mind in a little while + // and add back the ability to set the system or compute over all + unsigned int sys = 0; + + auto n_vars = elem->n_vars(sys); + + for (decltype(n_vars) var = 0; var < n_vars; var++) + _local_num_dofs += elem->n_dofs(sys, var); + + //} + } + + virtual void onInternalSide(const Elem * elem, unsigned int side) override + { + if (elem->neighbor(side)->processor_id() != _this_pid) + { + _local_num_partition_sides++; + + // Build the side so we can compute its volume + auto side_elem = elem->build_side(side); + _local_partition_surface_area += side_elem->volume(); + } + } + + void join(const WBElementLoop & y) + { + _local_num_elems += y._local_num_elems; + _local_num_dofs += y._local_num_dofs; + _local_num_partition_sides += y._local_num_partition_sides; + _local_partition_surface_area += y._local_partition_surface_area; + } + + dof_id_type _local_num_elems; + dof_id_type _local_num_dofs; + dof_id_type _local_num_partition_sides; + Real _local_partition_surface_area; + + processor_id_type _this_pid; +}; + +class WBNodeLoop : public ThreadedNodeLoop +{ +public: + WBNodeLoop(FEProblemBase & fe_problem) + : ThreadedNodeLoop(fe_problem), + _local_num_nodes(0), + _local_num_dofs(0) + { + } + + WBNodeLoop(ThreadedNodeLoop & x, Threads::split split) + : ThreadedNodeLoop(x, split), + _local_num_nodes(0), + _local_num_dofs(0) + { + } + + virtual void onNode(ConstNodeRange::const_iterator & node_it) + { + auto & node = *(*node_it); + + _local_num_nodes++; + + unsigned int sys = 0; + + auto n_vars = node.n_vars(sys); + + for (decltype(n_vars) var = 0; var < n_vars; var++) + _local_num_dofs += node.n_dofs(sys, var); + } + + void join(WBNodeLoop & y) + { + _local_num_nodes += y._local_num_nodes; + _local_num_dofs += y._local_num_dofs; + } + + dof_id_type _local_num_nodes; + dof_id_type _local_num_dofs; +}; + +} // End of blank namespace + +void +WorkBalance::execute() +{ + auto & mesh = _fe_problem.mesh(); + + // Get all of the Elem info first + auto wb_el = WBElementLoop(mesh); + + Threads::parallel_reduce(*mesh.getActiveLocalElementRange(), wb_el); + + _local_num_elems = wb_el._local_num_elems; + _local_num_dofs = wb_el._local_num_dofs; + _local_num_partition_sides = wb_el._local_num_partition_sides; + _local_partition_surface_area = wb_el._local_partition_surface_area; + + // Now Node info + auto wb_nl = WBNodeLoop(_fe_problem); + + Threads::parallel_reduce(*mesh.getLocalNodeRange(), wb_nl); + + _local_num_nodes = wb_nl._local_num_nodes; + _local_num_dofs += wb_nl._local_num_dofs; +} + +void +WorkBalance::finalize() +{ + // Gather the results down to processor 0 + _communicator.gather(0, static_cast(_local_num_elems), _num_elems); + _communicator.gather(0, static_cast(_local_num_nodes), _num_nodes); + _communicator.gather(0, static_cast(_local_num_dofs), _num_dofs); + _communicator.gather(0, static_cast(_local_num_partition_sides), _num_partition_sides); + _communicator.gather(0, _local_partition_surface_area, _partition_surface_area); + + // Fill in the PID column - this just makes plotting easier + _pid.resize(_num_elems.size()); + std::iota(_pid.begin(), _pid.end(), 0); +} diff --git a/test/tests/vectorpostprocessors/work_balance/gold/work_balance_out_work_balance_0000.csv b/test/tests/vectorpostprocessors/work_balance/gold/work_balance_out_work_balance_0000.csv new file mode 100644 index 000000000000..fe6261602d29 --- /dev/null +++ b/test/tests/vectorpostprocessors/work_balance/gold/work_balance_out_work_balance_0000.csv @@ -0,0 +1,3 @@ +pid,num_elems,num_nodes,num_dofs,num_partition_sides,partition_surface_area +0,50,67,67,12,1.2 +1,50,54,54,12,1.2 diff --git a/test/tests/vectorpostprocessors/work_balance/tests b/test/tests/vectorpostprocessors/work_balance/tests new file mode 100644 index 000000000000..c2e421ff19a5 --- /dev/null +++ b/test/tests/vectorpostprocessors/work_balance/tests @@ -0,0 +1,9 @@ +[Tests] + [./test] + type = 'CSVDiff' + input = 'work_balance.i' + csvdiff = 'work_balance_out_work_balance_0000.csv' + min_parallel = 2 + max_parallel = 2 + [../] +[] diff --git a/test/tests/vectorpostprocessors/work_balance/work_balance.i b/test/tests/vectorpostprocessors/work_balance/work_balance.i new file mode 100644 index 000000000000..592bd7bb3172 --- /dev/null +++ b/test/tests/vectorpostprocessors/work_balance/work_balance.i @@ -0,0 +1,51 @@ +[Mesh] + type = GeneratedMesh + dim = 2 + nx = 10 + ny = 10 +[] + +[Variables] + [./u] + [../] +[] + +[Kernels] + [./diff] + type = Diffusion + variable = u + [../] +[] + +[BCs] + [./left] + type = DirichletBC + variable = u + boundary = left + value = 0 + [../] + [./right] + type = DirichletBC + variable = u + boundary = right + value = 1 + [../] +[] + +[Executioner] + type = Steady + solve_type = 'PJFNK' + petsc_options_iname = '-pc_type -pc_hypre_type' + petsc_options_value = 'hypre boomeramg' +[] + +[VectorPostprocessors] + [./work_balance] + type = WorkBalance + execute_on = initial + [] +[] + +[Outputs] + csv = true +[]