diff --git a/src/dfChemistryModel/Make/files b/src/dfChemistryModel/Make/files index f7cada43..f130bf76 100644 --- a/src/dfChemistryModel/Make/files +++ b/src/dfChemistryModel/Make/files @@ -1,3 +1,13 @@ +loadBalancing/ChemistryProblem.C +loadBalancing/ChemistrySolution.C +loadBalancing/ChemistryLoad.C +loadBalancing/LoadBalancerBase.C +loadBalancing/SendBuffer.C +loadBalancing/RecvBuffer.C +loadBalancing/algorithms_DLB.C +loadBalancing/runtime_assert.C +loadBalancing/LoadBalancer.C + makeDfChemistryModels.C LIB = $(FOAM_USER_LIBBIN)/libdfChemistryModel \ No newline at end of file diff --git a/src/dfChemistryModel/dfChemistryModel.C b/src/dfChemistryModel/dfChemistryModel.C index 78a6014a..32bcf76c 100644 --- a/src/dfChemistryModel/dfChemistryModel.C +++ b/src/dfChemistryModel/dfChemistryModel.C @@ -25,6 +25,8 @@ License #include "dfChemistryModel.H" #include "UniformField.H" +#include "clockTime.H" +#include "loadBalancing/runtime_assert.H" // * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * // @@ -80,7 +82,20 @@ Foam::dfChemistryModel::dfChemistryModel mesh_, dimensionedScalar(dimEnergy/dimVolume/dimTime, 0) ), - torchSwitch_(lookupOrDefault("torch", false)) + torchSwitch_(lookupOrDefault("torch", false)), + cpuTimes_ + ( + IOobject + ( + "cellCpuTimes", + mesh_.time().timeName(), + mesh_, + IOobject::NO_READ, + IOobject::AUTO_WRITE + ), + mesh_, + scalar(0.0) + ) { if(torchSwitch_) { @@ -191,7 +206,8 @@ Foam::scalar Foam::dfChemistryModel::solve } else { - result = canteraSolve(deltaT); + //result = canteraSolve(deltaT); + result = solve_loadBalance(deltaT); } return result; } @@ -610,4 +626,295 @@ void Foam::dfChemistryModel::correctThermo() } } +template +void Foam::dfChemistryModel::solveSingle +( + ChemistryProblem& problem, ChemistrySolution& solution +) +{ + scalar timeLeft = problem.deltaT; + scalarList c0 = problem.c; + + // Timer begins + clockTime time; + time.timeIncrement(); + + Cantera::Reactor react; + const scalar Ti = problem.Ti; + const scalar pi = problem.pi; + const scalarList Y = problem.Y; + + + CanteraGas_->setState_TPY(Ti, pi, Y.begin()); + CanteraGas_->getConcentrations(c0.begin()); // value --> c0 + + react.insert(mixture_.CanteraSolution()); + react.setEnergy(0); // keep T const before and after sim.advance. this will give you a little improvement + Cantera::ReactorNet sim; + sim.addReactor(react); + setNumerics(sim); + + sim.advance(problem.deltaT); + + // get new concentrations + CanteraGas_->getConcentrations(cTemp_.begin()); // value --> cTemp_ + problem.c = cTemp_; + + + solution.c_increment = (problem.c - c0) / problem.deltaT; + //solution.deltaTChem = min(problem.deltaTChem, this->deltaTChemMax_); + + // Timer ends + solution.cpuTime = time.timeIncrement(); + + solution.cellid = problem.cellid; + solution.rhoi = problem.rhoi; +} + + + +template +template +Foam::DynamicList +Foam::dfChemistryModel::getProblems +( + const DeltaTType& deltaT +) +{ + const scalarField& T = T_; + const scalarField& p = p_; + const scalarField& rho = rho_; + + + DynamicList solved_problems; + + //solved_problems.resize(p.size(), ChemistryProblem(mixture_.nSpecies())); + + scalarField massFraction(mixture_.nSpecies()); + //scalarField concentration(mixture_.nSpecies()); + + label counter = 0; + forAll(T, celli) + { + + //if(T[celli] > this->Treact()) + { + //for(label i = 0; i < mixture_.nSpecies(); i++) + //{ + //concentration[i] = rho[celli] * this->Y_[i][celli] / this->specieThermo_[i].W(); + // massFraction[i] = this->Y_[i][celli]; + //} + + ChemistryProblem problem; + //problem.c = concentration; + problem.Y = massFraction; + problem.Ti = T[celli]; + problem.pi = p[celli]; + problem.rhoi = rho[celli]; + //problem.deltaTChem = this->deltaTChem_[celli]; + problem.deltaT = deltaT[celli]; + //problem.cpuTime = cpuTimes_[celli]; + problem.cellid = celli; + + + + solved_problems[counter] = problem; + counter++; + + + } + //else + //{ + // for(label i = 0; i < this->nSpecie(); i++) + // { + // this->RR_[i][celli] = 0; + // } + //} + + } + + //the real size is set here + solved_problems.setSize(counter); + + return solved_problems; +} + + +template +Foam::DynamicList +Foam::dfChemistryModel::solveList +( + UList& problems +) +{ + DynamicList solutions( + problems.size(), ChemistrySolution(mixture_.nSpecies())); + + for(label i = 0; i < problems.size(); ++i) + { + solveSingle(problems[i], solutions[i]); + } + return solutions; +} + + +template +Foam::RecvBuffer +Foam::dfChemistryModel::solveBuffer +( + RecvBuffer& problems +) +{ + // allocate the solutions buffer + RecvBuffer solutions; + + for(auto& p : problems) + { + solutions.append(solveList(p)); + } + return solutions; +} + + + +template +Foam::scalar +Foam::dfChemistryModel::updateReactionRates +( + const RecvBuffer& solutions +) +{ + scalar deltaTMin = great; + + for(const auto& array : solutions) + { + for(const auto& solution : array) + { + + // for(label j = 0; j < mixture_.nSpecies(); j++) + // { + // this->RR_[j][solution.cellid] = + // solution.c_increment[j] * this->specieThermo_[j].W(); + // } + + // deltaTMin = min(solution.deltaTChem, deltaTMin); + + // this->deltaTChem_[solution.cellid] = + // min(solution.deltaTChem, this->deltaTChemMax_); + + cpuTimes_[solution.cellid] = solution.cpuTime; + } + } + + return deltaTMin; +} + + +template +void Foam::dfChemistryModel::updateReactionRate +( + const ChemistrySolution& solution, const label& i +) +{ + //for(label j = 0; j < mixture_.nSpecies(); j++) + { + //this->RR_[j][i] = solution.c_increment[j] * this->specieThermo_[j].W(); + } + //this->deltaTChem_[i] = min(solution.deltaTChem, this->deltaTChemMax_); +} + + +template +Foam::LoadBalancer +Foam::dfChemistryModel::createBalancer() +{ + const IOdictionary chemistryDict_tmp + ( + IOobject + ( + "CanteraTorchProperties", + thermo_.db().time().constant(), + thermo_.db(), + IOobject::MUST_READ, + IOobject::NO_WRITE, + false + ) + ); + + return LoadBalancer(chemistryDict_tmp); +} + + + +template +template +Foam::scalar Foam::dfChemistryModel::solve_loadBalance +( + const DeltaTType& deltaT +) +{ + // CPU time analysis + clockTime timer; + scalar t_getProblems(0); + scalar t_updateState(0); + scalar t_balance(0); + scalar t_solveBuffer(0); + scalar t_unbalance(0); + + if(!this->chemistry_) + { + return great; + } + + timer.timeIncrement(); + DynamicList allProblems = getProblems(deltaT); + t_getProblems = timer.timeIncrement(); + + RecvBuffer incomingSolutions; + + if(balancer_.active()) + { + timer.timeIncrement(); + balancer_.updateState(allProblems); + t_updateState = timer.timeIncrement(); + + timer.timeIncrement(); + auto guestProblems = balancer_.balance(allProblems); + auto ownProblems = balancer_.getRemaining(allProblems); + t_balance = timer.timeIncrement(); + + timer.timeIncrement(); + auto ownSolutions = solveList(ownProblems); + auto guestSolutions = solveBuffer(guestProblems); + t_solveBuffer = timer.timeIncrement(); + + timer.timeIncrement(); + incomingSolutions = balancer_.unbalance(guestSolutions); + incomingSolutions.append(ownSolutions); + t_unbalance = timer.timeIncrement(); + } + else + { + timer.timeIncrement(); + incomingSolutions.append(solveList(allProblems)); + t_solveBuffer = timer.timeIncrement(); + } + + if(balancer_.log()) + { + balancer_.printState(); + cpuSolveFile_() << setw(22) + << this->time().timeOutputValue()< RR_; hashedWordList species_; @@ -110,6 +116,12 @@ public IOdictionary scalar Tact_; scalar Qdotact_; + // Load balancing object + LoadBalancer balancer_; + // Field containing chemistry CPU time information + volScalarField cpuTimes_; + // A file to output the balancing stats + autoPtr cpuSolveFile_; // Private Member Functions @@ -178,6 +190,32 @@ public: // update T, psi, mu, alpha, rhoD, hai (if needed) void correctThermo(); + + void solveSingle(ChemistryProblem& problem, ChemistrySolution& solution); + + template + DynamicList getProblems(const DeltaTType& deltaT); + + //- Solve a list of chemistry problems and return a list of solutions + DynamicList + solveList(UList& problems); + + //- Solve the problem buffer coming from the balancer + RecvBuffer + solveBuffer(RecvBuffer& problems); + + //- Update the reaction rate of cell i + virtual void + updateReactionRate(const ChemistrySolution& solution, const label& i); + + //- Update the reaction rates from a list of solutions + scalar updateReactionRates(const RecvBuffer& solutions); + + //- Create a load balancer object + LoadBalancer createBalancer(); + + template + scalar solve_loadBalance(const DeltaTType& deltaT); }; diff --git a/src/dfChemistryModel/loadBalancing/ChemistryLoad.C b/src/dfChemistryModel/loadBalancing/ChemistryLoad.C new file mode 100644 index 00000000..4c33d7bd --- /dev/null +++ b/src/dfChemistryModel/loadBalancing/ChemistryLoad.C @@ -0,0 +1,29 @@ +/*---------------------------------------------------------------------------*\ + ========= | + \\ / F ield | DLBFoam: Dynamic Load Balancing + \\ / O peration | for fast reactive simulations + \\ / A nd | + \\/ M anipulation | 2020, Aalto University, Finland +------------------------------------------------------------------------------- +License + This file is part of DLBFoam library, derived from OpenFOAM. + + https://github.com/blttkgl/DLBFoam + + OpenFOAM is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + OpenFOAM is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + You should have received a copy of the GNU General Public License + along with OpenFOAM. If not, see . + +\*---------------------------------------------------------------------------*/ + +#include "ChemistryLoad.H" +namespace Foam{ + +} \ No newline at end of file diff --git a/src/dfChemistryModel/loadBalancing/ChemistryLoad.H b/src/dfChemistryModel/loadBalancing/ChemistryLoad.H new file mode 100644 index 00000000..acc77411 --- /dev/null +++ b/src/dfChemistryModel/loadBalancing/ChemistryLoad.H @@ -0,0 +1,106 @@ +/*---------------------------------------------------------------------------*\ + ========= | + \\ / F ield | DLBFoam: Dynamic Load Balancing + \\ / O peration | for fast reactive simulations + \\ / A nd | + \\/ M anipulation | 2020, Aalto University, Finland +------------------------------------------------------------------------------- +License + This file is part of DLBFoam library, derived from OpenFOAM. + + https://github.com/blttkgl/DLBFoam + + OpenFOAM is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + OpenFOAM is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + You should have received a copy of the GNU General Public License + along with OpenFOAM. If not, see . + +Class + Foam::ChemistryLoad + +Description + A generic load object that stores a rank and a value and enables comparison + as well as serialization through the OStream and IStream operators. + +\*---------------------------------------------------------------------------*/ + +#ifndef ChemistryLoad_H +#define ChemistryLoad_H + +#include "Istream.H" +#include "Ostream.H" + +namespace Foam +{ + +// A struct containing rank and load value of each processor +struct ChemistryLoad +{ + // rank ID of the process + label rank; + + // the load value indicating how busy the processor is + scalar value; + + ChemistryLoad() = default; + + ChemistryLoad(label _rank, scalar _value) : rank(_rank), value(_value) + { + } + + // overload comparisons for sorting + bool operator==(const ChemistryLoad& rhs) const + { + return value == rhs.value; + } + bool operator!=(const ChemistryLoad& rhs) const + { + return !(value == rhs.value); + } + bool operator<=(const ChemistryLoad& rhs) const + { + return value <= rhs.value; + } + bool operator>=(const ChemistryLoad& rhs) const + { + return value >= rhs.value; + } + bool operator<(const ChemistryLoad& rhs) const + { + return value < rhs.value; + } + bool operator>(const ChemistryLoad& rhs) const + { + return value > rhs.value; + } +}; + +//- Serialization for send OStream +static inline Ostream& operator<<(Ostream& os, const ChemistryLoad& l) +{ + + os << l.rank; + os << l.value; + + return os; +} + +//- Get a serialized load from Istream +static inline Istream& operator>>(Istream& is, ChemistryLoad& l) +{ + + is >> l.rank; + is >> l.value; + + return is; +} + +} // namespace Foam + +#endif \ No newline at end of file diff --git a/src/dfChemistryModel/loadBalancing/ChemistryProblem.C b/src/dfChemistryModel/loadBalancing/ChemistryProblem.C new file mode 100644 index 00000000..7f4a9498 --- /dev/null +++ b/src/dfChemistryModel/loadBalancing/ChemistryProblem.C @@ -0,0 +1,29 @@ +/*---------------------------------------------------------------------------*\ + ========= | + \\ / F ield | DLBFoam: Dynamic Load Balancing + \\ / O peration | for fast reactive simulations + \\ / A nd | + \\/ M anipulation | 2020, Aalto University, Finland +------------------------------------------------------------------------------- +License + This file is part of DLBFoam library, derived from OpenFOAM. + + https://github.com/blttkgl/DLBFoam + + OpenFOAM is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + OpenFOAM is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + You should have received a copy of the GNU General Public License + along with OpenFOAM. If not, see . + +\*---------------------------------------------------------------------------*/ + +#include "ChemistryProblem.H" +namespace Foam{ + +} \ No newline at end of file diff --git a/src/dfChemistryModel/loadBalancing/ChemistryProblem.H b/src/dfChemistryModel/loadBalancing/ChemistryProblem.H new file mode 100644 index 00000000..720340ae --- /dev/null +++ b/src/dfChemistryModel/loadBalancing/ChemistryProblem.H @@ -0,0 +1,108 @@ +/*---------------------------------------------------------------------------*\ + ========= | + \\ / F ield | DLBFoam: Dynamic Load Balancing + \\ / O peration | for fast reactive simulations + \\ / A nd | + \\/ M anipulation | 2020, Aalto University, Finland +------------------------------------------------------------------------------- +License + This file is part of DLBFoam library, derived from OpenFOAM. + + https://github.com/blttkgl/DLBFoam + + OpenFOAM is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + OpenFOAM is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + You should have received a copy of the GNU General Public License + along with OpenFOAM. If not, see . + +Class + Foam::ChemistryProblem + +Description + A small object containing everything required for solving the reaction rate + using the ODE solver. These are passed around in the load balancer. + +\*---------------------------------------------------------------------------*/ + +#ifndef ChemistryProblem_H +#define ChemistryProblem_H + +#include "volFields.H" + +namespace Foam +{ + +struct ChemistryProblem +{ + + ChemistryProblem() = default; + ChemistryProblem(label nSpecie) + : c(nSpecie), Y(nSpecie), Ti(0), pi(0), rhoi(0), deltaTChem(0), deltaT(0), cellid(0) + { + } + + scalarList c; + scalarList Y; + scalar Ti; + scalar pi; + scalar rhoi; + scalar deltaTChem; + scalar deltaT; + scalar cpuTime; + label cellid; + + // TODO: implement! + bool operator==(const ChemistryProblem& rhs) const + { + return false; + } + + bool operator!=(const ChemistryProblem& rhs) const + { + return !(*this == rhs); + } +}; + +//- Serialization for send +static inline Ostream& operator<<(Ostream& os, const ChemistryProblem& p) +{ + + os << p.c; + os << p.Y; + os << p.Ti; + os << p.pi; + os << p.rhoi; + os << p.deltaTChem; + os << p.deltaT; + os << p.cpuTime; + os << p.cellid; + + return os; +} + +//- Get a serialized problem from IStream +static inline Istream& operator>>(Istream& is, ChemistryProblem& p) +{ + + is >> p.c; + is >> p.Y; + is >> p.Ti; + is >> p.pi; + is >> p.rhoi; + is >> p.deltaTChem; + is >> p.deltaT; + is >> p.cpuTime; + is >> p.cellid; + + return is; +} + +} // namespace Foam + +#endif \ No newline at end of file diff --git a/src/dfChemistryModel/loadBalancing/ChemistrySolution.C b/src/dfChemistryModel/loadBalancing/ChemistrySolution.C new file mode 100644 index 00000000..16c21cec --- /dev/null +++ b/src/dfChemistryModel/loadBalancing/ChemistrySolution.C @@ -0,0 +1,29 @@ +/*---------------------------------------------------------------------------*\ + ========= | + \\ / F ield | DLBFoam: Dynamic Load Balancing + \\ / O peration | for fast reactive simulations + \\ / A nd | + \\/ M anipulation | 2020, Aalto University, Finland +------------------------------------------------------------------------------- +License + This file is part of DLBFoam library, derived from OpenFOAM. + + https://github.com/blttkgl/DLBFoam + + OpenFOAM is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + OpenFOAM is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + You should have received a copy of the GNU General Public License + along with OpenFOAM. If not, see . + +\*---------------------------------------------------------------------------*/ + +#include "ChemistrySolution.H" +namespace Foam{ + +} \ No newline at end of file diff --git a/src/dfChemistryModel/loadBalancing/ChemistrySolution.H b/src/dfChemistryModel/loadBalancing/ChemistrySolution.H new file mode 100644 index 00000000..e803cea2 --- /dev/null +++ b/src/dfChemistryModel/loadBalancing/ChemistrySolution.H @@ -0,0 +1,92 @@ +/*---------------------------------------------------------------------------*\ + ========= | + \\ / F ield | DLBFoam: Dynamic Load Balancing + \\ / O peration | for fast reactive simulations + \\ / A nd | + \\/ M anipulation | 2020, Aalto University, Finland +------------------------------------------------------------------------------- +License + This file is part of DLBFoam library, derived from OpenFOAM. + + https://github.com/blttkgl/DLBFoam + + OpenFOAM is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + OpenFOAM is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + You should have received a copy of the GNU General Public License + along with OpenFOAM. If not, see . + +Class + Foam::ChemistrySolution + +Description + A small object containing everything required for updating the reaction rate + and the chemistry time step. These are passed around in the load balancer. + +\*---------------------------------------------------------------------------*/ + +#ifndef ChemistrySolution_H +#define ChemistrySolution_H + +#include "volFields.H" + +namespace Foam +{ + +struct ChemistrySolution +{ + + ChemistrySolution() = default; + + ChemistrySolution(label nspecie) + : c_increment(nspecie, 0.0), deltaTChem(0.0), cellid(0), rhoi(0.0) + { + } + + bool operator==(const ChemistrySolution& rhs) const + { + return false; + } + + bool operator!=(const ChemistrySolution& rhs) const + { + return !(*this == rhs); + } + + scalarField c_increment; // = (c_{i+1} - c_{i}) / deltaT; + scalar deltaTChem; + scalar cpuTime; + label cellid; + scalar rhoi; +}; + +//- Serialization for send +static inline Ostream& operator<<(Ostream& os, const ChemistrySolution& s) +{ + os << s.c_increment; + os << s.deltaTChem; + os << s.cpuTime; + os << s.cellid; + os << s.rhoi; + return os; +} + +//- Get a serialized solution from IStream +static inline Istream& operator>>(Istream& is, ChemistrySolution& s) +{ + is >> s.c_increment; + is >> s.deltaTChem; + is >> s.cpuTime; + is >> s.cellid; + is >> s.rhoi; + return is; +} + +} // namespace Foam + +#endif \ No newline at end of file diff --git a/src/dfChemistryModel/loadBalancing/LoadBalancer.C b/src/dfChemistryModel/loadBalancing/LoadBalancer.C new file mode 100644 index 00000000..37826f59 --- /dev/null +++ b/src/dfChemistryModel/loadBalancing/LoadBalancer.C @@ -0,0 +1,196 @@ +/*---------------------------------------------------------------------------*\ + ========= | + \\ / F ield | DLBFoam: Dynamic Load Balancing + \\ / O peration | for fast reactive simulations + \\ / A nd | + \\/ M anipulation | 2020, Aalto University, Finland +------------------------------------------------------------------------------- +License + This file is part of DLBFoam library, derived from OpenFOAM. + + https://github.com/blttkgl/DLBFoam + + OpenFOAM is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + OpenFOAM is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + You should have received a copy of the GNU General Public License + along with OpenFOAM. If not, see . + +\*---------------------------------------------------------------------------*/ + +#include "LoadBalancer.H" + +void +Foam::LoadBalancer::updateState( + const DynamicList& problems) +{ + auto myLoad = computeLoad(problems); + auto allLoads = allGather(myLoad); + auto operations = getOperations(allLoads, myLoad); + auto info = operationsToInfo(operations, problems, myLoad); + + setState(info); +} + +Foam::LoadBalancerBase::BalancerState +Foam::LoadBalancer::operationsToInfo( + const std::vector& operations, + const DynamicList& problems, + const ChemistryLoad& myLoad) +{ + BalancerState info; + + if(isSender(operations, myLoad.rank)) + { + double sum = 0.0; + std::vector times; + for(const auto& op : operations) + { + info.destinations.push_back(op.to); + sum += op.value; + times.push_back(op.value); + } + info.nProblems = timesToProblemCounts(times, problems); + + label total = std::accumulate(info.nProblems.begin(), info.nProblems.end(), 0); + info.nRemaining = problems.size() - total; + } + + // receiver + else + { + for(const auto& op : operations) + { + info.sources.push_back(op.from); + } + info.nProblems = {}; + info.nRemaining = problems.size(); + } + + + return info; +} + +std::vector +Foam::LoadBalancer::timesToProblemCounts( + const std::vector& times, + const DynamicList& problems) +{ + + std::vector counts; + counts.reserve(times.size() + 1); + auto begin = problems.begin(); + + for(const auto& time : times) + { + scalar sum(0); + auto operation = [&](const ChemistryProblem& problem) + { + sum += problem.cpuTime; + return sum <= time; + }; + auto count = count_while(begin, problems.end(), operation); + begin += count; + counts.push_back(count); + } + + + return counts; +} + +std::vector +Foam::LoadBalancer::getOperations( + DynamicList& loads, const ChemistryLoad& myLoad) +{ + + double globalMean = getMean(loads); + + std::vector operations; + + std::sort(loads.begin(), loads.end()); + + auto sender = loads.end() - 1; + auto receiver = loads.begin(); + + while(sender != receiver) + { + double send_value = std::min( + sender->value - globalMean, globalMean - receiver->value); + + Operation operation{sender->rank, receiver->rank, send_value}; + if(sender->rank == myLoad.rank || receiver->rank == myLoad.rank) + { + operations.push_back(operation); + } + sender->value -= send_value; + receiver->value += send_value; + + if(std::abs(sender->value - globalMean) < SMALL) + { + sender--; + } + else + { + receiver++; + } + } + + // explicitly filter very small operations + std::vector large; + for(const auto& op : operations) + { + if(op.value > 0.01 * globalMean) + { + large.push_back(op); + } + } + + runtime_assert( + !((isSender(operations, myLoad.rank) && + isReceiver(operations, myLoad.rank))), + "Only sender or receiver should be possible."); + + runtime_assert( + std::abs(getMean(loads) - globalMean) < 1E-7, "Vanishing load"); + + return large; +} + +bool +Foam::LoadBalancer::isSender( + const std::vector& operations, int rank) +{ + if(operations.size() == 0) + { + return false; + } + + for(const auto& op : operations) + { + if(op.from != rank) + { + return false; + } + } + return true; +} + +bool +Foam::LoadBalancer::isReceiver( + const std::vector& operations, int rank) +{ + for(const auto& op : operations) + { + if(op.to != rank) + { + return false; + } + } + return true; +} + diff --git a/src/dfChemistryModel/loadBalancing/LoadBalancer.H b/src/dfChemistryModel/loadBalancing/LoadBalancer.H new file mode 100644 index 00000000..06c008c0 --- /dev/null +++ b/src/dfChemistryModel/loadBalancing/LoadBalancer.H @@ -0,0 +1,140 @@ +/*---------------------------------------------------------------------------*\ + ========= | + \\ / F ield | DLBFoam: Dynamic Load Balancing + \\ / O peration | for fast reactive simulations + \\ / A nd | + \\/ M anipulation | 2020, Aalto University, Finland +------------------------------------------------------------------------------- +License + This file is part of DLBFoam library, derived from OpenFOAM. + + https://github.com/blttkgl/DLBFoam + + OpenFOAM is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + OpenFOAM is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + You should have received a copy of the GNU General Public License + along with OpenFOAM. If not, see . + +Class + Foam::LoadBalancer + +Description + Extends the base class LoadBalancerBase by implementing a + balancing algorithm which tries to set the global mean load to each rank. + +SourceFiles + LoadBalancer.C +\*---------------------------------------------------------------------------*/ + +#ifndef LoadBalancer_H +#define LoadBalancer_H + +#include "ChemistryProblem.H" +#include "IOdictionary.H" +#include "LoadBalancerBase.H" +#include "Switch.H" +#include "algorithms_DLB.H" +#include "runTimeSelectionTables.H" +#include "scalarField.H" + +#include +#include + +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // + +namespace Foam +{ + +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // + +class LoadBalancer : public LoadBalancerBase +{ + + +public: + struct Operation + { + int from, to; // ranks + double value; + }; + + LoadBalancer() = default; + + LoadBalancer(const dictionary& dict) + : LoadBalancerBase(), dict_(dict), + coeffsDict_(dict.subDict("loadbalancing")), + active_(coeffsDict_.lookupOrDefault("active", true)), + log_(coeffsDict_.lookupOrDefault("log", false)) + { + } + + // Destructor + virtual ~LoadBalancer() = default; + + //- Given a list of problems, update the balancer state member + virtual void updateState(const DynamicList& problems); + + //- Is load balancing active? + bool active() const + { + return active_; + } + + //- Is load balancing logged? + bool log() const + { + return log_; + } + + + +protected: + + //- Get the operations for this rank that would minimize the load to + // global mean + static std::vector getOperations( + DynamicList& loads, const ChemistryLoad& myLoad); + + //- Convert the operations to send and receive info to handle balancing + static BalancerState operationsToInfo( + const std::vector& operations, + const DynamicList& problems, + const ChemistryLoad& myLoad); + + + //- Convert the vector of cpu times to number of problems for the rank + static std::vector