From 0afc60d4366874c4413a7185dd990c219e45b802 Mon Sep 17 00:00:00 2001 From: maorz1998 Date: Tue, 21 Feb 2023 16:57:25 +0800 Subject: [PATCH 1/3] add new loadBalancing algorithm --- .gitignore | 1 + .../constant/CanteraTorchProperties | 1 + .../constant/CanteraTorchProperties | 1 + .../constant/CanteraTorchProperties | 1 + .../constant/CanteraTorchProperties | 1 + .../constant/CanteraTorchProperties | 1 + .../constant/CanteraTorchProperties | 1 + .../constant/CanteraTorchProperties | 1 + .../constant/CanteraTorchProperties | 1 + .../constant/CanteraTorchProperties | 1 + .../constant/CanteraTorchProperties | 1 + .../constant/CanteraTorchProperties | 1 + .../constant/CanteraTorchProperties | 1 + .../constant/CanteraTorchProperties | 1 + .../constant/CanteraTorchProperties | 1 + .../constant/CanteraTorchProperties | 1 + .../constant/CanteraTorchProperties | 1 + .../constant/CanteraTorchProperties | 1 + .../constant/CanteraTorchProperties | 1 + .../constant/CanteraTorchProperties | 1 + .../constant/CanteraTorchProperties | 1 + .../constant/CanteraTorchProperties | 1 + .../constant/CanteraTorchProperties | 1 + .../loadBalancing/LoadBalancer.C | 70 ++++++++++++++++++- .../loadBalancing/LoadBalancer.H | 19 ++++- 25 files changed, 110 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 8b01c19b..eeb717b6 100644 --- a/.gitignore +++ b/.gitignore @@ -69,3 +69,4 @@ libtorch*.zip* __pycache__/ lib/ bin/ +.vscode/ diff --git a/examples/df0DFoam/zeroD_cubicReactor/CH4/pytorchIntegrator/constant/CanteraTorchProperties b/examples/df0DFoam/zeroD_cubicReactor/CH4/pytorchIntegrator/constant/CanteraTorchProperties index d198a8b5..13fc340b 100644 --- a/examples/df0DFoam/zeroD_cubicReactor/CH4/pytorchIntegrator/constant/CanteraTorchProperties +++ b/examples/df0DFoam/zeroD_cubicReactor/CH4/pytorchIntegrator/constant/CanteraTorchProperties @@ -49,6 +49,7 @@ loadbalancing { active true; log false; + algorithm allAverage;//headTail; } diff --git a/examples/df0DFoam/zeroD_cubicReactor/H2/cvodeIntegrator/constant/CanteraTorchProperties b/examples/df0DFoam/zeroD_cubicReactor/H2/cvodeIntegrator/constant/CanteraTorchProperties index c7a63826..03374614 100644 --- a/examples/df0DFoam/zeroD_cubicReactor/H2/cvodeIntegrator/constant/CanteraTorchProperties +++ b/examples/df0DFoam/zeroD_cubicReactor/H2/cvodeIntegrator/constant/CanteraTorchProperties @@ -49,6 +49,7 @@ loadbalancing { active true; log false; + algorithm allAverage;//headTail; } diff --git a/examples/df0DFoam/zeroD_cubicReactor/H2/libtorchIntegrator/constant/CanteraTorchProperties b/examples/df0DFoam/zeroD_cubicReactor/H2/libtorchIntegrator/constant/CanteraTorchProperties index 0fe1354c..604b1e92 100644 --- a/examples/df0DFoam/zeroD_cubicReactor/H2/libtorchIntegrator/constant/CanteraTorchProperties +++ b/examples/df0DFoam/zeroD_cubicReactor/H2/libtorchIntegrator/constant/CanteraTorchProperties @@ -52,6 +52,7 @@ loadbalancing { active true; log false; + algorithm allAverage;//headTail; } diff --git a/examples/df0DFoam/zeroD_cubicReactor/H2/pytorchIntegrator/constant/CanteraTorchProperties b/examples/df0DFoam/zeroD_cubicReactor/H2/pytorchIntegrator/constant/CanteraTorchProperties index ebf63603..b7246472 100644 --- a/examples/df0DFoam/zeroD_cubicReactor/H2/pytorchIntegrator/constant/CanteraTorchProperties +++ b/examples/df0DFoam/zeroD_cubicReactor/H2/pytorchIntegrator/constant/CanteraTorchProperties @@ -49,6 +49,7 @@ loadbalancing { active true; log false; + algorithm allAverage;//headTail; } diff --git a/examples/dfHighSpeedFoam/oneD_detonationH2/constant/CanteraTorchProperties b/examples/dfHighSpeedFoam/oneD_detonationH2/constant/CanteraTorchProperties index 8577a66f..e0248ca5 100755 --- a/examples/dfHighSpeedFoam/oneD_detonationH2/constant/CanteraTorchProperties +++ b/examples/dfHighSpeedFoam/oneD_detonationH2/constant/CanteraTorchProperties @@ -44,6 +44,7 @@ loadbalancing { active true; log false; + algorithm allAverage;//headTail; } diff --git a/examples/dfHighSpeedFoam/oneD_reactiveShockTube/constant/CanteraTorchProperties b/examples/dfHighSpeedFoam/oneD_reactiveShockTube/constant/CanteraTorchProperties index 6e761171..6e51dd9b 100755 --- a/examples/dfHighSpeedFoam/oneD_reactiveShockTube/constant/CanteraTorchProperties +++ b/examples/dfHighSpeedFoam/oneD_reactiveShockTube/constant/CanteraTorchProperties @@ -44,6 +44,7 @@ loadbalancing { active true; log false; + algorithm allAverage;//headTail; } diff --git a/examples/dfHighSpeedFoam/twoD_detonationH2/constant/CanteraTorchProperties b/examples/dfHighSpeedFoam/twoD_detonationH2/constant/CanteraTorchProperties index c864d995..e3821dde 100755 --- a/examples/dfHighSpeedFoam/twoD_detonationH2/constant/CanteraTorchProperties +++ b/examples/dfHighSpeedFoam/twoD_detonationH2/constant/CanteraTorchProperties @@ -44,6 +44,7 @@ loadbalancing { active true; log false; + algorithm allAverage;//headTail; } diff --git a/examples/dfLowMachFoam/oneD_freelyPropagation/CH4/pytorchIntegrator/constant/CanteraTorchProperties b/examples/dfLowMachFoam/oneD_freelyPropagation/CH4/pytorchIntegrator/constant/CanteraTorchProperties index 6dfb79a4..8873b201 100644 --- a/examples/dfLowMachFoam/oneD_freelyPropagation/CH4/pytorchIntegrator/constant/CanteraTorchProperties +++ b/examples/dfLowMachFoam/oneD_freelyPropagation/CH4/pytorchIntegrator/constant/CanteraTorchProperties @@ -44,6 +44,7 @@ loadbalancing { active true; log false; + algorithm allAverage;//headTail; } diff --git a/examples/dfLowMachFoam/oneD_freelyPropagation/H2/cvodeIntegrator/constant/CanteraTorchProperties b/examples/dfLowMachFoam/oneD_freelyPropagation/H2/cvodeIntegrator/constant/CanteraTorchProperties index 8e1a85d2..2b6e27ad 100644 --- a/examples/dfLowMachFoam/oneD_freelyPropagation/H2/cvodeIntegrator/constant/CanteraTorchProperties +++ b/examples/dfLowMachFoam/oneD_freelyPropagation/H2/cvodeIntegrator/constant/CanteraTorchProperties @@ -44,6 +44,7 @@ loadbalancing { active true; log false; + algorithm allAverage;//headTail; } diff --git a/examples/dfLowMachFoam/oneD_freelyPropagation/H2/libtorchIntegrator/constant/CanteraTorchProperties b/examples/dfLowMachFoam/oneD_freelyPropagation/H2/libtorchIntegrator/constant/CanteraTorchProperties index 0fe1354c..604b1e92 100644 --- a/examples/dfLowMachFoam/oneD_freelyPropagation/H2/libtorchIntegrator/constant/CanteraTorchProperties +++ b/examples/dfLowMachFoam/oneD_freelyPropagation/H2/libtorchIntegrator/constant/CanteraTorchProperties @@ -52,6 +52,7 @@ loadbalancing { active true; log false; + algorithm allAverage;//headTail; } diff --git a/examples/dfLowMachFoam/oneD_freelyPropagation/H2/pytorchIntegrator/constant/CanteraTorchProperties b/examples/dfLowMachFoam/oneD_freelyPropagation/H2/pytorchIntegrator/constant/CanteraTorchProperties index aa558072..cb52df76 100644 --- a/examples/dfLowMachFoam/oneD_freelyPropagation/H2/pytorchIntegrator/constant/CanteraTorchProperties +++ b/examples/dfLowMachFoam/oneD_freelyPropagation/H2/pytorchIntegrator/constant/CanteraTorchProperties @@ -44,6 +44,7 @@ loadbalancing { active true; log false; + algorithm allAverage;//headTail; } diff --git a/examples/dfLowMachFoam/threeD_reactingTGV/H2/cvodeIntegrator/constant/CanteraTorchProperties b/examples/dfLowMachFoam/threeD_reactingTGV/H2/cvodeIntegrator/constant/CanteraTorchProperties index 8e1a85d2..2b6e27ad 100644 --- a/examples/dfLowMachFoam/threeD_reactingTGV/H2/cvodeIntegrator/constant/CanteraTorchProperties +++ b/examples/dfLowMachFoam/threeD_reactingTGV/H2/cvodeIntegrator/constant/CanteraTorchProperties @@ -44,6 +44,7 @@ loadbalancing { active true; log false; + algorithm allAverage;//headTail; } diff --git a/examples/dfLowMachFoam/threeD_reactingTGV/H2/libtorchIntegrator/constant/CanteraTorchProperties b/examples/dfLowMachFoam/threeD_reactingTGV/H2/libtorchIntegrator/constant/CanteraTorchProperties index 0fe1354c..604b1e92 100644 --- a/examples/dfLowMachFoam/threeD_reactingTGV/H2/libtorchIntegrator/constant/CanteraTorchProperties +++ b/examples/dfLowMachFoam/threeD_reactingTGV/H2/libtorchIntegrator/constant/CanteraTorchProperties @@ -52,6 +52,7 @@ loadbalancing { active true; log false; + algorithm allAverage;//headTail; } diff --git a/examples/dfLowMachFoam/threeD_reactingTGV/H2/pytorchIntegrator/constant/CanteraTorchProperties b/examples/dfLowMachFoam/threeD_reactingTGV/H2/pytorchIntegrator/constant/CanteraTorchProperties index aa558072..cb52df76 100644 --- a/examples/dfLowMachFoam/threeD_reactingTGV/H2/pytorchIntegrator/constant/CanteraTorchProperties +++ b/examples/dfLowMachFoam/threeD_reactingTGV/H2/pytorchIntegrator/constant/CanteraTorchProperties @@ -44,6 +44,7 @@ loadbalancing { active true; log false; + algorithm allAverage;//headTail; } diff --git a/examples/dfLowMachFoam/twoD_HIT_flame/CH4/pytorchIntegrator/constant/CanteraTorchProperties b/examples/dfLowMachFoam/twoD_HIT_flame/CH4/pytorchIntegrator/constant/CanteraTorchProperties index 6dfb79a4..8873b201 100644 --- a/examples/dfLowMachFoam/twoD_HIT_flame/CH4/pytorchIntegrator/constant/CanteraTorchProperties +++ b/examples/dfLowMachFoam/twoD_HIT_flame/CH4/pytorchIntegrator/constant/CanteraTorchProperties @@ -44,6 +44,7 @@ loadbalancing { active true; log false; + algorithm allAverage;//headTail; } diff --git a/examples/dfLowMachFoam/twoD_SandiaD_EDC_Trans/constant/CanteraTorchProperties b/examples/dfLowMachFoam/twoD_SandiaD_EDC_Trans/constant/CanteraTorchProperties index 54cefa64..817b4304 100755 --- a/examples/dfLowMachFoam/twoD_SandiaD_EDC_Trans/constant/CanteraTorchProperties +++ b/examples/dfLowMachFoam/twoD_SandiaD_EDC_Trans/constant/CanteraTorchProperties @@ -44,6 +44,7 @@ loadbalancing { active true; log false; + algorithm allAverage;//headTail; } diff --git a/examples/dfLowMachFoam/twoD_reactingTGV/CH4/pytorchIntegrator/constant/CanteraTorchProperties b/examples/dfLowMachFoam/twoD_reactingTGV/CH4/pytorchIntegrator/constant/CanteraTorchProperties index 6dfb79a4..8873b201 100644 --- a/examples/dfLowMachFoam/twoD_reactingTGV/CH4/pytorchIntegrator/constant/CanteraTorchProperties +++ b/examples/dfLowMachFoam/twoD_reactingTGV/CH4/pytorchIntegrator/constant/CanteraTorchProperties @@ -44,6 +44,7 @@ loadbalancing { active true; log false; + algorithm allAverage;//headTail; } diff --git a/examples/dfLowMachFoam/twoD_tripleFlame/H2/cvodeIntegrator/constant/CanteraTorchProperties b/examples/dfLowMachFoam/twoD_tripleFlame/H2/cvodeIntegrator/constant/CanteraTorchProperties index 8e1a85d2..2b6e27ad 100644 --- a/examples/dfLowMachFoam/twoD_tripleFlame/H2/cvodeIntegrator/constant/CanteraTorchProperties +++ b/examples/dfLowMachFoam/twoD_tripleFlame/H2/cvodeIntegrator/constant/CanteraTorchProperties @@ -44,6 +44,7 @@ loadbalancing { active true; log false; + algorithm allAverage;//headTail; } diff --git a/examples/dfLowMachFoam/twoD_tripleFlame/H2/libtorchIntegrator/constant/CanteraTorchProperties b/examples/dfLowMachFoam/twoD_tripleFlame/H2/libtorchIntegrator/constant/CanteraTorchProperties index 0fe1354c..604b1e92 100644 --- a/examples/dfLowMachFoam/twoD_tripleFlame/H2/libtorchIntegrator/constant/CanteraTorchProperties +++ b/examples/dfLowMachFoam/twoD_tripleFlame/H2/libtorchIntegrator/constant/CanteraTorchProperties @@ -52,6 +52,7 @@ loadbalancing { active true; log false; + algorithm allAverage;//headTail; } diff --git a/examples/dfLowMachFoam/twoD_tripleFlame/H2/pytorchIntegrator/constant/CanteraTorchProperties b/examples/dfLowMachFoam/twoD_tripleFlame/H2/pytorchIntegrator/constant/CanteraTorchProperties index aa558072..cb52df76 100644 --- a/examples/dfLowMachFoam/twoD_tripleFlame/H2/pytorchIntegrator/constant/CanteraTorchProperties +++ b/examples/dfLowMachFoam/twoD_tripleFlame/H2/pytorchIntegrator/constant/CanteraTorchProperties @@ -44,6 +44,7 @@ loadbalancing { active true; log false; + algorithm allAverage;//headTail; } diff --git a/examples/dfSprayFoam/aachenBomb/constant/CanteraTorchProperties b/examples/dfSprayFoam/aachenBomb/constant/CanteraTorchProperties index ce9f5b7f..9ffa283c 100644 --- a/examples/dfSprayFoam/aachenBomb/constant/CanteraTorchProperties +++ b/examples/dfSprayFoam/aachenBomb/constant/CanteraTorchProperties @@ -44,6 +44,7 @@ loadbalancing { active true; log false; + algorithm allAverage;//headTail; } diff --git a/examples/dfSprayFoam/sydneySprayBurner/constant/CanteraTorchProperties b/examples/dfSprayFoam/sydneySprayBurner/constant/CanteraTorchProperties index 1059de47..5ef6c6c5 100644 --- a/examples/dfSprayFoam/sydneySprayBurner/constant/CanteraTorchProperties +++ b/examples/dfSprayFoam/sydneySprayBurner/constant/CanteraTorchProperties @@ -44,6 +44,7 @@ loadbalancing { active true; log false; + algorithm allAverage;//headTail; } diff --git a/src/dfChemistryModel/loadBalancing/LoadBalancer.C b/src/dfChemistryModel/loadBalancing/LoadBalancer.C index d7a5d094..edb9973c 100644 --- a/src/dfChemistryModel/loadBalancing/LoadBalancer.C +++ b/src/dfChemistryModel/loadBalancing/LoadBalancer.C @@ -33,7 +33,23 @@ Foam::LoadBalancer::updateState( if (Pstream::myProcNo(comm) == -1) return; auto myLoad = computeLoad(problems, comm); auto allLoads = allGather(myLoad, comm); - auto operations = getOperations(allLoads, myLoad); + std::vector operations; + if (algorithm_ == "allAverage") + { + operations = getOperations(allLoads, myLoad); + if (log_) + { + Info << "now perform load balance with allAverage (DLB) algorithm" << endl; + } + } + else + { + operations = getOperationsRedezVous(allLoads, myLoad); + if (log_) + { + Info << "now perform load balance with headTail (RedezVous) algorithm" << endl; + } + } auto info = operationsToInfo(operations, problems, myLoad); setState(info); @@ -163,6 +179,58 @@ Foam::LoadBalancer::getOperations( return large; } +//perform load balance with Redez-vous algorithm +std::vector +Foam::LoadBalancer::getOperationsRedezVous( + DynamicList& loads, const ChemistryLoad& myLoad) +{ + double globalMean = getMean(loads); //calculate the mean load + + std::vector operations; + + std::sort(loads.begin(), loads.end()); //sort the loads + + auto sender = loads.end() - 1; //the last load greater than the mean load + auto receiver = loads.begin(); + + for (int i = 0; i < int(loads.size()/2); ++i) + { + //std::cout << "sender: " << sender->rank << " receiver: " << receiver->rank << std::endl; + + double send_value = (sender->value - receiver->value) / 2; // calculate the send value + + Operation operation{sender->rank, receiver->rank, send_value}; + if(sender->rank == myLoad.rank || receiver->rank == myLoad.rank) + { + operations.push_back(operation); // if send or recv rank related to my rank, add to operations + } + sender->value -= send_value; // update the load + receiver->value += send_value; // update the load + + sender--; + receiver++; + } + + // explicitly filter very small operations + std::vector large; + for(const auto& op:operations) + { + if(op.value > 0.01 * globalMean) //if send value is larger than 1% of global mean + { + large.push_back(op); // add to large operations + } + } + runtime_assert( + !((isSender(operations, myLoad.rank) && + isReceiver(operations, myLoad.rank))), + "Only sender or receiver should be possible."); + + runtime_assert( + std::abs(getMean(loads) - globalMean) < 1E-7, "Vanishing load"); + + return large; //return large operations +} + bool Foam::LoadBalancer::isSender( const std::vector& operations, int rank) diff --git a/src/dfChemistryModel/loadBalancing/LoadBalancer.H b/src/dfChemistryModel/loadBalancing/LoadBalancer.H index 90fceb76..427b3b12 100644 --- a/src/dfChemistryModel/loadBalancing/LoadBalancer.H +++ b/src/dfChemistryModel/loadBalancing/LoadBalancer.H @@ -71,8 +71,17 @@ public: : LoadBalancerBase(), dict_(dict), coeffsDict_(dict.subDict("loadbalancing")), active_(coeffsDict_.lookupOrDefault("active", true)), - log_(coeffsDict_.lookupOrDefault("log", false)) + log_(coeffsDict_.lookupOrDefault("log", false)), + algorithm_(coeffsDict_.lookup("algorithm")) { + if ((algorithm_ != "allAverage") && (algorithm_ != "headTail")) + { + FatalError + << "in loadBalancing Settings, unknown algorithm type " + << algorithm_ << nl + << " Valid types are: allAverage or headTail." + << exit(FatalError); + } } // Destructor @@ -103,6 +112,11 @@ protected: static std::vector getOperations( DynamicList& loads, const ChemistryLoad& myLoad); + //- Get the operations for this rank that would minimize the load to + // global mean with Redez-vous algorithm + static std::vector getOperationsRedezVous( + DynamicList& loads, const ChemistryLoad& myLoad); + //- Convert the operations to send and receive info to handle balancing static BalancerState operationsToInfo( const std::vector& operations, @@ -128,6 +142,9 @@ private: // Is load balancing logged? Switch log_; + // chose the appropriate load balancing algorithm + const word algorithm_; + //- Check if the rank is a sender static bool isSender(const std::vector& operations, int rank); From 20df5e38decc67c84f3a6a4665cd4a942a3d6fd0 Mon Sep 17 00:00:00 2001 From: maorz1998 Date: Wed, 22 Feb 2023 21:18:37 +0800 Subject: [PATCH 2/3] modify compile options of dfCombustionModel --- src/dfCombustionModels/Make/options | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/src/dfCombustionModels/Make/options b/src/dfCombustionModels/Make/options index edeb1a3d..d909e604 100644 --- a/src/dfCombustionModels/Make/options +++ b/src/dfCombustionModels/Make/options @@ -11,21 +11,9 @@ EXE_INC = -std=c++14 \ -I$(LIB_SRC)/TurbulenceModels/compressible/lnInclude \ -I$(LIB_SRC)/finiteVolume/lnInclude \ -I$(LIB_SRC)/meshTools/lnInclude \ - -I$(LIB_SRC)/lagrangian/basic/lnInclude \ - -I$(DF_SRC)/lagrangian/intermediate/lnInclude \ - -I$(LIB_SRC)/lagrangian/intermediate/lnInclude \ - -I$(DF_SRC)/lagrangian/spray/lnInclude \ - -I$(LIB_SRC)/lagrangian/spray/lnInclude \ - -I$(LIB_SRC)/thermophysicalModels/basic/lnInclude \ - -I$(DF_SRC)/thermophysicalModels/thermophysicalProperties/lnInclude \ - -I$(DF_SRC)/thermophysicalModels/SLGThermo/lnInclude \ - -I$(LIB_SRC)/regionModels/regionModel/lnInclude \ - -I$(LIB_SRC)/regionModels/surfaceFilmModels/lnInclude \ -I$(DF_SRC)/dfCanteraMixture/lnInclude \ -I$(DF_SRC)/dfChemistryModel/lnInclude \ -I$(LIB_SRC)/Pstream/mpi \ - -I$(LIB_SRC)/ODE/lnInclude \ - -I$(LIB_SRC)/functionObjects/field/lnInclude \ -I$(CANTERA_ROOT)/include \ $(if $(LIBTORCH_ROOT),-I$(LIBTORCH_ROOT)/include,) \ $(if $(LIBTORCH_ROOT),-I$(LIBTORCH_ROOT)/include/torch/csrc/api/include,) \ @@ -39,17 +27,8 @@ LIB_LIBS = \ -lmeshTools \ -L$(DF_LIBBIN) \ -ldfCompressibleTurbulenceModels \ - -ldfLagrangianIntermediate \ - -ldfLagrangianTurbulence \ - -ldfLagrangianSpray \ - -ldfFluidThermophysicalModels \ - -ldfThermophysicalProperties \ - -ldfSLGThermo \ - -lregionModels \ - -lsurfaceFilmModels \ -ldfCanteraMixture \ -ldfChemistryModel \ - -lODE \ $(CANTERA_ROOT)/lib/libcantera.so \ $(if $(LIBTORCH_ROOT),$(LIBTORCH_ROOT)/lib/libtorch.so,) \ $(if $(LIBTORCH_ROOT),$(LIBTORCH_ROOT)/lib/libc10.so,) \ From 4d0c0cdb3f27364e91f24f85c4554a161875c825 Mon Sep 17 00:00:00 2001 From: maorz1998 Date: Thu, 23 Feb 2023 09:52:13 +0800 Subject: [PATCH 3/3] modity test --- .../H2/cvodeSolver/constant/CanteraTorchProperties | 1 + 1 file changed, 1 insertion(+) diff --git a/test/dfLowMachFoam/twoD_reactingTGV/H2/cvodeSolver/constant/CanteraTorchProperties b/test/dfLowMachFoam/twoD_reactingTGV/H2/cvodeSolver/constant/CanteraTorchProperties index abaf3755..1fdeff4c 100644 --- a/test/dfLowMachFoam/twoD_reactingTGV/H2/cvodeSolver/constant/CanteraTorchProperties +++ b/test/dfLowMachFoam/twoD_reactingTGV/H2/cvodeSolver/constant/CanteraTorchProperties @@ -44,5 +44,6 @@ loadbalancing { active false; //log true; + algorithm allAverage; } // ************************************************************************* //