From caddbde635b9d481342bbe9b61a744f037877921 Mon Sep 17 00:00:00 2001 From: maorz1998 Date: Sun, 13 Nov 2022 14:56:05 +0800 Subject: [PATCH] modify install procedure & modify settings in CanteraTorchProperties --- .gitignore | 1 + applications/solvers/df0DFoam/EEqn.H | 4 + applications/solvers/df0DFoam/Make/options | 1 + applications/solvers/df0DFoam/YEqn.H | 15 +- applications/solvers/df0DFoam/createFields.H | 4 +- applications/solvers/df0DFoam/df0DFoam.C | 30 ++- .../solvers/dfLowMachFoam/createFields.H | 3 +- .../solvers/dfLowMachFoam/dfLowMachFoam.C | 22 ++- configure.sh | 177 +++++++++++++++++ .../constant/CanteraTorchProperties | 27 +-- .../H2/pytorchIntegrator/inference.py | 14 +- .../H2/pytorchIntegrator/Allrun | 2 +- .../pytorchIntegrator/system/decomposeParDict | 2 +- install.sh | 182 +----------------- mechanisms/H2/pytorchDNN/settings1.json | 85 -------- mechanisms/H2/pytorchDNN/settings2.json | 85 -------- mechanisms/H2/pytorchDNN/settings3.json | 85 -------- src/dfChemistryModel/dfChemistryModel.C | 92 +++++---- src/dfChemistryModel/dfChemistryModel.H | 2 + 19 files changed, 306 insertions(+), 527 deletions(-) create mode 100644 configure.sh delete mode 100644 mechanisms/H2/pytorchDNN/settings1.json delete mode 100644 mechanisms/H2/pytorchDNN/settings2.json delete mode 100644 mechanisms/H2/pytorchDNN/settings3.json diff --git a/.gitignore b/.gitignore index c8affc1c..8cbd18ed 100644 --- a/.gitignore +++ b/.gitignore @@ -66,3 +66,4 @@ polyMesh/ thirdParty libtorch*.zip* .DS_Store +__pycache__/ diff --git a/applications/solvers/df0DFoam/EEqn.H b/applications/solvers/df0DFoam/EEqn.H index faba8642..735aae8d 100644 --- a/applications/solvers/df0DFoam/EEqn.H +++ b/applications/solvers/df0DFoam/EEqn.H @@ -1,5 +1,9 @@ { volScalarField& he = thermo.he(); if (constProp == "volume") he[0] = u0 + p[0]/rho[0]; + start = std::clock(); chemistry.correctThermo(); + end = std::clock(); + time_monitor_corrThermo += double(end - start) / double(CLOCKS_PER_SEC); + } diff --git a/applications/solvers/df0DFoam/Make/options b/applications/solvers/df0DFoam/Make/options index 9f2607c1..5c3df1b5 100644 --- a/applications/solvers/df0DFoam/Make/options +++ b/applications/solvers/df0DFoam/Make/options @@ -13,6 +13,7 @@ EXE_INC = -std=c++14 \ -I$(LIB_SRC)/finiteVolume/lnInclude \ -I$(LIB_SRC)/meshTools/lnInclude \ -I$(LIB_SRC)/sampling/lnInclude \ + -I$(LIB_SRC)/Pstream/mpi \ -I$(LIB_SRC)/dynamicFvMesh/lnInclude \ -I$(DF_SRC)/CanteraMixture/lnInclude \ -I$(DF_SRC)/dfChemistryModel/lnInclude \ diff --git a/applications/solvers/df0DFoam/YEqn.H b/applications/solvers/df0DFoam/YEqn.H index f7c8f1de..2d8a2bd1 100644 --- a/applications/solvers/df0DFoam/YEqn.H +++ b/applications/solvers/df0DFoam/YEqn.H @@ -10,10 +10,20 @@ tmp> mvConvection ); { - chemistry.solve(mesh.time().deltaTValue()); + { + std::chrono::steady_clock::time_point start = std::chrono::steady_clock::now(); + chemistry.solve(mesh.time().deltaTValue()); + label flag_mpi_init; + MPI_Initialized(&flag_mpi_init); + if(flag_mpi_init) MPI_Barrier(PstreamGlobals::MPI_COMM_FOAM); + std::chrono::steady_clock::time_point stop = std::chrono::steady_clock::now(); + std::chrono::duration processingTime = std::chrono::duration_cast>(stop - start); + time_monitor_chem += processingTime.count(); + } volScalarField Yt(0.0*Y[0]); + start = std::clock(); forAll(Y, i) { if (i != inertIndex) @@ -43,4 +53,7 @@ tmp> mvConvection Y[inertIndex] = scalar(1) - Yt; Y[inertIndex].max(0.0); + + end = std::clock(); + time_monitor_Y += double(end - start) / double(CLOCKS_PER_SEC); } \ No newline at end of file diff --git a/applications/solvers/df0DFoam/createFields.H b/applications/solvers/df0DFoam/createFields.H index fd529a3b..e5d35066 100644 --- a/applications/solvers/df0DFoam/createFields.H +++ b/applications/solvers/df0DFoam/createFields.H @@ -116,4 +116,6 @@ if } volScalarField& he = thermo.he(); -scalar u0 = he[0] - p[0]/rho[0]; \ No newline at end of file +scalar u0 = he[0] - p[0]/rho[0]; + +const Switch log_ = CanteraTorchProperties.subDict("TorchSettings").lookupOrDefault("log", false); \ No newline at end of file diff --git a/applications/solvers/df0DFoam/df0DFoam.C b/applications/solvers/df0DFoam/df0DFoam.C index 3c2465e1..f5d79ef3 100644 --- a/applications/solvers/df0DFoam/df0DFoam.C +++ b/applications/solvers/df0DFoam/df0DFoam.C @@ -50,6 +50,7 @@ Description //#include "fvOptions.H" #include "localEulerDdtScheme.H" #include "fvcSmooth.H" +#include "PstreamGlobals.H" // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // @@ -73,6 +74,11 @@ int main(int argc, char *argv[]) #include "createFields.H" #include "createFieldRefs.H" #include "createRhoUfIfPresent.H" + + double time_monitor_chem=0; + double time_monitor_Y=0; + double time_monitor_corrThermo=0; + clock_t start, end; turbulence->validate(); @@ -107,9 +113,29 @@ int main(int argc, char *argv[]) runTime.write(); + Info<< "========Time Spent in diffenet parts========"<< endl; + Info<< "Chemical sources = " << time_monitor_chem << " s" << endl; + Info<< "Species Equations = " << time_monitor_Y << " s" << endl; + Info<< "thermo & Trans Properties = " << time_monitor_corrThermo << " s" << endl; + Info<< "============================================"<::dfChemistryModel useDNN = false; } - torchSwitch_ = this->lookupOrDefault("torch", false); - gpu_ = this->lookupOrDefault("GPU", false), + torchSwitch_ = this->subDict("TorchSettings").lookupOrDefault("torch", false); + gpu_ = this->subDict("TorchSettings").lookupOrDefault("GPU", false), + gpulog_ = this->subDict("TorchSettings").lookupOrDefault("log", false), torchModelName_ = this->lookupOrDefault("torchModel", word("")); - torchModelName1_ = this->lookupOrDefault("torchModel1", word("")); - torchModelName2_ = this->lookupOrDefault("torchModel2", word("")); - torchModelName3_ = this->lookupOrDefault("torchModel3", word("")); - - // thresholds adopted for selecting the networks - Tact1_ = this->subDict("torchParameters1").lookupOrDefault("Tact", 700); - Qdotact1_ = this->subDict("torchParameters1").lookupOrDefault("Qdotact", 1e9); - - Tact2_ = this->subDict("torchParameters2").lookupOrDefault("Tact", 700); - Qdotact2_ = this->subDict("torchParameters2").lookupOrDefault("Qdotact", 1e9); - - Tact3_ = this->subDict("torchParameters3").lookupOrDefault("Tact", 700); - Qdotact3_ = this->subDict("torchParameters3").lookupOrDefault("Qdotact", 1e9); + torchModelName1_ = this->subDict("TorchSettings").lookupOrDefault("torchModel1", word("")); + torchModelName2_ = this->subDict("TorchSettings").lookupOrDefault("torchModel2", word("")); + torchModelName3_ = this->subDict("TorchSettings").lookupOrDefault("torchModel3", word("")); // set the number of cores slaved by each GPU card - coresPerGPU = this->subDict("torchParameters1").lookupOrDefault("coresPerGPU", 8); - GPUsPerNode = this->subDict("torchParameters1").lookupOrDefault("GPUsPerNode", 4); + coresPerGPU = this->subDict("TorchSettings").lookupOrDefault("coresPerGPU", 8); + GPUsPerNode = this->subDict("TorchSettings").lookupOrDefault("GPUsPerNode", 4); // initialization the Inferencer (if use multi GPU) if(torchSwitch_) @@ -174,20 +165,11 @@ Foam::dfChemistryModel::dfChemistryModel { useDNN = false; } - gpu_ = this->lookupOrDefault("GPU", false), - - torchSwitch_ = this->lookupOrDefault("torch", false); + gpu_ = this->subDict("TorchSettings").lookupOrDefault("GPU", false), + torchSwitch_ = this->subDict("TorchSettings").lookupOrDefault("torch", false); + gpulog_ = this->subDict("TorchSettings").lookupOrDefault("log", false), - Tact1_ = this->subDict("torchParameters1").lookupOrDefault("Tact", 700); - Qdotact1_ = this->subDict("torchParameters1").lookupOrDefault("Qdotact", 1e9); - - Tact2_ = this->subDict("torchParameters2").lookupOrDefault("Tact", 700); - Qdotact2_ = this->subDict("torchParameters2").lookupOrDefault("Qdotact", 1e9); - - Tact3_ = this->subDict("torchParameters3").lookupOrDefault("Tact", 700); - Qdotact3_ = this->subDict("torchParameters3").lookupOrDefault("Qdotact", 1e9); - - coresPerGPU = this->subDict("torchParameters1").lookupOrDefault("coresPerGPU", 8); + coresPerGPU = this->subDict("TorchSettings").lookupOrDefault("coresPerGPU", 8); time_allsolve_ = 0; time_submaster_ = 0; @@ -401,8 +383,8 @@ Foam::dfChemistryModel::getGPUProblems scalar pi = p_[cellI]; scalar rhoi = rho_[cellI]; - // if T < 1000, set RR=0 - if (T_[cellI] < Tact1_) + // if T < 700, set RR=0 + if (T_[cellI] < 700) { Qdot_[cellI] = 0; for (size_t i = 0; i < CanteraGas_->nSpecies(); i++) @@ -424,21 +406,21 @@ Foam::dfChemistryModel::getGPUProblems problem.rhoi = rhoi; // choose DNN module - if ((Qdot_[cellI] < Qdotact2_) && (T_[cellI] < Tact2_) && ( T_[cellI] >= Tact1_))//choose1 + if ((Qdot_[cellI] < 3e7) && (T_[cellI] < 2000) && ( T_[cellI] >= 700))//choose1 { problem.DNNid = 0; problemList.append(problem); selectDNN_[cellI]=0; continue; } - if(((Qdot_[cellI] >= Qdotact2_) && (T_[cellI] < Tact2_)&&(T_[cellI] >= Tact1_))||((Qdot_[cellI] > Qdotact3_) && T_[cellI] > Tact2_)) //choose2 + if(((Qdot_[cellI] >= 3e7) && (T_[cellI] < 2000)&&(T_[cellI] >= 700))||((Qdot_[cellI] > 7e8) && T_[cellI] > 2000)) //choose2 { problem.DNNid = 1; problemList.append(problem); selectDNN_[cellI]=1; continue; } - if ((Qdot_[cellI] < Qdotact3_) && (T_[cellI] >= Tact2_) && (Qdot_[cellI]!=0)) //choose3 + if ((Qdot_[cellI] < 7e8) && (T_[cellI] >= 2000) && (Qdot_[cellI]!=0)) //choose3 { problem.DNNid = 2; problemList.append(problem); @@ -555,9 +537,12 @@ void Foam::dfChemistryModel::getDNNinputs cellIDBuffer = {cellIDList0Buffer, cellIDList1Buffer, cellIDList2Buffer}; problemCounter = {problemCounter0, problemCounter1, problemCounter2}; - std::cout<<"inputsDNN0 = "<::solve_DNN( recv >> problemBuffer[i]; //recv previous send problem and append to problemList problemSize += problemBuffer[i].size(); } - Info << "problemSize = " << problemSize << endl; + if (gpulog_) + { + Info << "problemSize = " << problemSize << endl; + } std::chrono::steady_clock::time_point stop3 = std::chrono::steady_clock::now(); std::chrono::duration processingTime3 = std::chrono::duration_cast>(stop3 - start3); @@ -1135,7 +1123,7 @@ Foam::scalar Foam::dfChemistryModel::solve_CVODE if(balancer_.active()) { - Info<<"Now use DLB algorithm!!"<::solve_CVODE } else { - Info<<"Now do not use DLB algorithm!!"<::getGPUProblems scalar rhoi = rho_[cellI]; // if T < 1000, set RR=0 - if (T_[cellI] < Tact1_) + if (T_[cellI] < 700) { Qdot_[cellI] = 0; for (size_t i = 0; i < CanteraGas_->nSpecies(); i++) @@ -1367,19 +1355,19 @@ Foam::dfChemistryModel::getGPUProblems problem.rhoi = rhoi; // choose DNN module - if ((Qdot_[cellI] < Qdotact2_) && (T_[cellI] < Tact2_) && ( T_[cellI] >= Tact1_))//choose1 + if ((Qdot_[cellI] < 3e7) && (T_[cellI] < 2000) && ( T_[cellI] >= 700))//choose1 { problem.DNNid = 0; problemList.append(problem); continue; } - if(((Qdot_[cellI] >= Qdotact2_) && (T_[cellI] < Tact2_)&&(T_[cellI] >= Tact1_))||((Qdot_[cellI] > Qdotact3_) && T_[cellI] > Tact2_)) //choose2 + if(((Qdot_[cellI] >= 3e7) && (T_[cellI] < 2000)&&(T_[cellI] >= 700))||((Qdot_[cellI] > 7e8) && T_[cellI] > 2000)) //choose2 { problem.DNNid = 1; problemList.append(problem); continue; } - if ((Qdot_[cellI] < Qdotact3_) && (T_[cellI] >= Tact2_) && (Qdot_[cellI]!=0)) //choose3 + if ((Qdot_[cellI] < 7e8) && (T_[cellI] >= 2000) && (Qdot_[cellI]!=0)) //choose3 { problem.DNNid = 2; problemList.append(problem); @@ -1487,9 +1475,12 @@ void Foam::dfChemistryModel::getDNNinputs cellIDBuffer = {cellIDList0Buffer, cellIDList1Buffer, cellIDList2Buffer}; problemCounter = {problemCounter0, problemCounter1, problemCounter2}; - std::cout<<"inputsDNN0 = "<::solve_DNN( recv >> problemBuffer[i]; //recv previous send problem and append to problemList problemSize += problemBuffer[i].size(); } - Info << "problemSize = " << problemSize << endl; + if (gpulog_) + { + Info << "problemSize = " << problemSize << endl; + } std::chrono::steady_clock::time_point stop3 = std::chrono::steady_clock::now(); std::chrono::duration processingTime3 = std::chrono::duration_cast>(stop3 - start3); diff --git a/src/dfChemistryModel/dfChemistryModel.H b/src/dfChemistryModel/dfChemistryModel.H index 3e931f08..d0ed9ae7 100644 --- a/src/dfChemistryModel/dfChemistryModel.H +++ b/src/dfChemistryModel/dfChemistryModel.H @@ -133,6 +133,7 @@ public IOdictionary Switch torchSwitch_; Switch gpu_; + Switch gpulog_; DNNInferencer DNNInferencer_; int coresPerGPU; int GPUsPerNode; @@ -173,6 +174,7 @@ public IOdictionary Switch torchSwitch_; Switch gpu_; + Switch gpulog_; scalar Tact1_; scalar Qdotact1_; scalar Tact2_;