From ed2bfe389e45b1f44b9e20990c648c3819251f48 Mon Sep 17 00:00:00 2001 From: Mathis Kelm Date: Thu, 21 Aug 2025 16:10:59 +0200 Subject: [PATCH 1/9] [two-scale-heat-conduction] Enable parallel macro dumux solver --- .../macro-dumux/appl/main.cc | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/two-scale-heat-conduction/macro-dumux/appl/main.cc b/two-scale-heat-conduction/macro-dumux/appl/main.cc index b8897e6c9..ae217db59 100644 --- a/two-scale-heat-conduction/macro-dumux/appl/main.cc +++ b/two-scale-heat-conduction/macro-dumux/appl/main.cc @@ -24,6 +24,7 @@ #include #include #include +#include #include "properties.hh" @@ -114,6 +115,8 @@ int main(int argc, char **argv) // get mesh coordinates std::vector coords; std::vector coupledElementIdxs; + std::vector overlapCoords; + std::vector overlapElementIdxs; // coordinate loop (created vectors are 1D) // these positions of cell centers are later communicated to precice @@ -122,15 +125,23 @@ int main(int argc, char **argv) auto fvGeometry = localView(*gridGeometry); fvGeometry.bindElement(element); for (const auto &scv : scvs(fvGeometry)) { - coupledElementIdxs.push_back(scv.elementIndex()); + if (element.partitionType() != Dune::OverlapEntity) + coupledElementIdxs.push_back(scv.elementIndex()); + else + overlapElementIdxs.push_back(scv.elementIndex()); const auto &pos = scv.center(); for (const auto p : pos) { - coords.push_back(p); + if (element.partitionType() != Dune::OverlapEntity) + coords.push_back(p); + else + overlapCoords.push_back(p); std::cout << p << " "; } std::cout << " ;" << std::endl; } } + coupledElementIdxs.insert(coupledElementIdxs.end(), overlapElementIdxs.begin(), + overlapElementIdxs.end()); std::cout << "Number of Coupled Cells:" << coupledElementIdxs.size() << std::endl; @@ -138,7 +149,7 @@ int main(int argc, char **argv) auto numberOfElements = coords.size() / couplingParticipant.getMeshDimensions(meshName); if (getParam("Precice.RunWithCoupling") == true) { - couplingParticipant.setMesh(meshName, coords); + couplingParticipant.setMesh(meshName, coords, overlapCoords); // couples between dumux element indices and preciceIndices; couplingParticipant.createIndexMapping(coupledElementIdxs); From 263263676cca74e6777b7f5ced048f76b2397887 Mon Sep 17 00:00:00 2001 From: Mathis Kelm Date: Thu, 21 Aug 2025 16:42:44 +0200 Subject: [PATCH 2/9] fixup clang-format --- two-scale-heat-conduction/macro-dumux/appl/main.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/two-scale-heat-conduction/macro-dumux/appl/main.cc b/two-scale-heat-conduction/macro-dumux/appl/main.cc index ae217db59..782ab3c5d 100644 --- a/two-scale-heat-conduction/macro-dumux/appl/main.cc +++ b/two-scale-heat-conduction/macro-dumux/appl/main.cc @@ -22,9 +22,9 @@ #include #include +#include #include #include -#include #include "properties.hh" @@ -141,7 +141,7 @@ int main(int argc, char **argv) } } coupledElementIdxs.insert(coupledElementIdxs.end(), overlapElementIdxs.begin(), - overlapElementIdxs.end()); + overlapElementIdxs.end()); std::cout << "Number of Coupled Cells:" << coupledElementIdxs.size() << std::endl; From e33ec9ca8fd1a842b54e14662ba1cb84193371a0 Mon Sep 17 00:00:00 2001 From: Mathis Kelm Date: Fri, 22 Aug 2025 15:40:37 +0200 Subject: [PATCH 3/9] [two-scale-heat-conduction] Don't pass overlap points to adapter --- .../macro-dumux/appl/main.cc | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/two-scale-heat-conduction/macro-dumux/appl/main.cc b/two-scale-heat-conduction/macro-dumux/appl/main.cc index 782ab3c5d..2518cd5fa 100644 --- a/two-scale-heat-conduction/macro-dumux/appl/main.cc +++ b/two-scale-heat-conduction/macro-dumux/appl/main.cc @@ -115,33 +115,25 @@ int main(int argc, char **argv) // get mesh coordinates std::vector coords; std::vector coupledElementIdxs; - std::vector overlapCoords; - std::vector overlapElementIdxs; // coordinate loop (created vectors are 1D) // these positions of cell centers are later communicated to precice std::cout << "Coordinates: " << std::endl; for (const auto &element : elements(leafGridView)) { + if (element.partitionType() == Dune::OverlapEntity) + continue; auto fvGeometry = localView(*gridGeometry); fvGeometry.bindElement(element); for (const auto &scv : scvs(fvGeometry)) { - if (element.partitionType() != Dune::OverlapEntity) - coupledElementIdxs.push_back(scv.elementIndex()); - else - overlapElementIdxs.push_back(scv.elementIndex()); + coupledElementIdxs.push_back(scv.elementIndex()); const auto &pos = scv.center(); for (const auto p : pos) { - if (element.partitionType() != Dune::OverlapEntity) - coords.push_back(p); - else - overlapCoords.push_back(p); + coords.push_back(p); std::cout << p << " "; } std::cout << " ;" << std::endl; } } - coupledElementIdxs.insert(coupledElementIdxs.end(), overlapElementIdxs.begin(), - overlapElementIdxs.end()); std::cout << "Number of Coupled Cells:" << coupledElementIdxs.size() << std::endl; @@ -149,7 +141,7 @@ int main(int argc, char **argv) auto numberOfElements = coords.size() / couplingParticipant.getMeshDimensions(meshName); if (getParam("Precice.RunWithCoupling") == true) { - couplingParticipant.setMesh(meshName, coords, overlapCoords); + couplingParticipant.setMesh(meshName, coords); // couples between dumux element indices and preciceIndices; couplingParticipant.createIndexMapping(coupledElementIdxs); From 1e3ee2b45986b9cdc4e6ecf4a3dd3db681115299 Mon Sep 17 00:00:00 2001 From: Mathis Kelm Date: Fri, 22 Aug 2025 15:02:35 +0200 Subject: [PATCH 4/9] [two-scale-heat-conduction] Store coupling data and exchange if parallel --- .../macro-dumux/appl/main.cc | 2 + .../macro-dumux/appl/problem.hh | 1 + .../macro-dumux/appl/spatialparams.hh | 61 +++++++++++++++---- 3 files changed, 51 insertions(+), 13 deletions(-) diff --git a/two-scale-heat-conduction/macro-dumux/appl/main.cc b/two-scale-heat-conduction/macro-dumux/appl/main.cc index 2518cd5fa..59df2d346 100644 --- a/two-scale-heat-conduction/macro-dumux/appl/main.cc +++ b/two-scale-heat-conduction/macro-dumux/appl/main.cc @@ -278,6 +278,8 @@ int main(int argc, char **argv) dt); couplingParticipant.readQuantityFromOtherSolver(meshName, readDataPorosity, dt); + // store coupling data in problem + problem->spatialParams().updateCouplingData(); } std::cout << "Solver starts" << std::endl; diff --git a/two-scale-heat-conduction/macro-dumux/appl/problem.hh b/two-scale-heat-conduction/macro-dumux/appl/problem.hh index 0ce1dbf5f..4aa8bc976 100644 --- a/two-scale-heat-conduction/macro-dumux/appl/problem.hh +++ b/two-scale-heat-conduction/macro-dumux/appl/problem.hh @@ -249,6 +249,7 @@ public: } } + private: Dumux::Precice::CouplingAdapter &couplingParticipant_; diff --git a/two-scale-heat-conduction/macro-dumux/appl/spatialparams.hh b/two-scale-heat-conduction/macro-dumux/appl/spatialparams.hh index f427d9939..b7743055b 100644 --- a/two-scale-heat-conduction/macro-dumux/appl/spatialparams.hh +++ b/two-scale-heat-conduction/macro-dumux/appl/spatialparams.hh @@ -18,6 +18,9 @@ #ifndef DUMUX_TEST_1PNI_SPATIAL_PARAMS_HH #define DUMUX_TEST_1PNI_SPATIAL_PARAMS_HH +#include + +#include #include #include @@ -48,8 +51,11 @@ public: using PermeabilityType = Scalar; OnePNISpatialParams(std::shared_ptr gridGeometry) - : ParentType(gridGeometry), - couplingParticipant_(Dumux::Precice::CouplingAdapter::getInstance()) {} + : ParentType(gridGeometry) + , couplingParticipant_(Dumux::Precice::CouplingAdapter::getInstance()) + , couplingData_(gridGeometry->numDofs()) + , couplingDataHandle_(this->gridGeometry().elementMapper(), couplingData_) + {} /*! * \brief Defines the intrinsic permeability \f$\mathrm{[m^2]}\f$. @@ -71,8 +77,7 @@ public: const ElementSolution &elemSol) const { if (getParam("Precice.RunWithCoupling") == true) - return couplingParticipant_.getScalarQuantityOnFace( - "macro-mesh", "porosity", scv.elementIndex()); + return couplingData_[scv.elementIndex()][0]; else return getParam("Problem.DefaultPorosity"); } @@ -87,14 +92,10 @@ public: DimWorldMatrix K; if (getParam("Precice.RunWithCoupling") == true) { - K[0][0] = couplingParticipant_.getScalarQuantityOnFace( - "macro-mesh", "k_00", scv.elementIndex()); - K[0][1] = couplingParticipant_.getScalarQuantityOnFace( - "macro-mesh", "k_01", scv.elementIndex()); - K[1][0] = couplingParticipant_.getScalarQuantityOnFace( - "macro-mesh", "k_10", scv.elementIndex()); - K[1][1] = couplingParticipant_.getScalarQuantityOnFace( - "macro-mesh", "k_11", scv.elementIndex()); + K[0][0] = couplingData_[scv.elementIndex()][1]; + K[0][1] = couplingData_[scv.elementIndex()][2]; + K[1][0] = couplingData_[scv.elementIndex()][3]; + K[1][1] = couplingData_[scv.elementIndex()][4]; } else { K[0][0] = getParam("Component.SolidThermalConductivity"); K[0][1] = 0.0; @@ -104,8 +105,42 @@ public: return K; } + void updateCouplingData() + { + for (const auto &element : elements(this->gridGeometry().gridView())) { + if (element.partitionType() == Dune::OverlapEntity) + continue; + auto fvGeometry = localView(this->gridGeometry()); + fvGeometry.bindElement(element); + for (const auto &scv : scvs(fvGeometry)) { + const auto elementIdx = scv.elementIndex(); + couplingData_[elementIdx][0] = + couplingParticipant_.getScalarQuantityOnFace("macro-mesh", "porosity", elementIdx); + couplingData_[elementIdx][1] = + couplingParticipant_.getScalarQuantityOnFace("macro-mesh", "k_00", elementIdx); + couplingData_[elementIdx][2] = + couplingParticipant_.getScalarQuantityOnFace("macro-mesh", "k_01", elementIdx); + couplingData_[elementIdx][3] = + couplingParticipant_.getScalarQuantityOnFace("macro-mesh", "k_10", elementIdx); + couplingData_[elementIdx][4] = + couplingParticipant_.getScalarQuantityOnFace("macro-mesh", "k_11", elementIdx); + } + } + // exchange coupling data with other ranks + if (this->gridGeometry().gridView().comm().size() > 1) { + this->gridGeometry().gridView().communicate(couplingDataHandle_, + Dune::InteriorBorder_All_Interface, Dune::ForwardCommunication); + } + } + private: - Dumux::Precice::CouplingAdapter &couplingParticipant_; + Dumux::Precice::CouplingAdapter &couplingParticipant_; + Dune::BlockVector> couplingData_; + Dumux::VectorCommDataHandleEqual< + typename GridGeometry::ElementMapper, + Dune::BlockVector>, + /* Entity codimension = */ 0> + couplingDataHandle_; }; } // end namespace Dumux From f9aa4be782ddcce48455046edbba8e4ef51e7640 Mon Sep 17 00:00:00 2001 From: Ishaan Desai Date: Fri, 22 Aug 2025 21:02:43 +0200 Subject: [PATCH 5/9] Remove whitespace --- two-scale-heat-conduction/macro-dumux/appl/problem.hh | 1 - 1 file changed, 1 deletion(-) diff --git a/two-scale-heat-conduction/macro-dumux/appl/problem.hh b/two-scale-heat-conduction/macro-dumux/appl/problem.hh index 4aa8bc976..0ce1dbf5f 100644 --- a/two-scale-heat-conduction/macro-dumux/appl/problem.hh +++ b/two-scale-heat-conduction/macro-dumux/appl/problem.hh @@ -249,7 +249,6 @@ public: } } - private: Dumux::Precice::CouplingAdapter &couplingParticipant_; From 3379f9af0ad254e907c636fe171f0f2a27ed447d Mon Sep 17 00:00:00 2001 From: Ishaan Desai Date: Sat, 23 Aug 2025 08:19:48 +0200 Subject: [PATCH 6/9] Fix clang-format --- .../macro-dumux/appl/spatialparams.hh | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/two-scale-heat-conduction/macro-dumux/appl/spatialparams.hh b/two-scale-heat-conduction/macro-dumux/appl/spatialparams.hh index b7743055b..5c9d6fbc2 100644 --- a/two-scale-heat-conduction/macro-dumux/appl/spatialparams.hh +++ b/two-scale-heat-conduction/macro-dumux/appl/spatialparams.hh @@ -51,11 +51,9 @@ public: using PermeabilityType = Scalar; OnePNISpatialParams(std::shared_ptr gridGeometry) - : ParentType(gridGeometry) - , couplingParticipant_(Dumux::Precice::CouplingAdapter::getInstance()) - , couplingData_(gridGeometry->numDofs()) - , couplingDataHandle_(this->gridGeometry().elementMapper(), couplingData_) - {} + : ParentType(gridGeometry), couplingParticipant_(Dumux::Precice::CouplingAdapter::getInstance()), couplingData_(gridGeometry->numDofs()), couplingDataHandle_(this->gridGeometry().elementMapper(), couplingData_) + { + } /*! * \brief Defines the intrinsic permeability \f$\mathrm{[m^2]}\f$. From 824f8811442406c7732a20e584962f98f03880b1 Mon Sep 17 00:00:00 2001 From: Ishaan Desai Date: Mon, 25 Aug 2025 07:17:22 +0200 Subject: [PATCH 7/9] Add changelog entry --- changelog-entries/663.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog-entries/663.md diff --git a/changelog-entries/663.md b/changelog-entries/663.md new file mode 100644 index 000000000..08d138fc0 --- /dev/null +++ b/changelog-entries/663.md @@ -0,0 +1 @@ +- Enable parallel execution of the participant macro-dumux in the two-scale heat conduction tutorial [#663](https://github.com/precice/tutorials/pull/663) From 1b9b712efc71bc3f6a8f4f0d620665ebd2c2ad73 Mon Sep 17 00:00:00 2001 From: Mathis Kelm Date: Mon, 25 Aug 2025 09:06:00 +0200 Subject: [PATCH 8/9] [two-scale-heat-conduction] Use Dune::Partitions to only couple interior points --- two-scale-heat-conduction/macro-dumux/appl/main.cc | 6 ++---- two-scale-heat-conduction/macro-dumux/appl/spatialparams.hh | 6 ++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/two-scale-heat-conduction/macro-dumux/appl/main.cc b/two-scale-heat-conduction/macro-dumux/appl/main.cc index 59df2d346..2e9c7dd25 100644 --- a/two-scale-heat-conduction/macro-dumux/appl/main.cc +++ b/two-scale-heat-conduction/macro-dumux/appl/main.cc @@ -22,7 +22,7 @@ #include #include -#include +#include #include #include @@ -119,9 +119,7 @@ int main(int argc, char **argv) // coordinate loop (created vectors are 1D) // these positions of cell centers are later communicated to precice std::cout << "Coordinates: " << std::endl; - for (const auto &element : elements(leafGridView)) { - if (element.partitionType() == Dune::OverlapEntity) - continue; + for (const auto &element : elements(leafGridView, Dune::Partitions::interior)) { auto fvGeometry = localView(*gridGeometry); fvGeometry.bindElement(element); for (const auto &scv : scvs(fvGeometry)) { diff --git a/two-scale-heat-conduction/macro-dumux/appl/spatialparams.hh b/two-scale-heat-conduction/macro-dumux/appl/spatialparams.hh index 5c9d6fbc2..d4d127067 100644 --- a/two-scale-heat-conduction/macro-dumux/appl/spatialparams.hh +++ b/two-scale-heat-conduction/macro-dumux/appl/spatialparams.hh @@ -18,7 +18,7 @@ #ifndef DUMUX_TEST_1PNI_SPATIAL_PARAMS_HH #define DUMUX_TEST_1PNI_SPATIAL_PARAMS_HH -#include +#include #include #include @@ -105,9 +105,7 @@ public: void updateCouplingData() { - for (const auto &element : elements(this->gridGeometry().gridView())) { - if (element.partitionType() == Dune::OverlapEntity) - continue; + for (const auto &element : elements(this->gridGeometry().gridView(), Dune::Partitions::interior)) { auto fvGeometry = localView(this->gridGeometry()); fvGeometry.bindElement(element); for (const auto &scv : scvs(fvGeometry)) { From 4d3e05baa1ad96e3ba0d24d0f71446675f9ae736 Mon Sep 17 00:00:00 2001 From: mathiskelm <114579716+mathiskelm@users.noreply.github.com> Date: Mon, 25 Aug 2025 09:11:37 +0200 Subject: [PATCH 9/9] Improve documentation Co-authored-by: Ishaan Desai --- two-scale-heat-conduction/macro-dumux/appl/spatialparams.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/two-scale-heat-conduction/macro-dumux/appl/spatialparams.hh b/two-scale-heat-conduction/macro-dumux/appl/spatialparams.hh index d4d127067..fc123d224 100644 --- a/two-scale-heat-conduction/macro-dumux/appl/spatialparams.hh +++ b/two-scale-heat-conduction/macro-dumux/appl/spatialparams.hh @@ -122,7 +122,7 @@ public: couplingParticipant_.getScalarQuantityOnFace("macro-mesh", "k_11", elementIdx); } } - // exchange coupling data with other ranks + // Trigger exchange of coupling data between neighboring ranks, if the domain is partitioned if (this->gridGeometry().gridView().comm().size() > 1) { this->gridGeometry().gridView().communicate(couplingDataHandle_, Dune::InteriorBorder_All_Interface, Dune::ForwardCommunication);