Decomposition stabilization

tests/Makefile

@@ -1,7 +1,9 @@
-CXX=CC
-CXXFLAGS= -O3 -std=c++11 -march=native -g -Wall
+CXX=mpic++
+# CXXFLAGS= -O3 -std=c++17 -ffast-math -march=native -g -Wall
+CXXFLAGS= -O3 -std=c++17 -march=native -g -Wall
+# CXXFLAGS= -O0 -std=c++17 -march=native -g -Wall
 
-all: benchmark test ddtest
+all: clean ddtest
 
 benchmark: benchmark.cpp ../fsgrid.hpp
 	$(CXX) $(CXXFLAGS) -o $@ $<
@@ -11,4 +13,4 @@ ddtest: ddtest.cpp
 	$(CXX) $(CXXFLAGS) -o $@ $<
 
 clean:
-	-rm test
+	-rm test ddtest benchmark

tests/benchmark.cpp

@@ -23,7 +23,8 @@
 
 template<class T, int stencil> void timeit(std::array<int32_t, 3> globalSize, std::array<bool, 3> isPeriodic, int iterations){
    double t1,t2;   
-   FsGrid<T ,stencil> testGrid(globalSize, MPI_COMM_WORLD, isPeriodic);
+   FsGridCouplingInformation gridCoupling;
+   FsGrid<T ,stencil> testGrid(globalSize, MPI_COMM_WORLD, isPeriodic, gridCoupling);
    int rank,size;
    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
    MPI_Comm_size(MPI_COMM_WORLD, &size);

tests/ddtest.cpp

@@ -25,51 +25,12 @@
 #include <math.h>
 #include <algorithm>
 #include <limits>
-
-
-void computeDomainDecomposition(const std::array<int, 3>& GlobalSize, int nProcs, std::array<int,3>& processDomainDecomposition) {
-
-   std::array<double, 3> systemDim;
-   std::array<double, 3 > processBox;
-   double optimValue = std::numeric_limits<double>::max();
-
-   for(int i = 0; i < 3; i++) {
-      systemDim[i] = (double)GlobalSize[i];
-   }
-   processDomainDecomposition = {1, 1, 1};
-
-   for (int i = 1; i <= std::min(nProcs, GlobalSize[0]); i++) {
-      processBox[0] = std::max(systemDim[0]/i, 1.0);
-      for (int j = 1; j <= std::min(nProcs, GlobalSize[1]) ; j++) {
-         if( i * j  > nProcs )
-            break;
-         processBox[1] = std::max(systemDim[1]/j, 1.0);
-         for (int k = 1; k <= std::min(nProcs, GlobalSize[2]); k++) {
-            if( i * j * k > nProcs )
-               break;
-            processBox[2] = std::max(systemDim[2]/k, 1.0);
-            double value = 
-               10 * processBox[0] * processBox[1] * processBox[2] + 
-               (i > 1 ? processBox[1] * processBox[2]: 0) +
-               (j > 1 ? processBox[0] * processBox[2]: 0) +
-               (k > 1 ? processBox[0] * processBox[1]: 0);
-
-            if(value < optimValue ){
-               optimValue = value;
-               processDomainDecomposition[0] = i;
-               processDomainDecomposition[1] = j;
-               processDomainDecomposition[2] = k;
-            }
-//            printf("%g: %d %d %d with box %g %g %g\n", value, i, j, k, processBox[0], processBox[1], processBox[2]);
-         }
-      }
-   }
-}
+#include "../fsgrid.hpp"
 
 int main(int argc, char **argv){
 
-   std::array<int,3>  sys;
-   std::array<int,3> processDomainDecomposition;
+   std::array<FsGridTools::FsSize_t,3> sys;
+   std::array<FsGridTools::Task_t,3> processDomainDecomposition;
 
    if(argc != 5) {
       printf("Usage %s size_x size_y size_z nProcesses\n", argv[0]);
@@ -82,8 +43,8 @@ int main(int argc, char **argv){
    sys[2] = atof(argv[3]);
    uint nProcs = atoi(argv[4]);
 
-   computeDomainDecomposition(sys, nProcs, processDomainDecomposition);
-   printf("DD of %d %d %d for %d processes is %d %d %d \n", 
+   FsGridTools::computeDomainDecomposition(sys, nProcs, processDomainDecomposition, 1, true);
+   printf("DD of %ld %ld %ld for %d processes is %ld %ld %ld \n", 
           sys[0], sys[1], sys[2], nProcs,
           processDomainDecomposition[0], processDomainDecomposition[1], processDomainDecomposition[2]);
 

tests/test-dd.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+vals=("256 256 256", "256 256 128", "256 128 256", "128 256 256")
+# vals=("1024 736 736")
+nprocs=(32)
+# nprocs=(2400)
+for val in "${nprocs[@]}"
+do
+ for dims in "${vals[@]}"
+  do
+ mpirun -n 1 ./ddtest $dims $val 
+#  mpirun -n 1 ./ddtest 256 256 128 $val
+#  mpirun -n 1 ./ddtest 256 128 256 $val
+#  mpirun -n 1 ./ddtest 128 256 256 $val
+ done
+
+done
+

tests/test.cpp

@@ -34,7 +34,8 @@ int main(int argc, char** argv) {
    std::array<int32_t, 3> globalSize{20,20,1};
    std::array<bool, 3> isPeriodic{false,false,true};
    {
-      FsGrid<int,1> testGrid(globalSize, MPI_COMM_WORLD, isPeriodic);
+      FsGridCouplingInformation gridCoupling;
+      FsGrid<int,1> testGrid(globalSize, MPI_COMM_WORLD, isPeriodic, gridCoupling);
 /*
       if(rank == 0) {
          std::cerr << " --- Test task mapping functions ---" << std::endl;
@@ -63,9 +64,9 @@ int main(int argc, char** argv) {
 
 
 
-      int z = 0;
-      for(int y = -1; y < localSize[1] + 1; y++){
-         for(int x = -1; x < localSize[0] + 1; x++){
+      FsGridTools::FsIndex_t z = 0;
+      for(FsGridTools::FsIndex_t y = -1; y < localSize[1] + 1; y++){
+         for(FsGridTools::FsIndex_t x = -1; x < localSize[0] + 1; x++){
             *(testGrid.get(x, y, z)) = rank;
          }
       }
@@ -75,11 +76,11 @@ int main(int argc, char** argv) {
       if(rank==1) {
          printf("local size %d %d %d\n", localSize[0], localSize[1], localSize[2]);
          printf("----------------------------------\n");
-         int z = 0;
+         FsGridTools::FsIndex_t z = 0;
          printf("z=%d\n", z);
-         for(int y = -1; y < localSize[1] + 1; y++){
+         for(FsGridTools::FsIndex_t y = -1; y < localSize[1] + 1; y++){
             printf("y=%d :", y);
-            for(int x = -1; x < localSize[0] +1;  x++){
+            for(FsGridTools::FsIndex_t x = -1; x < localSize[0] +1;  x++){
                printf("%d ", *(testGrid.get(x, y, z)));
             }
             printf("\n");
@@ -132,8 +133,8 @@ int main(int argc, char** argv) {
 
          // We are going to send data for all 8×8 Cells
          testGrid.setupForTransferIn(globalSize[0]*globalSize[1]);
-         for(int y=0; y<globalSize[1]; y++) {
-            for(int x=0; x<globalSize[0]; x++) {
+         for(FsGridTools::FsSize_t y=0; y<globalSize[1]; y++) {
+            for(FsGridTools::FsSize_t x=0; x<globalSize[0]; x++) {
                fillData[y*globalSize[0] + x] = x*y;
 
                testGrid.transferDataIn(y*globalSize[0]+x,&fillData[y*globalSize[0]+x]);
@@ -150,9 +151,9 @@ int main(int argc, char** argv) {
       for(int i=0; i<size; i++) {
          if(i == rank) {
             std::cerr << "Contents of Task #" << rank << ": " << std::endl;
-            std::array<int32_t,3> localSize = testGrid.getLocalSize();
-            for(int y=0; y<localSize[1]; y++) {
-               for(int x=0; x<localSize[0]; x++) {
+            std::array<FsGridTools::FsIndex_t,3> localSize = testGrid.getLocalSize();
+            for(FsGridTools::FsIndex_t y=0; y<localSize[1]; y++) {
+               for(FsGridTools::FsIndex_t x=0; x<localSize[0]; x++) {
                   std::cerr << *testGrid.get(x,y,0) << ", ";
                }
                std::cerr << std::endl;
@@ -168,8 +169,8 @@ int main(int argc, char** argv) {
          returnedData.resize(globalSize[0]*globalSize[1]);
 
          testGrid.setupForTransferOut(globalSize[0]*globalSize[1]);
-         for(int y=0; y<globalSize[1]; y++) {
-            for(int x=0; x<globalSize[0]; x++) {
+         for(FsGridTools::FsSize_t y=0; y<globalSize[1]; y++) {
+            for(FsGridTools::FsSize_t x=0; x<globalSize[0]; x++) {
                testGrid.transferDataOut(y*globalSize[0]+x,&returnedData[y*globalSize[0]+x]);
             }
          }
@@ -182,8 +183,8 @@ int main(int argc, char** argv) {
       if(rank == 0) {
          std::cerr << " --------- " << std::endl;
          std::cerr << "Returned array contents:" << std::endl;
-         for(int y=0; y<globalSize[1]; y++) {
-            for(int x=0; x<globalSize[0]; x++) {
+         for(FsGridTools::FsSize_t y=0; y<globalSize[1]; y++) {
+            for(FsGridTools::FsSize_t x=0; x<globalSize[0]; x++) {
                std::cerr << returnedData[y*globalSize[0]+x] << ", ";
             }
             std::cerr << std::endl;