Merge pull request #42 from E3SM-Project/develop

Develop
E3SM-Project · Aug 27, 2021 · e732872 · e732872
2 parents bf3edb7 + d399712
commit e732872
Show file tree

Hide file tree

Showing 10 changed files with 58 additions and 38 deletions.
diff --git a/methods/islet/figures/run-accuracy.sh b/methods/islet/figures/run-accuracy.sh
@@ -2,6 +2,8 @@ cat $0
 
 exe=../../slmm/slmmir
 
+mkdir -p tmp
+
 ctr=0
 function run {
     ctr=$(expr $ctr + 1)

diff --git a/methods/islet/figures/run-img-filament.sh b/methods/islet/figures/run-img-filament.sh
@@ -1,10 +1,12 @@
 cat $0
 exe=../../slmm/slmmir
 
+mkdir -p filament-imgs
+
 function run {
-    cmd="$exe -method pcsl -ode nondivergent -ic slottedcylinders -T 12 -ne $ne -nsteps $nsteps -timeint $timeint -nonunimesh 0 -np $np -dmc eh -mono $cdrglb -lim caas -lauritzen -we $we -io internal -o ../data/mar21/filament-imgs/ne$ne-np$np-nstep$nsteps-$timeint-$cdrglb-pr$prefine -res 256 -rit -prefine $prefine $d2c"
+    cmd="$exe -method pcsl -ode nondivergent -ic slottedcylinders -T 12 -ne $ne -nsteps $nsteps -timeint $timeint -nonunimesh 0 -np $np -dmc eh -mono $cdrglb -lim caas -lauritzen -we $we -io internal -o filament-imgs/ne$ne-np$np-nstep$nsteps-$timeint-$cdrglb-pr$prefine -res 256 -rit -prefine $prefine $d2c"
     echo "cmd> $cmd"
-    eval "OMP_NUM_THREADS=1 $cmd"
+    eval "OMP_NUM_THREADS=48 KMP_AFFINITY=balanced $cmd"
 }
 
 d2c="-d2c -io-nodss"

diff --git a/methods/islet/figures/run-mixing.sh b/methods/islet/figures/run-mixing.sh
@@ -2,6 +2,8 @@ cat $0
 
 exe=../../slmm/slmmir
 
+mkdir -p mixing-0
+
 ctr=0
 function run {
     ctr=$(expr $ctr + 1)

diff --git a/methods/islet/readme.txt b/methods/islet/readme.txt
@@ -14,7 +14,7 @@ results in this paper. We used Kokkos version 3.1
 
 For the methods/islet programs, on a standard Linux system with GNU compiler suite,
     ln -s make.inc.gnu make.inc
-    make
+    make -j8
 
 The program "cslunstab" demonstrates the unstable classical cubic interpolation
 semi-Lagrangian instances. Running it should produce no output, as in this case
@@ -30,26 +30,15 @@ This produces output of the following form:
     min_np  8
     min_np  7
     min_np  6
-    meam1 4.9e-15 mcV 9.6e+01 mdef 1.0e+00 w>0 1 wtr  9.44e+00 npm  7.27e-06  7.99e-06  1.62e-05 pum  5.41e-08 | np  8 subnp 6 6 7 6 offst 0 0 0 1
-    meam1 4.4e-15 mcV 1.3e+02 mdef 1.0e+00 w>0 1 wtr  7.48e+00 npm  1.08e-05  9.61e-06  1.62e-05 pum  1.60e-08 | np  8 subnp 6 6 6 6 offst 0 0 0 1
-    min_np  5
-    meam1 4.9e-15 mcV 1.3e+02 mdef 1.0e+00 w>0 1 wtr  8.31e+00 npm  7.21e-06  8.39e-06  1.62e-05 pum  1.10e-06 | np  8 subnp 5 7 7 6 offst 0 0 0 1
-    meam1 4.7e-15 mcV 2.5e+02 mdef 1.0e+00 w>0 1 wtr  1.06e+01 npm  1.46e-05  1.42e-05  2.16e-05 pum  5.78e-09 | np  8 subnp 5 5 7 6 offst 0 0 0 1
-    meam1 4.2e-15 mcV 4.1e+02 mdef 1.0e+00 w>0 1 wtr  1.89e+01 npm  5.08e-05  5.51e-05  8.70e-05 pum  3.04e-09 | np  8 subnp 5 6 5 6 offst 0 0 1 1
+    meam1 4.7e-15 mcV 1.3e+02 mdef 1.0e+00 w>0 1 wtr  7.48e+00 npm  1.08e-05  9.61e-06  1.62e-05 pum  4.38e-05 | np  8 subnp 6 6 6 6 offst 0 0 0 1
+    meam1 4.0e-15 mcV 9.6e+01 mdef 1.0e+00 w>0 1 wtr  9.44e+00 npm  7.27e-06  7.99e-06  1.62e-05 pum  1.82e-07 | np  8 subnp 6 6 7 6 offst 0 0 0 1
+    count 42592
     np  8
-    min_np  6 max_np  8
-    min_np  6 max_np  7
-    meam1  1.55e-15 w>0 1 wtr 9.30e+00 npm 7.51e-06 8.07e-06 1.62e-05 pum  2.71e-08 | np  8 subnp 6 6 7 6 nodes | 0 1 2 3 5 6 | 0 1 2 3 4 5 | 0 1 2 3 4 5 6 | 1 2 3 4 5 6
-    meam1  1.78e-15 w>0 1 wtr 9.60e+00 npm 7.71e-06 8.20e-06 1.62e-05 pum  4.29e-09 | np  8 subnp 6 6 7 6 nodes | 0 1 2 3 4 6 | 0 1 2 3 4 6 | 0 1 2 3 4 5 6 | 1 2 3 4 5 6
-    meam1  1.78e-15 w>0 1 wtr 9.58e+00 npm 7.74e-06 8.21e-06 1.62e-05 pum  3.65e-08 | np  8 subnp 6 6 7 6 nodes | 0 1 2 3 4 7 | 0 1 2 3 4 6 | 0 1 2 3 4 5 6 | 1 2 3 4 5 6
+    NsbSearchAtom::eval 4096/42592 (  9.6%)
+    meam1  4.88e-15 w>0 1 wtr 5.93e+00 npm 1.89e-05 1.60e-05 2.13e-05 pum  1.58e-07 | np  8 subnp 6 6 6 6 nodes | 0 1 2 4 5 7 | 0 1 2 3 5 6 | 0 1 2 3 5 6 | 0 2 3 4 5 7
+    meam1  4.66e-15 w>0 1 wtr 6.94e+00 npm 1.34e-05 1.13e-05 1.62e-05 pum  1.39e-07 | np  8 subnp 6 6 6 6 nodes | 0 1 2 3 5 7 | 0 1 2 3 4 7 | 0 1 2 3 4 7 | 1 2 3 4 5 6
     ...
-    meam1  8.88e-16 w>0 1 wtr 8.74e+00 npm 9.32e-06 1.05e-05 2.13e-05 pum  1.17e-09 | np  8 subnp 6 6 7 6 nodes | 0 1 2 3 5 7 | 0 1 2 3 4 6 | 0 1 2 3 4 5 6 | 0 2 3 4 5 7
-    meam1  1.55e-15 w>0 1 wtr 8.83e+00 npm 9.45e-06 1.05e-05 2.13e-05 pum  7.66e-10 | np  8 subnp 6 6 7 6 nodes | 0 1 2 3 4 7 | 0 1 2 3 4 6 | 0 1 2 3 4 6 7 | 0 2 3 4 5 7
-    min_np  6 max_np  6
-    meam1  1.78e-15 w>0 1 wtr 7.15e+00 npm 1.24e-05 1.07e-05 1.62e-05 pum  1.02e-09 | np  8 subnp 6 6 6 6 nodes | 0 1 2 3 4 6 | 0 1 2 3 4 6 | 0 1 2 3 4 6 | 1 2 3 4 5 6
-    min_np  7 max_np  8
-    min_np  7 max_np  7
-    min_np  8 max_np  8
+    meam1  4.44e-15 w>0 1 wtr 8.55e+00 npm 6.07e-06 7.62e-06 1.62e-05 pum  1.12e-07 | np  8 subnp 6 7 7 6 nodes | 0 1 2 3 4 5 | 0 1 2 3 4 6 7 | 0 1 2 3 4 5 6 | 1 2 3 4 5 6
 
 In this output, each line beginning with "meam1" corresponds to a t.p.s. basis.
 "meam1" means "maximum eigenvalue amplitude minus 1", and the following value
@@ -59,11 +48,12 @@ unused entries. Next is "w>0", which reports that all basis weights are > 0.
 the lambda_max^PUM value. After the "|" is the encoding of the basis, either
 o.n.s. ("offst" is in the encoding) or general n.s.
 
-For the methods/slmm/slmmir program, modify make.inc to point to your Kokkos
+For the methods/slmm/slmmir program, modify make.inc.gnu to point to your Kokkos
 installation, then
-    make
+    ln -s make.inc.gnu make.inc
+    make -j16
 Optionally run regression tests:
-    python2 slmm_runtests.py
+    OMP_NUM_THREADS=16 KMP_AFFINITY=balanced python2 slmm_runtests.py
 Bash scripts in the methods/islet/figures directory call the slmmir program.
 
 We use the language hy to create the figures. hy is a Lisp that compiles to

diff --git a/methods/slmm/Makefile b/methods/slmm/Makefile
@@ -1,9 +1,10 @@
 include make.inc
 
-F90 = gfortran
+SIQK=../../siqk
+BUILD_DIAG=1
 
-CXXFLAGS += $(opt) -Wall -pedantic -fopenmp -std=c++11 -I$(SIQK) -I$(PWD) -I$(KOKKOS)/include -DSIQK_TIME -Wno-unused-function -fPIC
-LDFLAGS += -fopenmp -L$(KOKKOS)/lib -lkokkos -ldl
+CXXFLAGS += -DRELAX_TIME -DSLMM_NP_GT_4 -DSLMM_NP_MAX=16 -fopenmp -std=c++11 -I$(SIQK) -I$(PWD) -I$(KOKKOS)/include -DSIQK_TIME -Wno-unused-function -fPIC
+LDFLAGS += -fopenmp
 
 OS_NAME := $(shell uname -s)
 

diff --git a/methods/slmm/make.inc b/methods/slmm/make.inc
diff --git a/methods/slmm/make.inc.gnu b/methods/slmm/make.inc.gnu
@@ -0,0 +1,15 @@
+KOKKOS= # fill in with path to Kokkos installation
+
+CXX=g++
+F90=gfortran
+CXXFLAGS=-O3 -g
+# newer versions of kokkos:
+#LDFLAGS=-L$(KOKKOS)/lib64 -lkokkoscore -ldl
+# older versions of kokkos:
+LDFLAGS=-L$(KOKKOS)/lib -lkokkos -ldl
+
+# Optional. Comment out if no TPL available.
+#NETCDF=
+
+LINK_LAPACK_BLAS=-llapack -lblas
+
diff --git a/methods/slmm/make.inc.icpc b/methods/slmm/make.inc.icpc
@@ -0,0 +1,15 @@
+KOKKOS= # path to kokkos install
+
+CXX=icpc
+F90=gfortran
+CXXFLAGS=-O3 -g -fp-model=source
+# newer versions of kokkos:
+LDFLAGS=-L$(KOKKOS)/lib64 -lkokkoscore -ldl
+# older versions of kokkos:
+#LDFLAGS=-L$(KOKKOS)/lib -lkokkos -ldl
+
+# Optional. Comment out if no TPL available.
+#NETCDF=
+
+LINK_LAPACK_BLAS=-mkl
+
diff --git a/methods/slmm/slmm_defs.hpp b/methods/slmm/slmm_defs.hpp
@@ -1,6 +1,9 @@
 #ifndef INCLUDE_SLMM_DEFS_HPP
 #define INCLUDE_SLMM_DEFS_HPP
 
+#ifdef _OPENMP
+# include <omp.h>
+#endif
 #include "siqk.hpp"
 #include "slmm_array.hpp"
 

diff --git a/methods/slmm/slmm_io.cpp b/methods/slmm/slmm_io.cpp
@@ -7,6 +7,7 @@
 using namespace netCDF;
 
 #include <stdexcept>
+#include <fstream>
 
 namespace slmm {
 namespace io {