Build on ALCF Theta
Cameron Smith edited this page Aug 8, 2017
·
16 revisions
gnu-sim.env
#!/bin/bash
module swap PrgEnv-intel PrgEnv-gnu
d=/gpfs/theta-fs1/home/cwsmith/develop
export CMAKE_PREFIX_PATH=$d/deps/install-gnu:$CMAKE_PREFIX_PATH
module use $d/deps/modules
module load simModSuite
export SIM_MPI=gnucraympich
Source the environment file:
source gnu-sim.env
core-config.sh
#!/bin/bash -ex
d=/gpfs/theta-fs1/home/cwsmith/develop
flags="-O3 "
cmake \
-DCMAKE_CXX_COMPILER=CC \
-DCMAKE_C_COMPILER=cc \
-DSCOREC_EXTRA_CXX_FLAGS="${flags} " \
-DENABLE_ZOLTAN=ON \
-DPCU_COMPRESS=ON \
-DMDS_SET_MAX=1024 \
-DMDS_ID_TYPE=long \
-DMPIRUN=$d/core/runTheta.sh \
-DMPIRUN_PROCFLAG="" \
-DIS_TESTING=True \
-DENABLE_SIMMETRIX=ON \
-DSIM_MPI=$SIM_MPI \
-DMESHES=$d/pumi-meshes \
../core
Using address sanitizer: core-config-asan.sh
#!/bin/bash -ex
d=/gpfs/theta-fs1/home/cwsmith/develop
flags="-O3 -fsanitize=address "
cmake \
-DCMAKE_CXX_COMPILER=CC \
-DCMAKE_C_COMPILER=cc \
-DSCOREC_EXTRA_CXX_FLAGS="${flags} " \
-DCMAKE_EXE_LINKER_FLAGS="-dynamic " \
-DENABLE_ZOLTAN=ON \
-DPCU_COMPRESS=ON \
-DMDS_SET_MAX=1024 \
-DMDS_ID_TYPE=long \
-DMPIRUN=$d/core/runTheta.sh \
-DMPIRUN_PROCFLAG="" \
-DIS_TESTING=True \
-DENABLE_SIMMETRIX=ON \
-DSIM_MPI=$SIM_MPI \
-DMESHES=$d/pumi-meshes \
../core
runTheta.sh
#!/bin/bash -x
processes=$1
exe=$2
myparam=${@:3}
aprun -n $processes -N $processes -j 1 $exe $myparam
Run cmake:
./core-config.sh
or
./core-config-asan.sh
Using Intel Compilers and MPICH (currently the only MPI choice available through modules) and Ben's cmake build:
/gpfs/theta-fs1/projects/UnsAdaptCFD_tesp/kjansen/cmake/v3.6.1/bin
Currently Loaded Modulefiles:
1) modules/3.2.10.4 5) craype/2.5.6 9) pmi/5.0.10-1.0000.11050.0.0.ari 13) job/1.5.5-3.58 17) atp/2.0.2
2) eswrap/2.0.9-2.1 6) cray-libsci/16.07.1 10) dmapp/7.1.0-12.37 14) dvs/2.7_0.9.0-2.148 18) PrgEnv-intel/6.0.3
3) intel/16.0.3.210 7) udreg/2.3.2-4.6 11) gni-headers/5.0.7-3.1 15) alps/6.1.6-20.1 19) craype-mic-knl
4) craype-network-aries 8) ugni/6.0.12-2.1 12) xpmem/0.1-4.5 16) rca/1.0.0-6.21 20) cray-mpich/7.4.2
#!/bin/bash -e
metis=/gpfs/theta-fs1/projects/UnsAdaptCFD_tesp/cwsmith/develop/deps/parmetis/parmetis-4.0.3/metis/
cmake \
-DCMAKE_INSTALL_PREFIX=$1 \
-DMETIS_PATH=$metis \
-DGKLIB_PATH=$metis/GKlib \
-DCMAKE_C_COMPILER=cc \
-DCMAKE_CXX_COMPILER=CC \
-DCMAKE_BUILD_TYPE=None \
-DCMAKE_C_FLAGS="-O2 -static " \
-DCMAKE_CXX_FLAGS="-O2 -static " \
..
#!/bin/bash -ex
unset CXXCPP
../configure \
--prefix=$1 \
--with-parmetis \
--with-parmetis-libdir=$PARMETIS_INSTALL_DIR/lib \
--with-parmetis-incdir=$PARMETIS_INSTALL_DIR/include \
--enable-mpi \
--disable-examples \
--host=x86_64-unknown-linux-gnu \
--with-gnumake \
--with-id-type=ulong \
CXXCPP='CC -E ' \
FC=ftn \
CC=cc \
CXX=CC
flags="-Wall -Winline -O2 -g "
make CC=cc clean
make bzip2 CC=cc CFLAGS="$flags"
make install PREFIX=$prefix
cmake configuration script using Intel compilers doConfigureIntel.sh
#!/bin/bash -ex
export CMAKE_PREFIX_PATH=/lus/theta-fs0/home/cwsmith/develop/deps/install-intel:$CMAKE_PREFIX_PATH
flags="-O3 -Wall"
cmake \
-DCMAKE_C_FLAGS="${flags} " \
-DCMAKE_CXX_FLAGS="${flags} " \
-DCMAKE_CXX_LINK_FLAGS="${flags} " \
-DCMAKE_CXX_COMPILER=CC \
-DCMAKE_C_COMPILER=cc \
-DENABLE_ZOLTAN=ON \
-DPCU_COMPRESS=ON \
-DMDS_SET_MAX=1024 \
-DMDS_ID_TYPE=long \
-DMPIRUN=/lus/theta-fs0/home/cwsmith/develop/core/runTheta.sh \
-DMPIRUN_PROCFLAG="" \
-DIS_TESTING=True \
-DMESHES=/lus/theta-fs0/home/cwsmith/develop/meshes \
../
cobalt job submission script for ctest submitCtest.sh
#!/bin/bash -ex
rundir=/lus/theta-fs0/home/cwsmith/develop/core/buildintel
qsub --cwd $rundir -q cache-quad -n 1 -t 10 -A UnsAdaptCFD_tesp /lus/theta-fs0/home/cwsmith/tools/cmake/v3.6.1/bin/ctest -VV
cray job execution script for each test
#!/bin/bash -x
processes=$1
exe=$2
myparam=${@:3}
aprun -n $processes -N $processes -j 1 $exe $myparam
All but three tests pass:
96% tests passed, 3 tests failed out of 68
Total Test time (real) = 490.48 sec
The following tests FAILED:
5 - bezierMisc (Failed)
6 - bezierRefine (Failed)
8 - bezierValidity (Failed)
test 5
Start 5: bezierMisc
5: Test command: /lus/theta-fs0/home/cwsmith/develop/core/runTheta.sh "1" "./bezierMisc"
5: Test timeout computed to be: 1500
5: + processes=1
5: + exe=./bezierMisc
5: + myparam=
5: + aprun -n 1 -N 1 -j 1 ./bezierMisc
5: bezierMisc: /lus/theta-fs0/home/cwsmith/develop/core/test/bezierMisc.cc:158: void testMatrixInverse(): Assertion `fabs(AiQR(i,j) - a_invdata[i][j]) < 1e-14' failed.
5: _pmiu_daemon(SIGCHLD): [NID 03044] [c3-1c2s9n0] [Thu Aug 18 19:41:37 2016] PE RANK 0 exit signal Aborted
5: Application 1583631 exit codes: 134
5: Application 1583631 resources: utime ~0s, stime ~1s, Rss ~11228, inblocks ~0, outblocks ~0
5/68 Test #5: bezierMisc .......................***Failed 5.39 sec
test 6
Start 6: bezierRefine
6: Test command: /lus/theta-fs0/home/cwsmith/develop/core/runTheta.sh "1" "./bezierRefine"
6: Test timeout computed to be: 1500
6: + processes=1
6: + exe=./bezierRefine
6: + myparam=
6: + aprun -n 1 -N 1 -j 1 ./bezierRefine
6: mesh verified in 0.000122 seconds
6:
6: MeshAdapt: version 2.0 !
6:
6: MeshAdapt: iteration 0
6:
6: MeshAdapt: refined 5 edges in 0.000260 seconds
6:
6: MeshAdapt: snapped in 0.000186 seconds: 2 targets, 2 non-layer snaps
6:
6: MeshAdapt: bad shapes down from 1 to 0 in 0.000210 seconds
6:
6: MeshAdapt: worst element quality is 7.500000e-01
... //omitted
6: MeshAdapt: Swapped 0 bad edges in 0.265236 seconds
6:
6: MeshAdapt: worst element quality is -7.993898e-01
6:
6: MeshAdapt: mesh adapted in 0.368478 seconds
6: number of tet 8 hex 0 prism 0 pyramid 0
6: mesh entity counts: v 10 e 25 f 24 r 8
6: bezierRefine: /lus/theta-fs0/home/cwsmith/develop/core/test/bezierRefine.cc:272: void test3D(): Assertion `numinvalid == 0' failed.
6: _pmiu_daemon(SIGCHLD): [NID 03044] [c3-1c2s9n0] [Thu Aug 18 19:41:45 2016] PE RANK 0 exit signal Aborted
6: Application 1583632 exit codes: 134
6: Application 1583632 resources: utime ~3s, stime ~1s, Rss ~14636, inblocks ~0, outblocks ~0
6/68 Test #6: bezierRefine .....................***Failed 8.57 sec
test 8
Start 8: bezierValidity
8: Test command: /lus/theta-fs0/home/cwsmith/develop/core/runTheta.sh "1" "./bezierValidity"
8: Test timeout computed to be: 1500
8: + processes=1
8: + exe=./bezierValidity
8: + myparam=
8: + aprun -n 1 -N 1 -j 1 ./bezierValidity
8: mesh verified in 0.000137 seconds
8: mesh verified in 0.000064 seconds
8: mesh verified in 0.000063 seconds
8: mesh verified in 0.000063 seconds
8: mesh verified in 0.000064 seconds
8: mesh verified in 0.000085 seconds
8: mesh verified in 0.000065 seconds
8: bezierValidity: /lus/theta-fs0/home/cwsmith/develop/core/test/bezierValidity.cc:391: void test3D(): Assertion `validityTag == 1' failed.
8: _pmiu_daemon(SIGCHLD): [NID 03044] [c3-1c2s9n0] [Thu Aug 18 19:41:56 2016] PE RANK 0 exit signal Aborted
8: Application 1583636 exit codes: 134
8: Application 1583636 resources: utime ~1s, stime ~1s, Rss ~11228, inblocks ~0, outblocks ~0
8/68 Test #8: bezierValidity ...................***Failed 5.59 sec