Building Albany on CCI DRP

Brian Granzow edited this page Nov 12, 2017 · 11 revisions

CCI DRP

Setup

module purge
module load cmake/3.3.2
module load gcc/6.3.0_1
module load openmpi/2.1.1_1

TPLS

tarballs/config scripts of all TPLs in /gpfs/u/home/ADDM/ADDMgnzw/barn-shared/tpls

zlib - version 1.2.8

# zlib v. 1.2.8
export tpls=/gpfs/u/home/ADDM/ADDMgnzw/scratch-shared/tpls
CC=mpicc CFLAGS="-O3 -fPIC" ./configure --64 --prefix=${tpls}

hdf5 - version 1.8.11

# hdf5 v 1.10.1
export tpls=/gpfs/u/home/ADDM/ADDMgnzw/scratch-shared/tpls
./configure CC=mpicc CXX=mpicxx FC=mpif90 \
CFLAGS="-O3 -fPIC -std=c99" CXXFLAGS="-O3 -fPIC" FCFLAGS="-O3 -fPIC" \
--enable-parallel \
--with-zlib=${tpls} \
--prefix=${tpls}

netcdf - version 4.3.2

# netcdf v 4.5.0
export tpls=/gpfs/u/home/ADDM/ADDMgnzw/scratch-shared/tpls
./configure CC=mpicc CXX=mpicxx FC=mpif90 \
CFLAGS="-fPIC -I${tpls}/include -O3 -march=native" \
CXXFLAGS="-fPIC -I${tpls}/include -O3 -march=native" \
FCFLAGS="-fPIC -I${tpls}/include -O3 -march=native" \
LDFLAGS="-fPIC -L${tpls}/lib -O3 -march=native" \
--prefix=${tpls} \
--disable-doxygen \
--enable-netcdf4

boost - version 1.55.0

# boost v 1.65.1
export tpls=/gpfs/u/home/ADDM/ADDMgnzw/scratch-shared/tpls
echo "using gcc : 6.3.0_1 : /gpfs/u/software/amd64-rhel6/compilers/gcc/6.3.0_1/bin/gcc ;" \
>> ~/user-config.jam
echo "using mpi : /gpfs/u/software/amd64-rhel6/mpi/openmpi/2.1.1_1/gcc-6.3.0_1/bin/mpicxx ;" \
>> ~/user-config.jam
./bootstrap.sh \
--with-libraries=signals,regex,filesystem,system,mpi,serialization,thread,program_options,exception \
--prefix=${tpls}

lapack - version 3.5.0

# lapack v 3.5.0
export tpls=/gpfs/u/home/ADDM/ADDMgnzw/scratch/tpls
mkdir build
cd build
ccmake -DCMAKE_INSTALL_PREFIX=${tpls} ..

parmetis - version 4.0.3

# parmetis v 4.0.3
export tpls=/gpfs/u/home/ADDM/ADDMgnzw/scratch/tpls
export metis=/gpfs/u/home/ADDM/ADDMgnzw/scratch/parmetis-4.0.3/metis
cd build
cmake \
 -D CMAKE_INSTALL_PREFIX=${tpls} \
 -D CMAKE_C_COMPILER=mpicc \
 -D CMAKE_CXX_COMPILER=mpicxx \
 -D METIS_PATH=${metis} \
 -D GKLIB_PATH=${metis}/GKlib \
..

Trilinos

#!/bin/bash
# Modify these paths for your system.
TOOLS=/gpfs/u/home/ADDM/ADDMgnzw/scratch-shared/tpls
MPIDIR=/gpfs/u/software/amd64-rhel6/mpi/openmpi/2.1.1_1/gcc-6.3.0_1/bin/mpicxx
TRIL_INSTALL_DIR=/gpfs/u/home/ADDM/ADDMgnzw/scratch-shared/trilinos
BOOSTDIR=$TOOLS
NETCDFDIR=$TOOLS
HDF5DIR=$TOOLS
ZLIBDIR=$TOOLS
PARMETISDIR=$TOOLS

cmake \
\
 -D Trilinos_DISABLE_ENABLED_FORWARD_DEP_PACKAGES=ON \
 -D CMAKE_INSTALL_PREFIX=$TRIL_INSTALL_DIR \
 -D CMAKE_BUILD_TYPE=NONE \
 -D TPL_ENABLE_MPI=ON \
 -D MPI_BASE_DIR=$MPIDIR \
 -D CMAKE_C_FLAGS="-O2 -g" \
 -D CMAKE_CXX_FLAGS="-O2 -g -Wno-deprecated-declarations -Wno-sign-compare" \
 -D CMAKE_VERBOSE_MAKEFILE=OFF \
 -D BUILD_SHARED_LIBS=ON \
 -D Trilinos_EXTRA_LINK_FLAGS="-ldl" \
 -D Trilinos_ENABLE_SECONDARY_TESTED_CODE=ON \
 -D CMAKE_INSTALL_RPATH_USE_LINK_PATH=True \
\
 -D Trilinos_ENABLE_ALL_PACKAGES=OFF \
 -D Trilinos_WARNINGS_AS_ERRORS_FLAGS="" \
 -D Trilinos_ENABLE_Teuchos=ON \
 -D Trilinos_ENABLE_Shards=ON \
 -D Trilinos_ENABLE_Sacado=ON \
 -D Trilinos_ENABLE_Epetra=ON \
 -D Trilinos_ENABLE_EpetraExt=ON \
 -D Trilinos_ENABLE_Ifpack=ON \
 -D Trilinos_ENABLE_AztecOO=ON \
 -D Trilinos_ENABLE_Amesos=ON \
 -D Trilinos_ENABLE_Anasazi=ON \
 -D Trilinos_ENABLE_Belos=ON \
 -D Trilinos_ENABLE_ML=ON \
 -D Trilinos_ENABLE_Phalanx=ON \
 -D Trilinos_ENABLE_MiniTensor=ON \
 -D Trilinos_ENABLE_NOX=ON \
 -D Trilinos_ENABLE_Stratimikos=ON \
 -D Trilinos_ENABLE_Thyra=ON \
 -D Trilinos_ENABLE_Rythmos=ON \
 -D Trilinos_ENABLE_Stokhos=OFF \
 -D Trilinos_ENABLE_Piro=ON \
 -D Trilinos_ENABLE_Teko=ON \
 -D Trilinos_ENABLE_STKIO=ON \
 -D Trilinos_ENABLE_STKMesh=ON \
 -D Trilinos_ENABLE_Tpetra=ON \
 -D Trilinos_ENABLE_Kokkos=ON \
 -D Kokkos_ENABLE_Pthread=OFF \
 -D HAVE_INTREPID_KOKKOSCORE=ON \
 -D Trilinos_ENABLE_Ifpack2=ON \
 -D Trilinos_ENABLE_Zoltan2=ON \
 -D Trilinos_ENABLE_MueLu=ON \
\
 -D Trilinos_ENABLE_Intrepid2=ON \
 -D Trilinos_ENABLE_Intrepid=ON \
 -D Intrepid2_ENABLE_KokkosDynRankView=ON \
\
 -D Trilinos_ENABLE_Amesos2=ON \
 -D Amesos2_ENABLE_KLU2=ON \
 -D Amesos2_ENABLE_SuperLUDist=OFF \
 -D Amesos2_ENABLE_Epetra=OFF \
\
 -D Trilinos_ENABLE_SEACAS=ON \
 -D Trilinos_ENABLE_SEACASIoss=ON \
 -D Trilinos_ENABLE_SEACASExodus=ON \
 -D SEACAS_ENABLE_SEACASSVDI=OFF \
 -D Trilinos_ENABLE_SEACASFastq=OFF \
 -D Trilinos_ENABLE_SEACASBlot=OFF \
 -D Trilinos_ENABLE_SEACASPLT=OFF \
\
 -D TPL_ENABLE_Matio=OFF \
 -D TPL_ENABLE_X11=OFF \
\
 -D TPL_ENABLE_Boost=ON \
 -D Boost_INCLUDE_DIRS="$BOOSTDIR/include" \
 -D Boost_LIBRARY_DIRS="$BOOSTDIR/lib" \
 -D TPL_ENABLE_BoostLib=ON \
 -D BoostLib_INCLUDE_DIRS="$BOOSTDIR/include" \
 -D BoostLib_LIBRARY_DIRS="$BOOSTDIR/lib" \
\
 -D TPL_ENABLE_Netcdf=ON \
 -D Netcdf_INCLUDE_DIRS="$NETCDFDIR/include" \
 -D Netcdf_LIBRARY_DIRS="$NETCDFDIR/lib" \
\
 -D TPL_ENABLE_HDF5=ON \
 -D HDF5_INCLUDE_DIRS="$HDF5DIR/include" \
 -D HDF5_LIBRARY_DIRS="$HDF5DIR/lib" \
\
 -D TPL_ENABLE_Zlib=ON \
 -D Zlib_INCLUDE_DIRS="$ZLIBDIR/include" \
 -D Zlib_LIBRARY_DIRS="$ZLIBDIR/lib" \
\
 -D TPL_ENABLE_BLAS=ON \
 -D TPL_BLAS_LIBRARIES="$TOOLS/lib/libblas.so" \
\
 -D TPL_ENABLE_LAPACK=ON \
 -D TPL_LAPACK_LIBRARIES="$TOOLS/lib/liblapack.so" \
\
 -D TPL_ENABLE_ParMETIS=ON \
 -D ParMETIS_INCLUDE_DIRS="$PARMETISDIR/include" \
 -D ParMETIS_LIBRARY_DIRS="$PARMETISDIR/lib" \
\
 -D TPL_ENABLE_METIS=ON \
 -D METIS_INCLUDE_DIRS="$PARMETISDIR/include" \
 -D METIS_LIBRARY_DIRS="$PARMETISDIR/lib" \
 -D Zoltan_ENABLE_ULLONG_IDS=ON \
 -D Teuchos_ENABLE_LONG_LONG_INT=ON \
\
 -D Trilinos_ENABLE_EXPLICIT_INSTANTIATION=ON \
 -D Tpetra_INST_FLOAT=OFF \
 -D Tpetra_INST_INT_INT=ON \
 -D Tpetra_INST_DOUBLE=ON \
 -D Tpetra_INST_COMPLEX_FLOAT=OFF \
 -D Tpetra_INST_COMPLEX_DOUBLE=OFF \
 -D Tpetra_INST_INT_UNSIGNED=OFF \
 -D Tpetra_INST_INT_LONG_LONG=ON \
\
..

SCOREC/core

TPL_INSTALL_DIR=/gpfs/u/home/ADDM/ADDMgnzw/scratch-shared/tpls
TRIL_INSTALL_DIR=/gpfs/u/home/ADDM/ADDMgnzw/scratch-shared/trilinos
CORE_INSTALL_DIR=/gpfs/u/home/ADDM/ADDMgnzw/scratch-shared/scorec

cmake \
-D CMAKE_C_COMPILER="mpicc" \
-D CMAKE_CXX_COMPILER="mpicxx" \
-D CMAKE_INSTALL_PREFIX=${CORE_INSTALL_DIR} \
-D BUILD_SHARED_LIBS=ON \
-D ENABLE_ZOLTAN=ON \
-D ZOLTAN_LIBRARY=${TRIL_INSTALL_DIR}/lib/libzoltan.so \
-D ZOLTAN_INCLUDE_DIR=${TRIL_INSTALL_DIR}/include \
-D PARMETIS_LIBRARY=${TPL_INSTALL_DIR}/lib/libparmetis.a \
-D METIS_LIBRARY=${TPL_INSTALL_DIR}/lib/libmetis.a \
-D PARMETIS_INCLUDE_DIR=${TPL_INSTALL_DIR}/include \
-D ENABLE_STK=ON \
-D ENABLE_STK_MESH=ON \
-D Trilinos_PREFIX=${TRIL_INSTALL_DIR} \
..
2>&1 | tee config_log

Albany

git clone https://github.com/gahansen/Albany
cd Albany
mkdir build
cd build
./do-config-albany.sh

where the ./do-config-albany.sh looks like:

(you will need to modify the ALB_DIR variable)

export TRI_DIR=/gpfs/u/home/ADDM/ADDMgnzw/scratch/trilinos/install
export ALB_DIR=/gpfs/u/home/ADDM/ADDMgnzw/barn-shared/albany
cmake \
 -D ENABLE_INSTALL=ON \
 -D CMAKE_INSTALL_PREFIX=${ALB_DIR} \
 -D ALBANY_TRILINOS_DIR=${TRI_DIR} \
 -D ENABLE_64BIT_INT=OFF \
 -D ENABLE_SCOREC=ON \
 -D ENABLE_AMP=ON \
 -D ENABLE_LCM=ON \
 -D ENABLE_CTM=ON \
 -D ENABLE_CHECK_FPE=OFF \
 -D ENABLE_ALBANY_EPETRA_EXE=OFF \
 -D ENABLE_LCM_SPECULATIVE=OFF \
 -D ENABLE_HYDRIDE=OFF \
 -D ENABLE_SG_MP=OFF \
 -D ENABLE_QCAD=OFF \
 -D ENABLE_MOR=OFF \
 -D ENABLE_STOKHOS=OFF \
 -D ENABLE_FELIX=OFF \
../

Job Submission

Before submitting a job ensure you have ssh'd to either drpfen01 or drpfen02 and loaded the modules here

sbatch -t 30 -p debug -n 1 srun.sh

This is what srun.sh contains:

#!/bin/bash -x 

# dirty hack
ln -s /usr/lib64/libslurm.so.30 libslurm.so.26
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$PWD

export LD_LIBRARY_PATH=/gpfs/u/home/ADDM/ADDMgnzw/barn-shared/albany/lib:$LD_LIBRARY_PATH
export LD_LIBRARY_PATH=/gpfs/u/home/ADDM/ADDMgnzw/scratch/tpls/lib:$LD_LIBRARY_PATH
export LD_LIBRARY_PATH=/gpfs/u/home/ADDM/ADDMgnzw/scratch/trilinos/install/lib:$LD_LIBRARY_PATH

export ALB=/gpfs/u/home/ADDM/ADDMgnzw/barn-shared/albany/bin/AlbanyT

srun hostname -s > /tmp//hosts.$SLURM_JOB_ID

if [ "x$SLURM_NPROCS" = "x" ]
then
  if [ "x$SLURM_NTASKS_PER_NODE" = "x" ]
  then
    SLURM_NTASKS_PER_NODE=1
  fi
  SLURM_NPROCS=`expr $SLURM_JOB_NUM_NODES \* $SLURM_NTASKS_PER_NODE`
fi

mpirun -hostfile /tmp/hosts.$SLURM_JOB_ID --bind-to core -np $SLURM_NPROCS ${ALB} input.xml

rm /tmp/hosts.$SLURM_JOB_ID
Clone this wiki locally
You can’t perform that action at this time.
You signed in with another tab or window. Reload to refresh your session. You signed out in another tab or window. Reload to refresh your session.
Press h to open a hovercard with more details.