Building Albany on Titan

Irina K. Tezaur edited this page Oct 13, 2016 · 3 revisions

Detailed build and run instructions with Serial, OpenMP and CUDA node (10/13/2016)

Please refer to SAND report written by Jerry Watkins, which can be found here.

Using GCC compiler + OpenMP/Serial node

  1. Building Trilinos:

1.1. unload/load next:

module unload cmake
module unload cray-hdf5
module unload cray-hdf5-parallel
module unload netcdf
module unload python
module unload cray-shmem
module unload cray-mpich cray-mpich2
module unload netcdf-hdf5parallel cray-netcdf-hdf5parallel cray-parallel-netcdf cray-hdf5-parallel
module unload boost
module unload dynamic-link
module unload pgi
module unload PrgEnv-cray PrgEnv-gnu PrgEnv-intel PrgEnv-pathscale PrgEnv-pgi

module load modules
module load cmake/2.8.11.2
module load PrgEnv-gnu
module unload gcc
module load cray-shmem
module load cray-mpich
module load gcc
module load python
module load boost/1.57.0
module load dynamic-link
module load cray-netcdf-hdf5parallel boost
module load cray-hdf5-parallel
module list

1.2 configure and build Trilinos. Configure script example:

 rm -f CMakeCache.txt
 rm -rf CMakeLists

 export TRILINOS_HOME=/ccs/home/ipdemes/TrilinosDir/Trilinos_github/Trilinos
 BOOSTDIR=/ccs/home/ikalash/Install/boost_1_55_0
 SUPERLUDIR=/ccs/home/ikalash/Install/SuperLU_4.3_2
 HWLOC_PATH=/ccs/home/ipdemes/install/hwloc_static
 BUILD_DIR=`pwd`

 EXTRA_ARGS=$@

 rm -rf CMakeFiles CMakeCache.txt

 cmake -D CMAKE_INSTALL_PREFIX:PATH=$BUILD_DIR/install \
  -D Boost_INCLUDE_DIRS:FILEPATH=$BOOSTDIR/include \
  -D Netcdf_LIBRARY_DIRS:FILEPATH="${NETCDF_DIR}/lib" \
  -D Netcdf_INCLUDE_DIRS:FILEPATH="${NETCDF_DIR}/include" \
  -D BoostLib_LIBRARY_DIRS:FILEPATH=$BOOSTDIR/lib \
  -D BoostLib_INCLUDE_DIRS:FILEPATH=$BOOSTDIR/include \
  -D TPL_ENABLE_BoostAlbLib:BOOL=ON \
  -D BoostAlbLib_INCLUDE_DIRS:FILEPATH=$BOOSTDIR/include \
  -D BoostAlbLib_LIBRARY_DIRS:FILEPATH=$BOOSTDIR/lib \
  -D CMAKE_BUILD_TYPE:STRING=RELEASE \
  -D Trilinos_WARNINGS_AS_ERRORS_FLAGS:STRING="" \
  -D Trilinos_ENABLE_ALL_PACKAGES:BOOL=OFF \
  -D Trilinos_ENABLE_ALL_OPTIONAL_PACKAGES:BOOL=OFF \
 \
  -D Trilinos_ENABLE_Fortran:BOOL=ON \
 \
  -D TPL_ENABLE_SuperLU:BOOL=ON \
  -D SuperLU_INCLUDE_DIRS:PATH=$SUPERLUDIR/SRC \
  -D TPL_SuperLU_LIBRARIES:PATH=$SUPERLUDIR/lib/libsuperlu_4.3.a \
 \
  -D Trilinos_ASSERT_MISSING_PACKAGES=OFF \
  -D Trilinos_ENABLE_Teuchos:BOOL=ON \
  -D HAVE_TEUCHOS_COMM_TIMERS=ON \
  -D Trilinos_ENABLE_Shards:BOOL=ON \
  -D Trilinos_ENABLE_Sacado:BOOL=ON \
  -D Trilinos_ENABLE_Epetra:BOOL=ON \
  -D Trilinos_ENABLE_Tpetra:BOOL=ON \
  -D Trilinos_ENABLE_EpetraExt:BOOL=ON \
  -D Trilinos_ENABLE_Ifpack:BOOL=ON \
  -D Trilinos_ENABLE_Ifpack2:BOOL=ON \
  -D Trilinos_ENABLE_AztecOO:BOOL=ON \
  -D Trilinos_ENABLE_Amesos:BOOL=ON \
  -D Trilinos_ENABLE_Amesos2:BOOL=ON \
  -D Trilinos_ENABLE_Anasazi:BOOL=ON \
  -D Trilinos_ENABLE_Belos:BOOL=ON \
  -D Trilinos_ENABLE_Phalanx:BOOL=ON \
  -D Phalanx_ENABLE_COMPILETIME_ARRAY_CHECK:BOOL=ON \
  -D Phalanx_KOKKOS_DEVICE_TYPE:STRING="OPENMP" \
  -D Phalanx_INDEX_SIZE_TYPE="UINT" \
  -D Tpetra_ENABLE_Kokkos_Refactor:BOOL=ON \
  -D KokkosClassic_DefaultNode:STRING="Kokkos::Compat::KokkosOpenMPWrapperNode" \
  -D Trilinos_ENABLE_TeuchosKokkosCompat:BOOL=ON \
  -D Trilinos_ENABLE_TpetraKernels:BOOL=ON \
  -D Trilinos_ENABLE_TeuchosKokkosComm:BOOL=ON \
  -D Trilinos_ENABLE_Kokkos:BOOL=ON \
  -D Trilinos_ENABLE_KokkosCore:BOOL=ON \
  -D Trilinos_ENABLE_KokkosClassic:BOOL=ON \
  -D Trilinos_ENABLE_KokkosContainers:BOOL=ON \
  -D Trilinos_ENABLE_KokkosLinAlg:BOOL=ON \
  -D Trilinos_ENABLE_KokkosCompat:BOOL=ON \
  -D Kokkos_ENABLE_OpenMP:BOOL=ON \
  -D Trilinos_ENABLE_OpenMP:BOOL=ON \
  -D TPL_ENABLE_OpenMP:BOOL=ON \
  -D TPL_HWLOC_LIBRARIES:PATHNAME="${HWLOC_PATH}/lib/libhwloc.a" \
  -D TPL_HWLOC_INCLUDE_DIRS:PATHNAME="${HWLOC_PATH}/include" \
  -D HWLOC_LIBRARY_DIRS="${HWLOC_PATH}/lib" \
  -D TPL_HWLOC_LIBRARIES="${HWLOC_PATH}/lib/libhwloc.a" \
  -D TPL_ENABLE_HWLOC:STRING=ON \
  -D Trilinos_ENABLE_Intrepid:BOOL=ON \
  -D HAVE_INTREPID_KOKKOSCORE:BOOL=ON \
  -D Trilinos_ENABLE_ML:BOOL=ON \
  -D Trilinos_ENABLE_MueLu:BOOL=ON \
  -D Trilinos_ENABLE_NOX:BOOL=ON \
  -D Trilinos_ENABLE_Stratimikos:BOOL=ON \
  -D Trilinos_ENABLE_Thyra:BOOL=ON \
  -D Trilinos_ENABLE_ThyraTpetraAdapters:BOOL=ON \
  -D Trilinos_ENABLE_TrilinosCouplings:BOOL=ON \
  -D Trilinos_ENABLE_Rythmos:BOOL=ON \
  -D Trilinos_ENABLE_OptiPack:BOOL=ON \
  -D Trilinos_ENABLE_GlobiPack:BOOL=ON \
  -D Trilinos_ENABLE_Stokhos:BOOL=ON \
  -D Trilinos_ENABLE_Isorropia:BOOL=ON\
  -D Trilinos_ENABLE_Piro:BOOL=ON \
  -D Trilinos_ENABLE_STKIO:BOOL=ON \
  -D Trilinos_ENABLE_STKMesh:BOOL=ON \
  -D Trilinos_ENABLE_SEACASExodus:BOOL=ON \
  -D Trilinos_ENABLE_Teko:BOOL=ON \
  -D TPL_FIND_SHARED_LIBS:BOOL=OFF \
 \
  -D Trilinos_ENABLE_TriKota:BOOL=OFF \
  -D TriKota_ENABLE_DakotaCMake:BOOL=OFF \
  -D DAKOTA_ENABLE_TESTS:BOOL=OFF \
  -D Boost_LIBRARY_DIRS:FILEPATH=$BOOSTDIR/lib \
 \
  -D Trilinos_ENABLE_SEACASIoss:BOOL=ON \
  -D Trilinos_ENABLE_ThreadPool:STRING="ON" \
  -D Trilinos_ENABLE_Pamgen:BOOL=ON \
  -D TPL_ENABLE_Netcdf:BOOL=ON \
  -D TPL_Netcdf_INCLUDE_DIRS:PATH="${NETCDF_DIR}/include" \
  -D Netcdf_LIBRARY_DIRS:PATH="${NETCDF_DIR}/lib" \
 \
  -D Trilinos_ENABLE_Mesquite:BOOL=OFF\
  -D Trilinos_ENABLE_Zoltan:BOOL=ON \
  -D Trilinos_ENABLE_Zoltan2:BOOL=ON\
  -D Zoltan_ENABLE_ULONG_IDS:BOOL=ON \
  -D ZOLTAN_BUILD_ZFDRIVE:BOOL=OFF \
  -D Zoltan2_ENABLE_Experimental:BOOL=ON\
  -D Trilinos_ENABLE_FEI:BOOL=OFF\
 \
  -D Trilinos_ENABLE_TESTS:BOOL=OFF \
  -D Piro_ENABLE_TESTS:BOOL=OFF \
  -D Trilinos_ENABLE_EXAMPLES:BOOL=OFF \
  -D TPL_ENABLE_MPI:BOOL=ON \
  -D TPL_ENABLE_Boost:BOOL=ON \
  -D TPL_ENABLE_BoostLib:BOOL=ON \
 \
  -D Anasazi_ENABLE_TEUCHOS_TIME_MONITOR:BOOL=ON \
  -D AztecOO_ENABLE_TEUCHOS_TIME_MONITOR:BOOL=ON \
  -D Belos_ENABLE_TEUCHOS_TIME_MONITOR:BOOL=ON \
  -D Ifpack_ENABLE_TEUCHOS_TIME_MONITOR:BOOL=ON \
  -D NOX_ENABLE_TEUCHOS_TIME_MONITOR:BOOL=ON \
  -D Phalanx_ENABLE_TEUCHOS_TIME_MONITOR:BOOL=ON \
  -D Rythmos_ENABLE_TEUCHOS_TIME_MONITOR:BOOL=ON \
  -D Stokhos_ENABLE_TEUCHOS_TIME_MONITOR:BOOL=ON \
  -D Stratimikos_ENABLE_TEUCHOS_TIME_MONITOR:BOOL=ON \
  -D Thyra_ENABLE_TEUCHOS_TIME_MONITOR:BOOL=ON \
  -D Trilinos_ENABLE_TEUCHOS_TIME_MONITOR:BOOL=ON \
 \
  -D CMAKE_VERBOSE_MAKEFILE:BOOL=OFF \
  -D Trilinos_VERBOSE_CONFIGURE:BOOL=OFF \
  -D CMAKE_CXX_FLAGS:STRING="-O2" \
  -D CMAKE_Fortran_FLAGS:STRING="-O2" \
  -D Trilinos_ENABLE_Export_Makefiles:BOOL=ON \
 \
  -D CMAKE_C_COMPILER:FILEPATH="cc" \
  -D CMAKE_CXX_COMPILER:FILEPATH="CC" \
  -D CMAKE_Fortran_COMPILER:FILEPATH="ftn" \
  -D CMAKE_SHARED_LINKER_FLAGS="/ccs/home/ipdemes/TrilinosDir/BuildTrilinos_Albany_OpenMP/install/lib/cmake/Trilinos/../../../lib /usr/lib64/libxml*.a /usr/lib64/libpciaccess.a  -lnuma" \
  -D TPL_BLAS_LIBRARIES:STRING="sci_gnu_mp" \
  -D TPL_LAPACK_LIBRARIES:STRING="sci_gnu_mp" \
  -DTPL_ENABLE_Pthread:BOOL=OFF \
  -DTPL_ENABLE_BinUtils:BOOL=OFF \
 \
  -D MPI_EXEC:FILEPATH=aprun \
  -D MPI_EXEC_MAX_NUMPROCS:STRING=4 \
  -D MPI_EXEC_NUMPROCS_FLAG:STRING=-n \
 \
 $EXTRA_ARGS \
 ${TRILINOS_HOME}
  1. Building Albany:

    2.1 use the same module script as for Trilinos

    2.2 use your usual Albany script to configure Albany (please see my script "do-configure" attached)

    2.3 building Albany is a little tricky, so here is my step-by-step instruction:

    make -j 4 Albany

    you'll get a linking error. To fix this do next:

    cd src/

    add " /usr/lib64/libxml*.a /usr/lib64/libpciaccess.a -static-libstdc++ -static-libgcc" to the end of the CMakeFiles/Albany.dir/link.txt -lnuma vi CMakeFiles/Albany.dir/link.txt

    copy the linking script from the CMakeFiles/Albany.dir/link.txt and paste into the terminal

    You need to repeat this step for every Albany executable.

Compiling Trilinos for CUDA on Titan:

  1. load modules: module unload cmake module unload cray-hdf5 module unload cray-hdf5-parallel module unload netcdf module unload python module unload cray-shmem module unload cray-mpich cray-mpich2 module unload netcdf-hdf5parallel cray-netcdf-hdf5parallel cray-parallel-netcdf cray-hdf5-parallel module unload boost module unload dynamic-link module unload pgi module unload PrgEnv-cray PrgEnv-gnu PrgEnv-intel PrgEnv-pathscale PrgEnv-pgi module unload cray-libsci cudatoolkit

    module load modules module load cmake/2.8.11.2 module load PrgEnv-gnu
    module unload gcc module load cray-shmem module load cray-mpich module swap cray-libsci cray-libsci/13.1.0 module swap cray-mpich cray-mpich/7.2.4 module load gcc/4.8.2 module load python module load boost/1.57.0 module load dynamic-link module load cray-netcdf-hdf5parallel boost module load cudatoolkit export CRAY_CPU_TARGET=x86-64 export CRAY_CUDA_PROXY=1 export CRAY_CPU_TARGET=istanbul module list

  2. Please have next example as a reference for the Trilinos configure script for CUDA on Titan:

    TRILINOS_HOME=/ccs/home/ipdemes/TrilinosDir/Trilinos_github/Trilinos TRILINSTALLDIR=/ccs/home/ipdemes/TrilinosDir/BuildTrilinos_Albany_CUDA/install HDF5DIR=/opt/cray/hdf5-parallel/1.8.14/gnu/4.8 BOOSTDIR=/sw/xk6/boost/1.57.0/cle5.2_gnu4.8.2/ SUPERLUDIR=/ccs/home/ikalash/Install/SuperLU_4.3_2 HWLOC_PATH=/ccs/home/ipdemes/install/hwloc_static

    export BOOST_ROOT=$BOOST_DIR

    EXTRA_ARGS=$@

    rm -rf CMakeFiles CMakeCache.txt

    cmake
    -D Trilinos_DISABLE_ENABLED_FORWARD_DEP_PACKAGES=ON
    -D CMAKE_INSTALL_PREFIX:PATH=${TRILINSTALLDIR}
    -D CMAKE_BUILD_TYPE:STRING=RELEASE \
    -D TPL_ENABLE_MPI:BOOL=ON
    -D CMAKE_VERBOSE_MAKEFILE:BOOL=OFF
    -D Trilinos_ENABLE_ALL_PACKAGES:BOOL=OFF
    -D Trilinos_WARNINGS_AS_ERRORS_FLAGS:STRING=""
    -D Teuchos_ENABLE_LONG_LONG_INT:BOOL=OFF

    -D TPL_FIND_SHARED_LIBS:BOOL=OFF
    -D Trilinos_ENABLE_Teuchos:BOOL=ON
    -D Trilinos_ENABLE_Shards:BOOL=ON
    -D Trilinos_ENABLE_Sacado:BOOL=ON
    -D Trilinos_ENABLE_Epetra:BOOL=ON
    -D Trilinos_ENABLE_EpetraExt:BOOL=ON
    -D Trilinos_ENABLE_Ifpack:BOOL=ON
    -D Trilinos_ENABLE_AztecOO:BOOL=ON
    -D Trilinos_ENABLE_Amesos:BOOL=ON
    -D Trilinos_ENABLE_Anasazi:BOOL=ON
    -D Trilinos_ENABLE_Belos:BOOL=ON
    -D Trilinos_ENABLE_ML:BOOL=ON
    -D Trilinos_ENABLE_Phalanx:BOOL=ON
    -D Phalanx_ENABLE_EXAMPLES:BOOL=OFF
    -D Phalanx_ENABLE_TESTS:BOOL=OFF
    -D Trilinos_ENABLE_Intrepid:BOOL=ON
    -D Trilinos_ENABLE_NOX:BOOL=ON
    -D Trilinos_ENABLE_Stratimikos:BOOL=ON
    -D Trilinos_ENABLE_Thyra:BOOL=ON
    -D Trilinos_ENABLE_Rythmos:BOOL=ON
    -D Trilinos_ENABLE_MOOCHO:BOOL=ON
    -D Trilinos_ENABLE_Stokhos:BOOL=ON
    -D Trilinos_ENABLE_Piro:BOOL=ON
    -D Trilinos_ENABLE_Teko:BOOL=OFF

    -D Trilinos_ENABLE_STKIO:BOOL=ON
    -D Trilinos_ENABLE_STKMesh:BOOL=ON
    -D TPL_ENABLE_Boost:BOOL=ON
    -D Boost_INCLUDE_DIRS:FILEPATH="$BOOSTDIR/include"
    -D Boost_LIBRARY_DIRS:FILEPATH="$BOOSTDIR/lib"
    -D TPL_ENABLE_BoostLib:BOOL=ON
    -D BoostLib_INCLUDE_DIRS:FILEPATH="$BOOSTDIR/include"
    -D BoostLib_LIBRARY_DIRS:FILEPATH="$BOOSTDIR/lib"
    -D CMAKE_C_COMPILER:FILEPATH="cc"
    -D CMAKE_CXX_COMPILER=/ccs/home/ipdemes/TrilinosDir/Albany_github/Albany/doc/nvcc_wrapper_Glen
    -D CMAKE_Fortran_COMPILER:FILEPATH="ftn"
    -D TPL_BLAS_LIBRARIES:STRING="sci_gnu_mp"
    -D TPL_LAPACK_LIBRARIES:STRING="sci_gnu_mp"
    -D CMAKE_CXX_FLAGS="-D_LZCNTINTRIN_H_INCLUDED"
    -D Trilinos_CXX11_FLAGS="-std=c++11"
    -D CMAKE_SHARED_LINKER_FLAGS="/ccs/home/ipdemes/TrilinosDir/BuildTrilinos_Albany_CUDA/install/lib/cmake/Trilinos /../../../lib /usr/lib64/libxml*.a /usr/lib64/libpciaccess.a"

    -D Trilinos_ENABLE_SEACASIoss:BOOL=ON
    -D TPL_ENABLE_Netcdf:BOOL=ON
    -D Netcdf_INCLUDE_DIRS:PATH="${NETCDF_DIR}/include"
    -D Netcdf_LIBRARY_DIRS:PATH="${NETCDF_DIR}/lib"
    -D TPL_ENABLE_HDF5:BOOL=ON
    -D HDF5_INCLUDE_DIRS:PATH="$HDF5DIR/include"
    -D HDF5_LIBRARY_DIRS:PATH="$HDF5DIR/lib"

    -D Trilinos_ENABLE_Tpetra:BOOL=ON
    -D Trilinos_ENABLE_Kokkos:BOOL=ON
    -D Trilinos_ENABLE_Ifpack:BOOL=ON
    -D Trilinos_ENABLE_Amesos2:BOOL=ON
    -D Trilinos_ENABLE_Zoltan2:BOOL=ON
    -D Trilinos_ENABLE_Ifpack2:BOOL=ON
    -D Trilinos_ENABLE_ML:BOOL=ON
    -D Trilinos_ENABLE_MueLu:BOOL=ON
    -D Amesos2_ENABLE_KLU2:BOOL=ON

    -D Trilinos_ENABLE_EXPLICIT_INSTANTIATION:BOOL=ON
    -D Tpetra_INST_INT_LONG_LONG:BOOL=OFF
    -D Tpetra_INST_INT_INT:BOOL=ON
    -D Tpetra_INST_DOUBLE:BOOL=ON
    -D Tpetra_INST_FLOAT:BOOL=OFF
    -D Tpetra_INST_COMPLEX_FLOAT:BOOL=OFF
    -D Tpetra_INST_COMPLEX_DOUBLE:BOOL=OFF
    -D Tpetra_INST_INT_LONG:BOOL=OFF
    -D Tpetra_INST_INT_UNSIGNED:BOOL=OFF
    -D Tpetra_INST_SERIAL=ON

    -D Trilinos_ENABLE_Kokkos:BOOL=ON
    -D Trilinos_ENABLE_KokkosCore:BOOL=ON
    -D Phalanx_KOKKOS_DEVICE_TYPE:STRING="CUDA"
    -D Phalanx_INDEX_SIZE_TYPE:STRING="INT"
    -D Phalanx_SHOW_DEPRECATED_WARNINGS:BOOL=OFF
    -D Kokkos_ENABLE_Serial:BOOL=ON
    -D Kokkos_ENABLE_OpenMP:BOOL=OFF
    -D Kokkos_ENABLE_Pthread:BOOL=OFF
    -D Trilinos_ENABLE_OpenMP:BOOL=OFF
    -D Kokkos_ENABLE_Cuda:BOOL=ON
    -D Kokkos_ENABLE_Cuda_UVM=ON
    -D TPL_ENABLE_CUDA:BOOL=ON
    -D TPL_ENABLE_CUSPARSE:BOOL=ON
    -D HAVE_INTREPID_KOKKOSCORE:BOOL=ON

    $EXTRA_ARGS
    ${TRILINOS_HOME}

  3. other steps look are similar to the ones for OpenMP

Clone this wiki locally
You can’t perform that action at this time.
You signed in with another tab or window. Reload to refresh your session. You signed out in another tab or window. Reload to refresh your session.
Press h to open a hovercard with more details.