T64 on Amos

Glen Hansen edited this page Nov 8, 2014 · 1 revision

Obtain Third Party Libraries (TPLs) for Trilinos

  • AMOS does not, in general, support file transfer from arbitrary websites.

  • On a different machine (not AMOS), download the TPLs

      mkdir tpls
      cd tpls
      wget http://zlib.net/zlib-1.2.8.tar.gz
      wget http://www.hdfgroup.org/ftp/HDF5/releases/hdf5-1.8.11/src/hdf5-1.8.11.tar.gz
      wget ftp://ftp.unidata.ucar.edu/pub/netcdf/netcdf-4.3.2.tar.gz
      wget http://sourceforge.net/projects/boost/files/boost/1.55.0/boost_1_55_0.tar.gz
      wget http://glaros.dtc.umn.edu/gkhome/fetch/sw/parmetis/parmetis-4.0.3.tar.gz
      wget http://www.netlib.org/lapack/lapack-3.5.0.tgz
      wget http://crd-legacy.lbl.gov/~xiaoye/SuperLU/superlu_4.3.tar.gz
    
  • scp the TPL tarballs to AMOS

      scp -r tpls cci:~/scratch/
    

Set up your build environment

  • On AMOS q,

      module load cmake/3.0.0
      module load gnu-4.7.2
    

Install TPLs for Trilinos

Building ZLib

  • Version 1.2.8

      cd scratch/tpls
      tar -xzvf zlib-1.2.8.tar.gz
      cd zlib-1.2.8
      CC=mpicc CFLAGS=-O3 ./configure --64 --static \
      --prefix=/gpfs/u/home/PGES/PGESgnzw/scratch/tpls
      make
      make install
    

Building HDF5

  • Version 1.8.11

      cd ~/scratch/tpls
      tar -xzvf hdf5-1.8.11.tar.gz
      cd hdf5-1.8.11
      ./configure CC=mpicc FC=mpif90 CXX=mpicxx \
      CXXFLAGS="-O3" CFLAGS="-O3" FCFLAGS="-O3" \
      --enable-parallel --enable-shared=no \
      --with-zlib=/gpfs/u/home/PGES/PGESgnzw/scratch/tpls \
      --prefix=/gpfs/u/home/PGES/PGESgnzw/scratch/tpls
      make -j 8
      make install
    

Building Netcdf

  • Version 4.3.2

      cd ~/scratch/tpls
      cd netcdf-4.3.2
    
  • Edit the file include/netcdf.h. Make the following edits near line 228:

      #define NC_MAX_DIMS    65536    /* max dimensions per file */
      #define NC_MAX_ATTRS    8192    
      #define NC_MAX_VARS   524288    /* max variables per file */
      #define NC_MAX_NAME   256 
      #define NC_MAX_VAR_DIMS      8     /* max per variable dimensions */
    
  • Configure and build

      mkdir build
      cd build
      export myprefix=/gpfs/u/home/PGES/PGESgnzw/scratch/tpls
      cmake \
      -DCMAKE_C_COMPILER=mpicc \
      -DCMAKE_C_FLAGS="-I${myprefix}/include -O2" \
      -DCMAKE_EXE_LINKER_FLAGS="-L${myprefix}/lib -O2" \
      -DCMAKE_INSTALL_PREFIX=${myprefix} \
      -DENABLE_DAP=OFF -DBUILD_SHARED_LIBS=OFF -DENABLE_FSYNC=OFF -DENABLE_CDMREMOTE=OFF \
      -DENABLE_DOXYGEN=OFF -DENABLE_NETCDF_4=ON -DCMAKE_BUILD_TYPE=RELEASE -DENABLE_EXAMPLES=OFF \
      -DENABLE_TESTS=OFF -DCMAKE_SKIP_INSTALL_RPATH=ON \
      -DZLIB_INCLUDE_DIRS=${myprefix}/include \
      -DZLIB_LIBRARY=${myprefix}/lib/libz.a \
      -DBUILD_UTILITIES=OFF \
      ..
      make -j 8
      make install
    

Building boost

  • Version 1.53.0

  • Build boost with the commands

      cd ~/scratch/tpls
      tar -xzvf boost_1_55_0.tar.gz
      cd boost_1_55_0
      echo "using gcc : 4.7.2 : /bgsys/drivers/ppcfloor/gnu-linux-4.7.2/bin/powerpc64-bgq-linux-g++ ;" \
      >> ./tools/build/v2/user-config.jam
      echo "using mpi : /bgsys/drivers/ppcfloor/comm/gcc/bin/mpicxx ;" \
      >> ./tools/build/v2/user-config.jam
      ./bootstrap.sh \
      --with-libraries=signals,regex,filesystem,system,mpi,serialization,thread,program_options,exception \
      --prefix=/gpfs/u/home/PGES/PGESgnzw/scratch/tpls
      ./b2 -j 8
      ./b2 -j 8 install
    

Building ParMetis

  • Version 4.0.3

      cd ~/scratch/tpls
      tar -xzvf parmetis-4.0.3.tar.gz 
      cd parmetis-4.0.3/
    
  • Edit 'metis.h' to use 'IDXTYPEWIDTH' of 64

      vi metis/include/metis.h 
      #define IDXTYPEWIDTH 64 
    
  • Build using CMake

      cd build
      export metis=/gpfs/u/home/PGES/PGESgnzw/scratch/tpls/parmetis-4.0.3/metis
      cmake \
      -DCMAKE_INSTALL_PREFIX=/gpfs/u/home/PGES/PGESgnzw/scratch/tpls \
      -DMETIS_PATH=$metis \
      -DGKLIB_PATH=$metis/GKlib \
      -DCMAKE_C_COMPILER=mpicc \
      -DCMAKE_CXX_COMPILER=mpicxx \
      ..
      make -j 8
      make install
      cp libmetis/libmetis.a /gpfs/u/home/PGES/PGESgnzw/scratch/tpls/lib
      cp $metis/include/metis.h /gpfs/u/home/PGES/PGESgnzw/scratch/tpls/include
    

Building BLAS/LAPACK

  • Version 3.5.0

      cd ~/scratch/tpls
      tar -xzvf lapack-3.5.0.tgz
      cd lapack-3.5.0
      mkdir build
      cd build
      ccmake ..
      edit CMAKE_INSTALL_PREFIX=/gpfs/u/home/PGES/PGESgnzw/scratch/tpls
      c
      g
      make -j 4
      make install
    

Building SuperLU

  • Version 4.3

      cd ~/scratch/tpls
      tar -xzvf superlu_4.3.tar.gz
      cd SuperLU_4.3/
      mkdir /gpfs/u/home/PGES/PGESgnzw/scratch/tpls/superlu
      mkdir /gpfs/u/home/PGES/PGESgnzw/scratch/tpls/superlu/include
      mkdir /gpfs/u/home/PGES/PGESgnzw/scratch/tpls/superlu/lib
    
  • Edit make.inc for your machine and environment

      PLAT = _linux
      SuperLUroot     = /gpfs/u/home/PGES/PGESgnzw/scratch/tpls/superlu
      SUPERLULIB      = $(SuperLUroot)/lib/libsuperlu_4.3.a
      TMGLIB          = libtmglib.a
      BLASDEF         = -DUSE_VENDOR_BLAS
      BLASLIB         = -L/gpfs/u/home/PGES/PGESgnzw/scratch/tpls/lib/libblas.a
      LIBS            = $(SUPERLULIB) $(BLASLIB)
      ARCH            = ar
      ARCHFLAGS       = cr
      RANLIB          = ranlib
      CC              = mpicc
      CFLAGS          = -DPRNTlevel=0 -O3
      NOOPTS          =
      FORTRAN         = mpif90
      FFLAGS          = -O3
      LOADER          = $(CC)
      LOADOPTS        =
      CDEFS           = -DNoChange
      MATLAB          =
    
  • Build SuperLU

      make
      cd SRC
      cp *.h /gpfs/u/home/PGES/PGESgnzw/scratch/tpls/superlu/include
      rm /gpfs/u/home/PGES/PGESgnzw/scratch/tpls/superlu/lib/libsuperlu_4.3.a dlamch.o slamch.o
    

Build Trilinos

  • do-configure script

      #!/bin/sh
    
      export TRILINOS_HOME=/gpfs/u/home/PGES/PGESgnzw/scratch/trilinos/src
      BUILD_DIR=`pwd`
      INSTALL_DIR=/gpfs/u/home/PGES/PGESgnzw/scratch/trilinos/install
      MPI_BASE_DIR=/bgsys/drivers/ppcfloor/comm/gcc
      TPL_DIR=/gpfs/u/home/PGES/PGESgnzw/scratch/tpls
      BOOST_DIR=$TPL_DIR
      ZLIB_DIR=$TPL_DIR
      SUPERLU_DIR=$TPL_DIR/superlu
      NETCDF=$TPL_DIR
      HDFDIR=$TPL_DIR
      PARMETISDIR=$TPL_DIR
      export BOOST_ROOT=$BOOST_DIR
      EXTRA_ARGS=$@
    
      cmake \
      -D Trilinos_EXTRA_REPOSITORIES:STRING=SCOREC \
      -D Trilinos_CONFIGURE_OPTIONS_FILE:FILEPATH=$TRILINOS_HOME/sampleScripts/AlbanySettings.cmake \
      -D CMAKE_BUILD_TYPE:STRING=NONE \
      \
      -D CMAKE_C_FLAGS:STRING="-O2" \
      -D CMAKE_CXX_FLAGS:STRING="-O2" \
      \
      -D TPL_ENABLE_MPI:BOOL=ON \
      -D MPI_BASE_DIR:PATH=$MPI_BASE_DIR \
      -D Trilinos_EXTRA_LINK_FLAGS:STRING="-ldl" \
      \
      -D TPL_ENABLE_Matio:BOOL=OFF \
      -D SEACAS_ENABLE_SEACASSVDI:BOOL=OFF \
      -D Trilinos_ENABLE_SEACASFastq:BOOL=OFF \
      -D Trilinos_ENABLE_SEACASBlot:BOOL=OFF \
      -D Trilinos_ENABLE_SEACASPLT:BOOL=OFF \
      -D TPL_ENABLE_X11:BOOL=OFF \
      \
      -D CMAKE_VERBOSE_MAKEFILE:BOOL=OFF \
      -D Trilinos_VERBOSE_CONFIGURE:BOOL=OFF \
      \
      -D Boost_INCLUDE_DIRS:PATH=$BOOST_DIR/include \
      -D BoostAlbLib_INCLUDE_DIRS:PATH=$BOOST_DIR/include \
      -D Boost_LIBRARY_DIRS:PATH=$BOOST_DIR/lib \
      -D BoostAlbLib_LIBRARY_DIRS:PATH=$BOOST_DIR/lib \
      \
      -D TPL_ENABLE_Netcdf:STRING=ON \
      -D Netcdf_INCLUDE_DIRS:PATH="${NETCDF}/include" \
      -D Netcdf_LIBRARY_DIRS:PATH="${NETCDF}/lib64" \
      -D TPL_Netcdf_LIBRARIES:STRING="$NETCDF/lib64/libnetcdf.a;$HDFDIR/lib/libhdf5_hl.a;$HDFDIR/lib/libhdf5.a;$
      \
      -D TPL_ENABLE_HDF5:STRING=ON \
      -D HDF5_INCLUDE_DIRS:PATH="${HDFDIR}/include" \
      -D HDF5_LIBRARY_DIRS:PATH="${HDFDIR}/lib" \
      -D TPL_HDF5_LIBRARIES:STRING="$HDFDIR/lib/libhdf5.a;$ZLIB_DIR/lib/libz.a" \
      \
      -D TPL_ENABLE_Zlib:STRING=ON \
      -D Zlib_INCLUDE_DIRS:PATH="${ZLIB_DIR}/include" \
      -D Zlib_LIBRARY_DIRS:PATH="${ZLIB_DIR}/lib" \
      \
      -D TPL_ENABLE_ParMETIS:STRING=ON \
      -D ParMETIS_INCLUDE_DIRS:PATH="${PARMETISDIR}/include" \
      -D ParMETIS_LIBRARY_DIRS:PATH="${PARMETISDIR}/lib" \
      \
      -D Trilinos_ENABLE_SCOREC:BOOL=ON \
      -D SCOREC_DISABLE_STRONG_WARNINGS:BOOL=ON \
      \
      -D Trilinos_ENABLE_ThyraTpetraAdapters:BOOL=ON \
      -D Trilinos_ENABLE_Ifpack2:BOOL=ON \
      -D Trilinos_ENABLE_Amesos2:BOOL=ON \
      -D Trilinos_ENABLE_MueLu:BOOL=ON \
      \
      -D Zoltan_ENABLE_ULONG_IDS:BOOL=ON \
      \
      -D Teuchos_ENABLE_COMPLEX:BOOL=OFF \
      \
      -D TPL_ENABLE_SuperLU:STRING=ON \
      -D SuperLU_INCLUDE_DIRS:PATH="${SUPERLU_DIR}/include" \
      -D SuperLU_LIBRARY_DIRS:PATH="${SUPERLU_DIR}/lib" \
      \
      -D Trilinos_ASSERT_MISSING_PACKAGES:BOOL=OFF \
      -D CMAKE_INSTALL_PREFIX:PATH=${INSTALL_DIR} \
      \
      -D TPL_ENABLE_BLAS:STRING=ON \
      -D TPL_BLAS_LIBRARIES:FILEPATH="${TPL_DIR}/lib/libblas.a" \
      \
      -D TPL_ENABLE_LAPACK:STRING=ON \
      -D TPL_LAPACK_LIBRARIES:FILEPATH="${TPL_DIR}/lib/liblapack.a" \
      \
      $EXTRA_ARGS \
      ${TRILINOS_HOME}
    
  • Build Trilinos

      make -j 4
      make install
    

Build Albany

  • CMake configure script:

      #!/bin/sh
      export TRILINOS_INSTALL_DIR=/gpfs/u/home/PGES/PGESgnzw/scratch/trilinos/install
      cmake \
      -D ALBANY_TRILINOS_DIR:FILEPATH="$TRILINOS_INSTALL_DIR" \
      -D ENABLE_LCM:BOOL=ON \
      -D ENABLE_QCAD:BOOL=OFF \
      -D ENABLE_LCM_SPECULATIVE:BOOL=OFF \
      -D ENABLE_LAME:BOOL=OFF \
      -D CMAKE_VERBOSE_MAKEFILE:BOOL=OFF \
      -D ENABLE_DEBUGGING:BOOL=OFF \
      -D ENABLE_CHECK_FPE:BOOL=OFF \
      -D ENABLE_HYDRIDE:BOOL=OFF \
      -D ENABLE_SCOREC:BOOL=ON \
      -D ENABLE_SEE:BOOL=OFF \
      -D ENABLE_FELIX:BOOL=OFF \
      -D ENABLE_MOR:BOOL=OFF \
      -D ENABLE_ALBANY_CI:BOOL=OFF \
      -D ENABLE_ASCR:BOOL=OFF \
      -D ENABLE_SG_MP:BOOL=OFF \
      -D ENABLE_64BIT_INT:BOOL=ON \
      -D ENABLE_ALBANY_EPETRA_EXE:BOOL=OFF \
      ../
    
  • Build Albany

      make -j 4
      make install
    
Clone this wiki locally
You can’t perform that action at this time.
You signed in with another tab or window. Reload to refresh your session. You signed out in another tab or window. Reload to refresh your session.
Press h to open a hovercard with more details.