Skip to content

Daint Clang

Jump to bottom Edit New page
John Biddiscombe edited this page Oct 3, 2017 · 30 revisions

Daint Clang

Please note that this page is intentionally written in such a way that chunks can be copied verbatim (including comments and section titles) and pasted directly into a bash shell. The idea is to make it as easy as possible to select the block you need use it directly. Usually following each of the blocks/section sequentially and skipping the ones you don't want/need.

#prerequisites

module unload PrgEnv-cray
module unload PrgEnv-gnu
module load cudatoolkit

# assuming clang is compiled and installed in /users/biddisco/apps/daint/llvm
export CLANG_ROOT=/users/biddisco/apps/daint/llvm
export CC=$CLANG_ROOT/bin/clang
export CXX=$CLANG_ROOT/bin/clang++
export CPP=$CLANG_ROOT/bin/clang-cpp
#
export PATH=$CLANG_ROOT/bin:$PATH
export LD_LIBRARY_PATH=$CLANG_ROOT/lib:$LD_LIBRARY_PATH
#
export CFLAGS=-fPIC
export CXXFLAGS="-fPIC -march=native -mtune=native -ffast-math -std=c++17 -stdlib=libc++ -I$CLANG_ROOT/include/c++/v1"
export LDFLAGS="-L$CLANG_ROOT/lib -rpath $CLANG_ROOT/lib"
export LDCXXFLAGS="$LDFLAGS -std=c++17 -stdlib=libc++"
#
# for cuda, (not yet tested 03/09/2017)
export CUDAFLAGS="--cuda-path=$CUDATOOLKIT_HOME \
 -L$CUDATOOLKIT_HOME/lib64 \
 -L$CUDATOOLKIT_HOME/extras/CUPTI/lib64 \
 -Wl,--as-needed -Wl,-lcupti -Wl,-lcudart -Wl,--no-as-needed \
 -L/opt/cray/nvidia/default/lib64 \
 -L/opt/cray/nvidia/default/lib64 \
 -lcuda"

# our build directory root for all clang compiled projects in one place 
export BUILD_ROOT=$SCRATCH/build/clang
# shortcut to go there
alias cdb='cd $BUILD_ROOT'

# Optional shortcuts for Apex output in HPX
alias APEX_OFF='export APEX_SCREEN_OUTPUT=0;export APEX_PROFILE=0;export APEX_OTF2=0'
alias APEX_ON='export APEX_SCREEN_OUTPUT=1;export APEX_PROFILE=1;export APEX_OTF2=1'

# Versions we will install
INSTALL_ROOT=$HOME/apps/daint/clang
HWLOC_VER=1.11.7
JEMALLOC_VER=5.0.1
OTF2_VER=2.0
BOOST_VER=1.65.0
BOOST_SUFFIX=1_65_0
BOOST_ROOT=$INSTALL_ROOT/boost/$BOOST_VER
PAPI_VER=5.5.1

#Compile clang (and other tools needed in one pass)

# Skip this step if clang is already built
# Note that you will need module load PrgEnv-gnu when building clang
cdb
git clone --depth 1 https://github.com/llvm-project/llvm-project-20170507 llvm-project
cd llvm-project
git config branch.master.rebase true
mkdir llvm-build && cd llvm-build
cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$CLANG_ROOT -DLLVM_ENABLE_PROJECTS="clang;libcxx;libcxxabi" -DLLVM_TARGETS_TO_BUILD="X86;NVPTX" ../llvm
make -j install

#Boost

cdb
wget http://vorboss.dl.sourceforge.net/project/boost/boost/$BOOST_VER/boost_$BOOST_SUFFIX.tar.gz
tar -xzf boost_$BOOST_SUFFIX.tar.gz
cd boost_$BOOST_SUFFIX
./bootstrap.sh 
./b2 toolset=clang cxxflags="$CXXFLAGS -D_LIBCPP_ENABLE_CXX17_REMOVED_AUTO_PTR" linkflags="$LDCXXFLAGS" --prefix=$BOOST_ROOT --layout=versioned threading=multi link=shared variant=release address-model=64 --without-mpi --without-python --without-graph --without-graph_parallel -j8 install

#cpptest (basic compiler/cmake/boost test project to check all ok)

# if this produces working binaries, then 99% of problems are solved
cdb
git clone https://github.com/biddisco/cpptest.git
cd cpptest
mkdir build && cd build
cmake \
 -DCMAKE_CXX_COMPILER=$CXX \
 -DCMAKE_CXX_FLAGS="$CXXFLAGS" \
 -DCMAKE_EXE_LINKER_FLAGS="$LDFLAGS" \
 -DBOOST_ROOT=$INSTALL_ROOT/boost/$BOOST_VER \
 -DMPI_C_LIBRARIES=/opt/cray/pe/mpt/default/gni/mpich-cray/8.6/lib/libmpich.so \
 -DMPI_INCLUDE_PATH=/opt/cray/pe/mpt/default/gni/mpich-cray/8.6/include \
 -DBoost_COMPILER=-clang60 \
 ..
make -j

#hwloc

cdb
wget --no-check-certificate https://www.open-mpi.org/software/hwloc/v1.11/downloads/hwloc-$HWLOC_VER.tar.gz
tar -xzf hwloc-$HWLOC_VER.tar.gz
cd hwloc-$HWLOC_VER
./configure --prefix=$INSTALL_ROOT/hwloc/$HWLOC_VER
make -j8 install

#jemalloc

cdb
wget https://github.com/jemalloc/jemalloc/releases/download/$JEMALLOC_VER/jemalloc-$JEMALLOC_VER.tar.bz2
tar -xjf jemalloc-$JEMALLOC_VER.tar.bz2
cd jemalloc-$JEMALLOC_VER
./autogen.sh
./configure --prefix=$INSTALL_ROOT/jemalloc/$JEMALLOC_VER
make -j8 -k install

#OTF2 (for APEX)

cdb
wget http://www.vi-hps.org/upload/packages/otf2/otf2-2.0.tar.gz
tar -xzf otf2-2.0.tar.gz
cd otf2-2.0/
./configure --prefix=$INSTALL_ROOT/otf2/$OTF2_VER CC=$CC CXX=$CXX --enable-shared
make -j8 -k install

#PAPI

cdb
wget http://icl.utk.edu/projects/papi/downloads/papi-${PAPI_VER}.tar.gz
tar -xzf papi-${PAPI_VER}.tar.gz
cd papi-${PAPI_VER}
./configure --prefix=$INSTALL_ROOT/papi/$PAPI_VER --enable-shared

#Vc-relwithdebinfo

cdb
mkdir vc
cd vc
cmake \
 -DCMAKE_INSTALL_PREFIX=$INSTALL_ROOT/vc-relwithdebinfo \
 -DCMAKE_BUILD_TYPE=RelWithDebInfo \
 -DCMAKE_CXX_COMPILER=$CXX \
 -DCMAKE_CXX_FLAGS="$CXXFLAGS" \
 -DBUILD_EXAMPLES=OFF \
 -DBUILD_TESTING=OFF \
 -DENABLE_MIC=OFF \
 /project/csvis/biddisco/src/Vc

#HPX-RelWithDebInfo

cdb
mkdir hpx-relwithdebinfo && cd hpx-relwithdebinfo
cmake \
 -DCMAKE_INSTALL_PREFIX=$INSTALL_ROOT/rdhpx \
 -DCMAKE_BUILD_TYPE=RelWithDebInfo \
 -DCMAKE_CXX_COMPILER=$CXX \
 -DCMAKE_CXX_FLAGS="$CXXFLAGS" \
 -DCMAKE_EXE_LINKER_FLAGS="$LDCXXFLAGS" \
 -DCMAKE_SHARED_LINKER_FLAGS="$LDCXXFLAGS" \
 -DHPX_WITH_CUDA=ON \
 -DHPX_WITH_CUDA_CLANG=ON \
 -DHPX_WITH_HWLOC:BOOL=ON \
 -DHWLOC_ROOT=$INSTALL_ROOT/hwloc/$HWLOC_VER \
 -DHPX_WITH_MALLOC=JEMALLOC \
 -DJEMALLOC_ROOT=$INSTALL_ROOT/jemalloc/$JEMALLOC_VER \
 -DBOOST_ROOT=$INSTALL_ROOT/boost/$BOOST_VER \
 -DBoost_COMPILER=-clang60 \
 -DOTF2_ROOT=$INSTALL_ROOT/otf2/$OTF2_VER \
 -DHPX_WITH_DATAPAR_VC=ON \
 -DHPX_WITH_DATAPAR_VC_NO_LIBRARY=ON \
 -DVc_DIR=$INSTALL_ROOT/vc-relwithdebinfo/lib/cmake/Vc \
 -DHPX_WITH_TESTS:BOOL=ON \
 -DHPX_WITH_TESTS_BENCHMARKS:BOOL=ON \
 -DHPX_WITH_TESTS_EXTERNAL_BUILD:BOOL=OFF \
 -DHPX_WITH_TESTS_HEADERS:BOOL=OFF \
 -DHPX_WITH_TESTS_REGRESSIONS:BOOL=ON \
 -DHPX_WITH_TESTS_UNIT:BOOL=ON \
 -DHPX_WITH_EXAMPLES:BOOL=ON \
 -DHPX_WITH_PARCELPORT_MPI:BOOL=ON \
 -DHPX_WITH_PAPI:BOOL=ON \
 -DHPX_WITH_APEX:BOOL=ON \
 -DAPEX_WITH_OTF2:BOOL=ON \
 -DHPX_WITH_THREAD_IDLE_RATES:BOOL=ON \
 -DHPX_WITH_MAX_CPU_COUNT=256 \
 -DHPX_WITH_MORE_THAN_64_THREADS=ON \
 -DHPX_WITH_TIMER_POOL:BOOL=OFF \
 -DMPI_C_LIBRARIES=/opt/cray/pe/mpt/default/gni/mpich-cray/8.6/lib/libmpich.so \
 -DMPI_CXX_LIBRARIES=/opt/cray/pe/mpt/default/gni/mpich-cray/8.6/lib/libmpich.so \
 -DMPI_INCLUDE_PATH=/opt/cray/pe/mpt/default/gni/mpich-cray/8.6/include \
 -DMPI_CXX_INCLUDE_PATH=/opt/cray/pe/mpt/default/gni/mpich-cray/8.6/include \
 /project/csvis/biddisco/src/hpx

#OctoTiger

cmake \
 -DCMAKE_BUILD_TYPE=RelWithDebInfo \
 -DCMAKE_CXX_COMPILER=$CXX \
 -DCMAKE_CXX_FLAGS="$CXXFLAGS" \
 -DCMAKE_EXE_LINKER_FLAGS="$LDCXXFLAGS -L/opt/nvidia/cudatoolkit8.0/8.0.54_2.2.8_ga620558-2.1/lib64" \
 -DCMAKE_SHARED_LINKER_FLAGS="$LDCXXFLAGS" \
 -DBoost_ADDITIONAL_VERSIONS=1.65 \
 -DBOOST_ROOT=$INSTALL_ROOT/boost/$BOOST_VER \
 -DBoost_COMPILER=-clang60 \
 -DHPX_DIR=$INSTALL_ROOT/hpx-relwithdebinfo/lib/cmake/HPX \
 -DMPI_C_LIBRARIES=/opt/cray/pe/mpt/7.5.2/gni/mpich-cray/8.4/lib/libmpich.so \
 -DMPI_INCLUDE_PATH=/opt/cray/pe/mpt/7.5.2/gni/mpich-cray/8.4/include \
 -DOCTOTIGER_WITH_SILO=OFF \
 /project/csvis/biddisco/src/octotiger

Below this point is not relevant for the hackathon

JB projects only

#HPX (assumes you have an existing hpx source dir, set correct path)

# note HPX_WITH_NETWORKING disabled here for JB projects
cdb
mkdir hpx && cd hpx 
cmake \
 -DCMAKE_INSTALL_PREFIX=$INSTALL_ROOT/hpx \
 -DCMAKE_BUILD_TYPE=Debug \
 -DCMAKE_CXX_COMPILER=$CXX \
 -DCMAKE_CXX_FLAGS="$CXXFLAGS" \
 -DCMAKE_EXE_LINKER_FLAGS="$LDCXXFLAGS" \
 -DCMAKE_SHARED_LINKER_FLAGS="$LDCXXFLAGS" \
 -DHPX_WITH_CUDA=On \
 -DHPX_WITH_CUDA_CLANG=On \
 -DHPX_WITH_HWLOC:BOOL=ON \
 -DHWLOC_ROOT=$INSTALL_ROOT/hwloc/$HWLOC_VER \
 -DHPX_WITH_MALLOC=JEMALLOC \
 -DJEMALLOC_ROOT=$INSTALL_ROOT/jemalloc/$JEMALLOC_VER \
 -DBOOST_ROOT=$INSTALL_ROOT/boost/$BOOST_VER \
 -DBoost_COMPILER=-clang60 \
 -DOTF2_ROOT=$INSTALL_ROOT/otf2/$OTF2_VER \
 -DHPX_WITH_TESTS:BOOL=ON \
 -DHPX_WITH_TESTS_BENCHMARKS:BOOL=ON \
 -DHPX_WITH_TESTS_EXTERNAL_BUILD:BOOL=OFF \
 -DHPX_WITH_TESTS_HEADERS:BOOL=OFF \
 -DHPX_WITH_TESTS_REGRESSIONS:BOOL=ON \
 -DHPX_WITH_TESTS_UNIT:BOOL=ON \
 -DHPX_WITH_EXAMPLES:BOOL=ON \
 -DHPX_WITH_NETWORKING:BOOL=OFF \
 -DHPX_WITH_PARCELPORT_MPI:BOOL=OFF \
 -DHPX_WITH_PAPI:BOOL=OFF \
 -DHPX_WITH_APEX:BOOL=OFF \
 -DAPEX_WITH_OTF2:BOOL=OFF \
 -DHPX_WITH_THREAD_IDLE_RATES:BOOL=OFF \
 -DHPX_WITH_MAX_CPU_COUNT=256 \
 -DHPX_WITH_MORE_THAN_64_THREADS=ON \
 -DHPX_WITH_TIMER_POOL:BOOL=OFF \
 -DHPX_WITH_IO_POOL:BOOL=OFF \
 /project/csvis/biddisco/src/hpx

#linear-algebra

cmake \
 -DCMAKE_BUILD_TYPE=RelWithDebInfo \
 -DCMAKE_CXX_COMPILER=$CXX \
 -DCMAKE_CXX_FLAGS="$CXXFLAGS" \
 -DCMAKE_EXE_LINKER_FLAGS="$LDCXXFLAGS -L/opt/nvidia/cudatoolkit8.0/8.0.54_2.2.8_ga620558-2.1/lib64" \
 -DCMAKE_SHARED_LINKER_FLAGS="$LDCXXFLAGS" \
 -DBoost_ADDITIONAL_VERSIONS=1.65 \
 -DBOOST_ROOT=$INSTALL_ROOT/boost/$BOOST_VER \
 -DBoost_COMPILER=-clang60 \
 -DHPX_DIR=$BUILD_ROOT/hpx/lib/cmake/HPX \
 -DMPI_C_LIBRARIES=/opt/cray/pe/mpt/7.5.2/gni/mpich-cray/8.4/lib/libmpich.so \
 -DMPI_INCLUDE_PATH=/opt/cray/pe/mpt/7.5.2/gni/mpich-cray/8.4/include \
 /scratch/snx1600/biddisco/src/hvtkm/linear_algebra
Add a custom footer
Add a custom sidebar
Clone this wiki locally