SIRIUS is not activated by default and is now compiled with and witho…

…ut GPU support if cuda is found Added a fix that avoid double initialization of sirius when farming is activated
cp2k · Oct 31, 2018 · 0fd58fd · 0fd58fd
1 parent 9a24425
commit 0fd58fd
Show file tree

Hide file tree

Showing 3 changed files with 95 additions and 65 deletions.
diff --git a/src/start/cp2k_runs.F b/src/start/cp2k_runs.F
@@ -263,6 +263,7 @@ RECURSIVE FUNCTION cp2k_run(input_declaration, input_file_name, output_unit, mpi
          ! But, we don't want to change the public f77_interface.
          ! TODO: refactor cp2k's startup code
          CALL dbcsr_finalize_lib(group=mpi_comm, output_unit=0)
+         CALL cp_sirius_finalize()
          CALL pw_cuda_finalize()
          CALL farming_run(input_declaration, root_section, para_env)
          CALL dbcsr_init_lib()

diff --git a/tools/toolchain/install_cp2k_toolchain.sh b/tools/toolchain/install_cp2k_toolchain.sh
@@ -290,6 +290,13 @@ with_mkl=__SYSTEM__
 with_openblas=__INSTALL__
 with_reflapack=__INSTALL__
 
+# sirius is not activated by default.
+with_sirius="__DONTUSE__"
+with_gsl="__DONTUSE__"
+with_spglib="__DONTUSE__"
+with_hdf5="__DONTUSE__"
+with_json_fortran="__DONTUSE__"
+
 # for MPI, we try to detect system MPI variant
 with_openmpi=__SYSTEM__
 with_mpich=__SYSTEM__
@@ -677,19 +684,10 @@ if [ "$with_sirius" = "__INSTALL__" ] ; then
     echo "                                                                         "
     echo " Sirius needs these libraries to work properly                           "
     echo " gsl, elpa, libxc, scalapack fftw, spglib, hdf5, and json                "
-    echo " they will be activated by default unless the right options are given to "
-    echo " install_cp2k_toolchain.sh. Please look at the help                      "
+    echo " they will not be activated by default unless the right options are      "
+    echo " given to install_cp2k_toolchain.sh. Please look at the help             "
     echo "                                                                         "
     echo "========================================================================="
-    [ "$with_gsl" = "__DONTUSE__" ] && with_gsl="__INSTALL__"
-    [ "$with_elpa" = "__DONTUSE__" ] && with_elpa="__INSTALL__"
-    [ "$with_libxc" = "__DONTUSE__" ] && with_libxc="__INSTALL__"
-    [ "$with_scalapack" = "__DONTUSE__" ] && with_scalapack="__INSTALL__"
-    [ "$with_fftw" = "__DONTUSE__" ] && with_fftw="__INSTALL__"
-    [ "$with_spglib" = "__DONTUSE__" ] && with_spglib="__INSTALL__"
-    [ "$with_hdf5" = "__DONTUSE__" ] && with_hdf5="__INSTALL__"
-    [ "$with_json_fortran" = "__DONTUSE__" ] && with_json_fortran="__INSTALL__"
-else
     if [ "$with_gsl" = "__DONTUSE__" ] ; then
         report_error "For SIRIUS to work you need a working gsl library use --with-gsl option to specify if you wish to install the library or specify its location."
         exit 1
@@ -719,7 +717,7 @@ else
         exit 1
     fi
     if [ "$with_json_fortran" = "__DONTUSE__"  ] ; then
-        report_error "For sirius intergration tro cp2k to work you need a working json-fortran library use --with-json option to specify if you wish to install it or specify its location."
+        report_error "For SIRIUS intergration tro cp2k to work you need a working json-fortran library use --with-json option to specify if you wish to install it or specify its location."
     exit 1
     fi
 fi

diff --git a/tools/toolchain/scripts/install_sirius.sh b/tools/toolchain/scripts/install_sirius.sh
@@ -12,12 +12,13 @@ with_sirius=${1:-__INSTALL__}
 if [ $MPI_MODE = no ] ; then
     if [ "$ENABLE_OMP" = "__FALSE__"] ; then
         report_warning $LINENO "MPI is disabled, skipping sirius installation"
-cat <<EOF > "${BUILDDIR}/setup_sirius"
-    with_sirius="__FALSE__"
+        cat <<EOF > "${BUILDDIR}/setup_sirius"
+with_sirius="__FALSE__"
 EOF
-      exit 0
+        exit 0
     fi
 fi
+
 [ -f "${BUILDDIR}/setup_sirius" ] && rm "${BUILDDIR}/setup_sirius"
 
 SIRIUS_CFLAGS=''
@@ -42,7 +43,7 @@ case "$with_sirius" in
         require_env FFTW_CFLAGS
         require_env ELPA_LDFLAGS
         require_env ELPA_LIBS
-        require_env ELPHA_CFLAGS
+        require_env ELPA_CFLAGS
         require_env GSL_LDFLAGS
         require_env GSL_CFLAGS
         require_env GSL_LIBS
@@ -77,32 +78,10 @@ case "$with_sirius" in
             [ -d sirius-${sirius_ver} ] && rm -rf sirius-${sirius_ver}
             tar -xzf SIRIUS-${sirius_ver}.tar.gz
             cd SIRIUS-${sirius_ver}
-            # rm -fr build
-            # mkdir build
-            # cd build
-            # if [ "$ENABLE_CUDA" = __TRUE__ ] ; then
-            #     cmake -DCMAKE_INSTALL_PREFIX=${pkg_install_dir} ..
-            # else
-            #     cmake -DCMAKE_INSTALL_PREFIX=${pkg_install_dir} -DUSE_CUDA=off ..
-            # fi
-            # make
-            # make install
-            # cd ../..
-            #touch "${install_lock_file}"
             cat <<EOF > make.inc
 CXX = \$(MPICXX)
 BASIC_CXX_OPT = -O3 -DNDEBUG -mtune=native -ftree-loop-vectorize ${MATH_CFLAGS}
 CXX_OPT = \$(BASIC_CXX_OPT) -fopenmp -std=c++11 -D__SCALAPACK -D__ELPA
-EOF
-             if [ "$ENABLE_CUDA" = __TRUE__ ] ; then
-                 cat <<EOF >> make.inc
-CXX_OPT := \$(CXX_OPT) -D__GPU -I${CUDA_SDK}/include -I${CUDA_ROOT}/include
-NVCC=nvcc -O3 -arch=sm_60
-LIBS := ${CUDA_LIBS}
-CXX_OPT := \$(CXX_OPT) ${CUDA_CFLAGS}
-EOF
-            fi
-            cat <<EOF >> make.inc
 CXX_OPT := \$(CXX_OPT) -I${PWD}/src
 CXX_OPT := \$(CXX_OPT) -I${PWD}/src/SDDK
 CXX_OPT := \$(CXX_OPT) ${ELPA_CFLAGS}
@@ -118,7 +97,7 @@ EOF
 
             # this a hack needs to be fixed permanently
             sed -i -e "s/: log/: /g" src/Makefile
-cat > src/version.hpp <<EOF
+            cat > src/version.hpp <<EOF
 #ifndef __VERSION_H__
 #define __VERSION_H__
 const char* const git_hash = "6ec392682d7ffda6090ee3124d6841adc6a5fd0d";
@@ -127,26 +106,71 @@ const char* const build_date = "Thu, 23 Aug 2018 13:58:38";
 #endif
 EOF
 
-             # a fix to indicate that spglib has changed the directory where the header file is located
-             sed -i -e "s/spglib\///g" src/Unit_cell/unit_cell_symmetry.hpp
-
-             # small patch to fix an error in sirius. It is already fixed in the develop branch. just waiting for a new release
-             sed -i -e "s/mpi_fin = call_mpi_fin/mpi_fin = *call_mpi_fin/g" src/sirius_api.cpp
-             sed -i -e "s/device_reset = call_device_reset/device_reset = *call_device_reset/g" src/sirius_api.cpp
-             sed -i -e "s/fftw_fin = call_fftw_fin/fftw_fin = *call_fftw_fin/g" src/sirius_api.cpp
-             make -C src
-             install -v -d ${pkg_install_dir}/include
-             install -v -d ${pkg_install_dir}/lib
-             cp -Rv src/* ${pkg_install_dir}/include
-             rm -f ${pkg_install_dir}/include/*.f90
-             #rm -f ${pkg_install_dir}/include/*.mod
-             rm -f ${pkg_install_dir}/include/*.o
-             install -v -m 644 src/*.a ${pkg_install_dir}/lib
-             install -v -m 644 src/*.mod ${pkg_install_dir}/lib
+            # a fix to indicate that spglib has changed the directory where the header file is located
+            sed -i -e "s/spglib\///g" src/Unit_cell/unit_cell_symmetry.hpp
+
+            # small patch to fix an error in sirius. It is already fixed in the develop branch. just waiting for a new release
+            sed -i -e "s/mpi_fin = call_mpi_fin/mpi_fin = *call_mpi_fin/g" src/sirius_api.cpp
+            sed -i -e "s/device_reset = call_device_reset/device_reset = *call_device_reset/g" src/sirius_api.cpp
+            sed -i -e "s/fftw_fin = call_fftw_fin/fftw_fin = *call_fftw_fin/g" src/sirius_api.cpp
+            make -C src
+            install -v -d ${pkg_install_dir}/include
+            install -v -d ${pkg_install_dir}/lib
+            cp -Rv src/* ${pkg_install_dir}/include
+            rm -f ${pkg_install_dir}/include/*.f90
+            #rm -f ${pkg_install_dir}/include/*.mod
+            rm -f ${pkg_install_dir}/include/*.o
+            install -v -m 644 src/*.a ${pkg_install_dir}/lib
+            install -v -m 644 src/*.mod ${pkg_install_dir}/lib
+
+            # now do we have cuda as well
+
+            if [ "$ENABLE_CUDA" = "__TRUE__" ] ; then
+                mv make.{inc,cpu}
+                touch cuda.txt
+                if [ -z "$CUDA_ROOT" ] ; then
+                    if [ -z "$CUDA_SDK" ] ; then
+                        CUDA_DIRECTORY=`command -v nvcc`
+                        CUDA_DIRECTORY=${CUDA_DIRECTORY%%"bin/nvcc"}
+                    else
+                        CUDA_DIRECTORY=$CUDA_SDK
+                    fi
+                else
+                    CUDA_DIRECTORY=$CUDA_ROOT
+                fi
+
+                cat <<EOF > make.inc
+CXX = \$(MPICXX)
+BASIC_CXX_OPT = -O3 -DNDEBUG -mtune=native -ftree-loop-vectorize ${MATH_CFLAGS}
+CXX_OPT = \$(BASIC_CXX_OPT) -fopenmp -std=c++11 -D__SCALAPACK -D__ELPA
+CXX_OPT := \$(CXX_OPT) -D__GPU -I${CUDA_DIRECTORY}include
+NVCC=nvcc -O3 -arch=sm_60
+LIBS := ${CUDA_LIBS}
+CXX_OPT := \$(CXX_OPT) ${CUDA_CFLAGS}
+CXX_OPT := \$(CXX_OPT) -I${PWD}/src
+CXX_OPT := \$(CXX_OPT) -I${PWD}/src/SDDK
+CXX_OPT := \$(CXX_OPT) ${ELPA_CFLAGS}
+CXX_OPT := \$(CXX_OPT) ${GSL_CFLAGS}
+CXX_OPT := \$(CXX_OPT) ${SPGLIB_CFLAGS}
+CXX_OPT := \$(CXX_OPT) ${HDF5_CFLAGS}
+MPI_FC = ${MPIFC}
+MPI_FC_OPT = -g -O2 -fopenmp -cpp
+
+LIBS := ${LIBXC_LDFLAGS} ${FFTW_LDFLAGS} ${ELPA_LDFLAGS} ${SCALAPACK_LDFLAGS} ${GSL_LDFLAGS} ${SPG_LDFLAGS}
+LIBS := \$(LIBS) ${SPGLIB_LIBS} ${FFTW_LIBS} ${GSL_LIBS} ${SCALAPACK_LIBS} ${ELPA_LIBS}
+EOF
+                #
+                make -C src clean
+                make -C src
+                install -v -d ${pkg_install_dir}/lib/cuda
+                install -v -m 644 src/*.a ${pkg_install_dir}/lib/cuda
+                install -v -m 644 src/*.mod ${pkg_install_dir}/lib/cuda
+                SIRIUS_CUDA_LDFLAGS="-L'${pkg_install_dir}/lib/cuda' -Wl,-rpath='${pkg_install_dir}/lib/cuda'"
+            fi
+            SIRIUS_CFLAGS="-I'${pkg_install_dir}/include'"
+            SIRIUS_LDFLAGS="-L'${pkg_install_dir}/lib' -Wl,-rpath='${pkg_install_dir}/lib'"
+            touch "${install_lock_file}"
         fi
-        SIRIUS_CFLAGS="-I'${pkg_install_dir}/include'"
-        SIRIUS_LDFLAGS="-L'${pkg_install_dir}/lib' -Wl,-rpath='${pkg_install_dir}/lib'"
-        touch "${install_lock_file}"
         ;;
     __SYSTEM__)
         echo "==================== Finding SIRIUS_DIST from system paths ===================="
@@ -198,24 +222,31 @@ EOF
 esac
 if [ "$with_sirius" != "__DONTUSE__" ] ; then
     SIRIUS_LIBS="-lsirius_f -lsirius"
+    SIRIUS_CUDA_LDFLAGS="-L'${pkg_install_dir}/lib/cuda' -Wl,-rpath='${pkg_install_dir}/lib/cuda'"
+    SIRIUS_LDFLAGS="-L'${pkg_install_dir}/lib' -Wl,-rpath='${pkg_install_dir}/lib'"
+    SIRIUS_CFLAGS="-I'${pkg_install_dir}/include'"
     if [ "$with_sirius" != "__SYSTEM__" ] ; then
         cat <<EOF > "${BUILDDIR}/setup_sirius"
 prepend_path LD_LIBRARY_PATH "$pkg_install_dir/lib"
+prepend_path LD_LIBRARY_PATH "$pkg_install_dir/lib/cuda"
 prepend_path LD_RUN_PATH "$pkg_install_dir/lib"
+prepend_path LD_RUN_PATH "$pkg_install_dir/lib/cuda"
 prepend_path LIBRARY_PATH "$pkg_install_dir/lib"
+prepend_path LIBRARY_PATH "$pkg_install_dir/lib/cuda"
 prepend_path CPATH "$pkg_install_dir/include"
 EOF
         cat "${BUILDDIR}/setup_sirius" >> $SETUPFILE
     fi
     cat <<EOF >> "${BUILDDIR}/setup_sirius"
-export SIRIUS_CFLAGS="-I'${pkg_install_dir}/include'"
-export SIRIUS_FFLAGS="-I'${pkg_install_dir}/include'"
-export SIRIUS_LDFLAGS="${SIRIUS_LDFLAGS}"
+export SIRIUS_CFLAGS="-I${pkg_install_dir}/include"
+export SIRIUS_FFLAGS="-I${pkg_install_dir}/include"
+export SIRIUS_LDFLAGS="-L'${pkg_install_dir}/lib' -Wl,-rpath='${pkg_install_dir}/lib'"
+export SIRIUS_CUDA_LDFLAGS="-L'${pkg_install_dir}/lib/cuda' -Wl,-rpath='${pkg_install_dir}/lib/cuda'"
 export SIRIUS_LIBS="${SIRIUS_LIBS}"
-export CP_DFLAGS="\${CP_DFLAGS} IF_MPI(IF_OMP(-D__SIRIUS|)|)"
-export CP_CFLAGS="\${CP_CFLAGS} IF_MPI(IF_OMP(-I'${pkg_install_dir}/include'|)|)"
-export CP_LDFLAGS="\${CP_LDFLAGS} IF_MPI(IF_OMP(${SIRIUS_LDFLAGS}|)|)"
-export CP_LIBS="IF_MPI(IF_OMP(${SIRIUS_LIBS}|)|) \${CP_LIBS}"
+export CP_DFLAGS="\${CP_DFLAGS} IF_MPI(IF_OMP("-D__SIRIUS"|)|)"
+export CP_CFLAGS="\${CP_CFLAGS} IF_MPI(IF_OMP("\${SIRIUS_CFLAGS}"|)|)"
+export CP_LDFLAGS="\${CP_LDFLAGS} IF_MPI(IF_OMP(IF_CUDA("\${SIRIUS_CUDA_LDFLAGS}"|"\${SIRIUS_LDFLAGS}")|)|)"
+export CP_LIBS="IF_MPI(IF_OMP("\${SIRIUS_LIBS}"|)|) \${CP_LIBS}"
 EOF
 fi
 cd "${ROOTDIR}"