From 18d6284ba0dcb60210f09ca8983d1768eea848c6 Mon Sep 17 00:00:00 2001 From: Levi Zhou <31941107+ZhouXY-PKU@users.noreply.github.com> Date: Thu, 6 Nov 2025 21:27:42 +0800 Subject: [PATCH 1/7] [Fix]Install libraries for PEXSI and update abacus build Added installation steps for GKlib, METIS, ParMETIS, SuperLU_DIST, and PEXSI in the Dockerfile. Updated the abacus build process to use a specific version. --- Dockerfile.intel | 109 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 107 insertions(+), 2 deletions(-) diff --git a/Dockerfile.intel b/Dockerfile.intel index 119cf4eb4b..0de7ceb8b7 100644 --- a/Dockerfile.intel +++ b/Dockerfile.intel @@ -74,14 +74,119 @@ RUN wget -q https://download.pytorch.org/libtorch/cpu/libtorch-cxx11-abi-shared- unzip -q /tmp/libtorch.zip -d /opt && rm -f /tmp/libtorch.zip ENV CMAKE_PREFIX_PATH=/opt/libtorch/share/cmake:${CMAKE_PREFIX_PATH} + +###### PEXSI PART: pexsi and gklib and metis and parmetis and superlu_dist +ENV GKLIB_VERSION="master" +ENV METIS_VERSION="master" +ENV PARMETIS_VERSION="main" +ENV SUPERLU_DIST_VERSION=7.2.0 +ENV PEXSI_VERSION=2.0.0 +ENV GKLIB_ROOT=/usr/local/gklib-${GKLIB_VERSION} +ENV METIS32_ROOT=/usr/local/metis32-${METIS_VERSION} +ENV PARMETIS32_ROOT=/usr/local/parmetis32-${PARMETIS_VERSION} +ENV PEXSI32_ROOT=/usr/local/pexsi32-${PEXSI_VERSION} +ENV SUPERLU_DIST32_ROOT=/usr/local/superlu_dist32-${SUPERLU_DIST_VERSION} + +# 2. install GKlib +RUN wget https://codeload.github.com/KarypisLab/GKlib/zip/refs/heads/${GKLIB_VERSION} -O GKlib-${GKLIB_VERSION}.zip && \ + unzip GKlib-${GKLIB_VERSION}.zip && \ + cd GKlib-${GKLIB_VERSION} && \ + make config shared=1 prefix=${GKLIB_ROOT} openmp=set && \ + make -j$(nproc) && \ + make install && \ + ls ${GKLIB_ROOT}/lib && \ + cp ${GKLIB_ROOT}/lib/libGKlib.so.0 ${GKLIB_ROOT}/lib/libGKlib.so && \ + cd / && rm -rf GKlib-${GKLIB_VERSION} GKlib-${GKLIB_VERSION}.zip +#.so file CANNOT be found otherwise. +# 2. install METIS +RUN export LD_LIBRARY_PATH=${GKLIB_ROOT}/lib:${LD_LIBRARY_PATH} && \ + wget https://codeload.github.com/KarypisLab/METIS/zip/refs/heads/${METIS_VERSION} -O METIS-${METIS_VERSION}.zip && \ + unzip METIS-${METIS_VERSION}.zip && \ + cd METIS-${METIS_VERSION} && \ + make config shared=1 prefix=${METIS32_ROOT} gklib_path=${GKLIB_ROOT} && \ + make -j$(nproc) && \ + make install && \ + cd / && rm -rf METIS-${METIS_VERSION} METIS-${METIS_VERSION}.zip + +# 3. install ParMETIS +RUN export LD_LIBRARY_PATH=${METIS32_ROOT}/lib:${GKLIB_ROOT}/lib:${LD_LIBRARY_PATH} && \ + wget https://codeload.github.com/KarypisLab/ParMETIS/zip/refs/heads/${PARMETIS_VERSION} -O ParMETIS-${PARMETIS_VERSION}.zip && \ + unzip ParMETIS-${PARMETIS_VERSION}.zip && \ + cd ParMETIS-${PARMETIS_VERSION} && \ + make config shared=1 prefix=${PARMETIS32_ROOT} gklib_path=${GKLIB_ROOT} metis_path=${METIS32_ROOT} && \ + make -j$(nproc) && \ + make install && \ + cd / && rm -rf ParMETIS-${PARMETIS_VERSION} ParMETIS-${PARMETIS_VERSION}.zip + +# 4. install SuperLU_DIST +RUN wget https://codeload.github.com/xiaoyeli/superlu_dist/tar.gz/refs/tags/v${SUPERLU_DIST_VERSION} -O v${SUPERLU_DIST_VERSION}.tar.gz && \ + tar -xzf v${SUPERLU_DIST_VERSION}.tar.gz && \ + cd superlu_dist-${SUPERLU_DIST_VERSION} && \ + mkdir build && cd build && \ + cmake .. \ + -DTPL_ENABLE_PARMETISLIB=ON \ + -DTPL_PARMETIS_LIBRARIES="${PARMETIS32_ROOT}/lib/libparmetis.so;${METIS32_ROOT}/lib/libmetis.so;${GKLIB_ROOT}/lib/libGKlib.so" \ + -DTPL_PARMETIS_INCLUDE_DIRS="${PARMETIS32_ROOT}/include;${METIS32_ROOT}/include;${GKLIB_ROOT}/include" \ + -DTPL_ENABLE_INTERNAL_BLASLIB=OFF \ + -DTPL_ENABLE_LAPACKLIB=ON \ + -DTPL_ENABLE_COMBBLASLIB=OFF \ + -DTPL_ENABLE_CUDALIB=OFF \ + -Denable_complex16=ON \ + -DXSDK_INDEX_SIZE=32 \ + -DBUILD_SHARED_LIBS=ON \ + -DCMAKE_INSTALL_PREFIX=${SUPERLU_DIST32_ROOT} \ + -DCMAKE_C_FLAGS="-O3 -fopenmp" \ + -DCMAKE_CXX_FLAGS="-O3 -fopenmp" \ + -DXSDK_ENABLE_Fortran=ON \ + -DCMAKE_Fortran_COMPILER=mpiifx && \ + make -j$(nproc) && \ + make install && \ + cd / && rm -rf superlu_dist-${SUPERLU_DIST_VERSION} v${SUPERLU_DIST_VERSION}.tar.gz + +### -DCMAKE_C_COMPILER=mpiicc \ +### -DCMAKE_CXX_COMPILER=mpiicpc \ +### -DCMAKE_CUDA_COMPILER=nvcc \ + +# 5. install PEXSI +RUN export LD_LIBRARY_PATH=${SUPERLU_DIST32_ROOT}/lib:${METIS32_ROOT}/lib:${PARMETIS32_ROOT}/lib:${GKLIB_ROOT}/lib:${LD_LIBRARY_PATH} && \ + export LIBRARY_PATH=${SUPERLU_DIST32_ROOT}/lib:${METIS32_ROOT}/lib:${PARMETIS32_ROOT}/lib:${GKLIB_ROOT}/lib:${LIBRARY_PATH} && \ + export PKG_CONFIG_PATH=${SUPERLU_DIST32_ROOT}/lib/pkgconfig:${METIS32_ROOT}/lib/pkgconfig:${PARMETIS32_ROOT}/lib/pkgconfig:${GKLIB_ROOT}/lib/pkgconfig:${PKG_CONFIG_PATH} && \ + export CPATH=${SUPERLU_DIST32_ROOT}/include:${PARMETIS32_ROOT}/include:${METIS32_ROOT}/include:${GKLIB_ROOT}/include:${CMAKE_PREFIX_PATH} && \ + export CMAKE_PREFIX_PATH=${SUPERLU_DIST32_ROOT}:${PARMETIS32_ROOT}:${METIS32_ROOT}:${GKLIB_ROOT}:${CMAKE_PREFIX_PATH} && \ + wget https://bitbucket.org/berkeleylab/pexsi/downloads/pexsi_v${PEXSI_VERSION}.tar.gz && \ + tar -xzf pexsi_v${PEXSI_VERSION}.tar.gz && \ + cd pexsi_v${PEXSI_VERSION} && \ + sed -i 's/^add_pexsi_f_example_exe/# add_pexsi_f_example_exe/g' fortran/CMakeLists.txt && \ + sed -i 's/^add_pexsi_example_exe/# add_pexsi_example_exe/g' examples/CMakeLists.txt && \ + sed -i 's/add_executable/# add_executable/g' fortran/CMakeLists.txt && \ + sed -i 's/add_executable/# add_executable/g' examples/CMakeLists.txt && \ + mkdir build && cd build && \ + cmake .. \ + -DCMAKE_INSTALL_PREFIX=${PEXSI32_ROOT} \ + -DPEXSI_ENABLE_OPENMP=ON \ + -DPEXSI_ENABLE_FORTRAN=OFF && \ + make pexsi -j$(nproc) && \ + make install && \ + cd / && rm -rf pexsi_v${PEXSI_VERSION} pexsi_v${PEXSI_VERSION}.tar.gz + +### -DCMAKE_C_COMPILER=mpiicc \ +### -DCMAKE_CXX_COMPILER=mpiicpc \ +###### END of PEXSI PART + # Clone and build abacus (optional during image build; keep for CI image) ADD https://api.github.com/repos/deepmodeling/abacus-develop/git/refs/heads/develop /dev/null RUN export LD_LIBRARY_PATH=${GKLIB_ROOT}/lib:${METIS32_ROOT}/lib:${PARMETIS32_ROOT}/lib:${SUPERLU32_DIST_ROOT}/lib:${PEXSI32_ROOT}/lib:${LD_LIBRARY_PATH} && \ export CPATH=${GKLIB_ROOT}/include:${METIS32_ROOT}/include:${PARMETIS32_ROOT}/include:${SUPERLU32_DIST_ROOT}/include:${PEXSI32_ROOT}/include:${CPATH} && \ - cd /tmp && git clone https://github.com/deepmodeling/abacus-develop.git --depth 1 && \ - cd abacus-develop && \ + export CMAKE_PREFIX_PATH=${PEXSI32_ROOT}:${SUPERLU_DIST32_ROOT}:${PARMETIS32_ROOT}:${METIS32_ROOT}:${GKLIB_ROOT}:${CMAKE_PREFIX_PATH} && \ + cd /tmp && wget https://github.com/deepmodeling/abacus-develop/archive/refs/tags/v3.9.0.17.zip -O abacus-develop-3.9.0.17.zip && \ + unzip abacus-develop-3.9.0.17.zip && \ + cd abacus-develop-3.9.0.17 && \ + sed -i '3i\#include "source_estate/elecstate_lcao.h"' source/source_lcao/edm.cpp && \ cmake -B build \ + -DMKL_SYCL=OFF \ + -DMKL_SYCL_LINK=OFF \ -DENABLE_MLALGO=ON \ + -DENABLE_PEXSI=ON \ -DENABLE_LIBXC=ON \ -DENABLE_LIBRI=ON \ -DENABLE_RAPIDJSON=ON \ From b80f538033b840cda6dcb880208a0bd146999518 Mon Sep 17 00:00:00 2001 From: Levi Zhou <31941107+ZhouXY-PKU@users.noreply.github.com> Date: Thu, 6 Nov 2025 21:32:34 +0800 Subject: [PATCH 2/7] Update build arguments for Intel and GNU toolchains --- .github/workflows/build_test_cmake.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_test_cmake.yml b/.github/workflows/build_test_cmake.yml index 22610fb454..2f4e39abbc 100644 --- a/.github/workflows/build_test_cmake.yml +++ b/.github/workflows/build_test_cmake.yml @@ -20,14 +20,14 @@ jobs: build_args: "-DENABLE_LIBXC=1 -DDENABLE_MLALGO=1 -DENABLE_LIBRI=1" name: "Build extra components with GNU toolchain" - tag: intel - build_args: "-DENABLE_LIBXC=1 -DENABLE_MLALGO=1 -DENABLE_LIBRI=1" + build_args: "-DENABLE_LIBXC=1 -DENABLE_PEXSI=1 -DENABLE_MLALGO=1 -DENABLE_LIBRI=1" name: "Build extra components with Intel toolchain" - tag: cuda build_args: "-DUSE_CUDA=1" name: "Build with CUDA support" - tag: gnu - build_args: "-DENABLE_LCAO=OFF" + build_args: "-DENABLE_LCAO=0" name: "Build without LCAO" - tag: gnu build_args: "-DUSE_ELPA=0 " From 618617e66e426430607f9d6f52eff5c11a410ffb Mon Sep 17 00:00:00 2001 From: Levi Zhou <31941107+ZhouXY-PKU@users.noreply.github.com> Date: Thu, 6 Nov 2025 21:45:31 +0800 Subject: [PATCH 3/7] Update Dockerfile.intel --- Dockerfile.intel | 2 -- 1 file changed, 2 deletions(-) diff --git a/Dockerfile.intel b/Dockerfile.intel index 0de7ceb8b7..3b3ddce842 100644 --- a/Dockerfile.intel +++ b/Dockerfile.intel @@ -183,8 +183,6 @@ RUN export LD_LIBRARY_PATH=${GKLIB_ROOT}/lib:${METIS32_ROOT}/lib:${PARMETIS32_RO cd abacus-develop-3.9.0.17 && \ sed -i '3i\#include "source_estate/elecstate_lcao.h"' source/source_lcao/edm.cpp && \ cmake -B build \ - -DMKL_SYCL=OFF \ - -DMKL_SYCL_LINK=OFF \ -DENABLE_MLALGO=ON \ -DENABLE_PEXSI=ON \ -DENABLE_LIBXC=ON \ From 7d8cd5e0b7a871f8e9f34fedea7b7cbc4cd2ce20 Mon Sep 17 00:00:00 2001 From: Levi Zhou <31941107+ZhouXY-PKU@users.noreply.github.com> Date: Thu, 6 Nov 2025 22:34:04 +0800 Subject: [PATCH 4/7] Update elecstate_lcao.cpp --- source/source_estate/elecstate_lcao.cpp | 28 +++++++++++++------------ 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/source/source_estate/elecstate_lcao.cpp b/source/source_estate/elecstate_lcao.cpp index 14562fca87..a9578b228a 100644 --- a/source/source_estate/elecstate_lcao.cpp +++ b/source/source_estate/elecstate_lcao.cpp @@ -1,6 +1,5 @@ -#include "elecstate_lcao.h" - -#include "cal_dm.h" +#include "source_estate/elecstate_lcao.h" +#include "source_estate/cal_dm.h" #include "source_base/timer.h" #include "source_estate/module_dm/cal_dm_psi.h" #include "source_hamilt/module_xc/xc_functional.h" @@ -31,9 +30,10 @@ double ElecStateLCAO>::get_spin_constrain_energy() return sc.cal_escon(); } -#ifdef __PEXSI template <> -void ElecStateLCAO::dm2Rho(std::vector pexsi_DM, std::vector pexsi_EDM) +void ElecStateLCAO::dm2Rho(std::vector pexsi_DM, + std::vector pexsi_EDM, + DensityMatrix* dm) { ModuleBase::timer::tick("ElecStateLCAO", "dm2Rho"); @@ -43,13 +43,15 @@ void ElecStateLCAO::dm2Rho(std::vector pexsi_DM, std::vectorget_DM()->pexsi_EDM = pexsi_EDM; +#ifdef __PEXSI + dm->pexsi_EDM = pexsi_EDM; +#endif for (int is = 0; is < nspin; is++) { - this->DM->set_DMK_pointer(is, pexsi_DM[is]); + dm->set_DMK_pointer(is, pexsi_DM[is]); } - DM->cal_DMR(); + dm->cal_DMR(); for (int is = 0; is < PARAM.inp.nspin; is++) { @@ -58,14 +60,14 @@ void ElecStateLCAO::dm2Rho(std::vector pexsi_DM, std::vectorDM->get_DMR_vector(), PARAM.inp.nspin, this->charge->rho); + ModuleGint::cal_gint_rho(dm->get_DMR_vector(), PARAM.inp.nspin, this->charge->rho); if (XC_Functional::get_ked_flag()) { for (int is = 0; is < PARAM.inp.nspin; is++) { ModuleBase::GlobalFunc::ZEROS(this->charge->kin_r[0], this->charge->nrxx); } - ModuleGint::cal_gint_tau(this->DM->get_DMR_vector(), PARAM.inp.nspin, this->charge->kin_r); + ModuleGint::cal_gint_tau(dm->get_DMR_vector(), PARAM.inp.nspin, this->charge->kin_r); } this->charge->renormalize_rho(); @@ -75,13 +77,13 @@ void ElecStateLCAO::dm2Rho(std::vector pexsi_DM, std::vector -void ElecStateLCAO>::dm2rho(std::vector*> pexsi_DM, - std::vector*> pexsi_EDM) +void ElecStateLCAO>::dm2Rho(std::vector*> pexsi_DM, + std::vector*> pexsi_EDM, + DensityMatrix, double>* dm) { ModuleBase::WARNING_QUIT("ElecStateLCAO", "pexsi is not completed for multi-k case"); } -#endif template class ElecStateLCAO; // Gamma_only case template class ElecStateLCAO>; // multi-k case From 0fb80c5fbbef19aaadb8428467ac13f0252a8514 Mon Sep 17 00:00:00 2001 From: Levi Zhou <31941107+ZhouXY-PKU@users.noreply.github.com> Date: Fri, 7 Nov 2025 11:44:52 +0800 Subject: [PATCH 5/7] Update Dockerfile.intel --- Dockerfile.intel | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Dockerfile.intel b/Dockerfile.intel index 3b3ddce842..7979a8232a 100644 --- a/Dockerfile.intel +++ b/Dockerfile.intel @@ -178,10 +178,8 @@ ADD https://api.github.com/repos/deepmodeling/abacus-develop/git/refs/heads/deve RUN export LD_LIBRARY_PATH=${GKLIB_ROOT}/lib:${METIS32_ROOT}/lib:${PARMETIS32_ROOT}/lib:${SUPERLU32_DIST_ROOT}/lib:${PEXSI32_ROOT}/lib:${LD_LIBRARY_PATH} && \ export CPATH=${GKLIB_ROOT}/include:${METIS32_ROOT}/include:${PARMETIS32_ROOT}/include:${SUPERLU32_DIST_ROOT}/include:${PEXSI32_ROOT}/include:${CPATH} && \ export CMAKE_PREFIX_PATH=${PEXSI32_ROOT}:${SUPERLU_DIST32_ROOT}:${PARMETIS32_ROOT}:${METIS32_ROOT}:${GKLIB_ROOT}:${CMAKE_PREFIX_PATH} && \ - cd /tmp && wget https://github.com/deepmodeling/abacus-develop/archive/refs/tags/v3.9.0.17.zip -O abacus-develop-3.9.0.17.zip && \ - unzip abacus-develop-3.9.0.17.zip && \ - cd abacus-develop-3.9.0.17 && \ - sed -i '3i\#include "source_estate/elecstate_lcao.h"' source/source_lcao/edm.cpp && \ + cd /tmp && git clone https://github.com/deepmodeling/abacus-develop.git --depth 1 && \ + cd abacus-develop && \ cmake -B build \ -DENABLE_MLALGO=ON \ -DENABLE_PEXSI=ON \ @@ -195,5 +193,9 @@ RUN export LD_LIBRARY_PATH=${GKLIB_ROOT}/lib:${METIS32_ROOT}/lib:${PARMETIS32_RO rm -rf /tmp/abacus-develop # -DMKL_SYCL=OFF \ # -DMKL_SYCL_DISTRIBUTED_DFT:BOOL=OFF \ + # cd /tmp && wget https://github.com/deepmodeling/abacus-develop/archive/refs/tags/v3.9.0.17.zip -O abacus-develop-3.9.0.17.zip && \ + # unzip abacus-develop-3.9.0.17.zip && \ + # cd abacus-develop-3.9.0.17 && \ + # sed -i '3i\#include "source_estate/elecstate_lcao.h"' source/source_lcao/edm.cpp && \ # Default entry CMD ["/bin/bash"] From 534958f1445f0b5a87c2a0c516b94c6d4a9a8b54 Mon Sep 17 00:00:00 2001 From: Levi Zhou <31941107+ZhouXY-PKU@users.noreply.github.com> Date: Fri, 7 Nov 2025 15:20:56 +0800 Subject: [PATCH 6/7] Update Dockerfile.intel --- Dockerfile.intel | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile.intel b/Dockerfile.intel index 7979a8232a..4a1a5b82f2 100644 --- a/Dockerfile.intel +++ b/Dockerfile.intel @@ -176,6 +176,7 @@ RUN export LD_LIBRARY_PATH=${SUPERLU_DIST32_ROOT}/lib:${METIS32_ROOT}/lib:${PARM # Clone and build abacus (optional during image build; keep for CI image) ADD https://api.github.com/repos/deepmodeling/abacus-develop/git/refs/heads/develop /dev/null RUN export LD_LIBRARY_PATH=${GKLIB_ROOT}/lib:${METIS32_ROOT}/lib:${PARMETIS32_ROOT}/lib:${SUPERLU32_DIST_ROOT}/lib:${PEXSI32_ROOT}/lib:${LD_LIBRARY_PATH} && \ + export PKG_CONFIG_PATH=${GKLIB_ROOT}/lib/pkgconfig:${METIS32_ROOT}/lib/pkgconfig:${PARMETIS32_ROOT}/lib/pkgconfig:${SUPERLU32_DIST_ROOT}/lib/pkgconfig:${PEXSI32_ROOT}/lib/pkgconfig:${PKG_CONFIG_PATH} && \ export CPATH=${GKLIB_ROOT}/include:${METIS32_ROOT}/include:${PARMETIS32_ROOT}/include:${SUPERLU32_DIST_ROOT}/include:${PEXSI32_ROOT}/include:${CPATH} && \ export CMAKE_PREFIX_PATH=${PEXSI32_ROOT}:${SUPERLU_DIST32_ROOT}:${PARMETIS32_ROOT}:${METIS32_ROOT}:${GKLIB_ROOT}:${CMAKE_PREFIX_PATH} && \ cd /tmp && git clone https://github.com/deepmodeling/abacus-develop.git --depth 1 && \ From d7e57b2952077b016aa3138dad838c58b68df134 Mon Sep 17 00:00:00 2001 From: Levi Zhou <31941107+ZhouXY-PKU@users.noreply.github.com> Date: Fri, 7 Nov 2025 16:18:46 +0800 Subject: [PATCH 7/7] Update build_test_cmake.yml --- .github/workflows/build_test_cmake.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_test_cmake.yml b/.github/workflows/build_test_cmake.yml index 2f4e39abbc..8eca97bafa 100644 --- a/.github/workflows/build_test_cmake.yml +++ b/.github/workflows/build_test_cmake.yml @@ -20,7 +20,7 @@ jobs: build_args: "-DENABLE_LIBXC=1 -DDENABLE_MLALGO=1 -DENABLE_LIBRI=1" name: "Build extra components with GNU toolchain" - tag: intel - build_args: "-DENABLE_LIBXC=1 -DENABLE_PEXSI=1 -DENABLE_MLALGO=1 -DENABLE_LIBRI=1" + build_args: "-DENABLE_LIBXC=1 -DENABLE_MLALGO=1 -DENABLE_LIBRI=1" name: "Build extra components with Intel toolchain" - tag: cuda