Merge branch 'develop' into spin_density_est_bug

config/build_olcf_crusher_ROCm.sh → config/build_olcf_frontier_ROCm.sh

@@ -1,28 +1,34 @@
 #!/bin/bash
 
-# Build script for test and development system crusher at OLCF
+# Build script for Frontier and its test and development system Crusher at OLCF
 # See https://github.com/QMCPACK/qmcpack/pull/4123 for more details on the module file if needed
 
 echo "Loading QMCPACK dependency modules for crusher"
 module unload PrgEnv-gnu PrgEnv-cray PrgEnv-amd PrgEnv-gnu-amd PrgEnv-cray-amd
 module unload amd amd-mixed gcc gcc-mixed cce cce-mixed
-module load PrgEnv-amd amd/5.4.0 gcc-mixed/11.2.0
+module load PrgEnv-amd amd/5.4.3
+module load  craype/2.7.16 # hard-coded version. 2.7.19 and 2.7.20 cause CC segfault.
 module unload cray-libsci
 module load cmake/3.22.2
 module load cray-fftw
 module load openblas/0.3.17-omp
 module load cray-hdf5-parallel
-module load boost/1.78.0
+
+# edit this line if you are not a member of mat151
+export BOOST_ROOT=/ccs/proj/mat151/opt/boost/boost_1_81_0
 
 module list >& module_list.txt
 
 TYPE=Release
-Compiler=rocm540
+Compiler=rocm543
 
 if [[ $# -eq 0 ]]; then
   source_folder=`pwd`
+elif [[ $# -eq 1 ]]; then
+  source_folder=$1
 else
   source_folder=$1
+  install_folder=$2
 fi
 
 if [[ -f $source_folder/CMakeLists.txt ]]; then
@@ -55,6 +61,11 @@ if [[ $name == *"cuda2hip"* ]]; then
 fi
 
 folder=build_crusher_${Compiler}_${name}
+
+if [[ -v install_folder ]]; then
+  CMAKE_FLAGS="$CMAKE_FLAGS -DCMAKE_INSTALL_PREFIX=$install_folder/$folder"
+fi
+
 echo "**********************************"
 echo "$folder"
 echo "$CMAKE_FLAGS"
@@ -66,7 +77,13 @@ cmake $CMAKE_FLAGS -DCMAKE_C_COMPILER=cc -DCMAKE_CXX_COMPILER=CC -DCMAKE_SYSTEM_
       -DCMAKE_C_FLAGS=--gcc-toolchain=/opt/cray/pe/gcc/11.2.0/snos -DCMAKE_CXX_FLAGS=--gcc-toolchain=/opt/cray/pe/gcc/11.2.0/snos \
       $source_folder
 fi
-make -j16
+
+if [[ -v install_folder ]]; then
+  make -j16 install && chmod -R -w $install_folder/$folder
+else
+  make -j16
+fi
+
 cd ..
 
 echo

config/build_olcf_summit_Clang.sh

@@ -34,7 +34,7 @@ module list >& module_list.txt
 
 TYPE=Release
 Machine=summit
-Compiler=Clang
+Compiler=Clang15
 
 if [[ $# -eq 0 ]]; then
   source_folder=`pwd`

docs/analyzing.rst

@@ -1061,9 +1061,38 @@ Using the qmc-fit tool for statistical time step extrapolation, trial wavefuncti
 -----------------------------------------------------------------------------------------------------------------
 
 The ``qmc-fit`` tool is used to provide statistical estimates of curve-fitting parameters based on QMCPACK data. 
-``qmc-fit`` is currently limited to estimating fitting parameters related to time step extrapolation and trial wavefunction
-optimization (optimal U for DFT+U, EXX fractions), it will eventually support many types of fitted curves (e.g., Morse
-potential binding curves and various equation-of-state fitting curves).
+``qmc-fit`` is currently estimates fitting parameters related to time step extrapolation and trial wavefunction
+optimization (optimal U for DFT+U, EXX fractions) and supports many types of fitted curves (e.g., Morse
+potential binding curves and various equation-of-state fitting curves). An overview of all supported input flags to 
+``qmc-fit`` can be obtained by typing ``qmc-fit -h`` at the command line: 
+
+::
+
+  >qmc-fit -h 
+    usage: qmc-fit [-h] [-f FIT_FUNCTION] (-t TIMESTEPS | -u HUBBARDS | --exx EXX | --eos EOS) [-s SERIES_START] [-e EQUILS] [-b REBLOCK_FACTORS] [--noplot] {ts,u,eos} scalar_files [scalar_files ...]
+
+    This utility provides a fit to the one-dimensional parameter scans of QMC observables. Currently, the functionality in place is to fit linear/quadratic polynomial fits to the timestep VMC/DMC studies and single parameter optimization of trial wavefunctions using DMC local
+    energies and quadratic, cubic and quartic fits and equation-of-state and Morse potential fits.
+
+    positional arguments:
+      {ts,u,eos}            One dimensional parameter used to fit QMC local energies. Options are ts for timestep and u for hubbard_u parameter fitting
+      scalar_files          Scalar files used in the fit. An explicit list of scalar files with space or a wildcard (e.g. dmc*/dmc.s001.scalar.dat) is acceptable.
+
+    optional arguments:
+      -h, --help            show this help message and exit
+      -f FIT_FUNCTION, --fit FIT_FUNCTION
+                            Fitting function, options are (for each fit type) ts:{linear, quadratic, sqrt} u:{cubic, quadratic, quartic} eos:{birch, morse, murnaghan, vinet}. (default: linear)
+      -t TIMESTEPS          Timesteps corresponding to scalar files, excluding any prior to --series_start (default: None)
+      -u HUBBARDS           Hubbard U values (eV) (default: None)
+      --exx EXX             EXX ratios (default: None)
+      --eos EOS             Structural parameter for EOS fitting (volume or distance) (default: None)
+      -s SERIES_START, --series_start SERIES_START
+                            Series number for first DMC run. Use to exclude prior VMC scalar files if they have been provided (default: None)
+      -e EQUILS, --equils EQUILS
+                            Equilibration lengths corresponding to scalar files, excluding any prior to --series_start. Can be a single value for all files. If not provided, equilibration periods will be estimated. (default: None)
+      -b REBLOCK_FACTORS, --reblock_factors REBLOCK_FACTORS
+                            Reblocking factors corresponding to scalar files, excluding any prior to --series_start. Can be a single value for all files. If not provided, reblocking factors will be estimated. (default: None)
+      --noplot              Do not show plots. (default: False)
 
 The jackknife statistical technique
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -1281,6 +1310,43 @@ An example of a cubic fit is given as below:
 
   Quadratic Hubbard-U fits to DMC data for a 24-atom supercell of monolayer FeCl\ :sub:`2`\  obtained with ``qmc-fit``.  DMC local energy minima are indicated by the red data point on the bottom halves of either panel.
 
+
+Performing equation-of-state and Morse potential binding curve fits 
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+For a systematic series of statistical data, such as QMC calculations performed at different interatomic distances, or at a series of volumes
+for an equation-of-states calculation, it is advised to perform jackknife fitting to determine quantities such as equilibrium distance, volume and 
+bulk moduli. For interatomic distances and equation of states fits to QMC calculations, ``qmc-fit`` has the capability to perform Morse and Birch, Murnaghan 
+and Vinet equation-of-state fits. In this example, we determine the equilibrium volume and bulk modulus of C-diamond using a 16 atom supercell using DMC and Murnaghan 
+equation-of-state fit. For the 16 atom supercell, we uniformly scan over the volumes between :math:`78.16` and :math:`99.62 A^3`. Assuming that all these DMC calculation 
+folders are located under the same parent folder and ordered from smaller to the large volume (e.g. dmc_78.16, dmc_80.65 ...), the following script can be used to make a
+Murnaghan fit to the DMC energies. 
+
+::
+
+  >qmc-fit eos -e 50 -b 6 --eos "78.16 80.65 83.20 85.80 88.46 91.16 93.92 96.74 99.62" --fit murnaghan dmc_*/dmc.s001.scalar.dat
+
+  fit function  : murnaghan
+  fitted formula: E_inf + B/Bp*V*((V_0/V)**Bp/(Bp-1)+1)-V_0*B/(Bp-1)
+  minimum_x: 89.00 +/- 0.12
+  e_inf: -91.2659 +/- 0.0012
+  B: 0.1053 +/- 0.0050
+  Bp: 0.000189 +/- 0.000011
+  pressure: -0.00000 +/- 0.00015
+
+Here, the minimum volume is reported as :math:`89.00 \pm 0.12A^3` consistent with the input volume units. Considering that this is a 16-atom cell, the per atom 
+quantity would be :math:`5.56 \pm 0.01 A^3` per C. Bulk modulus, B, is reported as :math:`0.1053 \pm 0.005 Ha/A^3`. In SI units, this bulk modulus value corresponds to 
+:math:`459 \pm 21` GPa. Different fitting functions are supported via the ``-f`` option. Currently supported options include ``Vinet`` , ``Murnaghan``, ``Birch`` and ``Morse``. 
+For more information and default options, please refer to ``qmc-fit -h``.
+
+.. _fig14:
+.. figure:: /figs/qmcfit_eos.png
+  :width: 400
+  :align: center
+
+  Murnaghan equation-of-state fits to DMC data for a 16-atom supercell of C-diamond obtained with ``qmc-fit``.  DMC structural minimum is indicated by the red data point with an error bar smaller than its marker size.
+
+
 .. _qdens:
 
 Using the qdens tool to obtain electron densities

docs/installation.rst

@@ -900,34 +900,114 @@ accelerators.
 Building QMCPACK
 ^^^^^^^^^^^^^^^^
 
-Note that these build instructions are preliminary as the
-software environment is subject to change. As of December 2018, the
-IBM XL compiler does not support C++14, so we currently use the
-gnu compiler.
+As of April 2023, LLVM Clang (>=15) is the only compiler, validated by QMCPACK developers,
+on Summit for OpenMP offloading computation to NVIDIA GPUs.
 
 For ease of reproducibility we provide build scripts for Summit.
 
 ::
 
   cd qmcpack
-  ./config/build_olcf_summit.sh
-  ls bin
+  ./config/build_olcf_summit_Clang.sh
+  ls build_*/bin
 
-Building Quantum ESPRESSO
-^^^^^^^^^^^^^^^^^^^^^^^^^
-We provide a build script for the v6.4.1 release of Quantum ESPRESSO (QE).
-The following can be used to build a CPU version of QE on Summit,
-placing the script in the external\_codes/quantum\_espresso directory.
+Running QMCPACK
+^^^^^^^^^^^^^^^
+Job script example with one MPI rank per GPU.
 
 ::
 
-  cd external_codes/quantum_espresso
-  ./build_qe_olcf_summit.sh
+  #!/bin/bash
+  # Begin LSF directives
+  #BSUB -P MAT151
+  #BSUB -J test
+  #BSUB -o tst.o%J
+  #BSUB -W 60
+  #BSUB -nnodes 1
+  #BSUB -alloc_flags smt1
+  # End LSF directives and begin shell commands
+
+  module load gcc/9.3.0
+  module load spectrum-mpi
+  module load cuda
+  module load essl
+  module load netlib-lapack
+  module load hdf5/1.10.7
+  module load fftw
+  # private module until OLCF provides a new llvm build
+  module use /gpfs/alpine/mat151/world-shared/opt/modules
+  module load llvm/release-15.0.0-cuda11.0
+
+  NNODES=$(((LSB_DJOB_NUMPROC-1)/42))
+  RANKS_PER_NODE=6
+  RS_PER_NODE=6
+
+  exe_path=/gpfs/alpine/mat151/world-shared/opt/qmcpack/release-3.16.0/build_summit_Clang_offload_cuda_real/bin
+
+  prefix=NiO-fcc-S1-dmc
+
+  export OMP_NUM_THREADS=7
+  jsrun -n $NNODES -a $RANKS_PER_NODE -c $((RANKS_PER_NODE*OMP_NUM_THREADS)) -g 6 -r 1 -d packed -b packed:$OMP_NUM_THREADS \
+        --smpiargs="-disable_gpu_hooks" $exe_path/qmcpack --enable-timers=fine $prefix.xml >& $prefix.out
+
+Installing on ORNL OLCF Frontier/Crusher
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Frontier is a HPE Cray EX supercomputer located at the Oak Ridge Leadership Computing Facility.
+Each Frontier compute node consists of [1x] 64-core AMD CPU with access to 512 GB of DDR4 memory.
+Each node also contains [4x] AMD MI250X, each with 2 Graphics Compute Dies (GCDs) for a total of 8 GCDs per node.
+Crusher is the test and development system of Frontier with exactly the same node architecture.
+
+Building QMCPACK
+^^^^^^^^^^^^^^^^
+
+As of April 2023, ROCm Clang (>= 5.3.0) is the only compiler, validated by QMCPACK developers,
+on Frontier for OpenMP offloading computation to AMD GPUs.
+
+For ease of reproducibility we provide build scripts for Frontier.
+
+::
+
+  cd qmcpack
+  ./config/build_olcf_frontier_ROCm.sh
+  ls build_*/bin
+
+Running QMCPACK
+^^^^^^^^^^^^^^^
+Job script example with one MPI rank per GPU.
+
+::
+
+  #!/bin/bash
+  #SBATCH -A MAT151
+  #SBATCH -J test
+  #SBATCH -o tst.o%J
+  #SBATCH -t 01:30:00
+  #SBATCH -N 1
+
+  echo "Loading QMCPACK dependency modules for crusher"
+  module unload PrgEnv-gnu PrgEnv-cray PrgEnv-amd PrgEnv-gnu-amd PrgEnv-cray-amd
+  module unload amd amd-mixed gcc gcc-mixed cce cce-mixed
+  module load PrgEnv-amd amd/5.4.3
+  module unload cray-libsci
+  module load cmake/3.22.2
+  module load cray-fftw
+  module load openblas/0.3.17-omp
+  module load cray-hdf5-parallel
+
+  exe_path=/lustre/orion/mat151/world-shared/opt/qmcpack/develop-20230411/build_crusher_rocm543_offload_cuda2hip_real/bin
+
+  prefix=NiO-fcc-S128-dmc
+
+  module list >& module_list.txt # record modules loaded at run
+  ldd $exe_path/qmcpack >& ldd.out # double check dynamic libraries
 
-Note that performance is
-not yet optimized although vendor libraries are
-used. Alternatively, the wavefunction files can be generated on
-another system and the converted HDF5 files copied over.
+  RANKS_PER_NODE=8
+  TOTAL_RANKS=$((SLURM_JOB_NUM_NODES * RANKS_PER_NODE))
+  THREAD_SLOTS=7
+  export OMP_NUM_THREADS=7 # change this to 1 if running with only 1 thread is intended.
+  srun -n $TOTAL_RANKS --ntasks-per-node=$RANKS_PER_NODE --gpus-per-task=1 -c $THREAD_SLOTS --gpu-bind=closest \
+       $exe_path/qmcpack --enable-timers=fine $prefix.xml >& $prefix.out
 
 Installing on NERSC Cori, Haswell Partition, Cray XC40
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~