diff --git a/.github/workflows/gfortran.yml b/.github/workflows/gfortran.yml index bb2940b8..1f3904cb 100644 --- a/.github/workflows/gfortran.yml +++ b/.github/workflows/gfortran.yml @@ -14,6 +14,9 @@ env: CURL_OPTS: -S --no-silent CODECOV_OPTIONS: -Z -X coveragepy -X xcode + # FFLAGS for building ABIN, applicable for most jobs + ABIN_FFLAGS: -O0 -fopenmp -Wall --coverage -ffpe-trap=invalid,zero,overflow,denormal + jobs: basic_build: @@ -25,7 +28,6 @@ jobs: gcc_v: [7, 8, 9] env: FC: gfortran - FFLAGS: -O0 -fopenmp -Wall --coverage -ffpe-trap=invalid,zero,overflow,denormal GCC_V: ${{ matrix.gcc_v}} CODECOV_NAME: ${{format('{0} GCC-{1}', github.job, matrix.gcc_v)}} @@ -56,6 +58,8 @@ jobs: - name: Build ABIN run: ./configure --pfunit ${HOME}/pfunit/build/installed/ && make + env: + FFLAGS: ${{ env.ABIN_FFLAGS }} - name: Run Unit tests run: make unittest @@ -132,7 +136,6 @@ jobs: matrix: gcc_v: [7] env: - FFLAGS: -O0 -fopenmp -Wall --coverage -ffpe-trap=invalid,zero,overflow,denormal GCC_V: ${{ matrix.gcc_v}} CODECOV_NAME: ${{format('{0} GCC-{1}', github.job, matrix.gcc_v)}} steps: @@ -156,6 +159,8 @@ jobs: - name: Build ABIN run: ./configure --pfunit ${HOME}/pfunit/build/installed/ --fftw && make + env: + FFLAGS: ${{ env.ABIN_FFLAGS }} - name: Run Unit tests run: make unittest @@ -174,9 +179,8 @@ jobs: fail-fast: false matrix: gcc_v: [7, 8, 9] - mpich_v: ["3.1.3", "3.3.2"] + mpich_v: ["3.3.2", "3.4.1"] env: - ABIN_FFLAGS: -O0 -fopenmp --coverage -ffpe-trap=invalid,zero,overflow,denormal # To speed-up MPICH build CFLAGS: -O0 GCC_V: ${{ matrix.gcc_v}} @@ -204,10 +208,9 @@ jobs: run: ./dev_scripts/install_mpich.sh ${HOME}/mpich ${MPICH_V} - name: build ABIN - run: | - export FFLAGS=${ABIN_FFLAGS} &&\ - ./configure --mpi ${HOME}/mpich/${MPICH_V}/install &&\ - make + run: ./configure --mpi ${HOME}/mpich/${MPICH_V}/install && make + env: + FFLAGS: ${{ env.ABIN_FFLAGS }} -g - name: test ABIN run: make test # We upload to Codecov from the OpenMPI build, @@ -227,7 +230,6 @@ jobs: matrix: gcc_v: [7] env: - ABIN_FFLAGS: -O0 --coverage -fopenmp -ffpe-trap=invalid,zero,overflow,denormal # To speed-up OpenMPI build CFLAGS: -O0 GCC_V: ${{ matrix.gcc_v}} @@ -255,10 +257,9 @@ jobs: run: ./dev_scripts/install_openmpi.sh ${HOME}/openmpi ${OPENMPI_V} ${OPENMPI_PATCH} - name: build ABIN - run: | - export FFLAGS=${ABIN_FFLAGS} &&\ - ./configure --mpi "${HOME}/openmpi/${OPENMPI_V}/install" &&\ - make + run: ./configure --mpi "${HOME}/openmpi/${OPENMPI_V}/install" && make + env: + FFLAGS: ${{ env.ABIN_FFLAGS }} - name: test ABIN run: make test - name: Codecov upload @@ -278,7 +279,6 @@ jobs: env: PLUMED_V: ${{ matrix.plumed_v}} GCC_V: ${{ matrix.gcc_v}} - ABIN_FFLAGS: -O0 -fopenmp --coverage -ffpe-trap=invalid,zero,overflow,denormal # Speeding up the Plumed build CFLAGS: -O0 CXXLAGS: -O0 @@ -312,10 +312,11 @@ jobs: - name: build ABIN run: | - export FFLAGS=${ABIN_FFLAGS} &&\ ./configure --plumed "${HOME}/plumed/${PLUMED_V}/install"\ --pfunit ~/pfunit/build/installed/ &&\ make + env: + FFLAGS: ${{ env.ABIN_FFLAGS }} - name: Run Unit tests run: make unittest diff --git a/codecov.yml b/codecov.yml index 839139fb..dc8cc0f5 100644 --- a/codecov.yml +++ b/codecov.yml @@ -27,7 +27,7 @@ coverage: patch: default: target: auto - threshold: 3.0% + threshold: 3% parsers: gcov: diff --git a/dev_scripts/install_mpich.sh b/dev_scripts/install_mpich.sh index 824a0e04..63cdef60 100755 --- a/dev_scripts/install_mpich.sh +++ b/dev_scripts/install_mpich.sh @@ -17,6 +17,10 @@ TAR_FILE="mpich-${MPICH_VERSION}.tar.gz" DOWNLOAD_URL="https://www.mpich.org/static/downloads/${MPICH_VERSION}/${TAR_FILE}" INSTALL_DIR="$MPICH_DIR/$MPICH_VERSION/install" +# Github Actions machines have two CPUs, per: +# https://docs.github.com/en/free-pro-team@latest/actions/reference/specifications-for-github-hosted-runners#supported-runners-and-hardware-resources +NCPUS=2 + if [[ -d $MPICH_DIR/$MPICH_VERSION ]];then echo "Found existing MPICH installation in $MPICH_DIR/$MPICH_VERSION" echo "Remove this folder if you want to reinstall" @@ -30,18 +34,23 @@ curl "$DOWNLOAD_URL" > $MPICH_DIR/$MPICH_VERSION/pkg/${TAR_FILE} cd $MPICH_DIR/$MPICH_VERSION/src && tar -xzf ../pkg/${TAR_FILE} && cd mpich-${MPICH_VERSION} # If you're building MPI for general use, not only for ABIN, -# you might want to change the configure options: -# --enable-fortran=yes Compile all versions of Fortran interfaces -# This option is needed for GitHub Actions build, I don't know why -# --with-hydra-pm=pmiserv --with-namepublisher=pmi -# Needed for MPI interface with TeraChem +# you might want change some of the configure options. +# --enable-fortran=all Compile all versions of Fortran interfaces +# In principle we don't need F77, but configure fails in that case. +# --with-namepublisher=pmi +# This compiles hydra_nameserver binary, needed for MPI interface with TeraChem +# +# Use the two rows below for a debug build/ +# export CFLAGS='-g -O0' +# --disable-fast --enable-g-option=all \ ./configure FC=gfortran CC=gcc \ - --enable-fortran=all --disable-cxx \ - --with-hydra-pm=pmiserv --with-namepublisher=pmi \ + --enable-fortran=all \ + --with-pm=hydra --with-device=ch3:nemesis \ + --with-namepublisher=pmi \ --enable-static --disable-shared \ --prefix=${INSTALL_DIR} 2>&1 |\ tee configure.log -make 2>&1 | tee make.log +make -j $NCPUS 2>&1 | tee make.log make install 2>&1 | tee make_install.log echo " diff --git a/interfaces/TERA/r.tera b/interfaces/TERA/r.tera deleted file mode 100755 index c5d75a28..00000000 --- a/interfaces/TERA/r.tera +++ /dev/null @@ -1,110 +0,0 @@ -#!/bin/bash -cd $(dirname $0) -timestep=$1 -ibead=$2 -input=input$ibead.com -natom=$(cat ../geom.dat.$ibead | wc -l ) - - -source ../SetEnvironment.sh TERACHEM # dev # uncomment this for development version - - -######## USER INPUT FOR TERACHEM ################ -parallel=0 #=1 if we execute ABIN in parallel -# TODO: checking for nproc > 0 and parallel=0 -# woulds only work if input is input.in -iqmmm=0 # 0 - QMMM off, 1- QMMM on -natqm=5 # number of qm atoms for qmmm jobs -numgpus=1 # number of gpus -cat > $input << EOF -basis 6-31g* -charge 0 -spinmult 2 -method ublyp -timings yes -EOF -######## END OF USER INPUT######## - - -#---------DO NOT MODIFY BELOW------# - -scrdir="./scratch$ibead" - -if [[ $parallel -eq "1" ]];then - gpuid="" - let gpu0=ibead-1 - let gpu0=gpu0*numgpus -else - gpu0=0 -fi - -for ((i=0;i> $input << EOF -scrdir $scrdir -coordinates input$ibead.xyz -units angstrom -gpus $numgpus $gpuid -run gradient -EOF - -#### PASSING THE WAVE FUNCTION -if [[ -e $scrdir/c0 ]];then - echo "guess $scrdir/c0" >> $input -fi -if [[ -e $scrdir/ca0 && -e $scrdir/cb0 ]];then - echo "guess $scrdir/ca0 $scrdir/cb0" >> $input -fi - -### CREATING THE INPUT GEOMETRIES ############ -if [[ $iqmmm -eq 1 ]];then - echo "qmmm input_mm$ibead.xyz" >> $input - let natmm=natom-natqm - cat > input_mm$ibead.xyz << EOF -$natmm - -EOF -tail -n $natmm ../geom.dat.$ibead >> input_mm$ibead.xyz - -else - let natqm=natom -fi - -echo "end" >> $input - - -cat > input$ibead.xyz << EOF -$natqm - -EOF -head -n $natqm ../geom.dat.$ibead >> input$ibead.xyz -############################################# - -###########LAUNCHING TERACHEM################ - -node=$(uname -n) -export OMP_NUM_THREADS=$numgpus #setting to number of gpus,used only for MKL library anyway i.e. for initial diagonalization - -$TERAEXE $input > $input.out - -#check whether all is ok -if [[ $? -eq 0 ]];then - cp $input.out $input.out.old -else - echo "WARNING: TeraChem calculation probably failed." - echo "See TERA/$input.out.error" - cp $input.out $input.out.error -fi - -########## EXTRACTING DATA ################ - -grep 'FINAL ENERGY' $input.out | awk '{print $3}' > ../engrad.dat.$ibead -grep -A$natqm 'dE/dX' $input.out|tail -$natqm >> ../engrad.dat.$ibead - -if [[ $iqmmm -eq 1 ]];then - grep -A$natmm 'MM / Point charge' $input.out|tail -$natmm >> ../engrad.dat.$ibead -fi - diff --git a/src/Makefile b/src/Makefile index dcb68ef7..fa539f8f 100755 --- a/src/Makefile +++ b/src/Makefile @@ -1,7 +1,7 @@ F_OBJS := modules.o fortran_interfaces.o utils.o io.o random.o arrays.o qmmm.o fftw_interface.o \ shake.o nosehoover.o gle.o transform.o potentials.o estimators.o ekin.o vinit.o plumed.o \ remd.o force_bound.o water.o force_cp2k.o sh_integ.o surfacehop.o landau_zener.o\ - force_mm.o force_tera.o force_terash.o force_abin.o en_restraint.o analyze_ext_template.o geom_analysis.o analysis.o \ + force_mm.o tera_mpi_api.o force_tera.o force_terash.o force_abin.o en_restraint.o analyze_ext_template.o geom_analysis.o analysis.o \ minimizer.o mdstep.o forces.o read_cmdline.o init.o C_OBJS := water_interface.o diff --git a/src/abin.F90 b/src/abin.F90 index 07eb1534..b051d9c0 100644 --- a/src/abin.F90 +++ b/src/abin.F90 @@ -43,8 +43,9 @@ program abin logical :: file_exists integer, dimension(8) :: time_start, time_end real(DP) :: total_cpu_time +#ifdef USE_MPI integer :: ierr -!$ integer :: nthreads, omp_get_max_threads +#endif call date_and_time(VALUES=time_start) @@ -52,7 +53,9 @@ program abin call init(dt) ! This cannot be in init because of the namelist 'system' - if (my_rank == 0) call clean_temp_files() + if (my_rank == 0) then + call clean_temp_files() + end if if (irest == 1 .and. (my_rank == 0 .or. iremd == 1)) then call archive_file('restart.xyz', it) @@ -66,9 +69,7 @@ program abin call lz_rewind(en_array_lz) end if -!$ nthreads = omp_get_max_threads() if (my_rank == 0) then -!$ write (*, *) 'Number of OpenMP threads used = ', nthreads write (*, '(A)') 'Job started at: '//trim(get_formatted_date_and_time(time_start)) write (*, *) '' end if @@ -111,7 +112,7 @@ program abin if (ipimd == 5) call lz_rewind(en_array_lz) ! if we use reference potential with RESPA - if (pot_ref /= 'none') then + if (pot_ref /= '_none_') then call force_clas(fxc, fyc, fzc, x, y, z, eclas, pot_ref) call force_clas(fxc_diff, fyc_diff, fzc_diff, x, y, z, eclas, pot) end if diff --git a/src/arrays.F90 b/src/arrays.F90 index 84fb61a9..640fc5d6 100644 --- a/src/arrays.F90 +++ b/src/arrays.F90 @@ -105,7 +105,7 @@ subroutine allocate_arrays(natomalloc, nwalkalloc) fxc = 0.0D0; fyc = fxc; fzc = fxc fxq = fxc; fyq = fxc; fzq = fxc transfxc = fxc; transfyc = fxc; transfzc = fxc - if (pot_ref /= 'none') then + if (pot_ref /= '_none_') then allocate (fxc_diff(natomalloc, nwalkalloc)) allocate (fyc_diff(natomalloc, nwalkalloc)) allocate (fzc_diff(natomalloc, nwalkalloc)) diff --git a/src/force_abin.F90 b/src/force_abin.F90 index 84337d70..f576ec6a 100644 --- a/src/force_abin.F90 +++ b/src/force_abin.F90 @@ -121,12 +121,11 @@ subroutine force_abin(x, y, z, fx, fy, fz, eclas, chpot, walkmax) !-----MAKE THE CALL----------! ISTATUS = system(chsystem) + ! TODO: Verify that the below is true even for GCC >= 7 ! Exit status 0 turns to 0 ! For some reason, exit status 1 turns to 256 - ! However, this one we get by default from bash, don't know why... - ! see this thread for explanation: - ! http://coding.derkeiler.com/Archive/Fortran/comp.lang.fortran/MAXUNITS07-01/msg00085.html - ! If the bash script wants to notify ABIN, it can use e.g. exit 2 + ! However, this one we get by default from BASH, I don't know why. + ! If the BASH script wants to notify ABIN, it can use e.g. exit 2 if (ISTATUS /= 0 .and. ISTATUS /= 256) then write (*, '(A)') 'ERROR during the execution of the ab initio external program.' write (*, '(A)') 'Please inspect the output files in& diff --git a/src/force_tera.F90 b/src/force_tera.F90 index a947140f..31714961 100644 --- a/src/force_tera.F90 +++ b/src/force_tera.F90 @@ -1,9 +1,7 @@ -module mod_terampi +module mod_force_tera ! ---------------------------------------------------------------- ! Interface for TeraChem based QM and QM/MM MD. -! Perform MPI communications with terachem. Requires MPI 2.0 or above to use -! So far, I was not able to make it work with OpenMPI. -! (but now that we use file based tera_port, it should work as well) +! Perform MPI communications with terachem. ! ! Currently supports: ! pure QM and ONIOM @@ -12,35 +10,21 @@ module mod_terampi ! Original Author: Andreas Goetz (agoetz@sdsc.edu) ! ! Date: November 2010 -! Modified by Dan Hollas hollas@vscht.cz -! Date: September 2014 - September 2015 +! Modified by Daniel Hollas hollas@vscht.cz ! ---------------------------------------------------------------- use mod_const, only: DP - implicit none - private - integer, parameter :: MAXTERASERVERS = 9 - integer, parameter :: MPI_TAG_ERROR = 13, MPI_TAG_EXIT = 0 - ! By default, take port name from a file - character(len=1024) :: teraport = '' - integer :: newcomms(MAXTERASERVERS) ! Communicator, initialized in mpi_init subroutine -! DH WARNING, initial hack, we do not support TeraChem-based QM/MM yet - integer :: natmm_tera = 0 - integer :: nteraservers = 1 - real(DP), allocatable :: mmcharges(:) - real(DP) :: mpi_sleep = 0.05 - public :: teraport, newcomms, mpi_sleep, nteraservers - public :: force_tera, natmm_tera + use mod_terampi #ifdef USE_MPI - ! TODO: Move handle_mpi_error to a dedicated MPI module - public :: handle_mpi_error - public :: finalize_terachem, initialize_terachem, connect_terachem + use mpi #endif + implicit none + private + public :: force_tera save contains subroutine force_tera(x, y, z, fx, fy, fz, eclas, walkmax) - use mod_const, only: DP, ANG use mod_utils, only: abinerror use mod_general, only: iqmmm use mod_interfaces, only: oniom @@ -48,34 +32,39 @@ subroutine force_tera(x, y, z, fx, fy, fz, eclas, walkmax) real(DP), intent(inout) :: fx(:, :), fy(:, :), fz(:, :) real(DP), intent(inout) :: eclas integer, intent(in) :: walkmax - integer :: iw, itera + integer :: iw, itc + integer :: tc_comm integer :: OMP_GET_THREAD_NUM -! DHnote: we cannot use Niklasson's propagator in TC if nwalk > 1 -! This is a responsibility of the user + ! DHnote: we cannot use Niklasson's propagator in TC if nwalk > 1 + ! This is a responsibility of the user if (modulo(walkmax, nteraservers) /= 0) then write (*, *) 'ERROR: Parameter "nwalk" must be divisible by "nteraservers"!' call abinerror("force_tera") end if - itera = 1 + itc = 1 -! NOTE: Parallelization accross TeraChem servers -!$OMP PARALLEL DO PRIVATE(itera) + ! Parallelization accross TeraChem servers +!$OMP PARALLEL DO PRIVATE(iw, itc, tc_comm) do iw = 1, walkmax ! map OMP thread to TC server -!$ itera = OMP_GET_THREAD_NUM() + 1 +!$ itc = OMP_GET_THREAD_NUM() + 1 #ifdef USE_MPI - call send_tera(x, y, z, iw, newcomms(itera)) + tc_comm = get_tc_communicator(itc) - call receive_tera(fx, fy, fz, eclas, iw, walkmax, newcomms(itera)) + call send_tera(x, y, z, iw, tc_comm) + + call receive_tera(fx, fy, fz, eclas, iw, walkmax, tc_comm) #endif ! ONIOM was not yet tested!! - if (iqmmm == 1) call oniom(x, y, z, fx, fy, fz, eclas, iw) + if (iqmmm == 1) then + call oniom(x, y, z, fx, fy, fz, eclas, iw) + end if end do !$OMP END PARALLEL DO @@ -84,417 +73,162 @@ end subroutine force_tera #ifdef USE_MPI - subroutine send_tera(x, y, z, iw, newcomm) - use mpi - use mod_const, only: DP, ANG - use mod_general, only: idebug, iremd, my_rank + subroutine send_tera(x, y, z, iw, tc_comm) use mod_system, only: names use mod_qmmm, only: natqm - use mod_utils, only: abinerror - use mod_interfaces, only: oniom real(DP), intent(in) :: x(:, :), y(:, :), z(:, :) - integer, intent(in) :: iw, newcomm - real(DP) :: coords(3, size(x, 1)) - character(len=2) :: names_qm(size(x, 1) + 6) - integer :: ierr, iat + integer, intent(in) :: iw, tc_comm - do iat = 1, natqm - coords(1, iat) = x(iat, iw) / ANG - coords(2, iat) = y(iat, iw) / ANG - coords(3, iat) = z(iat, iw) / ANG - end do + ! We send these data to TC each step + call send_natom(natqm, tc_comm) - ! ----------------------------------------- - ! Begin sending data each step to terachem - ! ----------------------------------------- + call send_atom_types_and_scrdir(names, natqm, iw, tc_comm, .true.) - ! Send natqm and the type of each qmatom - if (idebug > 1) then - write (6, '(/, a, i0)') 'Sending natqm = ', natqm - call flush (6) - end if - call MPI_Send(natqm, 1, MPI_INTEGER, 0, 2, newcomm, ierr) - call handle_mpi_error(ierr) + call send_coordinates(x, y, z, natqm, iw, tc_comm) - do iat = 1, natqm - names_qm(iat) = names(iat) - end do - if (idebug > 1) then - write (6, '(/,a)') 'Sending QM atom types: ' - write (*, *) (names_qm(iat), iat=1, natqm) - call flush (6) - end if - ! DH WARNING: this will not work for iw>199 - ! not really tested for iw>99 - ! TODO: refactor this mess - write (names_qm(natqm + 1), '(A2)') '++' - write (names_qm(natqm + 2), '(A2)') 'sc' - if (iw > 99) then - write (names_qm(natqm + 3), '(A1,I1)') 'r', 1 - write (names_qm(natqm + 4), '(I2.2)') iw - 100 - else - write (names_qm(natqm + 3), '(A1,I1)') 'r', 0 - write (names_qm(natqm + 4), '(I2.2)') iw - end if + ! NOT IMPLEMENTED ! + !if (natmm_tera > 0) then + ! call send_mm_data(x, y, z, iw, tc_comm) + !end if + end subroutine send_tera - ! REMD HACK - if (iremd == 1) then - write (names_qm(natqm + 5), '(I2.2)') my_rank - write (names_qm(natqm + 6), '(A2)') '++' - else - write (names_qm(natqm + 5), '(A2)') '++' - end if +! QM/MM via TC-MPI interface is currently not +! implemented so excluding this code from compilation. +#if 0 + subroutine send_mm_data(x, y, z, iw, tc_comm) + use mod_const, only: ANG + use mod_general, only: idebug + use mod_qmmm, only: natqm + real(DP), intent(in) :: x(:, :), y(:, :), z(:, :) + integer, intent(in) :: iw, comm + real(DP) :: coords(3, size(x, 1)) + integer :: ierr, iat + real(DP), intent(in) :: coords(:, :) - call MPI_Send(names_qm, 2 * natqm + 12, MPI_CHARACTER, 0, 2, newcomm, ierr) - call handle_mpi_error(ierr) + call send_natom(natmm_tera) - ! Send QM coordinate array if (idebug > 1) then - write (6, '(a)') 'Sending QM coords: ' - do iat = 1, natqm - write (6, *) 'Atom ', iat, ': ', coords(:, iat) - call flush (6) - end do + write (6, '(a)') 'Sending charges: ' end if - call MPI_Send(coords, natqm * 3, MPI_DOUBLE_PRECISION, 0, 2, newcomm, ierr) + call MPI_Send(mmcharges, natmm_tera, MPI_DOUBLE_PRECISION, 0, TC_TAG, tc_comm, ierr) call handle_mpi_error(ierr) - if (natmm_tera > 0) then - - do iat = 1, natmm_tera - coords(1, iat) = x(iat + natqm, iw) / ANG - coords(2, iat) = y(iat + natqm, iw) / ANG - coords(3, iat) = z(iat + natqm, iw) / ANG - end do - - ! Send natmm and the charge of each atom - if (idebug > 1) then - write (6, '(a, i0)') 'Sending natmm = ', natmm_tera - call flush (6) - end if - call MPI_Send(natmm_tera, 1, MPI_INTEGER, 0, 2, newcomm, ierr) - call handle_mpi_error(ierr) - - if (idebug > 1) then - write (6, '(a)') 'Sending charges: ' - end if - call MPI_Send(mmcharges, natmm_tera, MPI_DOUBLE_PRECISION, 0, 2, newcomm, ierr) - call handle_mpi_error(ierr) - - ! Send MM point charge coordinate array - if (idebug > 1) then - write (6, '(a)') 'Sending charges coords: ' - end if - - call MPI_Send(coords, 3 * natmm_tera, MPI_DOUBLE_PRECISION, 0, 2, newcomm, ierr) - call handle_mpi_error(ierr) + ! Send MM point charge coordinate array + if (idebug > 1) then + write (6, '(a)') 'Sending MM coordinates...' end if + do iat = 1, natmm_tera + coords(1, iat) = x(iat + natqm, iw) / ANG + coords(2, iat) = y(iat + natqm, iw) / ANG + coords(3, iat) = z(iat + natqm, iw) / ANG + end do + call MPI_Send(coords, 3 * natmm_tera, MPI_DOUBLE_PRECISION, 0, TC_TAG, tc_comm, ierr) + call handle_mpi_error(ierr) + end subroutine send_mm_data +#endif - end subroutine send_tera - - subroutine receive_tera(fx, fy, fz, eclas, iw, walkmax, newcomm) - use mpi - use mod_const, only: DP, ANG - use mod_general, only: idebug + subroutine receive_tera(fx, fy, fz, eclas, iw, walkmax, tc_comm) + use mod_const, only: ANG + use mod_general, only: idebug, it, nwrite + use mod_io, only: print_charges, print_dipoles use mod_qmmm, only: natqm use mod_utils, only: abinerror real(DP), intent(inout) :: fx(:, :), fy(:, :), fz(:, :) real(DP), intent(inout) :: eclas - integer, intent(in) :: iw, walkmax, newcomm - ! TODO: make qmcharges global variable + integer, intent(in) :: iw, walkmax + integer, intent(in) :: tc_comm real(DP) :: qmcharges(size(fx, 1)) real(DP) :: dxyz_all(3, size(fx, 1)) real(DP) :: escf ! SCF energy real(DP) :: dipmom(4, 3) ! Dipole moment {x, y, z, |D|}, {QM, MM, TOT} integer :: status(MPI_STATUS_SIZE) integer :: ierr, iat - logical :: ltest - character*50 :: chsys_sleep - ! ----------------------------------- - ! Begin receiving data from terachem - ! ----------------------------------- - - ! DH TODO: we need to somehow make sure that we don't wait forever if - ! terachem crashes - ! At this moment, this is ensured at the bash script level - - ! DH reduce cpu usage comming from MPI_Recv() via system call to 'sleep'. - ! Not elegant, but MPICH apparently does not currently provide better solution. - ! Based according to an answer here: - ! http://stackoverflow.com/questions/14560714/probe-seems-to-consume-the-cpu - - ltest = .false. - write (chsys_sleep, '(A6, F10.4)') 'sleep ', mpi_sleep - do while (.not. ltest) - call MPI_IProbe(MPI_ANY_SOURCE, MPI_ANY_TAG, newcomm, ltest, status, ierr) - call handle_mpi_error(ierr) - call system(chsys_sleep) - end do - ! Energy + call wait_for_terachem(tc_comm) + + ! Begin receiving data from TeraChem + if (idebug > 2) then - write (6, '(a)') 'Waiting to receive scf energy from TeraChem...' + write (6, '(a)') 'Waiting to receive energy from TeraChem...' call flush (6) end if - call MPI_Recv(escf, 1, MPI_DOUBLE_PRECISION, MPI_ANY_SOURCE, MPI_ANY_TAG, newcomm, status, ierr) + call MPI_Recv(escf, 1, MPI_DOUBLE_PRECISION, MPI_ANY_SOURCE, & + & MPI_ANY_TAG, tc_comm, status, ierr) call handle_mpi_error(ierr) + call check_recv_count(status, 1, MPI_DOUBLE_PRECISION) ! Checking for TAG=1, which means that SCF did not converge if (status(MPI_TAG) == 1) then - write (*, *) 'GOT TAG 1 from TeraChem: SCF probably did not converge.' + write (*, *) 'Got TAG 1 from TeraChem: SCF probably did not converge.' call abinerror('force_tera') end if if (idebug > 1) then - write (6, '(a,es15.6)') 'Received scf energy from server:', escf + write (6, '(A,ES15.6)') 'Received SCF energy from server:', escf call flush (6) end if - ! Charges (Mulliken or other) if (idebug > 2) then write (6, '(a)') 'Waiting to receive charges...' end if - call MPI_Recv(qmcharges(:), natqm, MPI_DOUBLE_PRECISION, MPI_ANY_SOURCE, MPI_ANY_TAG, newcomm, status, ierr) + call MPI_Recv(qmcharges(:), natqm, MPI_DOUBLE_PRECISION, MPI_ANY_SOURCE, & + & MPI_ANY_TAG, tc_comm, status, ierr) call handle_mpi_error(ierr) - if (idebug > 2) then - write (6, '(a)') 'Received the following charges from server:' - do iat = 1, natqm - write (6, *) 'Atom ', iat, ': ', qmcharges(iat) - end do - call flush (6) + call check_recv_count(status, natqm, MPI_DOUBLE_PRECISION) + if (modulo(it, nwrite) == 0 .and. nteraservers == 1) then + call print_charges(qmcharges, iw) end if - ! Dipole moment if (idebug > 2) then write (6, '(a)') 'Waiting to receive dipole moment...' end if ! QM dipole moment - call MPI_Recv(dipmom(:, 1), 4, MPI_DOUBLE_PRECISION, MPI_ANY_SOURCE, MPI_ANY_TAG, newcomm, status, ierr) + call MPI_Recv(dipmom(:, 1), 4, MPI_DOUBLE_PRECISION, MPI_ANY_SOURCE, & + & MPI_ANY_TAG, tc_comm, status, ierr) call handle_mpi_error(ierr) + call check_recv_count(status, 4, MPI_DOUBLE_PRECISION) if (idebug > 1) then write (6, '(a,4es15.6)') 'Received QM dipole moment from server:', dipmom(:, 1) call flush (6) end if + ! TODO: Attach dipoles to global electronic structure type + ! and print them elsewhere. Right now when we run concurrent + ! TC servers, the printing is not deterministic. + if (modulo(it, nwrite) == 0 .and. nteraservers == 1) then + call print_dipoles(dipmom(:, 1), iw, 1) + end if + ! MM dipole moment, disabled for now -! call MPI_Recv( dipmom(:,2), 4, MPI_DOUBLE_PRECISION, MPI_ANY_SOURCE, MPI_ANY_TAG, newcomm, status, ierr ) -! if ( idebug > 1 ) then -! write(6,'(a,4es15.6)') 'Received MM dipole moment from server:', dipmom(:,2) -! call flush(6) -! end if - ! TOT dipole moment -! call MPI_Recv( dipmom(:,3), 4, MPI_DOUBLE_PRECISION, MPI_ANY_SOURCE, MPI_ANY_TAG, newcomm, status, ierr ) -! if ( idebug > 1 ) then -! write(6,'(a,4es15.6)') 'Received TOT dipole moment from server:', dipmom(:,3) -! call flush(6) -! end if +! call MPI_Recv( dipmom(:,2), 4, MPI_DOUBLE_PRECISION, MPI_ANY_SOURCE, MPI_ANY_TAG, tc_comm, status, ierr ) +! call MPI_Recv( dipmom(:,3), 4, MPI_DOUBLE_PRECISION, MPI_ANY_SOURCE, MPI_ANY_TAG, tc_comm, status, ierr ) ! QM gradients if (idebug > 1) then - write (*, '(a)') 'Waiting to receive gradients...' + write (*, '(A)') 'Waiting to receive gradients...' end if - call MPI_Recv(dxyz_all, 3 * (natqm + natmm_tera), MPI_DOUBLE_PRECISION, & - MPI_ANY_SOURCE, MPI_ANY_TAG, newcomm, status, ierr) + call MPI_Recv(dxyz_all, 3 * natqm, MPI_DOUBLE_PRECISION, & + MPI_ANY_SOURCE, MPI_ANY_TAG, tc_comm, status, ierr) call handle_mpi_error(ierr) + call check_recv_count(status, 3 * natqm, MPI_DOUBLE_PRECISION) if (idebug > 1) then - write (6, '(a)') 'Received the following gradients from server:' - do iat = 1, natqm + natmm_tera + write (6, '(A)') 'Received the following gradients from server:' + do iat = 1, natqm write (6, *) 'Atom ', iat, ': ', dxyz_all(:, iat) end do call flush (6) end if -!!$OMP CRITICAL - do iat = 1, natqm + natmm_tera + + do iat = 1, natqm fx(iat, iw) = -dxyz_all(1, iat) fy(iat, iw) = -dxyz_all(2, iat) fz(iat, iw) = -dxyz_all(3, iat) end do -!!$OMP END CRITICAL + ! TODO: Divide by walkmax in forces.xyz !$OMP ATOMIC eclas = eclas + escf / walkmax end subroutine receive_tera - subroutine connect_terachem(itera) - use mpi - use mod_utils, only: abinerror - use mod_general, only: iremd, my_rank, idebug - integer, intent(in) :: itera - character(len=MPI_MAX_PORT_NAME) :: port_name - integer :: ierr, newcomm, iost - real(DP) :: timer - character(len=1024) :: server_name, portfile - character(len=1) :: chtera - - ! ----------------------------------- - ! Look for server_name, get port name - ! After 60 seconds, exit if not found - ! ----------------------------------- - port_name = '' - - if (itera > MAXTERASERVERS) then - write (*, *) 'ERROR: We currently support only ', MAXTERASERVERS, 'TC servers!' - write (*, *) 'Shutting down...' - write (*, *) 'Running TC servers might not be shutdown properly!' - call abinerror('force_tera') - end if - - timer = MPI_WTIME() - - ! This allows us to retry failed MPI_LOOKUP_NAME() call - call MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_RETURN, ierr) - call handle_mpi_error(ierr) - - write (chtera, '(I1)') itera - - if (iremd == 1) write (chtera, '(I1)') my_rank + 1 - if (teraport /= '') then - server_name = trim(teraport)//'.'//trim(chtera) - write (6, '(2a)') 'Looking up TeraChem server under name:', trim(server_name) - call flush (6) - - do - - call MPI_LOOKUP_NAME(trim(server_name), MPI_INFO_NULL, port_name, ierr) - if (ierr == MPI_SUCCESS) then - ! This sometimes happens, I have no idea why. - if (len_trim(port_name) == 0) then - write (6, '(a)') 'Found empty port, retrying...' - call system('sleep 1') - else - write (6, '(2a)') 'Found port: ', trim(port_name) - call flush (6) - exit - end if - else - ! Let's wait a bit - ! Too many calls to MPI_LOOKUP_NAME can crash the hydra_nameserver process - if (idebug > 1) write (6, '(A)') 'Waiting for TC port' - call system('sleep 1') - end if - - if ((MPI_WTIME() - timer) > 60) then ! Time out after 60 seconds - write (*, *) 'Port "'//trim(server_name)//'" not found. Timed out after 60 seconds.' - call abinerror("connect_to_terachem") - end if - - end do - - else - - portfile = 'port.txt.'//chtera - write (6, '(A)') 'Reading TeraChem port name from file '//trim(portfile) - call system('sync') ! flush HDD buffer, not sure how portable this is - open (500, file=portfile, action="read", status="old", iostat=iost) - if (iost /= 0) then - write (*, *) 'WARNING: Cannot open file '//trim(portfile) - write (*, *) 'Will wait for 10s and try again...' - call system('sleep 10') - open (500, file=portfile, action="read", status="old") - end if - read (500, '(A)', iostat=iost) port_name - if (iost /= 0) then - write (*, *) 'ERROR when reading file '//trim(portfile) - stop 1 - end if - close (500) - - end if - - write (6, '(a)') 'Establishing connection to TeraChem...' - ! ---------------------------------------- - ! Establish new communicator via port name - ! ---------------------------------------- - call flush (6) - call MPI_COMM_CONNECT(port_name, MPI_INFO_NULL, 0, MPI_COMM_SELF, newcomm, ierr) - call handle_mpi_error(ierr) - write (6, '(a,i0)') 'Established a new communicator:', newcomm - - newcomms(itera) = newcomm - - end subroutine connect_terachem - - subroutine initialize_terachem() - use mpi - use mod_qmmm, only: natqm - use mod_system, only: names - use mod_utils, only: abinerror - integer :: ierr, itera - - if (natmm_tera > 0) then - allocate (mmcharges(natmm_tera)) - end if - - do itera = 1, nteraservers - write (*, *) 'Sending initial number of QM atoms to TeraChem.' - call MPI_Send(natqm, 1, MPI_INTEGER, 0, 2, newcomms(itera), ierr) - call handle_mpi_error(ierr) - end do - - do itera = 1, nteraservers - write (*, *) 'Sending initial QM atom names to TeraChem.' - call MPI_Send(names, 2 * natqm, MPI_CHARACTER, 0, 2, newcomms(itera), ierr) - call handle_mpi_error(ierr) - end do - - if (mpi_sleep <= 0) then - write (*, *) 'WARNING: Parameter "mpi_sleep" must be positive!' - write (*, *) 'Setting it back to default value' - mpi_sleep = 0.05 - end if - - end subroutine initialize_terachem - - subroutine finalize_terachem(error_code) - use mpi - integer, intent(in) :: error_code - integer :: ierr, itera - integer :: empty = 0 - - do itera = 1, nteraservers - - write (*, *) 'Shutting down TeraChem server; id=', itera - if (error_code == 0) then - call MPI_Send(empty, 1, MPI_INTEGER, 0, MPI_TAG_EXIT, newcomms(itera), ierr) - else - call MPI_Send(empty, 1, MPI_INTEGER, 0, MPI_TAG_ERROR, newcomms(itera), ierr) - end if - if (ierr /= 0) then - write (*, *) 'I got a MPI Error when I tried to shutdown TeraChem server id =', itera - write (*, *) 'Please, verify manually that the TeraChem server was terminated.' - write (*, *) 'The error code was:', ierr - end if - - end do - end subroutine finalize_terachem - - ! TODO: call this after each MPI call - subroutine handle_mpi_error(mpi_err, status, datatype, expected_count) - use mpi - use mod_utils, only: abinerror - integer, intent(in) :: mpi_err - integer, intent(in), optional :: status(MPI_STATUS_SIZE) - integer, intent(in), optional :: datatype, expected_count - integer :: received_count, ierr - - if (present(status) .and. status(MPI_TAG) == MPI_TAG_ERROR) then - write (*, *) 'TeraChem sent an ERROR TAG. Exiting...' - call abinerror('TeraChem ERROR') - end if - - if (present(expected_count)) then - ! Compare the length of received message and what we expected - call MPI_GET_COUNT(status, datatype, received_count, ierr) - if (received_count /= expected_count) then - write (*, *) 'Received message of unexpected size' - call abinerror('MPI ERROR') - end if - end if - - if (mpi_err /= 0) then - write (*, *) 'Unspecified MPI Error, code:', ierr - call abinerror('MPI ERROR') - end if - end subroutine handle_mpi_error +! USE_MPI #endif -end module mod_terampi - +end module mod_force_tera diff --git a/src/force_terash.F90 b/src/force_terash.F90 index b9d186f7..b1d3588a 100644 --- a/src/force_terash.F90 +++ b/src/force_terash.F90 @@ -1,17 +1,16 @@ module mod_terampi_sh -!---------------------------------------------------------------- -! Interface for TeraChem based Surface Hopping. -! Based on the FMS interface from FMS90 (TerachemModule.f90) -! -! Original Authors: Basile Curchod, J. Snyder and Ed Hohenstein -!---------------------------------------------------------------- + !---------------------------------------------------------------- + ! Interface for TeraChem based Surface Hopping. + ! Based on the FMS interface from FMS90 (TerachemModule.f90) + ! + ! Original Authors: Basile Curchod, J. Snyder and Ed Hohenstein + !---------------------------------------------------------------- use mod_const, only: DP + use mod_terampi, only: TC_TAG implicit none private -#ifdef USE_MPI - public :: init_terash -#endif - public :: force_terash, finalize_terash + public :: force_terash + public :: init_terash, finalize_terash public :: write_wfn, read_wfn, move_new2old_terash, move_old2new_terash real(DP), allocatable :: CIvecs(:, :), MO(:, :), blob(:), NAC(:) real(DP), allocatable :: CIvecs_old(:, :), MO_old(:, :), blob_old(:) @@ -21,33 +20,29 @@ module mod_terampi_sh contains +#ifdef USE_MPI subroutine force_terash(x, y, z, fx, fy, fz, eclas) use mod_const, only: DP - use mod_terampi, only: newcomms + use mod_terampi, only: get_tc_communicator real(DP), intent(in) :: x(:, :), y(:, :), z(:, :) real(DP), intent(inout) :: fx(:, :), fy(:, :), fz(:, :) real(DP), intent(inout) :: eclas + integer :: tc_comm - ! for SH, we use only one TC server... - ! might be changes if we ever implement more elaborate SH schemes -#ifdef USE_MPI - call send_terash(x, y, z, fx, fy, fz, newcomms(1)) + tc_comm = get_tc_communicator(1) - call receive_terash(fx, fy, fz, eclas, newcomms(1)) -#else - write (*, *) "FATAL ERROR: ABIN not compiled with MPI, cannot connect to TeraChem" - stop 1 -#endif + ! For SH we use only one TC server. + call send_terash(x, y, z, tc_comm) + call receive_terash(fx, fy, fz, eclas, tc_comm) end subroutine force_terash -#ifdef USE_MPI - - subroutine receive_terash(fx, fy, fz, eclas, newcomm) + subroutine receive_terash(fx, fy, fz, eclas, tc_comm) + use mod_terampi, only: wait_for_terachem use mod_const, only: DP, ANG use mod_array_size, only: NSTMAX use mod_general, only: idebug, natom, en_restraint, ipimd - use mod_terampi, only: mpi_sleep, handle_mpi_error + use mod_terampi, only: handle_mpi_error, check_recv_count use mod_qmmm, only: natqm use mod_utils, only: abinerror use mod_io, only: print_charges, print_dipoles, print_transdipoles @@ -57,36 +52,26 @@ subroutine receive_terash(fx, fy, fz, eclas, newcomm) use mpi real(DP), intent(inout) :: fx(:, :), fy(:, :), fz(:, :) real(DP), intent(inout) :: eclas - integer, intent(in) :: newcomm + integer, intent(in) :: tc_comm real(DP) :: dip(NSTMAX * 3), tdip((NSTMAX - 1) * 3) ! Dipole moment {x, y, z, |D|}, {QM, MM, TOT} real(DP) :: qmcharges(size(fx, 1)) integer :: status(MPI_STATUS_SIZE) integer :: ierr, iat, iw, ist1, ist2, itrj, ipom, i - logical :: ltest - character*50 :: chsys_sleep itrj = 1 iw = 1 -! Receive energies from TC - if (idebug > 0) write (*, '(a)') 'Receiving energies from TC.' - - ! DH reduce cpu usage comming from MPI_Recv() via system call to 'sleep'. - ! Not elegant, but MPICH apparently does not currently provide better solution. - ! Based according to an answer here: - ! http://stackoverflow.com/questions/14560714/probe-seems-to-consume-the-cpu - - ltest = .false. - write (chsys_sleep, '(A6, F10.4)') 'sleep ', mpi_sleep - do while (.not. ltest) - call MPI_IProbe(MPI_ANY_SOURCE, MPI_ANY_TAG, newcomm, ltest, status, ierr) - call system(chsys_sleep) - end do + call wait_for_terachem(tc_comm) -! DH WARNING this will only work if itrj = 1 +! Receive energies from TC + if (idebug > 0) then + write (*, '(a)') 'Receiving energies from TC.' + end if + ! DH WARNING this will only work if itrj = 1 call MPI_Recv(en_array, nstate, MPI_DOUBLE_PRECISION, & - MPI_ANY_SOURCE, MPI_ANY_TAG, newcomm, status, ierr) + MPI_ANY_SOURCE, MPI_ANY_TAG, tc_comm, status, ierr) call handle_mpi_error(ierr) + call check_recv_count(status, nstate, MPI_DOUBLE_PRECISION) eclas = en_array(istate(itrj), itrj) @@ -101,8 +86,9 @@ subroutine receive_terash(fx, fy, fz, eclas, newcomm) if (idebug > 0) write (*, '(a)') 'Receiving transition dipoles from TC.' call MPI_Recv(TDip, (nstate - 1) * 3, & - MPI_DOUBLE_PRECISION, MPI_ANY_SOURCE, MPI_ANY_TAG, newcomm, status, ierr) + MPI_DOUBLE_PRECISION, MPI_ANY_SOURCE, MPI_ANY_TAG, tc_comm, status, ierr) call handle_mpi_error(ierr) + call check_recv_count(status, (nstate - 1) * 3, MPI_DOUBLE_PRECISION) ! do i=1, nstate-1 ! T_FMS%ElecStruc%TransDipole(i+1,:)=TDip(3*(i-1)+1:3*(i-1)+3) ! end do @@ -115,64 +101,75 @@ subroutine receive_terash(fx, fy, fz, eclas, newcomm) ! Receive dipole moment from TC if (idebug > 0) write (*, '(a)') 'Receiving dipole moments from TC.' call MPI_Recv(Dip, nstate * 3, & - MPI_DOUBLE_PRECISION, MPI_ANY_SOURCE, MPI_ANY_TAG, newcomm, status, ierr) + MPI_DOUBLE_PRECISION, MPI_ANY_SOURCE, MPI_ANY_TAG, tc_comm, status, ierr) call handle_mpi_error(ierr) + call check_recv_count(status, nstate * 3, MPI_DOUBLE_PRECISION) call print_dipoles(Dip, iw, nstate) ! Receive partial charges from TC if (idebug > 0) write (*, '(a)') 'Receiving atomic charges from TC.' - call MPI_Recv(qmcharges, natqm, MPI_DOUBLE_PRECISION, MPI_ANY_SOURCE, MPI_ANY_TAG, newcomm, status, ierr) + call MPI_Recv(qmcharges, natqm, MPI_DOUBLE_PRECISION, MPI_ANY_SOURCE, MPI_ANY_TAG, tc_comm, status, ierr) call handle_mpi_error(ierr) + call check_recv_count(status, natqm, MPI_DOUBLE_PRECISION) call print_charges(qmcharges, istate(itrj)) ! Receive MOs from TC if (idebug > 0) write (*, '(a)') 'Receiving MOs from TC.' call MPI_Recv(MO, nbf * nbf, MPI_DOUBLE_PRECISION, MPI_ANY_SOURCE, & - MPI_ANY_TAG, newcomm, status, ierr) + MPI_ANY_TAG, tc_comm, status, ierr) call handle_mpi_error(ierr) + call check_recv_count(status, nbf * nbf, MPI_DOUBLE_PRECISION) ! T_FMS%ElecStruc%OldOrbitals=MO if (idebug > 0) write (*, '(a)') 'Receiving CI vectors from TC.' call MPI_Recv(CIvecs, nstate * civec, & - MPI_DOUBLE_PRECISION, MPI_ANY_SOURCE, MPI_ANY_TAG, newcomm, status, ierr) + MPI_DOUBLE_PRECISION, MPI_ANY_SOURCE, MPI_ANY_TAG, tc_comm, status, ierr) call handle_mpi_error(ierr) + call check_recv_count(status, nstate * civec, MPI_DOUBLE_PRECISION) - if (idebug > 0) write (*, *) "Receiving wavefunction overlap via MPI." - call MPI_Recv(SMatrix, nstate * nstate, MPI_DOUBLE, MPI_ANY_SOURCE, MPI_ANY_TAG, newcomm, status, ierr); + if (idebug > 0) write (*, *) "Receiving wavefunction overlap." + call MPI_Recv(SMatrix, nstate * nstate, MPI_DOUBLE_PRECISION, MPI_ANY_SOURCE, MPI_ANY_TAG, tc_comm, status, ierr); call handle_mpi_error(ierr) + call check_recv_count(status, nstate * nstate, MPI_DOUBLE_PRECISION) ! Should change the following according to what is done in TeraChem i = Check_CIVector(CIvecs, CIvecs_old, civec, nstate) CIVecs_old = Civecs - if (idebug > 0) write (*, '(a)') 'Receiving blob from TC.' + if (idebug > 0) write (*, '(a)') 'Receiving blob.' call MPI_Recv(blob, blobsize, & - MPI_DOUBLE_PRECISION, MPI_ANY_SOURCE, MPI_ANY_TAG, newcomm, status, ierr) + MPI_DOUBLE_PRECISION, MPI_ANY_SOURCE, MPI_ANY_TAG, tc_comm, status, ierr) call handle_mpi_error(ierr) + call check_recv_count(status, blobsize, MPI_DOUBLE_PRECISION) - if (idebug > 0) write (*, '(a)') 'Receiving gradients and NACME from TC.' + ! TODO: Extract all this to a function. + if (idebug > 0) then + write (*, '(A)') 'Receiving gradients and NACME.' + end if do ist1 = 1, nstate do ist2 = ist1, nstate - if (idebug > 0) write (*, '(a,i3,i3)') 'Receiving derivatives between states.' & - , ist1, ist2 + if (idebug > 0) then + write (*, '(A,i0,i0)') 'Receiving derivatives between states ', ist1, ist2 + end if ! NOTE: We do not filter here based on tocalc because TC always sends the whole ! derivative matrix, including zero elements, see 'terachem/fms.cpp:' ! Is TC sending zero arrays for NAC that we did not want it to compute??? call MPI_Recv(NAC, 3 * natom, MPI_DOUBLE_PRECISION, & - MPI_ANY_SOURCE, MPI_ANY_TAG, newcomm, status, ierr) + MPI_ANY_SOURCE, MPI_ANY_TAG, tc_comm, status, ierr) call handle_mpi_error(ierr) + call check_recv_count(status, 3 * natom, MPI_DOUBLE_PRECISION) if (idebug > 0) write (*, *) (NAC(i), i=1, 3 * natom) ipom = 1 + ! GRADIENTS if (ist1 == ist2 .and. istate(itrj) == ist1) then - ! GRADIENTS do iat = 1, natom fx(iat, iw) = -NAC(ipom) fy(iat, iw) = -NAC(ipom + 1) @@ -217,7 +214,8 @@ subroutine receive_terash(fx, fy, fz, eclas, newcomm) end subroutine receive_terash - subroutine send_terash(x, y, z, vx, vy, vz, newcomm) + subroutine send_terash(x, y, z, tc_comm) + use mod_terampi, only: send_coordinates use mod_array_size, only: NSTMAX use mod_const, only: DP, ANG, AUTOFS use mod_terampi, only: handle_mpi_error @@ -228,24 +226,15 @@ subroutine send_terash(x, y, z, vx, vy, vz, newcomm) use mod_sh, only: istate, tocalc, ignore_state use mpi real(DP), intent(in) :: x(:, :), y(:, :), z(:, :) - real(DP), intent(inout) :: vx(:, :), vy(:, :), vz(:, :) - integer, intent(in) :: newcomm + integer, intent(in) :: tc_comm real(DP) :: bufdoubles(100) - real(DP) :: qmcoords(3, size(x, 1)), vels(3, size(vx, 1)) - integer :: ierr, iw, iat, itrj, i, ist1, ist2 + real(DP) :: vels(3, size(x, 1)) + integer :: ierr, iw, itrj, i, ist1, ist2 integer :: bufints(NSTMAX * (NSTMAX - 1) / 2 + NSTMAX) integer, parameter :: FMSInit = 0 itrj = 1 iw = 1 - do iat = 1, natqm - qmcoords(1, iat) = x(iat, iw) - qmcoords(2, iat) = y(iat, iw) - qmcoords(3, iat) = z(iat, iw) - vels(1, iat) = vx(iat, iw) - vels(2, iat) = vy(iat, iw) - vels(3, iat) = vz(iat, iw) - end do ! Send ESinit bufints(1) = FMSinit @@ -261,7 +250,7 @@ subroutine send_terash(x, y, z, vx, vy, vz, newcomm) bufints(11) = 0 ! first_call, not used bufints(12) = 0 ! FMSRestart, not used - call MPI_Send(bufints, 12, MPI_INTEGER, 0, 2, newcomm, ierr) + call MPI_Send(bufints, 12, MPI_INTEGER, 0, TC_TAG, tc_comm, ierr) call handle_mpi_error(ierr) ! The following bit is not in FMS code @@ -296,94 +285,88 @@ subroutine send_terash(x, y, z, vx, vy, vz, newcomm) write (*, *) 'Sending derivative matrix logic.' write (*, *) (bufints(i), i=1, nstate * (nstate - 1) / 2 + nstate) end if - call MPI_SSend(bufints, nstate * (nstate - 1) / 2 + nstate, MPI_INTEGER, 0, 2, newcomm, ierr) + call MPI_SSend(bufints, nstate * (nstate - 1) / 2 + nstate, MPI_INTEGER, 0, TC_TAG, tc_comm, ierr) call handle_mpi_error(ierr) ! temporary hack bufdoubles(1) = sim_time ! * AUtoFS !* dt ! Send Time - call MPI_Send(bufdoubles, 1, MPI_DOUBLE_PRECISION, 0, 2, newcomm, ierr) + call MPI_Send(bufdoubles, 1, MPI_DOUBLE_PRECISION, 0, TC_TAG, tc_comm, ierr) call handle_mpi_error(ierr) -! Send coordinates - call MPI_Send(qmcoords, 3 * natom, MPI_DOUBLE_PRECISION, 0, 2, newcomm, ierr) - call handle_mpi_error(ierr) - if (idebug > 0) write (*, '(a)') 'Sent coordinates to TeraChem.' + call send_coordinates(x, y, z, natqm, iw, tc_comm) ! Send previous diabatic MOs if (idebug > 0) write (*, *) 'Sending previous orbitals.', nbf * nbf - call MPI_Send(MO, nbf * nbf, MPI_DOUBLE_PRECISION, 0, 2, newcomm, ierr) + call MPI_Send(MO, nbf * nbf, MPI_DOUBLE_PRECISION, 0, TC_TAG, tc_comm, ierr) call handle_mpi_error(ierr) ! Send previous CI vecs if (idebug > 0) write (*, *) 'Sending CI vector of size ', civec * nstate - call MPI_Send(CIvecs, civec * nstate, MPI_DOUBLE_PRECISION, 0, 2, newcomm, ierr) + call MPI_Send(CIvecs, civec * nstate, MPI_DOUBLE_PRECISION, 0, TC_TAG, tc_comm, ierr) call handle_mpi_error(ierr) if (idebug > 0) write (*, *) 'Sending blob.' - call MPI_Send(blob, blobsize, MPI_DOUBLE_PRECISION, 0, 2, newcomm, ierr) + call MPI_Send(blob, blobsize, MPI_DOUBLE_PRECISION, 0, TC_TAG, tc_comm, ierr) call handle_mpi_error(ierr) if (idebug > 0) write (*, *) 'Sending velocities' -! Only needed for numerical NACME, so send 0 instead for now + ! Only needed for numerical NACME, so send 0 instead for now vels = 0.0D0 - call MPI_Send(vels, 3 * natom, MPI_DOUBLE_PRECISION, 0, 2, newcomm, ierr) + call MPI_Send(vels, 3 * natom, MPI_DOUBLE_PRECISION, 0, TC_TAG, tc_comm, ierr) call handle_mpi_error(ierr) -! Imaginary velocities for FMS, not needed here, sending zeros... - call MPI_SSend(vels, 3 * natom, MPI_DOUBLE_PRECISION, 0, 2, newcomm, ierr) + ! Imaginary velocities for FMS, not needed here, sending zeros... + call MPI_SSend(vels, 3 * natom, MPI_DOUBLE_PRECISION, 0, TC_TAG, tc_comm, ierr) call handle_mpi_error(ierr) - if (idebug > 0) write (*, *) 'Succesfully sent all data to TeraChem-FMS' - + if (idebug > 0) then + write (*, *) 'Succesfully sent all data to TeraChem-FMS' + end if end subroutine send_terash subroutine init_terash(x, y, z) + use mpi use mod_const, only: DP, ANG use mod_general, only: idebug, DP, natom use mod_system, only: names use mod_qmmm, only: natqm use mod_sh_integ, only: nstate - use mod_terampi, only: newcomms, natmm_tera, handle_mpi_error - use mpi + use mod_terampi, only: get_tc_communicator, & + handle_mpi_error, & + send_natom, & + send_atom_types_and_scrdir, & + send_coordinates real(DP), intent(in) :: x(:, :), y(:, :), z(:, :) - real(DP) :: qmcoords(3, size(x, 1)) integer :: status(MPI_STATUS_SIZE) - integer :: ierr, iat, iw, newcomm - integer, parameter :: FMSinit = 1 + integer :: ierr, iw, tc_comm + integer, parameter :: FMS_INIT = 1 + logical, parameter :: send_scrdir = .false. integer :: bufints(3) + ! QMMM currently not supported + integer, parameter :: natmm_tera = 0 ! use only one TC server ! - newcomm = newcomms(1) + tc_comm = get_tc_communicator(1) iw = 1 - do iat = 1, natom - qmcoords(1, iat) = x(iat, iw) - qmcoords(2, iat) = y(iat, iw) - qmcoords(3, iat) = z(iat, iw) - end do - bufints(1) = FMSinit - bufints(2) = natom - natmm_tera + bufints(1) = FMS_INIT + bufints(2) = natqm bufints(3) = natmm_tera - call MPI_SSend(bufints, 3, MPI_INTEGER, 0, 2, newcomm, ierr) - call handle_mpi_error(ierr) - if (idebug > 0) write (*, '(a)') 'Sent initial FMSinit.' - -! Send atom types - call MPI_Send(names, 2 * natqm, MPI_CHARACTER, 0, 2, newcomm, ierr) + call MPI_SSend(bufints, 3, MPI_INTEGER, 0, TC_TAG, tc_comm, ierr) call handle_mpi_error(ierr) - if (idebug > 0) write (*, '(a)') 'Sent initial atom types.' + if (idebug > 0) then + write (*, '(a)') 'Sent initial FMSinit.' + end if -! Send coordinates - call MPI_Send(qmcoords, 3 * natom, MPI_DOUBLE_PRECISION, 0, 2, newcomm, ierr) - call handle_mpi_error(ierr) - if (idebug > 0) write (*, '(a)') 'Sent initial coordinates to TeraChem.' + call send_atom_types_and_scrdir(names, natqm, iw, tc_comm, send_scrdir) -!-- START RECEIVING INFO FROM TeraChem ------! + call send_coordinates(x, y, z, natqm, iw, tc_comm) -! Receive nbf,CI length and blob size + ! START RECEIVING INFO FROM TeraChem. + ! Receive nbf, CI length and blob size call MPI_Recv(bufints, 3, MPI_INTEGER, MPI_ANY_SOURCE, & - MPI_ANY_TAG, newcomm, status, ierr) + MPI_ANY_TAG, tc_comm, status, ierr) call handle_mpi_error(ierr) civec = bufints(1) @@ -402,9 +385,9 @@ subroutine init_terash(x, y, z) allocate (SMatrix(nstate * nstate)) blob = 0.0D0 blob_old = 0.0D0 - end subroutine init_terash +! USE_MPI #endif subroutine finalize_terash() @@ -523,4 +506,24 @@ subroutine move_old2new_terash blob = blob_old end subroutine move_old2new_terash +#ifndef USE_MPI + subroutine init_terash(x, y, z) + use mod_utils, only: not_compiled_with + real(DP), intent(inout) :: x(:, :), y(:, :), z(:, :) + ! Assignments just to squash compiler warnings + x = 0.0D0; y = 0.0D0; z = 0.0D0 + call not_compiled_with('MPI', 'init_terash') + end subroutine init_terash + + subroutine force_terash(x, y, z, fx, fy, fz, eclas) + use mod_const, only: DP + use mod_utils, only: not_compiled_with + real(DP), intent(in) :: x(:, :), y(:, :), z(:, :) + real(DP), intent(inout) :: fx(:, :), fy(:, :), fz(:, :) + real(DP), intent(inout) :: eclas + ! Assignments just to squash compiler warnings + fx = x; fy = y; fz = z; eclas = 0.0D0 + call not_compiled_with('MPI', 'force_terash') + end subroutine force_terash +#endif end module mod_terampi_sh diff --git a/src/forces.F90 b/src/forces.F90 index 39e86334..91b0dbe6 100644 --- a/src/forces.F90 +++ b/src/forces.F90 @@ -83,7 +83,7 @@ subroutine force_clas(fx, fy, fz, x, y, z, energy, chpot) if (iplumed == 1) call force_plumed(transx, transy, transz, fxab, fyab, fzab, eclas) ! For reference potential and ring-polymer contraction - if (pot_ref /= 'none' .and. chpot == pot) then + if (pot_ref /= '_none_' .and. chpot == pot) then ! fxab now holds the full potential, ! but we need the difference force on the output fx = fxab; fy = fyab; fz = fzab @@ -164,7 +164,7 @@ subroutine force_wrapper(x, y, z, fx, fy, fz, e_pot, chpot, walkmax) use mod_harmon, only: force_harmon, force_2dho, force_morse, force_doublewell use mod_splined_grid use mod_cp2k, only: force_cp2k - use mod_terampi, only: force_tera + use mod_force_tera, only: force_tera use mod_terampi_sh, only: force_terash implicit none real(DP), intent(in) :: x(:, :), y(:, :), z(:, :) diff --git a/src/init.F90 b/src/init.F90 index 54274336..68255be0 100644 --- a/src/init.F90 +++ b/src/init.F90 @@ -44,35 +44,38 @@ subroutine init(dt) use mod_terampi use mod_terampi_sh #ifdef USE_MPI - use mpi, only: MPI_COMM_WORLD, MPI_Init, MPI_Comm_Rank, MPI_Comm_Size, MPI_Barrier + use mpi #endif implicit none real(DP), intent(out) :: dt real(DP) :: masses(MAXTYPES) real(DP) :: rans(10) - integer :: iw, iat, natom_xyz, imol, shiftdihed = 1, iost - integer :: error, getpid, nproc = 1, ipom - character(len=2) :: massnames(MAXTYPES), atom + integer :: ipom, iw, iat, natom_xyz, imol, iost + integer :: shiftdihed + ! Number of OpenMP processes, read from ABIN input + ! WARNING: We do NOT use OMP_NUM_THREADS environment variable! + integer :: nproc + integer :: getPID +!$ integer :: omp_get_max_threads + character(len=2) :: massnames(MAXTYPES) + character(len=2) :: atom character(len=200) :: chinput, chcoords, chveloc character(len=200) :: chiomsg, chout - character(len=20) :: xyz_units = 'angstrom' + character(len=20) :: xyz_units character(len=60) :: chdivider character(len=60) :: mdtype + character(len=1024) :: tc_server_name logical :: file_exists logical :: rem_comvel, rem_comrot -! real(DP) :: wnw=5.0d-5 - ! Used for MPI calls integer :: ierr integer :: irand -!$ integer :: nthreads, omp_get_max_threads -! wnw "optimal" frequency for langevin (inose=3) namelist /general/ natom, pot, ipimd, mdtype, istage, inormalmodes, nwalk, nstep, icv, ihess, imini, nproc, iqmmm, & nwrite, nwritex, nwritev, nwritef, dt, irandom, nabin, irest, nrest, anal_ext, & isbc, rb_sbc, kb_sbc, gamm, gammthr, conatom, mpi_sleep, narchive, xyz_units, & dime, ncalc, idebug, enmini, rho, iknow, watpot, iremd, iplumed, plumedfile, & en_restraint, en_diff, en_kk, restrain_pot, & - pot_ref, nstep_ref, nteraservers, cp2k_mpi_beads + pot_ref, nstep_ref, nteraservers, max_wait_time, cp2k_mpi_beads #ifdef USE_MPI namelist /remd/ nswap, nreplica, deltaT, Tmax, temp_list @@ -87,7 +90,7 @@ subroutine init(dt) Nshake, ishake1, ishake2, shake_tol namelist /sh/ istate_init, nstate, substep, deltae, integ, inac, nohop, phase, decoh_alpha, popthr, ignore_state, & - nac_accu1, nac_accu2, popsumthr, energydifthr, energydriftthr, adjmom, revmom, natmm_tera, & + nac_accu1, nac_accu2, popsumthr, energydifthr, energydriftthr, adjmom, revmom, & dE_S0S1_thr, correct_decoherence namelist /lz/ initstate_lz, nstate_lz, nsinglet_lz, ntriplet_lz, deltaE_lz, energydifthr_lz @@ -95,16 +98,19 @@ subroutine init(dt) namelist /qmmm/ natqm, natmm, q, rmin, eps, attypes chcoords = 'mini.dat' + xyz_units = 'angstrom' chinput = 'input.in' chveloc = '' + tc_server_name = '' mdtype = '' dt = -1 - error = 0 + nproc = 1 iplumed = 0 + shiftdihed = 1 chdivider = "######################################################" - call get_cmdline(chinput, chcoords, chveloc, teraport) + call get_cmdline(chinput, chcoords, chveloc, tc_server_name) ! READING MAIN INPUT open (150, file=chinput, status='OLD', delim='APOSTROPHE', action="READ") @@ -123,7 +129,26 @@ subroutine init(dt) call init_cp2k() #ifdef USE_MPI else - call MPI_INIT(ierr) + ! TODO: Move this to a mpi_wrapper module + if (pot == "_tera_" .and. nteraservers > 1) then + ! We will be calling TS servers concurently + ! via OpenMP parallelization, hence we need MPI_Init_thread(). + ! https://www.mpi-forum.org/docs/mpi-3.1/mpi31-report/node303.htm + call MPI_Init_thread(MPI_THREAD_MULTIPLE, ipom, ierr) + if (ipom /= MPI_THREAD_MULTIPLE) then + write (*, *) 'Provided safety level is not MPI_THREAD_MULTIPLE' + write (*, '(A,I1,A,I1)') 'Requested ', MPI_THREAD_MULTIPLE, 'got:', ipom + call abinerror('init') + end if + ! nproc is used to initialize OpenMP threads below. + if (nproc /= nteraservers) then + nproc = nteraservers + end if + else + ! TODO: Check whether MPI is already initialized + ! This can happen when using internal CP2K interface. + call MPI_Init(ierr) + end if if (ierr /= 0) then write (*, *) 'Bad signal from MPI_INIT:', ierr stop 1 @@ -131,42 +156,29 @@ subroutine init(dt) #endif end if - ! We need to connect to TeraChem as soon as possible - ! because we want to shut down TeraChem nicely in case something goes wrong. + ! Set OpenMP parallelization + ! Currently only used in PIMD for trivial + ! parallelization over PI beads. + ! Note that scaling is actually not so great + ! since SCF timings will vary for different beads, + ! which decreases thread utilization. +!$ call OMP_set_num_threads(nproc) + #ifdef USE_MPI - ! TODO: Add explicit checks for ierr for all MPI calls! + ! TODO: Move this to an mpi_wrapper module call MPI_Comm_rank(MPI_COMM_WORLD, my_rank, ierr) call MPI_Comm_size(MPI_COMM_WORLD, mpi_world_size, ierr) + ! TODO: allow mpi_world_size > 1 only for REMD if (my_rank == 0 .and. mpi_world_size > 1) then write (*, '(A,I3)') 'Number of MPI processes = ', mpi_world_size end if - if (pot == '_tera_' .or. restrain_pot == '_tera_') then - if (nwalk > 1) then - write (*, *) 'WARNING: You are using PIMD with direct TeraChem interface.' - write (*, *) 'You should have "integrator regular" in & -& the TeraChem input file' - end if - write (*, *) 'Number of TeraChem servers = ', nteraservers - do ipom = 1, nteraservers - call connect_terachem(ipom) - end do - - if (nproc /= nteraservers) then - write (*, *) 'WARNING: parameter "nproc" must equal "nteraservers"' - write (*, *) 'Setting nproc = ', nteraservers - nproc = nteraservers - end if - end if - - call MPI_Barrier(MPI_COMM_WORLD, ierr) +#endif -#else - if (pot == '_tera_') then - write (*, *) 'FATAL ERROR: This version was not compiled with MPI support.' - write (*, *) 'You cannot use the direct MPI interface to TeraChem.' - call abinerror('init') + ! We need to connect to TeraChem as soon as possible, + ! because we want to shut down TeraChem nicely in case something goes wrong. + if (pot == '_tera_' .or. restrain_pot == '_tera_') then + call initialize_terachem_interface(trim(tc_server_name)) end if -#endif if (en_restraint >= 1) then call en_rest_init() @@ -269,7 +281,9 @@ subroutine init(dt) ! This line is super important, ! cause we actually use natqm in many parts of the code - if (iqmmm == 0 .and. pot /= 'mm') natqm = natom + if (iqmmm == 0 .and. pot /= 'mm') then + natqm = natom + end if if (irest == 1) then readnhc = 1 !readnhc has precedence before initNHC @@ -287,7 +301,7 @@ subroutine init(dt) scaleveloc = 0 end if - ! for future multiple time step integration in SH +! for future multiple time step integration in SH dt0 = dt ! We have to initialize here, because we read them from input @@ -314,17 +328,17 @@ subroutine init(dt) rem_comvel = .true. end if - ! allocate all basic arrays and set them to 0.0d0 +! allocate all basic arrays and set them to 0.0d0 call allocate_arrays(natom, nwalk + 1) - ! Ehrenfest require larger array since gradients for all of the states are need - ! TODO: We should really make this differently.. +! Ehrenfest require larger array since gradients for all of the states are need +! TODO: We should really make this differently.. if (ipimd == 4) call allocate_ehrenfest(natom, nstate) if (iplumed == 1) then call plumed_init() end if - ! READING GEOMETRY +! READING GEOMETRY read (111, *) do iat = 1, natom @@ -355,7 +369,7 @@ subroutine init(dt) ! the namelist system does not need to be present read (150, system, iostat=iost, iomsg=chiomsg) rewind (150) - ! check, whether we hit End-Of-File or other error + !check, whether we hit End-Of-File or other error if (IS_IOSTAT_END(iost)) then !fortran intrinsic for EOF continue else if (iost /= 0) then @@ -415,22 +429,19 @@ subroutine init(dt) end if close (150) - ! END OF READING INPUT +!--END OF READING INPUT--------------- -!$ call OMP_set_num_threads(nproc) -!$ nthreads = omp_get_max_threads() - -#ifdef USE_MPI if (pot == '_tera_' .or. restrain_pot == '_tera_') then - call initialize_terachem() - if (ipimd == 2 .or. ipimd == 4 .or. ipimd == 5) call init_terash(x, y, z) + call initialize_tc_servers() + if (ipimd == 2 .or. ipimd == 4 .or. ipimd == 5) then + call init_terash(x, y, z) + end if end if -#endif - ! HERE WE CHECK FOR ERRORS IN INPUT +!-----HERE WE CHECK FOR ERRORS IN INPUT----------- call check_inputsanity() - ! resetting number of walkers to 1 in case of classical simulation +! resetting number of walkers to 1 in case of classical simulation if (ipimd == 0) then if (my_rank == 0) then write (*, *) 'Using velocity Verlet integrator' @@ -442,7 +453,7 @@ subroutine init(dt) ! algorithm as well end if - ! for surface hopping and ehrenfest +!for surface hopping and ehrenfest if (ipimd == 2 .or. ipimd == 4 .or. ipimd == 5) then nwalk = ntraj !currently 1 md = 2 ! velocity verlet @@ -459,7 +470,7 @@ subroutine init(dt) md = 3 end if - if (pot_ref /= 'none') then + if (pot_ref /= '_none_') then md = 4 write (*, '(A)') 'Using Multiple Time-Step RESPA integrator!' write (*, '(A)') "Reference (cheap) potential is "//trim(pot_ref) @@ -487,7 +498,7 @@ subroutine init(dt) cvhess_cumul = 0.0D0 end if - ! SHAKE initialization, determining the constrained bond lenghts +! SHAKE initialization, determining the constrained bond lenghts if (nshake /= 0) then if (my_rank == 0) then write (*, *) 'Setting distances for SHAKE from XYZ coordinates' @@ -495,11 +506,11 @@ subroutine init(dt) call shake_init(x, y, z) end if - if (pot == 'mmwater' .or. pot_ref == 'mmwater')then + if (pot == 'mmwater' .or. pot_ref == 'mmwater') then call check_water(natom, names) end if - ! MUST BE BEFORE RESTART DUE TO ARRAY ALOCATION +! MUST BE BEFORE RESTART DUE TO ARRAY ALOCATION if (my_rank /= 0) then call srand(irandom) do ipom = 0, my_rank @@ -507,11 +518,10 @@ subroutine init(dt) end do end if - ! initialize prng, maybe rewritten during restart - ! call vranf(rans,0,irandom) - call gautrg(rans, 0, irandom) +! call vranf(rans,0,IRandom) !initialize prng,maybe rewritten during restart + call gautrg(rans, 0, IRandom) !initialize prng, maybe rewritten during restart - ! THERMOSTAT INITIALIZATION +! THERMOSTAT INITIALIZATION if (inose == 1) then call nhc_init() else if (inose == 2) then @@ -525,18 +535,18 @@ subroutine init(dt) call abinerror('init') end if - ! performing RESTART from restart.xyz +! performing RESTART from restart.xyz if (irest == 1) call restin(x, y, z, vx, vy, vz, it) if (pot == '2dho') then f = 0 !temporary hack end if if (nchain > 1) then - f = 0 ! what about nchain=1? - ! what about massive thermostat? + f = 0 !what about nchain=1? + ! what about massive therm? end if - ! SETTING initial velocities according to the Maxwell-Boltzmann distribution +! SETTING initial velocities according to the Maxwell-Boltzmann distribution if (irest == 0 .and. chveloc == '') then ! TODO: GLE thermostat, initialize momenta in gle_init if (temp0 >= 0) then @@ -546,7 +556,7 @@ subroutine init(dt) end if end if - ! Reading velocities from file +! Reading velocities from file if (chveloc /= '' .and. irest == 0) then ! TODO: move the following to a separate function open (500, file=chveloc, status='OLD', action="READ") @@ -577,7 +587,7 @@ subroutine init(dt) end if - ! END OF READING VELOCITIES +! END OF READING VELOCITIES-------------------- ! doing this here so that we can do it even when reading velocities from file if (rem_comvel) call remove_comvel(vx, vy, vz, am, rem_comvel) @@ -621,12 +631,13 @@ subroutine init(dt) if (iqmmm == 3 .or. pot == 'mm') write (*, nml=qmmm) write (*, *) end if - call flush (6) #ifdef USE_MPI call MPI_Barrier(MPI_COMM_WORLD, ierr) + write (*, '(A,I0,A,I0)') 'MPI rank: ', my_rank, ' PID: ', GetPID() +#else + write (*, '(A,I0)') 'Process ID (PID): ', GetPID() #endif - pid = GetPID() - if (my_rank == 0) write (*, *) 'Pid of the current proccess is:', pid +!$ write (*, '(A,I0)') 'Number of OpenMP threads: ', omp_get_max_threads() ! Open files for writing ! TODO: It's strange that we're passing these random params here... @@ -638,8 +649,13 @@ subroutine init(dt) subroutine check_inputsanity() use mod_chars, only: chknow + integer :: error +!$ integer :: nthreads, omp_get_max_threads + + error = 0 ! We should exclude all non-abinitio options, but whatever.... +!$ nthreads = omp_get_max_threads() !$ if (nthreads > 1 .and. (ipimd /= 1 .and. pot /= '_cp2k_')) then !$ write (*, *) 'Number of threads is ', nthreads !$ write (*, *) 'ERROR: Parallel execution is currently only supported with ab initio PIMD (ipimd=1)' @@ -654,14 +670,14 @@ subroutine check_inputsanity() end if if (irest == 1 .and. chveloc /= '') then - ! write(*,*)'ERROR: Input velocities are not compatible with irest=1.' + ! write(*,*)'ERROR: Input velocities are not compatible with irest=1.' write (*, *) 'WARNING: Input velocities from file'//trim(chveloc)//' will be ignored!' write (*, *) 'Velocities will be taken from restart file because irest=1.' - ! write(*,*)chknow - ! if(iknow.ne.1) error=1 + ! write(*,*)chknow + ! if(iknow.ne.1) error=1 end if - ! Check, whether input variables don't exceeds array limits + !-----Check, whether input variables don't exceeds array limits if (ntraj > ntrajmax) then write (*, *) 'Maximum number of trajectories is:' write (*, *) ntrajmax @@ -698,7 +714,7 @@ subroutine check_inputsanity() write (*, *) 'Adjust variable ndistmax in modules.f90' error = 1 end if - ! HERE we check for errors in input. +!----------HERE we check for errors in input. if (pot /= '_cp2k_') then if (nproc > nwalk) then write (*, *) 'ERROR: Nproc greater than nwalk. That does not make sense.' @@ -710,11 +726,12 @@ subroutine check_inputsanity() error = 1 end if if (modulo(nwalk, nproc) /= 0) then - write (*, *) 'ERROR: Nwalk is not divisible by nproc. This is not a wise usage of your computer time.' + write (*, *) 'ERROR: Nwalk is not divisible by the number of OpenMP threads.' + write (*, *) 'This is not a wise usage of your computer time.' error = 1 end if end if - if (pot == 'none') then + if (pot == '_none_') then write (*, *) 'FATAL: Variable "pot" not specified.Exiting now...' error = 1 end if @@ -956,8 +973,7 @@ subroutine check_inputsanity() error = 1 end if if (nyosh <= 1 .and. inose == 1) then - write (*, *) 'It is strongly reccommended to use Suzuki-Yoshida scheme'//& - &' when using Nose-Hoover thermostat (nyosh=3 or 7).' + write (*, *) 'It is strongly reccommended to use Suzuki-Yoshida scheme when using Nose-Hoover thermostat (nyosh=3 or 7).' write (*, *) iknow, error, chknow if (iknow /= 1) error = 1 end if @@ -1082,6 +1098,7 @@ subroutine print_logo() print '(a)', ' /_/ \_\ |_____/ |_| |_| \_|' print '(a)', ' ' +! TODO: Pass version as compiler parameter print '(a)', ' version 1.1' print '(a)', ' D. Hollas, J. Suchan, O. Svoboda, M. Oncak, P. Slavicek' print '(a)', ' ' @@ -1108,12 +1125,11 @@ end subroutine print_runtime_info end subroutine init -! TODO: Move to a separate file. subroutine finish(error_code) use mod_arrays, only: deallocate_arrays use mod_general use mod_files, only: MAXUNITS - use mod_nhc, only: inose, finalize_nhc + use mod_nhc !, only: finalize_nhc use mod_gle, only: finalize_gle use mod_estimators, only: h use mod_harmon, only: hess @@ -1192,13 +1208,13 @@ subroutine finish(error_code) call finalize_cp2k() end if - ! At last, we terminate MPI processes - ! TODO: We should have an MPI module handling this - ! TODO: We should check whether MPI was initialized with MPI_Init - ! before we attempt to call MPI_Finalize(). +! At last, we terminate MPI processes +! TODO: We should have an MPI module handling this +! TODO: We should check whether MPI was initialized with MPI_Init +! before we attempt to call MPI_Finalize(). #ifdef USE_MPI if (iremd == 1 .or. pot == '_tera_' .or. pot == '_cp2k_') then - if (error_code == 0 .and. pot /= "_cp2k_") then + if (error_code == 0 .and. pot /= "_cp2k_" .or. pot == '_tera_') then call MPI_Finalize(ierr) if (ierr /= MPI_SUCCESS) then write (*, '(A)') 'Bad signal from MPI_FINALIZE: ', ierr diff --git a/src/io.F90 b/src/io.F90 index b9143931..50f3d0b1 100644 --- a/src/io.F90 +++ b/src/io.F90 @@ -1,19 +1,26 @@ module mod_io use mod_const, only: DP implicit none - character(len=50) :: formats(10) + ! TODO: Use more descriptive names for these character(len=50) :: format1 = '(1I8)', format2 = '(1E12.4)', format3 = '(3E12.4)' character(len=50) :: format4 = '(1F8.4)', format5 = '(I8,I3)' contains + ! TODO: Create a ElectronicStructure type + ! and add reading and printint routines as type-bound procedures. + ! The instance of this type should be an array + ! corresponding to the number of beads. subroutine print_charges(charges, iw_ist) use mod_files, only: UCHARGES use mod_general, only: natom, it use mod_const, only: DP + ! Index iw_ist is indexing either PI beads + ! or electronic state in Surface Hopping. integer, intent(in) :: iw_ist real(KIND=DP), intent(in) :: charges(:) integer :: iat + ! TODO: Print time in fs, not time step write (UCHARGES, format5, advance='no') it, iw_ist do iat = 1, natom write (UCHARGES, format4, advance='no') charges(iat) @@ -30,8 +37,11 @@ subroutine print_dipoles(dipoles, iw, nstates) real(KIND=DP) :: total_dip integer :: ind - write (UDIP, format1, advance='no') it, iw + ! TODO: Print time in fs, not time step + write (UDIP, format5, advance='no') it, iw + ! TODO: Why the heck are we printing total dipmom fist? + ! Most QM program print Dx, Dy, Dz components first. do ind = 1, 3 * nstates, 3 total_dip = dipoles(ind)**2 + dipoles(ind + 1)**2 + dipoles(ind + 2)**2 total_dip = dsqrt(total_dip) @@ -42,7 +52,6 @@ subroutine print_dipoles(dipoles, iw, nstates) write (UDIP, format3, advance="no") dipoles(ind), dipoles(ind + 1), dipoles(ind + 2) end do write (UDIP, *) - end subroutine print_dipoles subroutine print_transdipoles(tdipoles, istate, ntdip) @@ -73,10 +82,10 @@ function read_forces(fx, fy, fz, num_atom, iw, funit) result(iost) integer, intent(in) :: iw, funit, num_atom integer :: iat, iost + iost = 0 ! For SH or EH, when we did not calculate forces... ! Needs to be rewritten anyway... if (iw < 1) then - iost = 0 return end if diff --git a/src/modules.F90 b/src/modules.F90 index 2c55c1ff..00597b66 100644 --- a/src/modules.F90 +++ b/src/modules.F90 @@ -68,9 +68,9 @@ module mod_general ! PIMD parameters, staging transformation, number of beads, NM transform integer :: istage = 0, nwalk = 1, inormalmodes = 0 ! Ab-initio potential - character(len=15) :: pot = 'none' - ! Ab initio potential for a reference in Multile time step propagator - character(len=15) :: pot_ref = 'none' + character(len=15) :: pot = '_none_' + ! Reference potential for a multiple time step propagator + character(len=15) :: pot_ref = '_none_' ! imini keyword is mostly deprecated integer :: imini = 0 ! number of time steps (length of simulation) @@ -197,7 +197,7 @@ subroutine mass_init(masses, massnames) allocate (am(natom)) am = -1.0D0 do i = 1, natom - select case(names(i)) + select case (names(i)) case ('H') am(i) = 1.008D0 case ('H1') @@ -606,6 +606,15 @@ subroutine files_init(isbc, phase, ndist, nang, ndih) end if end if + if (ipimd /= 2 .and. pot == '_tera_') then + open (UCHARGES, file=chfiles(UCHARGES), access=chaccess, action='write') + write (UCHARGES, *) '# Atomic Mulliken charges from current electronic state' + write (UCHARGES, *) '# Time_step Bead_index ', (names(i), i=1, natom) + + open (UDIP, file=chfiles(UDIP), access=chaccess, action='write') + write (UDIP, *) '# Time |D| Dx Dy Dz' + end if + if (isbc == 1) then open (URADIUS, file=chfiles(URADIUS), access=chaccess, action='write') write (URADIUS, *) '#TimeStep Radius[ANG] approximate density[kg.m^3]' diff --git a/src/read_cmdline.F90 b/src/read_cmdline.F90 index 36772b10..bcac8b53 100644 --- a/src/read_cmdline.F90 +++ b/src/read_cmdline.F90 @@ -24,9 +24,9 @@ subroutine print_help() print '(a)', '' end subroutine print_help - subroutine get_cmdline(chinput, chcoords, chveloc, tc_port) + subroutine get_cmdline(chinput, chcoords, chveloc, tc_server_name) use mod_utils, only: abinerror, file_exists_or_exit - character(len=*), intent(inout) :: chinput, chcoords, chveloc, tc_port + character(len=*), intent(inout) :: chinput, chcoords, chveloc, tc_server_name character(len=len(chinput)) :: arg integer :: i @@ -59,8 +59,7 @@ subroutine get_cmdline(chinput, chcoords, chveloc, tc_port) case ('-M') i = i + 1 call get_command_argument(i, arg) - read (arg, '(A)') tc_port - tc_port = trim(tc_port) + read (arg, '(A)') tc_server_name case default write (*, '(A)') 'Invalid command line argument '//arg call print_help() diff --git a/src/tera_mpi_api.F90 b/src/tera_mpi_api.F90 new file mode 100644 index 00000000..3de95e76 --- /dev/null +++ b/src/tera_mpi_api.F90 @@ -0,0 +1,435 @@ +! Common subroutines for MPI interface with TeraChem +module mod_terampi + use mod_const, only: DP +#ifdef USE_MPI + use mpi +#endif + implicit none + private + integer, parameter :: MPI_TAG_EXIT = 0 + integer, parameter :: MPI_TAG_ERROR = 13 + integer, parameter :: MPI_TAG_GRADIENT = 2 + ! Default tag that we send to TC + integer, parameter :: TC_TAG = MPI_TAG_GRADIENT + ! By default, take port name from a file + character(len=*), parameter :: TC_PORT_FILE_NAME = 'port.txt.' + + integer :: nteraservers = 1 + ! How long do we wait for TC port [seconds] + real(DP) :: max_wait_time = 30 + ! Sleep interval while waiting for TC calculation to finish. + real(DP) :: mpi_sleep = 0.05 + + public :: nteraservers + public :: mpi_sleep, max_wait_time + public :: TC_TAG +#ifdef USE_MPI + integer, allocatable :: tc_comms(:) + + ! TODO: Move handle_mpi_error and check_recv_count to a dedicated MPI module + public :: handle_mpi_error, check_recv_count + public :: get_tc_communicator + public :: wait_for_terachem + public :: finalize_terachem + public :: send_natom, send_atom_types_and_scrdir, send_coordinates +#endif + public :: initialize_tc_servers, initialize_terachem_interface + save + +contains + +#ifdef USE_MPI + + subroutine initialize_terachem_interface(tc_server_name) + use mod_general, only: nwalk + character(len=*) :: tc_server_name + integer :: i, ierr + + if (nwalk > 1) then + write (*, '(A)') 'WARNING: You are using PIMD with direct TeraChem interface.' + write (*, '(A)') 'You should have "integrator regular" in the TeraChem input file' + end if + write (*, '(A,I0)') 'Number of TeraChem servers: ', nteraservers + + if (mpi_sleep <= 0) then + write (*, *) 'WARNING: Parameter "mpi_sleep" must be positive!' + write (*, *) 'Setting it back to default value' + mpi_sleep = 0.05D0 + end if + + allocate (tc_comms(nteraservers)) + tc_comms = MPI_COMM_NULL + + ! Setting MPI_ERRORS_RETURN error handler allows us to retry + ! failed MPI_LOOKUP_NAME() call. It also allows us + ! to send the exit signal to TeraChem upon encoutering an error. + + ! It might also be a good idea to write our own error handler by MPI_Errorhandler_Create() + ! https://www.open-mpi.org/doc/current/man3/MPI_Comm_create_errhandler.3.php + ! so that we don't have to call handle_mpi_error() after each MPI call. + ! This error handler should call abinerror() and if possible should try send + ! the error shutdown MPI_Send to TC (though we'd need to make sure we don't + ! enter some weird endless loop!). + call MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_RETURN, ierr) + call handle_mpi_error(ierr) + + ! Parallel calls to MPI_Comm_connect often lead to segfault, + ! maybe a bug in MPICH. Commenting out until we debug further. + !!$OMP PARALLEL DO PRIVATE(i) + do i = 1, nteraservers + call connect_tc_server(trim(tc_server_name), i) + end do + !!$OMP END PARALLEL DO + end subroutine initialize_terachem_interface + + subroutine connect_tc_server(tc_server_name, itera) + use mod_utils, only: abinerror + ! TODO: Figure out how to handle REMD + ! use mod_general, only: iremd, my_rank + character(len=*), intent(in) :: tc_server_name + integer, intent(in) :: itera + character(len=MPI_MAX_PORT_NAME) :: port_name + integer :: ierr, newcomm + character(len=1024) :: server_name + character(len=1024) :: portfile + + if (tc_server_name /= '') then + + write (server_name, '(A,I0)') trim(adjustl(tc_server_name))//'.', itera + call lookup_port_via_nameserver(trim(server_name), port_name) + + else + + write (portfile, '(A,I0)') TC_PORT_FILE_NAME, itera + call read_tc_port_from_file(trim(portfile), port_name) + + end if + + write (6, '(2A)') 'Found TeraChem port: ', trim(port_name) + write (6, '(A)') 'Establishing connection...' + call flush (6) + + ! Establish new communicator via port name + call MPI_Comm_connect(trim(port_name), MPI_INFO_NULL, 0, MPI_COMM_SELF, newcomm, ierr) + call handle_mpi_error(ierr) + write (6, '(A)') 'Connection established!' + + tc_comms(itera) = newcomm + end subroutine connect_tc_server + + integer function get_tc_communicator(itera) result(tc_comm) + integer, intent(in) :: itera + tc_comm = tc_comms(itera) + end function get_tc_communicator + + ! Look for server_name via MPI nameserver, get port name + subroutine lookup_port_via_nameserver(server_name, port_name) + use mod_general, only: idebug + use mod_utils, only: abinerror + character(len=*), intent(in) :: server_name + character(len=MPI_MAX_PORT_NAME), intent(out) :: port_name + real(DP) :: timer + integer :: ierr + + port_name = '' + + write (*, '(2A)') 'Looking up TeraChem server under name:', server_name + call flush (6) + + timer = MPI_WTIME() + + do + + call MPI_LOOKUP_NAME(server_name, MPI_INFO_NULL, port_name, ierr) + if (ierr == MPI_SUCCESS) then + ! Workaround for a bug in hydra_nameserver for MPICH versions < 3.3 + if (len_trim(port_name) == 0) then + write (*, '(A)') 'Found empty port, retrying...' + else + exit + end if + end if + + ! Timeout after max_wait_time seconds + if ((MPI_WTIME() - timer) > max_wait_time) then + write (*, *) 'Server name '//server_name//' not found.' + call abinerror("lookup_port_via_nameserver") + end if + + ! Let's wait a bit since too many calls + ! to MPI_LOOKUP_NAME() can crash the hydra_nameserver process + if (idebug > 1) then + write (*, '(A)') 'Waiting for TC port' + end if + ! TODO: Try out how long should we sleep here. + call system('sleep 0.5') + + end do + + end subroutine lookup_port_via_nameserver + + ! Read TeraChem port from a file. + ! TeraChem prints it's port to STDOUT, where the launch script + ! can grep it into a file, which is read here. + ! This is more portable then the nameserver approach, + ! but have to rely on HDD. + subroutine read_tc_port_from_file(portfile, port_name) + use mod_utils, only: abinerror + character(len=*), intent(in) :: portfile + character(len=MPI_MAX_PORT_NAME), intent(out) :: port_name + integer :: iunit, iost + real(DP) :: timer + + write (*, '(A)') 'Reading TeraChem port name from file '//portfile + port_name = '' + timer = MPI_WTIME() + + do + open (newunit=iunit, file=portfile, action="read", status="old", iostat=iost) + if (iost == 0) then + exit + end if + + if ((MPI_WTIME() - timer) > max_wait_time) then + write (*, '(A)') 'ERROR: Could not open file '//portfile + call abinerror('read_tc_port_from_file') + end if + + write (*, '(A)') 'WARNING: Cannot open file '//portfile + call system('sleep 0.5') + + end do + + read (iunit, '(A)', iostat=iost) port_name + if (iost /= 0) then + write (*, '(A)') 'ERROR reading file '//portfile + call abinerror('read_tc_port_from_file') + end if + + close (iunit, status='delete') + end subroutine read_tc_port_from_file + + subroutine initialize_tc_servers() + use mod_qmmm, only: natqm + use mod_system, only: names + integer :: itera + + !$OMP PARALLEL DO + do itera = 1, nteraservers + call send_natom(natqm, tc_comms(itera)) + + call send_atom_types_and_scrdir(names, natqm, 0, tc_comms(itera), .false.) + end do + !$OMP END PARALLEL DO + end subroutine initialize_tc_servers + + subroutine finalize_terachem(abin_error_code) + integer, intent(in) :: abin_error_code + integer :: itera, ierr, empty + + ! Make sure we send MPI_TAG_EXIT to all servers. + call MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_RETURN, ierr) + do itera = 1, nteraservers + + write (*, '(A,I0)') 'Shutting down TeraChem server id = ', itera + if (abin_error_code == 0) then + call MPI_Send(empty, 0, MPI_INTEGER, 0, MPI_TAG_EXIT, tc_comms(itera), ierr) + else + call MPI_Send(empty, 0, MPI_INTEGER, 0, MPI_TAG_ERROR, tc_comms(itera), ierr) + end if + if (ierr /= MPI_SUCCESS) then + write (*, '(A,I0)') 'MPI ERROR during shutdown of TeraChem server id = ', itera + write (*, '(A)') 'Verify manually that the TeraChem server was terminated.' + call print_mpi_error(ierr) + end if + + call MPI_Comm_free(tc_comms(itera), ierr) + if (ierr /= MPI_SUCCESS) then + call print_mpi_error(ierr) + end if + + end do + + deallocate (tc_comms) + end subroutine finalize_terachem + + subroutine print_mpi_error(mpi_err) + integer, intent(in) :: mpi_err + character(len=MPI_MAX_ERROR_STRING) :: error_string + integer :: ierr, result_len + + call MPI_Error_string(mpi_err, error_string, result_len, ierr) + if (ierr == MPI_SUCCESS) then + write (*, '(A)') trim(error_string) + end if + end subroutine print_mpi_error + + subroutine handle_mpi_error(mpi_err) + use mod_utils, only: abinerror + integer, intent(in) :: mpi_err + + if (mpi_err /= MPI_SUCCESS) then + call print_mpi_error(mpi_err) + call abinerror('handle_mpi_error') + end if + end subroutine handle_mpi_error + + ! TODO: Move this to mpi_wrapper module + subroutine check_recv_count(mpi_status, expected_count, datatype) + use mod_utils, only: abinerror + integer, intent(in) :: mpi_status(:) + integer, intent(in) :: expected_count + integer, intent(in) :: datatype ! e.g. MPI_INTEGER + integer :: recv_count + integer :: ierr + + call MPI_Get_count(mpi_status, datatype, recv_count, ierr) + call handle_mpi_error(ierr) + if (recv_count /= expected_count) then + write (*, *) 'ERROR: MPI_Recv failed' + write (*, '(A,I0,A,I0)') 'Received ', recv_count, ' bytes, expected ', expected_count + call abinerror('check_recv_count') + end if + end subroutine check_recv_count + + subroutine wait_for_terachem(tc_comm) + integer, intent(in) :: tc_comm + character(len=20) :: chsys_sleep + integer :: status(MPI_STATUS_SIZE) + integer :: ierr + logical :: ready + ! TODO: we need to somehow make sure that + ! we don't wait forever if TeraChem crashes. + ! At this moment, this is ensured at the BASH script level. + + ! The idea here is to reduce the CPU usage of MPI_Recv() via system call to 'sleep'. + ! In most MPI implementations, MPI_Recv() is actively polling the other end + ! (in this case TeraChem) and consumes a whole CPU core. That's clearly wasteful, + ! since we're waiting for a long time for the ab initio result. + + ! Some implementation provide an option to change this behaviour, + ! but I didn't figure out any for MPICH so we rather inelegantly call system sleep. + ! Based according to an answer here: + ! http://stackoverflow.com/questions/14560714/probe-seems-to-consume-the-cpu + ready = .false. + write (chsys_sleep, '(A6, F10.4)') 'sleep ', mpi_sleep + do while (.not. ready) + call MPI_IProbe(MPI_ANY_SOURCE, MPI_ANY_TAG, tc_comm, ready, status, ierr) + call handle_mpi_error(ierr) + call system(trim(chsys_sleep)) + end do + end subroutine wait_for_terachem + + subroutine append_scrdir_name(buffer, offset, iw, remd_replica) + use mod_general, only: iremd + character(len=*), intent(inout) :: buffer + integer, intent(in) :: offset + integer, intent(in) :: iw + integer, intent(in) :: remd_replica + integer :: i + + i = offset + write (buffer(i + 1:i + 12), '(A8, I4.4)') '++scrdir', iw + if (iremd == 1) then + write (buffer(i + 13:i + 23), '(A4, I4.4, A2)') 'rank', remd_replica, '++' + else + write (buffer(i + 13:i + 14), '(A2)') '++' + end if + end subroutine append_scrdir_name + + subroutine send_natom(num_atom, tc_comm) + use mod_general, only: idebug + integer, intent(in) :: num_atom + integer, intent(in) :: tc_comm + integer :: ierr + + ! Send natqm and the type of each qmatom + if (idebug > 1) then + write (6, '(A, I0)') 'Sending number of atoms = ', num_atom + call flush (6) + end if + call MPI_Send(num_atom, 1, MPI_INTEGER, 0, TC_TAG, tc_comm, ierr) + call handle_mpi_error(ierr) + end subroutine send_natom + + subroutine send_atom_types_and_scrdir(at_names, num_atom, iw, tc_comm, send_scrdir) + use mod_general, only: my_rank, idebug + character(len=2), intent(in) :: at_names(:) + logical, intent(in) :: send_scrdir + integer, intent(in) :: num_atom + integer, intent(in) :: iw + integer, intent(in) :: tc_comm + integer, parameter :: MAX_SCRDIR_LEN = 30 + character(len=2*num_atom + MAX_SCRDIR_LEN) :: buffer + integer :: ierr, offset, iat + integer :: num_char + + buffer = '' + offset = 1 + do iat = 1, num_atom + write (buffer(offset:offset + 1), '(A2)') at_names(iat) + offset = offset + 2 + end do + num_char = num_atom * 2 + + if (send_scrdir) then + call append_scrdir_name(buffer, num_atom * 2, iw, my_rank) + num_char = len_trim(buffer) + end if + + if (idebug > 1) then + write (6, '(A)') 'Sending QM atom types: ' + write (*, '(A)') trim(buffer) + call flush (6) + end if + + call MPI_Send(buffer, num_char, MPI_CHARACTER, 0, TC_TAG, tc_comm, ierr) + call handle_mpi_error(ierr) + end subroutine send_atom_types_and_scrdir + + subroutine send_coordinates(x, y, z, num_atom, iw, tc_comm) + use mod_general, only: idebug + use mod_const, only: ANG + real(DP), intent(in) :: x(:, :), y(:, :), z(:, :) + integer, intent(in) :: num_atom + integer, intent(in) :: iw + integer, intent(in) :: tc_comm + real(DP), allocatable :: coords(:, :) + integer :: ierr, iat + + allocate (coords(3, num_atom)) + do iat = 1, num_atom + coords(1, iat) = x(iat, iw) / ANG + coords(2, iat) = y(iat, iw) / ANG + coords(3, iat) = z(iat, iw) / ANG + end do + + if (idebug > 1) then + write (6, '(A)') 'Sending QM coords: ' + do iat = 1, num_atom + write (6, *) 'Atom ', iat, ': ', coords(:, iat) + call flush (6) + end do + end if + call MPI_Send(coords, num_atom * 3, MPI_DOUBLE_PRECISION, 0, TC_TAG, tc_comm, ierr) + call handle_mpi_error(ierr) + end subroutine send_coordinates + +#else + + subroutine initialize_tc_servers() + use mod_utils, only: not_compiled_with + call not_compiled_with('MPI', 'initialize_tc_servers') + end subroutine initialize_tc_servers + + subroutine initialize_terachem_interface(tc_server_name) + use mod_utils, only: not_compiled_with + character(len=*), intent(in) :: tc_server_name + write (*, *) 'TC_SERVER_NAME=', tc_server_name + call not_compiled_with('MPI', 'initialize_terachem_interface') + end subroutine initialize_terachem_interface + +! USE_MPI +#endif + +end module mod_terampi diff --git a/tests/.gitignore b/tests/.gitignore index da1ad04c..48b3e7d1 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -11,7 +11,7 @@ */geom.dat.00* */geom_mm.dat.00* REMD/DFTB/OUT* -*/abin.out +*/abin.out* REMD/restart.xyz.??.* */restart.xyz.old */restart.xyz.0? @@ -21,6 +21,17 @@ REMD/restart.xyz.??.* */ERROR SH_BUTCHER/geom.?.?.? CP2K*/WATER* + # PLUMED backup files bck.* + +# Files from TERAPI* tests +tc_server? +tc_server +tc.out* +port.txt* +TC_ERROR? +ABIN_ERROR? +scrdir???? + !mini.xyz diff --git a/tests/CMDLINE/ERROR.ref b/tests/CMDLINE/ERROR.ref new file mode 100644 index 00000000..749f4b92 --- /dev/null +++ b/tests/CMDLINE/ERROR.ref @@ -0,0 +1,2 @@ + FATAL ERROR encountered in subroutine: get_cmdline + Check standard output for further information. diff --git a/tests/CMDLINE/test.sh b/tests/CMDLINE/test.sh new file mode 100755 index 00000000..18f8b8c5 --- /dev/null +++ b/tests/CMDLINE/test.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -euo pipefail + +rm -f ERROR abin.out +if [[ $1 = "clean" ]];then + exit 0 +fi + +ABINEXE=$1 + +# Test that ABIN fails with invalid cmdline argument +$ABINEXE -invalid > abin.out || true + +# Test that ABIN prints help, without an error +$ABINEXE -h >> abin.out || echo "ERROR when printing help" >> ERROR diff --git a/tests/TERAPI-FAILS/ABIN_ERROR1.ref b/tests/TERAPI-FAILS/ABIN_ERROR1.ref new file mode 100644 index 00000000..b2a86acd --- /dev/null +++ b/tests/TERAPI-FAILS/ABIN_ERROR1.ref @@ -0,0 +1,2 @@ + FATAL ERROR encountered in subroutine: force_tera + Check standard output for further information. diff --git a/tests/TERAPI-FAILS/ABIN_ERROR2.ref b/tests/TERAPI-FAILS/ABIN_ERROR2.ref new file mode 100644 index 00000000..a0efcec7 --- /dev/null +++ b/tests/TERAPI-FAILS/ABIN_ERROR2.ref @@ -0,0 +1,2 @@ + FATAL ERROR encountered in subroutine: check_inputsanity + Check standard output for further information. diff --git a/tests/TERAPI-FAILS/ABIN_ERROR3.ref b/tests/TERAPI-FAILS/ABIN_ERROR3.ref new file mode 100644 index 00000000..a0efcec7 --- /dev/null +++ b/tests/TERAPI-FAILS/ABIN_ERROR3.ref @@ -0,0 +1,2 @@ + FATAL ERROR encountered in subroutine: check_inputsanity + Check standard output for further information. diff --git a/tests/TERAPI-FAILS/ABIN_ERROR4.ref b/tests/TERAPI-FAILS/ABIN_ERROR4.ref new file mode 100644 index 00000000..13aaa5ab --- /dev/null +++ b/tests/TERAPI-FAILS/ABIN_ERROR4.ref @@ -0,0 +1,2 @@ + FATAL ERROR encountered in subroutine: handle_mpi_error + Check standard output for further information. diff --git a/tests/TERAPI-FAILS/ABIN_ERROR5.ref b/tests/TERAPI-FAILS/ABIN_ERROR5.ref new file mode 100644 index 00000000..84d490e8 --- /dev/null +++ b/tests/TERAPI-FAILS/ABIN_ERROR5.ref @@ -0,0 +1,2 @@ + FATAL ERROR encountered in subroutine: lookup_port_via_nameserver + Check standard output for further information. diff --git a/tests/TERAPI-FAILS/ABIN_ERROR6.ref b/tests/TERAPI-FAILS/ABIN_ERROR6.ref new file mode 100644 index 00000000..3620bb3a --- /dev/null +++ b/tests/TERAPI-FAILS/ABIN_ERROR6.ref @@ -0,0 +1,2 @@ + FATAL ERROR encountered in subroutine: read_tc_port_from_file + Check standard output for further information. diff --git a/tests/TERAPI-FAILS/ABIN_ERROR7.ref b/tests/TERAPI-FAILS/ABIN_ERROR7.ref new file mode 100644 index 00000000..3620bb3a --- /dev/null +++ b/tests/TERAPI-FAILS/ABIN_ERROR7.ref @@ -0,0 +1,2 @@ + FATAL ERROR encountered in subroutine: read_tc_port_from_file + Check standard output for further information. diff --git a/tests/TERAPI-FAILS/ABIN_ERROR8.ref b/tests/TERAPI-FAILS/ABIN_ERROR8.ref new file mode 100644 index 00000000..69c2791f --- /dev/null +++ b/tests/TERAPI-FAILS/ABIN_ERROR8.ref @@ -0,0 +1,2 @@ + FATAL ERROR encountered in subroutine: check_recv_count + Check standard output for further information. diff --git a/tests/TERAPI-FAILS/TC_ERROR1.ref b/tests/TERAPI-FAILS/TC_ERROR1.ref new file mode 100644 index 00000000..b37e4648 --- /dev/null +++ b/tests/TERAPI-FAILS/TC_ERROR1.ref @@ -0,0 +1 @@ + what(): Client sent an error tag. diff --git a/tests/TERAPI-FAILS/TC_ERROR2.ref b/tests/TERAPI-FAILS/TC_ERROR2.ref new file mode 100644 index 00000000..b37e4648 --- /dev/null +++ b/tests/TERAPI-FAILS/TC_ERROR2.ref @@ -0,0 +1 @@ + what(): Client sent an error tag. diff --git a/tests/TERAPI-FAILS/TC_ERROR3.ref b/tests/TERAPI-FAILS/TC_ERROR3.ref new file mode 100644 index 00000000..d83d66bd --- /dev/null +++ b/tests/TERAPI-FAILS/TC_ERROR3.ref @@ -0,0 +1,3 @@ +tc.out3.1: what(): Client sent an error tag. +tc.out3.2: what(): Client sent an error tag. +tc.out3.3: what(): Client sent an error tag. diff --git a/tests/TERAPI-FAILS/TC_ERROR4.ref b/tests/TERAPI-FAILS/TC_ERROR4.ref new file mode 100644 index 00000000..b37e4648 --- /dev/null +++ b/tests/TERAPI-FAILS/TC_ERROR4.ref @@ -0,0 +1 @@ + what(): Client sent an error tag. diff --git a/tests/TERAPI-FAILS/TC_ERROR8.ref b/tests/TERAPI-FAILS/TC_ERROR8.ref new file mode 100644 index 00000000..b37e4648 --- /dev/null +++ b/tests/TERAPI-FAILS/TC_ERROR8.ref @@ -0,0 +1 @@ + what(): Client sent an error tag. diff --git a/tests/TERAPI-FAILS/input.in.valid b/tests/TERAPI-FAILS/input.in.valid new file mode 100644 index 00000000..ba7f7f0f --- /dev/null +++ b/tests/TERAPI-FAILS/input.in.valid @@ -0,0 +1,22 @@ +&general +mpi_sleep=-1 + +pot='_tera_' +watpot=1 +ipimd=0, +nstep=1, +dt=40., +irandom=13131313, + +nwrite=1, +nwritef=1, +nwritev=1, +nwritex=1, +nrest=1, +idebug=3 +/ + +&nhcopt +inose=0, +temp=0.0d0 +/ diff --git a/tests/TERAPI-FAILS/input.in1 b/tests/TERAPI-FAILS/input.in1 new file mode 120000 index 00000000..fe06409c --- /dev/null +++ b/tests/TERAPI-FAILS/input.in1 @@ -0,0 +1 @@ +input.in.valid \ No newline at end of file diff --git a/tests/TERAPI-FAILS/input.in2 b/tests/TERAPI-FAILS/input.in2 new file mode 100644 index 00000000..e89d7e30 --- /dev/null +++ b/tests/TERAPI-FAILS/input.in2 @@ -0,0 +1,18 @@ +Intentionally invalid ABIN input +(requires PIMD without thermostat) + +&general +mpi_sleep=-1 + +pot='_tera_' +ipimd=1, +nstep=1, +dt=20., +irandom=13131313, +idebug=3 +/ + +&nhcopt +inose=0, +temp=0.0d0 +/ diff --git a/tests/TERAPI-FAILS/input.in3 b/tests/TERAPI-FAILS/input.in3 new file mode 100644 index 00000000..df2eb99e --- /dev/null +++ b/tests/TERAPI-FAILS/input.in3 @@ -0,0 +1,20 @@ +Intentionally invalid ABIN input +nteraservers > nwalk + +&general +pot='_tera_' +ipimd=1, +istage=1 +nwalk=2 +nteraservers=3 + +dt=20 +nstep=1, +irandom=13131313, +idebug=3 +/ + +&nhcopt +inose=1, +temp=300 +/ diff --git a/tests/TERAPI-FAILS/input.in4 b/tests/TERAPI-FAILS/input.in4 new file mode 120000 index 00000000..fe06409c --- /dev/null +++ b/tests/TERAPI-FAILS/input.in4 @@ -0,0 +1 @@ +input.in.valid \ No newline at end of file diff --git a/tests/TERAPI-FAILS/input.in5 b/tests/TERAPI-FAILS/input.in5 new file mode 100644 index 00000000..aba17f58 --- /dev/null +++ b/tests/TERAPI-FAILS/input.in5 @@ -0,0 +1,11 @@ +&general +max_wait_time=0.5 + +pot='_tera_' +ipimd=0, +nstep=1, +dt=40., +irandom=13131313, + +idebug=3 +/ diff --git a/tests/TERAPI-FAILS/input.in6 b/tests/TERAPI-FAILS/input.in6 new file mode 100644 index 00000000..aba17f58 --- /dev/null +++ b/tests/TERAPI-FAILS/input.in6 @@ -0,0 +1,11 @@ +&general +max_wait_time=0.5 + +pot='_tera_' +ipimd=0, +nstep=1, +dt=40., +irandom=13131313, + +idebug=3 +/ diff --git a/tests/TERAPI-FAILS/input.in7 b/tests/TERAPI-FAILS/input.in7 new file mode 120000 index 00000000..fe06409c --- /dev/null +++ b/tests/TERAPI-FAILS/input.in7 @@ -0,0 +1 @@ +input.in.valid \ No newline at end of file diff --git a/tests/TERAPI-FAILS/input.in8 b/tests/TERAPI-FAILS/input.in8 new file mode 100644 index 00000000..ba7f7f0f --- /dev/null +++ b/tests/TERAPI-FAILS/input.in8 @@ -0,0 +1,22 @@ +&general +mpi_sleep=-1 + +pot='_tera_' +watpot=1 +ipimd=0, +nstep=1, +dt=40., +irandom=13131313, + +nwrite=1, +nwritef=1, +nwritev=1, +nwritex=1, +nrest=1, +idebug=3 +/ + +&nhcopt +inose=0, +temp=0.0d0 +/ diff --git a/tests/TERAPI-FAILS/mini.xyz b/tests/TERAPI-FAILS/mini.xyz new file mode 100644 index 00000000..77afbffe --- /dev/null +++ b/tests/TERAPI-FAILS/mini.xyz @@ -0,0 +1,5 @@ +3 + +O 0.000 0.118 0.000 +h 0.757 -0.472 0.000 +h -0.757 -0.472 0.000 diff --git a/tests/TERAPI-FAILS/tc_server.cpp b/tests/TERAPI-FAILS/tc_server.cpp new file mode 100644 index 00000000..b757cf00 --- /dev/null +++ b/tests/TERAPI-FAILS/tc_server.cpp @@ -0,0 +1,62 @@ +#include +#include + +#include "../tc_mpi_api.h" + +using namespace std; + +int main(int argc, char* argv[]) +{ + char *serverName = NULL; + + // Due to a bug in hydra_nameserver, it crashes + // when multiple TC servers call `MPI_Unpublish_name()` + // Hence, we want to allow invoking without this parameter, + // in which case TC server will just print the port to stdin, + // where it could be grepped and passed via file to ABIN, + // and it will never call MPI_Publish_name/MPI_Unpublish_name + // NOTE: This behaviour is different from real TC, + // which has default serverName and will always try to publish it. + if (argc > 2) { + printf("Only one cmdline argument supported, , but you provided more!"); + throw std::runtime_error("Incorrect invocation"); + } + + if (argc == 2) { + serverName = new char[1024]; + strcpy(serverName, argv[1]); + } + + TCServerMock tc = TCServerMock(serverName); + + if (serverName) { + delete[] serverName; + } + + tc.initializeCommunication(); + + tc.receiveNumAtoms(); + tc.receiveAtomTypes(); + + int loop_counter = 0; + int MAX_LOOP_COUNT = 100; + // Will go through this loop until MPI client gives an exit signal. + while (true) { + + int status = tc.receive(); + if (status == MPI_TAG_EXIT) { + break; + } + + tc.send(); + + // This is just a precaution, we don't want endless loop! + loop_counter++; + if (loop_counter > MAX_LOOP_COUNT) { + printf("Maximum number of steps exceeded!\n"); + return(1); + } + } + + return(0); +} diff --git a/tests/TERAPI-FAILS/tc_server1.cpp b/tests/TERAPI-FAILS/tc_server1.cpp new file mode 100644 index 00000000..e9d279fa --- /dev/null +++ b/tests/TERAPI-FAILS/tc_server1.cpp @@ -0,0 +1,81 @@ +#include +#include + +#include "../tc_mpi_api.h" + +// HERE we test how ABIN responds if we send the +// MPI_SCF_DIE (i.e. when SCF does not converge). +// ABIN should stop immediately. + +int main(int argc, char* argv[]) +{ + char *serverName = NULL; + + // Due to a bug in hydra_nameserver, it crashes + // when multiple TC servers call `MPI_Unpublish_name()` + // Hence, we want to allow invoking without this parameter, + // in which case TC server will just print the port to stdin, + // where it could be grepped and passed via file to ABIN, + // and it will never call MPI_Publish_name/MPI_Unpublish_name + // NOTE: This behaviour is different from real TC, + // which has default serverName and will always try to publish it. + if (argc > 2) { + printf("Only one cmdline argument supported, , but you provided more!"); + throw std::runtime_error("Incorrect invocation"); + } + + if (argc == 2) { + serverName = new char[1024]; + strcpy(serverName, argv[1]); + } + + TCServerMock tc = TCServerMock(serverName); + + if (serverName) { + delete[] serverName; + } + + tc.initializeCommunication(); + + tc.receiveNumAtoms(); + tc.receiveAtomTypes(); + + int loop_counter = 0; + int MAX_LOOP_COUNT = 100; + // Will go through this loop until MPI client gives an exit signal. + while (true) { + + int status = tc.receiveBeginLoop(); + if (status == MPI_TAG_EXIT) { + break; + } + + tc.receiveAtomTypesAndScrdir(); + tc.receiveCoordinates(); + + // Energies and gradients from qTIP4PF potential + double energy = tc.getWaterGradients(); + + // Let's simulate that SCF did not converge. + int MPI_SCF_DIE = 1; + tc.sendSCFEnergy(energy, MPI_SCF_DIE); + + tc.sendQMCharges(); + + tc.sendQMDipoleMoment(); + + // NOTE: In the real TC interface, gradients are sent + // conditionally only if they are requested. + // But we always request them in ABIN (see tc.receive). + tc.sendQMGradients(); + + // This is just a precaution, we don't want endless loop! + loop_counter++; + if (loop_counter > MAX_LOOP_COUNT) { + printf("Maximum number of steps exceeded!\n"); + return(1); + } + } + + return(0); +} diff --git a/tests/TERAPI-FAILS/tc_server4.cpp b/tests/TERAPI-FAILS/tc_server4.cpp new file mode 100644 index 00000000..fb78feb9 --- /dev/null +++ b/tests/TERAPI-FAILS/tc_server4.cpp @@ -0,0 +1,56 @@ +#include +#include + +#include "../tc_mpi_api.h" + +using namespace std; + +int main(int argc, char* argv[]) +{ + char *serverName = NULL; + + // Due to a bug in hydra_nameserver, it crashes + // when multiple TC servers call `MPI_Unpublish_name()` + // Hence, we want to allow invoking without this parameter, + // in which case TC server will just print the port to stdin, + // where it could be grepped and passed via file to ABIN, + // and it will never call MPI_Publish_name/MPI_Unpublish_name + // NOTE: This behaviour is different from real TC, + // which has default serverName and will always try to publish it. + if (argc > 2) { + printf("Only one cmdline argument supported, , but you provided more!"); + throw std::runtime_error("Incorrect invocation"); + } + + if (argc == 2) { + serverName = new char[1024]; + strcpy(serverName, argv[1]); + } + + TCServerMock tc = TCServerMock(serverName); + + if (serverName) { + delete[] serverName; + } + + tc.initializeCommunication(); + + tc.receiveNumAtoms(); + tc.receiveAtomTypes(); + + int status = tc.receive(); + if (status == MPI_TAG_EXIT) { + throw std::runtime_error("unexpected exit tag"); + } + + // At this point ABIN expects SCF energy. Let's send + // two doubles instead of one to throw it off! + printf("Sending invalid data, muhehehe!\n"); + MPI_Comm *abinComm = tc.getABINCommunicator(); + double energies[2] = {1, 2}; + MPI_Send(energies, 2, MPI_DOUBLE, 0, 0, *abinComm); + + // Calling receive since we're expecting MPI_ERROR_TAG from ABIN. + tc.receive(); + return(0); +} diff --git a/tests/TERAPI-FAILS/tc_server8.cpp b/tests/TERAPI-FAILS/tc_server8.cpp new file mode 100644 index 00000000..fa3617f4 --- /dev/null +++ b/tests/TERAPI-FAILS/tc_server8.cpp @@ -0,0 +1,56 @@ +#include +#include + +#include "../tc_mpi_api.h" + +using namespace std; + +int main(int argc, char* argv[]) +{ + char *serverName = NULL; + + // Due to a bug in hydra_nameserver, it crashes + // when multiple TC servers call `MPI_Unpublish_name()` + // Hence, we want to allow invoking without this parameter, + // in which case TC server will just print the port to stdin, + // where it could be grepped and passed via file to ABIN, + // and it will never call MPI_Publish_name/MPI_Unpublish_name + // NOTE: This behaviour is different from real TC, + // which has default serverName and will always try to publish it. + if (argc > 2) { + printf("Only one cmdline argument supported, , but you provided more!"); + throw std::runtime_error("Incorrect invocation"); + } + + if (argc == 2) { + serverName = new char[1024]; + strcpy(serverName, argv[1]); + } + + TCServerMock tc = TCServerMock(serverName); + + if (serverName) { + delete[] serverName; + } + + tc.initializeCommunication(); + + tc.receiveNumAtoms(); + tc.receiveAtomTypes(); + + int status = tc.receive(); + if (status == MPI_TAG_EXIT) { + throw std::runtime_error("unexpected exit tag"); + } + + // At this point ABIN expects SCF energy. Let's send + // zero doubles instead of one to throw it off! + printf("Sending invalid (zero bytes) data, muhehehe!\n"); + MPI_Comm *abinComm = tc.getABINCommunicator(); + double energies[1] = {1}; + MPI_Send(energies, 0, MPI_DOUBLE, 0, 0, *abinComm); + + // Calling receive since we're expecting MPI_ERROR_TAG from ABIN. + tc.receive(); + return(0); +} diff --git a/tests/TERAPI-FAILS/test.sh b/tests/TERAPI-FAILS/test.sh new file mode 100755 index 00000000..1e34f731 --- /dev/null +++ b/tests/TERAPI-FAILS/test.sh @@ -0,0 +1,59 @@ +#/bin/bash + +# The goal here is to test various failure modes +# and how TC and ABIN respond to them. + +# We're testing multiple things in this single test, +# and we will be collecting ABIN and TC error messages +# and compare them with the reference. +# This is not a perfect approach, but let's see how it works. +# Also, Codecov coverage will help us determine +# that we have hit all the paths. +set -euo pipefail + +export ABINEXE=$1 + +source ../test_tc_server_utils.sh + +set_default_vars +set_mpich_vars + +# If $1 = "clean"; exit early. +rm -f TC_ERROR? ABIN_ERROR? ${TCOUT}* ${ABINOUT}* +if ! clean_output_files $1; then + exit 0 +fi + +# Exit early for OpenMPI build. +check_for_openmpi + +# We relaunch the nameserver in each subtest +# due to the bug in hydra_nameserver +#launch_hydra_nameserver $MPICH_HYDRA + +# Compile default TC server +$MPICXX $TCSRC -Wall -o $TCEXE + +#trap cleanup INT ABRT TERM EXIT + +echo "########### SUBTEST 1 ###################" +./test1.sh +echo "########### SUBTEST 2 ###################" +./test2.sh || true +echo "########### SUBTEST 3 ###################" +./test3.sh +echo "########### SUBTEST 4 ###################" +./test4.sh +echo "########### SUBTEST 5 ###################" +./test5.sh +echo "########### SUBTEST 6 ###################" +./test6.sh +echo "########### SUBTEST 7 ###################" +./test7.sh +echo "########### SUBTEST 8 ###################" +./test8.sh + +# TODO: Check how tc_server handles bad input +# (again, we'll need a modified version) +# Basically we should test every assertion +# in the TCServerMock code. diff --git a/tests/TERAPI-FAILS/test1.sh b/tests/TERAPI-FAILS/test1.sh new file mode 100755 index 00000000..e7a48609 --- /dev/null +++ b/tests/TERAPI-FAILS/test1.sh @@ -0,0 +1,47 @@ +#/bin/bash + +# Test a scenario in which SCF does not converge +# and TeraChem sends the MPI_SCF_DIE tag to ABIN. +# In that case, ABIN should stop with an error. + +set -euo pipefail + +source ../test_tc_server_utils.sh + +IDX=1 +ABININ=input.in$IDX +ABINOUT=${ABINOUT}$IDX +TCOUT=${TCOUT}$IDX +TCSRC="../tc_mpi_api.cpp ../../water_potentials/qtip4pf.cpp tc_server$IDX.cpp" +TCEXE=tc_server$IDX + +# Compile fake TC server +$MPICXX $TCSRC -Wall -o $TCEXE + +launch_hydra_nameserver $MPICH_HYDRA + +hostname=$HOSTNAME +MPIRUN="$MPIRUN -nameserver $hostname -n 1" + +TC_PORT="test1.$$" +ABIN_CMD="$ABINEXE -i $ABININ -x $ABINGEOM -M $TC_PORT" +TC_CMD="./$TCEXE $TC_PORT.1" + +$MPIRUN $TC_CMD > $TCOUT 2>&1 || true & +tcpid=$! + +$MPIRUN $ABIN_CMD > $ABINOUT 2>&1 || true & +abinpid=$! + +function cleanup { + kill -9 $tcpid $abinpid > /dev/null 2>&1 || true + grep 'what()' $TCOUT > TC_ERROR$IDX + if [[ -f ERROR ]];then + mv ERROR ABIN_ERROR$IDX + fi + exit 0 +} + +trap cleanup INT ABRT TERM EXIT + +check_running_processes $abinpid $tcpid diff --git a/tests/TERAPI-FAILS/test2.sh b/tests/TERAPI-FAILS/test2.sh new file mode 100755 index 00000000..b6cab508 --- /dev/null +++ b/tests/TERAPI-FAILS/test2.sh @@ -0,0 +1,41 @@ +#/bin/bash + +# Test that ABIN sends TC error tag early +# when it fails during parsing its own input, +# so that the TC server exits gracefully. + +set -euo pipefail + +source ../test_tc_server_utils.sh + +IDX=2 +ABININ=input.in$IDX +ABINOUT=${ABINOUT}$IDX +TCOUT=${TCOUT}$IDX + +launch_hydra_nameserver $MPICH_HYDRA +hostname=$HOSTNAME +MPIRUN="$MPIRUN -nameserver $hostname -n 1" + +TC_PORT="test$IDX.$$" +ABIN_CMD="$ABINEXE -i $ABININ -x $ABINGEOM -M $TC_PORT" +TC_CMD="./$TCEXE $TC_PORT.1" + +$MPIRUN $TC_CMD > $TCOUT 2>&1 || true & +tcpid=$! + +$MPIRUN $ABIN_CMD > $ABINOUT 2>&1 || true & +abinpid=$! + +function cleanup { + kill -9 $tcpid $abinpid > /dev/null 2>&1 || true + grep 'what()' $TCOUT > TC_ERROR$IDX + if [[ -f ERROR ]];then + mv ERROR ABIN_ERROR$IDX + fi + exit 0 +} + +trap cleanup INT ABRT TERM EXIT + +check_running_processes $abinpid $tcpid diff --git a/tests/TERAPI-FAILS/test3.sh b/tests/TERAPI-FAILS/test3.sh new file mode 100755 index 00000000..78c5d724 --- /dev/null +++ b/tests/TERAPI-FAILS/test3.sh @@ -0,0 +1,42 @@ +#/bin/bash + +# Test that ABIN stops if nteraservers > nbeads + +set -euo pipefail +source ../test_tc_server_utils.sh + +IDX=3 +ABININ=input.in$IDX +ABINOUT=${ABINOUT}$IDX +TCOUT=${TCOUT}$IDX +N_TERA_SERVERS=$(egrep --only-matching 'nteraservers\s*=\s*[0-9]' $ABININ | egrep -o [0-9]) + +MPIRUN="$MPIRUN -n 1" +ABIN_CMD="$ABINEXE -i $ABININ -x $ABINGEOM" + +declare -A job_pids +for ((itera=1;itera<=N_TERA_SERVERS;itera++)) { + $MPIRUN ./$TCEXE > $TCOUT.$itera 2>&1 & + job_pids[$itera]=$! +} +sleep 1 +# Grep port names from TC outputs, pass to ABIN via a file. +for ((itera=1;itera<=N_TERA_SERVERS;itera++)) { + grep 'port name' $TCOUT.$itera | awk -F"port name: " '{print $2;exit}' > $TC_PORT_FILE.$itera +} + +$MPIRUN $ABIN_CMD > $ABINOUT 2>&1 & +job_pids[$(expr $N_TERA_SERVERS + 1)]=$! + +function cleanup { + kill -9 ${job_pids[@]} > /dev/null 2>&1 || true + grep 'what()' $TCOUT.* > TC_ERROR$IDX + if [[ -f ERROR ]];then + mv ERROR ABIN_ERROR$IDX + fi + exit 0 +} + +trap cleanup INT ABRT TERM EXIT + +check_running_processes ${job_pids[@]} diff --git a/tests/TERAPI-FAILS/test4.sh b/tests/TERAPI-FAILS/test4.sh new file mode 100755 index 00000000..0cd8b8c0 --- /dev/null +++ b/tests/TERAPI-FAILS/test4.sh @@ -0,0 +1,47 @@ +#/bin/bash + +# Test how ABIN handles MPI error. +# Concretely, how it handles if TC sends more data +# then expected (see tc_server4.cpp). + +set -euo pipefail + +source ../test_tc_server_utils.sh + +IDX=4 +ABININ=input.in$IDX +ABINOUT=${ABINOUT}$IDX +TCOUT=${TCOUT}$IDX +TCSRC="../tc_mpi_api.cpp ../../water_potentials/qtip4pf.cpp tc_server$IDX.cpp" +TCEXE=tc_server$IDX + +# Compile fake TC server +$MPICXX $TCSRC -Wall -o $TCEXE + +launch_hydra_nameserver $MPICH_HYDRA + +hostname=$HOSTNAME +MPIRUN="$MPIRUN -nameserver $hostname -n 1" + +TC_PORT="test$IDX.$$" +ABIN_CMD="$ABINEXE -i $ABININ -x $ABINGEOM -M $TC_PORT" +TC_CMD="./$TCEXE $TC_PORT.1" + +$MPIRUN $TC_CMD > $TCOUT 2>&1 || true & +tcpid=$! + +$MPIRUN $ABIN_CMD > $ABINOUT 2>&1 || true & +abinpid=$! + +function cleanup { + kill -9 $tcpid $abinpid > /dev/null 2>&1 || true + grep 'what()' $TCOUT > TC_ERROR$IDX + if [[ -f ERROR ]];then + mv ERROR ABIN_ERROR$IDX + fi + exit 0 +} + +trap cleanup INT ABRT TERM EXIT + +check_running_processes $abinpid $tcpid diff --git a/tests/TERAPI-FAILS/test5.sh b/tests/TERAPI-FAILS/test5.sh new file mode 100755 index 00000000..93f887cc --- /dev/null +++ b/tests/TERAPI-FAILS/test5.sh @@ -0,0 +1,35 @@ +#/bin/bash + +# Test that ABIN timeouts gracefully +# if it cannot find TC port via hydra_nameserver. + +set -euo pipefail + +source ../test_tc_server_utils.sh + +IDX=5 +ABININ=input.in$IDX +ABINOUT=${ABINOUT}$IDX + +launch_hydra_nameserver $MPICH_HYDRA + +hostname=$HOSTNAME +#MPIRUN="$MPIRUN -nameserver $hostname -n 1" + +TC_PORT="test$IDX.$$" +ABIN_CMD="$ABINEXE -i $ABININ -x $ABINGEOM -M $TC_PORT" + +$MPIRUN $ABIN_CMD > $ABINOUT 2>&1 || true & +abinpid=$! + +function cleanup { + kill -9 $abinpid > /dev/null 2>&1 || true + if [[ -f ERROR ]];then + mv ERROR ABIN_ERROR$IDX + fi + exit 0 +} + +trap cleanup INT ABRT TERM EXIT + +check_running_processes $abinpid diff --git a/tests/TERAPI-FAILS/test6.sh b/tests/TERAPI-FAILS/test6.sh new file mode 100755 index 00000000..84ed4394 --- /dev/null +++ b/tests/TERAPI-FAILS/test6.sh @@ -0,0 +1,33 @@ +#/bin/bash + +# Test that ABIN timeouts gracefully +# if it cannot open file with the TC port. + +set -euo pipefail + +source ../test_tc_server_utils.sh + +IDX=6 +ABININ=input.in$IDX +ABINOUT=${ABINOUT}$IDX + +launch_hydra_nameserver $MPICH_HYDRA + +MPIRUN="$MPIRUN -n 1" + +ABIN_CMD="$ABINEXE -i $ABININ -x $ABINGEOM" + +$MPIRUN $ABIN_CMD > $ABINOUT 2>&1 || true & +abinpid=$! + +function cleanup { + kill -9 $abinpid > /dev/null 2>&1 || true + if [[ -f ERROR ]];then + mv ERROR ABIN_ERROR$IDX + fi + exit 0 +} + +trap cleanup INT ABRT TERM EXIT + +check_running_processes $abinpid diff --git a/tests/TERAPI-FAILS/test7.sh b/tests/TERAPI-FAILS/test7.sh new file mode 100755 index 00000000..ffea5789 --- /dev/null +++ b/tests/TERAPI-FAILS/test7.sh @@ -0,0 +1,36 @@ +#/bin/bash + +# Test that ABIN exits gracefully +# when reading from empty TC port file. +touch port.txt.1 + +set -euo pipefail + +source ../test_tc_server_utils.sh + +IDX=7 +ABININ=input.in$IDX +ABINOUT=${ABINOUT}$IDX + +launch_hydra_nameserver $MPICH_HYDRA + +MPIRUN="$MPIRUN -n 1" + +ABIN_CMD="$ABINEXE -i $ABININ -x $ABINGEOM" + + +$MPIRUN $ABIN_CMD > $ABINOUT 2>&1 || true & +abinpid=$! + +function cleanup { + kill -9 $abinpid > /dev/null 2>&1 || true + rm -f port.txt.1 + if [[ -f ERROR ]];then + mv ERROR ABIN_ERROR$IDX + fi + exit 0 +} + +trap cleanup INT ABRT TERM EXIT + +check_running_processes $abinpid diff --git a/tests/TERAPI-FAILS/test8.sh b/tests/TERAPI-FAILS/test8.sh new file mode 100755 index 00000000..b0af1b9c --- /dev/null +++ b/tests/TERAPI-FAILS/test8.sh @@ -0,0 +1,47 @@ +#/bin/bash + +# Test how ABIN handles MPI error. +# Concretely, how it handles if TC sends less data +# than expected (see tc_server8.cpp). + +set -euo pipefail + +source ../test_tc_server_utils.sh + +IDX=8 +ABININ=input.in$IDX +ABINOUT=${ABINOUT}$IDX +TCOUT=${TCOUT}$IDX +TCSRC="../tc_mpi_api.cpp ../../water_potentials/qtip4pf.cpp tc_server$IDX.cpp" +TCEXE=tc_server$IDX + +# Compile fake TC server +$MPICXX $TCSRC -Wall -o $TCEXE + +launch_hydra_nameserver $MPICH_HYDRA + +hostname=$HOSTNAME +MPIRUN="$MPIRUN -nameserver $hostname -n 1" + +TC_PORT="test$IDX.$$" +ABIN_CMD="$ABINEXE -i $ABININ -x $ABINGEOM -M $TC_PORT" +TC_CMD="./$TCEXE $TC_PORT.1" + +$MPIRUN $TC_CMD > $TCOUT 2>&1 || true & +tcpid=$! + +$MPIRUN $ABIN_CMD > $ABINOUT 2>&1 || true & +abinpid=$! + +function cleanup { + kill -9 $tcpid $abinpid > /dev/null 2>&1 || true + grep 'what()' $TCOUT > TC_ERROR$IDX + if [[ -f ERROR ]];then + mv ERROR ABIN_ERROR$IDX + fi + exit 0 +} + +trap cleanup INT ABRT TERM EXIT + +check_running_processes $abinpid $tcpid diff --git a/tests/TERAPI-PIMD-PARALLEL/est_energy.dat.ref b/tests/TERAPI-PIMD-PARALLEL/est_energy.dat.ref new file mode 100644 index 00000000..701cea3c --- /dev/null +++ b/tests/TERAPI-PIMD-PARALLEL/est_energy.dat.ref @@ -0,0 +1,2 @@ + # Time[fs] E-potential E-primitive E-virial CumulAvg_prim CumulAvg_vir + 0.24 0.7962477515E-03 0.7962477515E-03 0.7962520021E-03 0.0000000000E+00 0.0000000000E+00 diff --git a/tests/TERAPI-PIMD-PARALLEL/forces.xyz.ref b/tests/TERAPI-PIMD-PARALLEL/forces.xyz.ref new file mode 100644 index 00000000..2bb720bf --- /dev/null +++ b/tests/TERAPI-PIMD-PARALLEL/forces.xyz.ref @@ -0,0 +1,14 @@ + 3 +net force: 0.00000E+00 0.00000E+00 0.00000E+00 torque force: 0.00000E+00 0.00000E+00 0.00000E+00 +O 0.0000000000E+00 -0.6945880791E-02 0.0000000000E+00 +H -0.2672605904E-02 0.3472940396E-02 0.0000000000E+00 +H 0.2672605904E-02 0.3472940396E-02 0.0000000000E+00 +O 0.0000000000E+00 -0.6916370894E-02 0.0000000000E+00 +H -0.2655760787E-02 0.3458185447E-02 0.0000000000E+00 +H 0.2655760787E-02 0.3458185447E-02 0.0000000000E+00 +O 0.0000000000E+00 -0.6906530387E-02 0.0000000000E+00 +H -0.2650142923E-02 0.3453265194E-02 0.0000000000E+00 +H 0.2650142923E-02 0.3453265194E-02 0.0000000000E+00 +O 0.0000000000E+00 -0.6916370894E-02 0.0000000000E+00 +H -0.2655760787E-02 0.3458185447E-02 0.0000000000E+00 +H 0.2655760787E-02 0.3458185447E-02 0.0000000000E+00 diff --git a/tests/TERAPI-PIMD-PARALLEL/input.in b/tests/TERAPI-PIMD-PARALLEL/input.in new file mode 100644 index 00000000..3eaf72bc --- /dev/null +++ b/tests/TERAPI-PIMD-PARALLEL/input.in @@ -0,0 +1,26 @@ +&general +pot='_tera_' +watpot=1 +ipimd=1, +istage=1 +nwalk=4, +nstep=1, +dt=10., +irandom=13131313, + +nwrite=1, +nwritef=1, +nwritev=1, +nwritex=1, +nrest=1, +idebug=3 + +nteraservers=4 + +iknow=1 ! We're running PIMD without thermostat ! +/ + +&nhcopt +inose=0, +temp=0.0, +/ diff --git a/tests/TERAPI-PIMD-PARALLEL/mini.xyz b/tests/TERAPI-PIMD-PARALLEL/mini.xyz new file mode 100644 index 00000000..77afbffe --- /dev/null +++ b/tests/TERAPI-PIMD-PARALLEL/mini.xyz @@ -0,0 +1,5 @@ +3 + +O 0.000 0.118 0.000 +h 0.757 -0.472 0.000 +h -0.757 -0.472 0.000 diff --git a/tests/TERAPI-PIMD-PARALLEL/movie.xyz.ref b/tests/TERAPI-PIMD-PARALLEL/movie.xyz.ref new file mode 100644 index 00000000..2892fae3 --- /dev/null +++ b/tests/TERAPI-PIMD-PARALLEL/movie.xyz.ref @@ -0,0 +1,20 @@ + 3 +Time step: 1 Sim. Time [au] 10.00 +O 0.00000000E+00 0.11797451E+00 0.00000000E+00 +H 0.75684348E+00 -0.47179770E+00 0.00000000E+00 +H -0.75684348E+00 -0.47179770E+00 0.00000000E+00 + 3 +Time step: 1 Sim. Time [au] 10.00 +O 0.00000000E+00 0.11796495E+00 0.00000000E+00 +H 0.75678478E+00 -0.47172184E+00 0.00000000E+00 +H -0.75678478E+00 -0.47172184E+00 0.00000000E+00 + 3 +Time step: 1 Sim. Time [au] 10.00 +O 0.00000000E+00 0.11796176E+00 0.00000000E+00 +H 0.75676522E+00 -0.47169655E+00 0.00000000E+00 +H -0.75676522E+00 -0.47169655E+00 0.00000000E+00 + 3 +Time step: 1 Sim. Time [au] 10.00 +O 0.00000000E+00 0.11796495E+00 0.00000000E+00 +H 0.75678478E+00 -0.47172184E+00 0.00000000E+00 +H -0.75678478E+00 -0.47172184E+00 0.00000000E+00 diff --git a/tests/TERAPI-PIMD-PARALLEL/restart.xyz.ref b/tests/TERAPI-PIMD-PARALLEL/restart.xyz.ref new file mode 100644 index 00000000..c2b9086a --- /dev/null +++ b/tests/TERAPI-PIMD-PARALLEL/restart.xyz.ref @@ -0,0 +1,1313 @@ + 1 10.000000000000000 + Cartesian Coordinates [au] + 0.0000000000000000 0.22293951200140324 0.0000000000000000 + 1.4302268996036975 -0.89156844370906640 0.0000000000000000 + -1.4302268996036975 -0.89156844370906640 0.0000000000000000 + 0.0000000000000000 0.22292144762229921 0.0000000000000000 + 1.4301159809857611 -0.89142508458144487 0.0000000000000000 + -1.4301159809857611 -0.89142508458144487 0.0000000000000000 + 0.0000000000000000 0.22291542616259788 0.0000000000000000 + 1.4300790081131156 -0.89137729820557099 0.0000000000000000 + -1.4300790081131156 -0.89137729820557099 0.0000000000000000 + 0.0000000000000000 0.22292144762229923 0.0000000000000000 + 1.4301159809857611 -0.89142508458144487 0.0000000000000000 + -1.4301159809857611 -0.89142508458144487 0.0000000000000000 + Cartesian Velocities [au] + 0.0000000000000000 -9.5635640087645337E-006 0.0000000000000000 + -5.8515535693924805E-005 7.5896557825507824E-005 0.0000000000000000 + 5.8515535693924805E-005 7.5896557825507824E-005 0.0000000000000000 + 0.0000000000000000 -1.3147792414895394E-005 0.0000000000000000 + -8.0439761551330711E-005 1.0434103712594810E-004 0.0000000000000000 + 8.0439761551330711E-005 1.0434103712594810E-004 0.0000000000000000 + 0.0000000000000000 -1.4341972858402675E-005 0.0000000000000000 + -8.7742741190639505E-005 1.1381806734205576E-004 0.0000000000000000 + 8.7742741190639505E-005 1.1381806734205576E-004 0.0000000000000000 + 0.0000000000000000 -1.3147792414895394E-005 0.0000000000000000 + -8.0439761551330711E-005 1.0434103712594812E-004 0.0000000000000000 + 8.0439761551330711E-005 1.0434103712594812E-004 0.0000000000000000 + Cumulative averages of various estimators + 1.3273231226437057E-006 + 0.0000000000000000 0.0000000000000000 + 0.0000000000000000 +PRNG STATE (OPTIONAL) + 595103133 36 + 0 0.49324585308429186 + 0.49629292000981806 + 0.46025861761134124 + 0.50251434707008613 + 0.19799855572176028 + 0.74317218699772170 + 5.8075842586035265E-002 + 0.27483363561092844 + 0.27719491285128939 + 0.48728943011535364 + 0.46801484136926064 + 0.39412014412518559 + 0.20773289712341381 + 0.25392142309214805 + 0.30829012729460459 + 0.50916255424382584 + 0.43676609038372405 + 0.73775390609469582 + 0.13824519782462730 + 0.95461716641537109 + 0.73809168750274878 + 0.39054313315828537 + 0.42848370315795847 + 0.51661028368821960 + 0.49645661158506371 + 0.32698440109840021 + 0.54104885709240591 + 0.85145718739972764 + 0.91786181279842438 + 0.16586520729799048 + 0.97300337411869364 + 0.10003251760135612 + 0.40295346298842460 + 0.84512132469547296 + 0.95463689512224548 + 2.5910727974245873E-002 + 0.96630622392773446 + 0.18383747726886313 + 0.92088133249984239 + 0.77113521596477241 + 0.47437965445285712 + 0.35130879751342547 + 0.20921294354998210 + 9.1374779842865905E-002 + 0.45186765734940693 + 0.39446020684599148 + 0.95716829603551545 + 0.87867833359246816 + 0.18555385917002098 + 0.80494863829938623 + 0.49589478880382387 + 4.7012530352265713E-002 + 0.98360638479730511 + 0.78896443148071427 + 0.47133181076921105 + 0.95542030148846990 + 0.22086832429566883 + 0.70605708650068877 + 0.46775882110550882 + 0.27314069065646507 + 0.81516711500871608 + 0.15044684215293458 + 0.88950536312020745 + 0.97394254611016606 + 0.85968018508344457 + 0.33301454477990688 + 0.53470345563750854 + 3.2741238003257678E-002 + 0.79340124117738853 + 6.8909814826419336E-002 + 0.44230400378664925 + 0.57034834634476539 + 0.70397596781939953 + 0.55076343247115744 + 5.1053705188852661E-002 + 0.58211370496187698 + 0.48159410839389594 + 0.90804116563111492 + 2.2367772917075257E-002 + 0.67557588472600827 + 0.88921451415548347 + 0.22390347494851781 + 0.57021288317992358 + 0.54176710102172976 + 0.14734870667805211 + 8.2660977967744742E-002 + 0.44563228441164782 + 0.35025334188112112 + 0.33003761522191155 + 0.19750126970936677 + 0.64965277034081126 + 0.75770326930313914 + 5.7049954076624942E-002 + 0.60589031047008390 + 0.16227326127837927 + 6.6519674582050214E-002 + 0.15168121130731649 + 4.8750680956111125E-003 + 0.48918163044671559 + 0.10650604919490192 + 0.48904627178482585 + 0.80901801758646386 + 7.1311012125914175E-002 + 0.55418150844065650 + 0.83438336051749928 + 0.99842587984902664 + 0.64940069478122453 + 6.3054804090619143E-002 + 0.57675469542413182 + 4.9812360626813046E-002 + 0.17272020649907915 + 0.80532406784800514 + 8.5617945919896243E-002 + 0.80418573083733236 + 0.88996023478180675 + 0.77586548514162601 + 0.45048214944494447 + 9.5693150066765043E-002 + 0.22961908735589986 + 0.40548063998852157 + 0.19054339047147195 + 0.73117401118166470 + 9.6009895921724819E-003 + 0.20263952293048249 + 0.79223910414447474 + 5.0685423566193322E-002 + 0.31043009335787275 + 0.46917293070848487 + 0.11351314271628610 + 0.19485346938860459 + 0.70366790666560775 + 0.54243607547062211 + 0.35584089742587111 + 0.93636699788146416 + 0.43156241726620337 + 0.29115796587234044 + 0.83966017964082695 + 0.39510852236282190 + 0.98199268024954378 + 0.87199612884721844 + 0.31106278617873073 + 0.66203481978063294 + 0.23628623649304359 + 0.95803214049719898 + 0.65635758421441537 + 0.79935779178625666 + 0.67263540020343271 + 0.58108476366842510 + 7.7543355398340452E-002 + 1.7388641777451141E-002 + 0.29707825098842378 + 0.38183422980812054 + 0.73884564959254817 + 0.67197842873743596 + 0.73599078841893473 + 0.99906414263992005 + 0.23719295414896990 + 0.29814696950393227 + 0.13291026961452701 + 0.58601680058732342 + 0.62154676634001405 + 0.25451339892073932 + 0.31260041196477673 + 9.7069412538008493E-002 + 0.50224763634274794 + 0.35619915554918080 + 0.23935156209466157 + 0.37336625771253651 + 1.2369698223213987E-002 + 0.34136559720325721 + 0.11017712492900600 + 0.74912064943983836 + 0.18070536750009580 + 0.49286434862546002 + 0.57986229638101960 + 0.98886002639123660 + 0.48226954608717776 + 0.69492656006587694 + 0.42707052418293756 + 0.33448971911181857 + 0.53908362255230813 + 0.75801351081264556 + 0.10915606885155427 + 0.21830299867344749 + 0.46813105023782242 + 0.39963358763756318 + 0.21433097842775339 + 0.52259855604177119 + 0.72075984093170575 + 0.50389637622084038 + 2.4373607876761127E-002 + 0.74425736913556406 + 0.46410665733550260 + 0.86585005715718211 + 0.28039344394222354 + 0.79624103219791564 + 0.15979878109718371 + 0.82072429653535295 + 8.6086495272265751E-002 + 0.70607868155129694 + 0.69337763768394467 + 2.1409348732930766E-002 + 0.88112760963274539 + 0.70721370676366035 + 0.91622662881070482 + 9.8614301866138732E-002 + 9.9281624680585878E-002 + 0.82472945885024984 + 5.1907069686841112E-002 + 0.84016769331044117 + 0.20102834979289241 + 0.16807787853937128 + 0.62876845789692126 + 0.99543089289756281 + 0.13494449856878177 + 0.35657143296177551 + 0.85555030185167169 + 0.20953289380073770 + 0.41657234820349487 + 0.91444812552706267 + 0.82654108856169728 + 0.92520408165309220 + 0.84504246589307286 + 0.37163925677521803 + 0.23218198459457540 + 0.60265004614421258 + 0.31387514993012289 + 0.41000329509889966 + 0.20527825677785927 + 0.98950365583741728 + 0.97283544247425624 + 0.73479550677496519 + 0.68875514001355143 + 0.95439256147139417 + 0.14216467179825898 + 4.3646151288456281E-002 + 0.42907170457118582 + 0.54868393131082982 + 0.27996658066949820 + 0.62004954089318431 + 0.68822117766336177 + 0.63095603117775667 + 0.27300518652981154 + 3.7281943031523213E-002 + 9.3959564720353939E-002 + 9.7169381291188017E-002 + 0.20633383159733043 + 0.67349556126177035 + 0.95821715134439245 + 0.92729407144461717 + 0.96671286063364903 + 0.66699092661017900 + 0.17740484432721004 + 4.4641688239241972E-002 + 0.64183374293180862 + 0.74168293360887461 + 3.9225728304700169E-002 + 0.89398416877949671 + 0.79520191889594116 + 0.54128317655580460 + 0.14402585464293693 + 0.20376813083881018 + 0.82505465637876796 + 0.63889554341918142 + 4.3467667240172148E-002 + 0.37088191979264096 + 0.96925518104082897 + 0.54220632535502489 + 0.97441976706182842 + 0.20385822206242921 + 0.37041552642137177 + 0.23492261365965561 + 4.3951053138179930E-002 + 0.12196236410233396 + 0.21740406042093596 + 0.80051545678352909 + 0.14381300568658162 + 0.72332072569340511 + 0.23572331665182489 + 0.44817553259780851 + 0.91616068740353640 + 0.10631561038890069 + 0.47916994476991803 + 0.48940447134899046 + 0.25479999529605024 + 0.88054897486578199 + 0.50909308440123624 + 0.55078336384144677 + 0.65523378672363108 + 0.29857621442787163 + 0.68784189038595045 + 0.83727214028596819 + 0.52409818150328036 + 0.36215662978765195 + 0.53395519406301162 + 0.28479322326557011 + 0.43781313941258304 + 0.61745902873723324 + 0.56316216922065365 + 0.53604193467990058 + 0.53438894391144132 + 0.80372400540051814 + 0.68022860258893303 + 0.55957512601506210 + 4.0534357922563657E-002 + 0.25441822157545957 + 0.82842845775856588 + 6.7044515733972077E-002 + 0.19964019745497907 + 0.46142423624182882 + 0.92460624211602038 + 0.84206872091855800 + 0.55937217407144502 + 0.62363003002528217 + 0.28150732082043817 + 0.41771500116203342 + 0.23251509984273611 + 0.54081739077343727 + 0.45141020334454041 + 0.32482625161830470 + 0.95410615754259354 + 0.24123251178570015 + 0.50821615087825478 + 0.15316740108570670 + 0.11205657700804750 + 0.39324624893320603 + 0.24856377911824978 + 0.20343745892967036 + 0.74472311381824596 + 0.40624323949095498 + 0.21404384330451620 + 0.38590679063393907 + 0.62884322579906282 + 0.29749345136205108 + 0.23387309019370406 + 0.89078383955289553 + 0.13558361493640092 + 0.48658804174344539 + 0.62737194564095589 + 0.78598225154866697 + 0.13137624163984540 + 0.58196012880768677 + 0.10018062919492365 + 0.14579136748324828 + 9.3305828765455345E-002 + 0.42693437972586779 + 0.16272711176750576 + 0.62265360319290863 + 0.40930881593081736 + 0.41091205064884662 + 0.82340461008796595 + 0.68701474294215714 + 0.93040886766721798 + 0.55248226680274470 + 0.60774571909526642 + 6.8810131601534152E-002 + 0.66020312901976297 + 7.3430509311354086E-002 + 0.25163218169350188 + 0.46899198341295545 + 0.41394940277212200 + 0.90919958122827893 + 0.21159826169762752 + 0.42380637387735476 + 0.45415712968395994 + 0.86560859659824985 + 0.54308418754210308 + 0.22768108114869534 + 0.63250210466362589 + 0.27603846678341171 + 8.7062595874559889E-002 + 0.84879094420920964 + 0.58808073063696042 + 0.78143208320231139 + 0.83978092577298469 + 0.45729783032871296 + 0.76173542435132191 + 0.17923980016979613 + 0.20103480137447605 + 0.93840830661349273 + 0.34711230919593916 + 0.58116366914373430 + 0.67699509330761387 + 0.29991363046530495 + 0.20923001835816279 + 9.4750828680822963E-002 + 0.23385249724359625 + 0.18997745559904544 + 0.54884521152075294 + 0.98965268505164872 + 0.83461958062056851 + 0.14622633500608373 + 0.98342229836803696 + 0.26111904008478248 + 8.3766894533169989E-002 + 0.26203549105587598 + 0.92050292818672474 + 0.12683696399781397 + 0.24038715562976165 + 0.79859373597746597 + 0.96855039688318456 + 0.92726143335753619 + 2.3000738910852192E-003 + 0.45250532143043642 + 0.28392395256364367 + 0.81347148322686280 + 2.6315498313508101E-002 + 0.79015433579546723 + 0.82125078165874044 + 0.29534291913652311 + 1.0330770504083375E-002 + 0.33648617676590575 + 0.78416281904277696 + 0.98798999860693471 + 0.80836934645357061 + 0.72253137488112174 + 0.80318806866686998 + 0.77159811022642444 + 0.61098275132179225 + 0.71318518273869103 + 0.82905619305262235 + 0.43927283573212250 + 0.91953182742218331 + 0.11989538298481506 + 0.35586690373822805 + 1.1183235533493985E-003 + 0.67342906618577558 + 7.2930855770394487E-002 + 0.75464020085112438 + 0.98386680411371330 + 0.77289896350504961 + 0.56275930457820778 + 0.51851366712667613 + 0.95504408477258806 + 0.85896574770908884 + 5.0489964233619844E-003 + 0.38869435627021431 + 8.2106162304569352E-003 + 7.4217648476267328E-002 + 0.71430827948955056 + 8.9488183561449830E-002 + 0.62715051038177094 + 0.34592413808423927 + 0.59324287364264805 + 0.60394280170542558 + 0.85355694015056116 + 0.74001415268891080 + 0.76706069755032757 + 0.82784832438217038 + 0.60858969865472190 + 0.98926852290765765 + 0.12724946149202054 + 0.95740681396613070 + 0.93885642671968839 + 0.30145730919106484 + 0.21078063365247601 + 0.21385139243814777 + 0.90224746667208322 + 0.82503164439856036 + 5.3814738656463135E-002 + 0.62543823810650423 + 0.83095986728138982 + 0.33169061153166624 + 0.81679911959487583 + 0.59425653215748753 + 0.78802643162518748 + 0.75990166426681682 + 0.32420649227556098 + 0.98686132380390390 + 0.98626672091111089 + 0.97791442860325617 + 0.78544875788069746 + 0.77419814985276503 + 0.78117220767950002 + 0.92994647466491998 + 0.21593818392123865 + 0.50239618473163006 + 0.15039641948255422 + 0.65776713738440407 + 0.33477708520495497 + 0.85625491258544884 + 0.36094993645332707 + 0.90993949942234664 + 0.34720649086859012 + 0.51404136294097214 + 0.79449456932379903 + 0.54251461546341417 + 0.98975088419386736 + 0.49885170704877524 + 4.4391629771411090E-002 + 0.94017216845187690 + 0.25151323682252524 + 0.49020425528447831 + 0.61440920982823144 + 0.46432078562274626 + 0.93425385111629566 + 0.54733002316946866 + 0.80123157826903579 + 0.13549204673158854 + 0.80100736915681026 + 0.42093770357993776 + 0.83049545121010127 + 0.57467921324095883 + 0.72135185311027072 + 0.14115987240230510 + 0.17994318854805513 + 0.59278210419826749 + 0.93969513145815853 + 0.93539545173686633 + 0.87699610068994005 + 0.30150166861178107 + 0.97399049562242013 + 0.35477529095203764 + 0.58610203938174266 + 0.40644477256361355 + 0.71793056595648608 + 0.66477589590931174 + 0.97927429696548529 + 0.36499696765057976 + 0.52802936784698673 + 0.52193872997570168 + 0.65870406107171320 + 0.61820421766104161 + 0.90040398303971614 + 0.29803533235775248 + 0.43464180099291738 + 0.87943860239172977 + 0.55864292921723191 + 0.13078376121440627 + 0.32540694310914731 + 0.34218271962835800 + 0.85482213249557404 + 0.94370033620028693 + 0.47511272032083696 + 0.47345305476497757 + 0.30221722705385190 + 0.90788085689764131 + 0.73088894292231998 + 0.17160524082518336 + 0.49465372870198721 + 0.96147695954616808 + 0.88148268494196458 + 0.55764885791823815 + 8.9901851454712300E-002 + 0.65913871676668379 + 0.72138163168975211 + 0.12502379216927295 + 0.71709960280769991 + 0.18777107318550890 + 0.10248155941998149 + 0.98617977224360587 + 0.68567655766027968 + 0.17195690648704343 + 0.43362203783085818 + 0.61442574093912583 + 0.22795440666493150 + 0.45159776477714786 + 0.35506644719964342 + 0.22091367714392263 + 0.19869581487634491 + 0.39823284736400311 + 0.48020328462351358 + 3.9444513547934434E-002 + 0.12587307223420652 + 0.44774786276711254 + 2.2021640108484064E-002 + 0.56493299073228442 + 0.73226615157794228 + 7.1254567266961999E-002 + 0.31994815977919089 + 0.31708865283838250 + 0.36656558366404113 + 0.16644798526758109 + 0.93486265063832974 + 0.71011988102743473 + 0.54926645936444274 + 0.58605107884734053 + 2.7772399667945535E-002 + 0.75909909283966215 + 0.92398273313319024 + 0.94833569515493821 + 0.97384004056465656 + 0.39397350717486646 + 0.27285621224040568 + 0.95653472169096787 + 0.70328398407865222 + 0.87180350219484026 + 6.6432874578069345E-002 + 0.69072941477110916 + 0.73734474714848730 + 0.80319825618801843 + 0.42729616541454263 + 5.4919787710240087E-002 + 0.15450456340634489 + 0.74034100276525905 + 0.96935078189734725 + 0.90443234384611060 + 0.62283710241925050 + 0.46785604484035659 + 0.46286005241418948 + 0.31773798027739275 + 0.63765611427091784 + 0.80328051712626092 + 0.19525323066950762 + 0.74756144304188865 + 0.92027981114775770 + 0.23242189664972557 + 0.56477479140523101 + 0.16691241907533438 + 0.39313171095178490 + 0.83634321388461430 + 0.85824982606269629 + 0.83048557891225272 + 0.20967159121899215 + 0.50465997873369872 + 0.69439147801213608 + 0.89473894925435360 + 0.15885310306500600 + 0.18406558912944249 + 0.44741975100717823 + 0.24417447469403086 + 0.23280846840193803 + 0.94881850753792207 + 0.93461161448051300 + 0.98519339601865497 + 0.38559516159581975 + 0.39606912367768388 + 3.1572368911323423E-002 + 0.59106852644135799 + 4.9805119209725746E-002 + 6.7688840788029836E-002 + 0.85560432841582923 + 0.65653631149139713 + 0.44057114606664527 + 4.3629071715606926E-002 + 0.85186666330847416 + 8.8802471360835966E-003 + 0.98174099879466681 + 2.5836412163702249E-002 + 0.52300842691343874 + 0.64894410785534262 + 0.56494700296770617 + 0.78858968385953077 + 3.7948233691199817E-002 + 0.20123976558154411 + 0.34807867093762468 + 0.81695343131666576 + 0.43351293926762935 + 0.84384262048552827 + 4.2577172366605964E-002 + 0.85361888738422920 + 0.95788831084358605 + 0.79807765581852763 + 0.91327714868102916 + 0.85284136018775314 + 0.55569980891232262 + 0.49083490210034242 + 0.43150479510447681 + 0.70729472554743467 + 0.86498769409025300 + 0.28848072947738501 + 0.16175396436556611 + 0.25688428814351738 + 0.34400203933857298 + 0.86244684696218243 + 0.64051358750307230 + 0.92926672202570515 + 0.22008928936727301 + 0.92435618627948912 + 0.73972259057434897 + 0.73416767574040165 + 0.76481953984293227 + 0.10591075720374121 + 0.97739621257275289 + 0.70589339623840175 + 0.52087884406080320 + 0.62461198422844078 + 0.28044763214094104 + 0.97068743889651898 + 0.53039819641475816 + 0.52483948157134819 + 0.31710449859349765 + 0.61502373095249752 + 0.48691376936032427 + 0.31241425478234319 + 0.47538384794357214 + 0.19715054098147888 + 0.88328283504554506 + 0.41738258089505820 + 0.49131169213141135 + 0.23547550930690520 + 0.69041567038755147 + 9.4628745392359548E-002 + 0.45154820637795723 + 0.25110076290679473 + 0.50124555005024973 + 0.56332580935005439 + 0.85765721095451042 + 0.59394939796868229 + 0.82372824154009194 + 0.12752887845827843 + 0.76413749367498696 + 0.90069522975852934 + 0.43607909818563684 + 0.96611608037018826 + 0.76994630144758958 + 0.13937315847451615 + 0.83244975735589222 + 0.66534691176449101 + 0.73242656767334324 + 0.86430061446515793 + 4.2056980144323575E-002 + 0.47900168812325816 + 0.20884876281339260 + 0.67552678318658366 + 0.74538444831178197 + 0.16351980201258698 + 0.97782578979865775 + 0.70761116689389780 + 0.92787506925407115 + 0.31626157696342716 + 0.29271893560768447 + 0.22480754433519579 + 0.41979954655704077 + 0.41116196821139184 + 0.27439289669445444 + 0.46883878934423961 + 0.11048974599079386 + 0.84023055802836666 + 0.33963728790632786 + 0.33246132254972238 + 0.30830043066972834 + 0.61320074002953362 + 0.83682518188176047 + 0.20590560052593787 + 0.17592274953600651 + 8.6346862720994011E-002 + 0.53554714921266466 + 0.38131617737101919 + 0.80932337908682328 + 0.13750965464620180 + 0.88575174572093474 + 0.12909987218158037 + 0.83251873127817788 + 0.24842627494136238 + 0.59683076582325967 + 0.48883168275802191 + 0.26625902271628021 + 0.40682078748693229 + 4.2754145616687111E-002 + 1.5114071010209784E-002 + 0.53437721266600491 + 0.71543972709202208 + 0.11686018319695179 + 0.21189493264208892 + 0.46511154443140512 + 0.12719825429922338 + 0.15157980707422425 + 0.43520946435180718 + 0.64153069898346970 + 0.37847826275947938 + 0.34488022400942242 + 1.9218596094404461E-002 + 0.55378127201960936 + 0.46399180569913767 + 0.53279711094453930 + 0.34199054770159520 + 0.60928719568182288 + 5.1042739039672824E-003 + 0.11368173665286108 + 0.99593573631802101 + 0.25335611153120041 + 0.16593627654569687 + 0.65036738048539178 + 0.91844469229673109 + 0.36588645515485752 + 0.42575656951619223 + 2.5948008105135756E-002 + 0.96553752554667938 + 0.10453811537968249 + 0.24916182732853542 + 0.13315022009238220 + 0.69797803367777789 + 0.83052361278405584 + 0.98353783475080903 + 0.46303658422554861 + 0.76821323829073052 + 0.20502108939150432 + 0.47510410940603265 + 0.95107594096149484 + 0.71316678992811688 + 0.89444924307004214 + 0.58122364358867173 + 0.63452067971237724 + 0.19035939388105660 + 0.60941755714207346 + 0.14823703378512221 + 0.56958247344090651 + 0.74383718236159879 + 0.53582182822951552 + 8.8118712314528835E-002 + 0.78735186159694592 + 0.10682429744509747 + 0.34889746129283239 + 0.27316917982397371 + 0.55020936546745247 + 0.19875822849681057 + 0.32727051053365130 + 0.10423212591144448 + 0.31047609908745599 + 0.27039021172527811 + 0.55134565163007210 + 0.98024063073614442 + 0.93304599528342891 + 5.7322413622490132E-002 + 9.8757603947650097E-002 + 0.10199126743695430 + 0.86290047098718148 + 0.74772555208704361 + 0.33560674163423343 + 7.0401022012212877E-002 + 0.31742669012715297 + 0.67553427505066566 + 0.63814134176603332 + 7.9834348807100497E-002 + 1.2513608927477549E-002 + 0.49244428138305452 + 1.2885830355461536E-002 + 0.78574658950779153 + 0.81226540440050599 + 0.61111607966359216 + 0.83461606111027464 + 0.98460264355779259 + 0.56666672379289196 + 3.2891494475933314E-002 + 0.81526318772061046 + 0.74199714281300899 + 0.32318822348783272 + 0.26285999044327824 + 9.5929780332578929E-002 + 0.34220067007473176 + 1.9889802059953610E-002 + 0.78488176387158504 + 0.83129726012966998 + 0.11245932704061801 + 0.34448869023827910 + 0.25034910384552944 + 0.26776978458689626 + 0.80637984526830309 + 0.10776415004085393 + 0.74385243239834153 + 0.54217682357920438 + 0.26669251756461776 + 0.54915979130428383 + 0.12673814774015213 + 0.21949275531181556 + 7.1380720677826304E-002 + 0.71886143071972342 + 0.53028125570763152 + 0.96570017853737866 + 0.83442533023564991 + 5.3762733444703059E-002 + 4.9926535455153243E-002 + 0.84241172350688487 + 0.54902115351462655 + 0.32045901028729418 + 0.97950703602923284 + 0.72804286298322296 + 0.58162277554273345 + 0.36128911712554768 + 0.94084764263085674 + 0.20746205049784194 + 0.49491212202195101 + 9.9624782102264220E-002 + 0.84587432233623616 + 0.75936085307670709 + 0.47998934585895725 + 0.30116224349299969 + 0.30309134003009319 + 0.68274399405911623 + 0.23239154807731666 + 0.82776191812662958 + 0.12872698483300127 + 0.26795550598453133 + 0.75973958764646810 + 0.19596665348190356 + 0.58401906663879544 + 0.58321919930093813 + 0.47019635167153950 + 0.70969636950427883 + 3.6987763860924616E-002 + 0.52712512496501063 + 0.86525578925498081 + 0.98764182352961427 + 0.32921808042189582 + 0.52332690527862979 + 0.70111792721126420 + 0.28244241559992389 + 3.3281502203404756E-002 + 0.44487603775949580 + 0.18870694862308923 + 0.65327822013202663 + 0.86304215202964940 + 0.69406828766550888 + 0.29018680399104468 + 0.20219921884479319 + 3.9787057060440389E-002 + 0.95461058087433770 + 0.44452620693538591 + 0.83873898049369799 + 0.86473192718546699 + 0.31256996801438319 + 0.36029421237918413 + 0.57620203302522555 + 0.99345519679777539 + 0.32074721641011905 + 0.37235741323139848 + 0.76115650217565900 + 4.3422824443165808E-002 + 0.62205970424373191 + 0.10669865052003757 + 0.56339179824833607 + 0.84008286585745040 + 0.78754420609820031 + 0.65426888804310579 + 0.63180694043031238 + 0.27379358059283021 + 0.52376982437886355 + 6.5628737063754272E-003 + 0.64381119211424931 + 0.53105016953987771 + 0.39386857664553077 + 0.64703913418516734 + 1.1410243733678271E-002 + 0.24476978596169019 + 0.49633517217042922 + 0.48278468524535256 + 9.1042016025070893E-002 + 0.23093119090698622 + 0.47154985607143018 + 0.83803143217707188 + 0.28971577849473817 + 0.65480573910208406 + 0.92414692946266896 + 0.75968153955727757 + 0.37535282600894249 + 0.24570338052825846 + 0.81945735509187045 + 7.4884898638600816E-002 + 0.37484470171379414 + 0.77891273362762092 + 2.5869671853797627E-002 + 0.93857063044766420 + 9.5879165930575994E-002 + 3.7519810481470017E-003 + 0.59141653656473636 + 0.13623968337627090 + 0.27304589165438742 + 0.66854607975901814 + 0.92761332260016260 + 0.27199437101797130 + 0.85100955192947225 + 0.95673207295768137 + 0.26699168962499797 + 0.51735911015909153 + 0.39926348765866493 + 0.62833487265679722 + 0.61030784341147637 + 0.45351774287358282 + 0.39071599079986186 + 0.19600530855928255 + 0.66411151984994632 + 0.18022452258902888 + 0.61703693681279503 + 0.84114228390776802 + 0.81330048042515202 + 0.47916226599670608 + 9.0463484149008622E-002 + 5.6510441728043759E-002 + 0.79256901137416591 + 0.67690113722304801 + 0.70870980115194016 + 0.59410263206372704 + 0.67276664667762631 + 0.94685030780890855 + 0.18017878875817672 + 0.49846175377805224 + 1.0900501685927821E-002 + 0.59778308466137986 + 0.66404830524820468 + 0.92313745314454820 + 0.31609444824582056 + 0.57297586549800528 + 0.24276090521697569 + 2.5336238987033965E-002 + 0.21722352990184746 + 0.34444630009062038 + 0.14813575696799930 + 0.29470476428820547 + 0.82647206348174862 + 0.86397162628984603 + 0.87253762249390476 + 0.99437831408630828 + 0.96611436633802228 + 0.86535980027614912 + 0.69833716481997854 + 4.3079072220191250E-002 + 0.50765982176574553 + 0.80666539779437585 + 5.2176252401643808E-002 + 0.75671390607858768 + 0.21419749381046316 + 0.70426965121627205 + 0.20699836748160649 + 0.21348292857078022 + 0.14277546900442317 + 0.93722724172188165 + 0.77110485349003710 + 0.14104322419489890 + 0.45596034255804341 + 0.17004193950362989 + 0.88899914280643344 + 2.2196643937938632E-002 + 0.93835843600527014 + 0.25706680177818697 + 0.77307646204329927 + 0.71809822519050215 + 0.72993867820703429 + 0.68004235380489320 + 0.42387308547439417 + 0.72245210178429886 + 0.34701690304286359 + 0.12182489990669154 + 0.50910218461131151 + 0.80371985724276485 + 0.84350836630640558 + 0.64527143750703431 + 0.22311490663459210 + 0.67078404720487939 + 0.99213295191400874 + 0.91995804396966108 + 0.33238785754262778 + 0.46942009324286715 + 0.37553893107884662 + 0.61018540998448856 + 0.46097967881933499 + 0.33339569286177095 + 0.57300834014620960 + 0.63104708963166445 + 0.91731549321855610 + 0.50681402887710902 + 0.47047084346093015 + 2.5512286327135314E-002 + 0.99612308445772513 + 0.74275349309682781 + 0.47591020530570560 + 0.40181828773070549 + 0.78958817331287179 + 4.7109378789517820E-002 + 7.6896431363802975E-002 + 0.52024921083216213 + 0.10480716966488046 + 0.42315609462054837 + 5.4088876400001595E-002 + 0.50191886444818934 + 0.57294879761911588 + 0.61273083454976174 + 0.30781987367103980 + 0.29015147800931018 + 2.1794091398266602E-002 + 0.63088344450125078 + 0.79661323318752508 + 0.17043623105301720 + 0.41032335358113770 + 0.92030312356238753 + 0.87396824733972167 + 0.74899509338264281 + 0.31838721599348929 + 0.86054367420573641 + 0.84019508502745310 + 0.83760455345749918 + 4.0671404166424452E-002 + 8.7429545787472307E-002 + 0.62751823707785803 + 0.86310241290249579 + 5.6408304683458255E-002 + 0.64132246071872601 + 5.7409004448881973E-002 + 0.45600569783792011 + 0.83987595170225404 + 0.91812015949614789 + 0.66968416911208450 + 0.10216675578983825 + 0.39572187272063175 + 0.92676645819906867 + 0.54303841682618881 + 0.39691687027067601 + 0.26140082137998988 + 0.88851342077088091 + 0.39595075046580419 + 0.33741583893651139 + 0.86202860286726590 + 0.43769682316377612 + 0.50440613242815147 + 0.76225606266665125 + 0.96186039885528274 + 0.43798261467920696 + 0.37437952559121612 + 0.96408342797511537 + 6.3187664110252229E-002 + 0.16024309193573316 + 0.18450941891428840 + 0.47306309311417749 + 0.35900644215697497 + 0.61470913305812047 + 0.10474165984858530 + 0.95917068078074053 + 0.88940297818810876 + 0.36746756311191220 + 0.62620203727022172 + 5.2412185597020766E-002 + 0.85704224052903655 + 5.1788849909250700E-002 + 0.93533060539331103 + 0.77906133022465696 + 0.82492165062861744 + 0.61374629857715846 + 0.68768843507729471 + 5.0853126173038277E-002 + 0.17810889780727379 + 0.29695389645936032 + 2.8190332292901132E-002 + 1.3266090115848783E-002 + 0.65968091915802063 + 0.55206255062179110 + 0.62410948926586585 + 6.5703440779245170E-002 + 0.78994459416097129 + 0.39692255819054978 + 0.21402161817816179 + 0.57677106604464967 + 0.83319482350367835 + 0.17790186849569523 + 0.73078871483060226 + 0.22665571786620475 + 0.74671159152102717 + 0.86438106596336439 + 0.95413637335751744 + 0.14620227943607134 + 5.7936809913204712E-002 + 0.98535172252024594 + 2.0865933557963956E-002 + 0.65052522272778290 + 0.83708978486930263 + 0.60027810148141469 + 0.85099195813480932 + 0.85443785633967906 + 0.73517901754262738 + 0.97714779185145417 + 0.26830391876777426 + 0.57658970543655030 + 0.73134209808441497 + 0.69279360310381577 + 0.22293560047807759 + 0.22272825397147145 + 0.88005286031377139 + 0.96564411627953461 + 0.87145362842802143 + 0.56230611494263272 + 0.93014355526995374 + 0.90883360596208718 + 0.45265482883941033 + 7.5890123196877823E-003 + 0.13891899600998414 + 0.19395797792807912 + 0.40822572570344562 + 0.91984262678367656 + 0.41035009880865658 + 0.21031624559043749 + 0.18100789306921783 + 0.79352208861650197 + 0.54701620377705495 + 0.43573916585428663 + 0.45475459895084569 + 0.56263253074866171 + 0.61249511196383466 + 1.7521048876169232E-002 + 0.34828113003582928 + 0.52769822167224234 + 9.2073402614058608E-002 + 0.67385854686032332 + 6.2394989126019595E-002 + 0.67562353970063427 + 0.47211798936650595 + 9.8665257294559439E-002 + 0.43082834190490971 + 0.14249350583460796 + 0.50526362688148296 + 0.38193182809615323 + 0.15678567120378872 + 0.84734656947749087 + 0.63450811669369500 + 0.69169034334883506 + 0.98033573532169171 + 0.32444405697311041 + 0.37046074085349545 + 0.89749076441217213 + 0.92944504213263102 + 0.30065964450140470 + 0.82602067292728165 + 0.82831342461735602 + 0.82782600616961588 + 0.35433425353576808 + 0.89621590808965834 + 0.53228509433131066 + 0.20190963812276053 + 0.67000526616971356 + 0.93890699885969653 + 0.10520203411017093 + 0.62848355019055191 + 0.33034544583888703 + 7.8076170394218991E-003 + 0.72417687182973012 + 0.98800541160080257 + 0.74327516345928402 + 0.92590555866150481 + 0.60352349719456910 + 0.15680962659125086 + 4.9434958757949232E-002 + 0.55686051658955904 + 0.97810570859908452 + 0.12872754429692890 + 0.95547914162817804 + 0.95573437656426208 + 0.12967304813757963 + 0.45951519961713316 + 0.68151881426853933 + 0.46566941554450381 + 0.62813314240559137 + 0.54948969763289313 + 0.77136885714444148 + 0.80877759415522377 + 0.65958877927092274 + 0.89058500654762440 + 0.90982465711328686 + 0.61749100212857400 + 0.54754063509364670 + 0.35713498382768094 + 2.4839537655822141E-002 + 0.44473647181794362 + 0.48318333042530526 + 0.15052230596724669 + 0.98645625807854387 + 0.16983403901664573 + 0.24743079769734777 + 0.75834503525907948 + 0.39212420611110943 + 0.64782771756848589 + 0.41577658200279899 + 0.15879365042174598 + 0.94978611458295603 + 0.66463121868276431 + 0.84009097217167650 + 0.90795928852065444 + 0.52724828450050865 + 0.38418444434347165 + 0.16407355312341210 + 8.8496510745926571E-002 + 0.50958525975331526 + 7.0102045493356968E-002 + 0.35537675549345948 + 0.93728436913066560 + 0.26853869097094929 + 5.9162881009825696E-002 + 0.46807856276806703 + 0.34526803158516728 + 0.77813731816475240 + 0.70936884855344218 + 0.36739517626832097 + 0.36261806234035987 diff --git a/tests/TERAPI-PIMD-PARALLEL/scrdir0001.ref b/tests/TERAPI-PIMD-PARALLEL/scrdir0001.ref new file mode 100644 index 00000000..e69de29b diff --git a/tests/TERAPI-PIMD-PARALLEL/scrdir0002.ref b/tests/TERAPI-PIMD-PARALLEL/scrdir0002.ref new file mode 100644 index 00000000..e69de29b diff --git a/tests/TERAPI-PIMD-PARALLEL/scrdir0003.ref b/tests/TERAPI-PIMD-PARALLEL/scrdir0003.ref new file mode 100644 index 00000000..e69de29b diff --git a/tests/TERAPI-PIMD-PARALLEL/scrdir0004.ref b/tests/TERAPI-PIMD-PARALLEL/scrdir0004.ref new file mode 100644 index 00000000..e69de29b diff --git a/tests/TERAPI-PIMD-PARALLEL/tc_server.cpp b/tests/TERAPI-PIMD-PARALLEL/tc_server.cpp new file mode 100644 index 00000000..725cd176 --- /dev/null +++ b/tests/TERAPI-PIMD-PARALLEL/tc_server.cpp @@ -0,0 +1,61 @@ +#include +#include + +#include "../tc_mpi_api.h" + +using namespace std; + +int main(int argc, char* argv[]) +{ + char *serverName = NULL; + + // Due to a bug in hydra_nameserver, it crashes + // when multiple TC servers call `MPI_Unpublish_name()` + // Hence, we want to allow invoking without this parameter, + // in which case TC server will just print the port to stdin, + // where it could be grepped and passed via file to ABIN, + // and it will never call MPI_Publish_name/MPI_Unpublish_name + // NOTE: This behaviour is different from real TC, + // which has default serverName and will always try to publish it. + if (argc > 2) { + printf("Only one cmdline argument supported, , but you provided more!"); + throw std::runtime_error("Incorrect invocation"); + } + + if (argc == 2) { + serverName = new char[1024]; + strcpy(serverName, argv[1]); + } + + TCServerMock tc = TCServerMock(serverName); + if (serverName) { + delete[] serverName; + } + + tc.initializeCommunication(); + + tc.receiveNumAtoms(); + tc.receiveAtomTypes(); + + int loop_counter = 0; + int MAX_LOOP_COUNT = 100; + // Will go through this loop until MPI client gives an exit signal. + while (true) { + + int status = tc.receive(); + if (status == MPI_TAG_EXIT) { + break; + } + + tc.send(); + + // This is just a precaution, we don't want endless loop! + loop_counter++; + if (loop_counter > MAX_LOOP_COUNT) { + printf("Maximum number of steps exceeded!\n"); + return(1); + } + } + + return(0); +} diff --git a/tests/TERAPI-PIMD-PARALLEL/temper.dat.ref b/tests/TERAPI-PIMD-PARALLEL/temper.dat.ref new file mode 100644 index 00000000..727c8ac4 --- /dev/null +++ b/tests/TERAPI-PIMD-PARALLEL/temper.dat.ref @@ -0,0 +1,2 @@ + # Time[fs] Temperature T-Average Conserved_quantity_of_thermostat + 0.24 0.42 0.42 diff --git a/tests/TERAPI-PIMD-PARALLEL/test.sh b/tests/TERAPI-PIMD-PARALLEL/test.sh new file mode 100755 index 00000000..4cdabed3 --- /dev/null +++ b/tests/TERAPI-PIMD-PARALLEL/test.sh @@ -0,0 +1,61 @@ +#/bin/bash +set -euo pipefail +# Useful for debugging +#set -x + +ABINEXE=$1 +source ../test_tc_server_utils.sh + +set_default_vars +set_mpich_vars +# If $1 = "clean"; exit early. +if ! clean_output_files $1; then + exit 0 +fi + +N_TERA_SERVERS=$(egrep --only-matching 'nteraservers\s*=\s*[0-9]' $ABININ | egrep -o [0-9]) + +# Exit early for OpenMPI build. +check_for_openmpi + +# Compiled the fake TC server +$MPICXX $TCSRC -Wall -o $TCEXE + +# NOTE: We very intentionally do NOT launch +# hydra_nameserver in this test since it cannot handle +# multiple TC servers due to a bug in MPI_Unpublish_name +# https://github.com/pmodels/mpich/issues/5058 +# +# Therefore, we pass the port_name to ABIN via files, see below. +#TC_SERVER_NAME="tcserver.$$" +#launch_hydra_nameserver $MPICH_HYDRA +#hostname=$HOSTNAME +#MPIRUN="$MPIRUN -nameserver $hostname -n 1" + +MPIRUN="$MPIRUN -n 1" + +ABIN_CMD="$ABINEXE -i $ABININ -x $ABINGEOM" # -M $TC_SERVER_NAME" + +let NUM_JOBS=N_TERA_SERVERS+1 +declare -A job_pids +for ((itera=1;itera<=N_TERA_SERVERS;itera++)) { + #$MPIRUN ./$TCEXE $TC_SERVER_NAME.$itera > $TCOUT.$itera 2>&1 & + $MPIRUN ./$TCEXE > $TCOUT.$itera 2>&1 & + job_pids[$itera]=$! +} +sleep 1 +# Grep port names from TC output, pass to ABIN via a file. +for ((itera=1;itera<=N_TERA_SERVERS;itera++)) { + grep 'port name' $TCOUT.$itera | awk -F"port name: " '{print $2;exit}' > $TC_PORT_FILE.$itera +} + +$MPIRUN $ABIN_CMD > $ABINOUT 2>&1 & +job_pids[$NUM_JOBS]=$! + +function cleanup { + kill -9 ${job_pids[@]} > /dev/null 2>&1 || true + exit 0 +} + +trap cleanup INT ABRT TERM EXIT +check_running_processes ${job_pids[@]} diff --git a/tests/TERAPI-PIMD-PARALLEL/velocities.xyz.ref b/tests/TERAPI-PIMD-PARALLEL/velocities.xyz.ref new file mode 100644 index 00000000..2eda254e --- /dev/null +++ b/tests/TERAPI-PIMD-PARALLEL/velocities.xyz.ref @@ -0,0 +1,14 @@ + 3 + Time step: 1 +O 0.0000000000E+00 -0.9563564009E-05 0.0000000000E+00 +H -0.5851553569E-04 0.7589655783E-04 0.0000000000E+00 +H 0.5851553569E-04 0.7589655783E-04 0.0000000000E+00 +O 0.0000000000E+00 -0.1314779241E-04 0.0000000000E+00 +H -0.8043976155E-04 0.1043410371E-03 0.0000000000E+00 +H 0.8043976155E-04 0.1043410371E-03 0.0000000000E+00 +O 0.0000000000E+00 -0.1434197286E-04 0.0000000000E+00 +H -0.8774274119E-04 0.1138180673E-03 0.0000000000E+00 +H 0.8774274119E-04 0.1138180673E-03 0.0000000000E+00 +O 0.0000000000E+00 -0.1314779241E-04 0.0000000000E+00 +H -0.8043976155E-04 0.1043410371E-03 0.0000000000E+00 +H 0.8043976155E-04 0.1043410371E-03 0.0000000000E+00 diff --git a/tests/TERAPI-PIMD/est_energy.dat.ref b/tests/TERAPI-PIMD/est_energy.dat.ref new file mode 100644 index 00000000..701cea3c --- /dev/null +++ b/tests/TERAPI-PIMD/est_energy.dat.ref @@ -0,0 +1,2 @@ + # Time[fs] E-potential E-primitive E-virial CumulAvg_prim CumulAvg_vir + 0.24 0.7962477515E-03 0.7962477515E-03 0.7962520021E-03 0.0000000000E+00 0.0000000000E+00 diff --git a/tests/TERAPI-PIMD/forces.xyz.ref b/tests/TERAPI-PIMD/forces.xyz.ref new file mode 100644 index 00000000..2bb720bf --- /dev/null +++ b/tests/TERAPI-PIMD/forces.xyz.ref @@ -0,0 +1,14 @@ + 3 +net force: 0.00000E+00 0.00000E+00 0.00000E+00 torque force: 0.00000E+00 0.00000E+00 0.00000E+00 +O 0.0000000000E+00 -0.6945880791E-02 0.0000000000E+00 +H -0.2672605904E-02 0.3472940396E-02 0.0000000000E+00 +H 0.2672605904E-02 0.3472940396E-02 0.0000000000E+00 +O 0.0000000000E+00 -0.6916370894E-02 0.0000000000E+00 +H -0.2655760787E-02 0.3458185447E-02 0.0000000000E+00 +H 0.2655760787E-02 0.3458185447E-02 0.0000000000E+00 +O 0.0000000000E+00 -0.6906530387E-02 0.0000000000E+00 +H -0.2650142923E-02 0.3453265194E-02 0.0000000000E+00 +H 0.2650142923E-02 0.3453265194E-02 0.0000000000E+00 +O 0.0000000000E+00 -0.6916370894E-02 0.0000000000E+00 +H -0.2655760787E-02 0.3458185447E-02 0.0000000000E+00 +H 0.2655760787E-02 0.3458185447E-02 0.0000000000E+00 diff --git a/tests/TERAPI-PIMD/input.in b/tests/TERAPI-PIMD/input.in new file mode 100644 index 00000000..b8b78f71 --- /dev/null +++ b/tests/TERAPI-PIMD/input.in @@ -0,0 +1,33 @@ +&general +pot='_tera_' +watpot=1 +ipimd=1, +istage=1 +nwalk=4, +nstep=1, +dt=10., +irandom=13131313, + +nwrite=1, +nwritef=1, +nwritev=1, +nwritex=1, +nrest=1, +idebug=3 + +! TODO: Set nteraservers=nwalk=4 +! For now we're running just one since I've seen +! random failures in Github CI when running 4 TC servers. +! Will need to dig deeper after I refactor the code, +! there might be some issue with OpenMP parallelization. +! For now, we're running single threaded so that I +! don't have to disable this test. +nteraservers=1 + +iknow=1 ! We're running PIMD without thermostat ! +/ + +&nhcopt +inose=0, +temp=0.0, +/ diff --git a/tests/TERAPI-PIMD/mini.xyz b/tests/TERAPI-PIMD/mini.xyz new file mode 100644 index 00000000..77afbffe --- /dev/null +++ b/tests/TERAPI-PIMD/mini.xyz @@ -0,0 +1,5 @@ +3 + +O 0.000 0.118 0.000 +h 0.757 -0.472 0.000 +h -0.757 -0.472 0.000 diff --git a/tests/TERAPI-PIMD/movie.xyz.ref b/tests/TERAPI-PIMD/movie.xyz.ref new file mode 100644 index 00000000..2892fae3 --- /dev/null +++ b/tests/TERAPI-PIMD/movie.xyz.ref @@ -0,0 +1,20 @@ + 3 +Time step: 1 Sim. Time [au] 10.00 +O 0.00000000E+00 0.11797451E+00 0.00000000E+00 +H 0.75684348E+00 -0.47179770E+00 0.00000000E+00 +H -0.75684348E+00 -0.47179770E+00 0.00000000E+00 + 3 +Time step: 1 Sim. Time [au] 10.00 +O 0.00000000E+00 0.11796495E+00 0.00000000E+00 +H 0.75678478E+00 -0.47172184E+00 0.00000000E+00 +H -0.75678478E+00 -0.47172184E+00 0.00000000E+00 + 3 +Time step: 1 Sim. Time [au] 10.00 +O 0.00000000E+00 0.11796176E+00 0.00000000E+00 +H 0.75676522E+00 -0.47169655E+00 0.00000000E+00 +H -0.75676522E+00 -0.47169655E+00 0.00000000E+00 + 3 +Time step: 1 Sim. Time [au] 10.00 +O 0.00000000E+00 0.11796495E+00 0.00000000E+00 +H 0.75678478E+00 -0.47172184E+00 0.00000000E+00 +H -0.75678478E+00 -0.47172184E+00 0.00000000E+00 diff --git a/tests/TERAPI-PIMD/restart.xyz.ref b/tests/TERAPI-PIMD/restart.xyz.ref new file mode 100644 index 00000000..c2b9086a --- /dev/null +++ b/tests/TERAPI-PIMD/restart.xyz.ref @@ -0,0 +1,1313 @@ + 1 10.000000000000000 + Cartesian Coordinates [au] + 0.0000000000000000 0.22293951200140324 0.0000000000000000 + 1.4302268996036975 -0.89156844370906640 0.0000000000000000 + -1.4302268996036975 -0.89156844370906640 0.0000000000000000 + 0.0000000000000000 0.22292144762229921 0.0000000000000000 + 1.4301159809857611 -0.89142508458144487 0.0000000000000000 + -1.4301159809857611 -0.89142508458144487 0.0000000000000000 + 0.0000000000000000 0.22291542616259788 0.0000000000000000 + 1.4300790081131156 -0.89137729820557099 0.0000000000000000 + -1.4300790081131156 -0.89137729820557099 0.0000000000000000 + 0.0000000000000000 0.22292144762229923 0.0000000000000000 + 1.4301159809857611 -0.89142508458144487 0.0000000000000000 + -1.4301159809857611 -0.89142508458144487 0.0000000000000000 + Cartesian Velocities [au] + 0.0000000000000000 -9.5635640087645337E-006 0.0000000000000000 + -5.8515535693924805E-005 7.5896557825507824E-005 0.0000000000000000 + 5.8515535693924805E-005 7.5896557825507824E-005 0.0000000000000000 + 0.0000000000000000 -1.3147792414895394E-005 0.0000000000000000 + -8.0439761551330711E-005 1.0434103712594810E-004 0.0000000000000000 + 8.0439761551330711E-005 1.0434103712594810E-004 0.0000000000000000 + 0.0000000000000000 -1.4341972858402675E-005 0.0000000000000000 + -8.7742741190639505E-005 1.1381806734205576E-004 0.0000000000000000 + 8.7742741190639505E-005 1.1381806734205576E-004 0.0000000000000000 + 0.0000000000000000 -1.3147792414895394E-005 0.0000000000000000 + -8.0439761551330711E-005 1.0434103712594812E-004 0.0000000000000000 + 8.0439761551330711E-005 1.0434103712594812E-004 0.0000000000000000 + Cumulative averages of various estimators + 1.3273231226437057E-006 + 0.0000000000000000 0.0000000000000000 + 0.0000000000000000 +PRNG STATE (OPTIONAL) + 595103133 36 + 0 0.49324585308429186 + 0.49629292000981806 + 0.46025861761134124 + 0.50251434707008613 + 0.19799855572176028 + 0.74317218699772170 + 5.8075842586035265E-002 + 0.27483363561092844 + 0.27719491285128939 + 0.48728943011535364 + 0.46801484136926064 + 0.39412014412518559 + 0.20773289712341381 + 0.25392142309214805 + 0.30829012729460459 + 0.50916255424382584 + 0.43676609038372405 + 0.73775390609469582 + 0.13824519782462730 + 0.95461716641537109 + 0.73809168750274878 + 0.39054313315828537 + 0.42848370315795847 + 0.51661028368821960 + 0.49645661158506371 + 0.32698440109840021 + 0.54104885709240591 + 0.85145718739972764 + 0.91786181279842438 + 0.16586520729799048 + 0.97300337411869364 + 0.10003251760135612 + 0.40295346298842460 + 0.84512132469547296 + 0.95463689512224548 + 2.5910727974245873E-002 + 0.96630622392773446 + 0.18383747726886313 + 0.92088133249984239 + 0.77113521596477241 + 0.47437965445285712 + 0.35130879751342547 + 0.20921294354998210 + 9.1374779842865905E-002 + 0.45186765734940693 + 0.39446020684599148 + 0.95716829603551545 + 0.87867833359246816 + 0.18555385917002098 + 0.80494863829938623 + 0.49589478880382387 + 4.7012530352265713E-002 + 0.98360638479730511 + 0.78896443148071427 + 0.47133181076921105 + 0.95542030148846990 + 0.22086832429566883 + 0.70605708650068877 + 0.46775882110550882 + 0.27314069065646507 + 0.81516711500871608 + 0.15044684215293458 + 0.88950536312020745 + 0.97394254611016606 + 0.85968018508344457 + 0.33301454477990688 + 0.53470345563750854 + 3.2741238003257678E-002 + 0.79340124117738853 + 6.8909814826419336E-002 + 0.44230400378664925 + 0.57034834634476539 + 0.70397596781939953 + 0.55076343247115744 + 5.1053705188852661E-002 + 0.58211370496187698 + 0.48159410839389594 + 0.90804116563111492 + 2.2367772917075257E-002 + 0.67557588472600827 + 0.88921451415548347 + 0.22390347494851781 + 0.57021288317992358 + 0.54176710102172976 + 0.14734870667805211 + 8.2660977967744742E-002 + 0.44563228441164782 + 0.35025334188112112 + 0.33003761522191155 + 0.19750126970936677 + 0.64965277034081126 + 0.75770326930313914 + 5.7049954076624942E-002 + 0.60589031047008390 + 0.16227326127837927 + 6.6519674582050214E-002 + 0.15168121130731649 + 4.8750680956111125E-003 + 0.48918163044671559 + 0.10650604919490192 + 0.48904627178482585 + 0.80901801758646386 + 7.1311012125914175E-002 + 0.55418150844065650 + 0.83438336051749928 + 0.99842587984902664 + 0.64940069478122453 + 6.3054804090619143E-002 + 0.57675469542413182 + 4.9812360626813046E-002 + 0.17272020649907915 + 0.80532406784800514 + 8.5617945919896243E-002 + 0.80418573083733236 + 0.88996023478180675 + 0.77586548514162601 + 0.45048214944494447 + 9.5693150066765043E-002 + 0.22961908735589986 + 0.40548063998852157 + 0.19054339047147195 + 0.73117401118166470 + 9.6009895921724819E-003 + 0.20263952293048249 + 0.79223910414447474 + 5.0685423566193322E-002 + 0.31043009335787275 + 0.46917293070848487 + 0.11351314271628610 + 0.19485346938860459 + 0.70366790666560775 + 0.54243607547062211 + 0.35584089742587111 + 0.93636699788146416 + 0.43156241726620337 + 0.29115796587234044 + 0.83966017964082695 + 0.39510852236282190 + 0.98199268024954378 + 0.87199612884721844 + 0.31106278617873073 + 0.66203481978063294 + 0.23628623649304359 + 0.95803214049719898 + 0.65635758421441537 + 0.79935779178625666 + 0.67263540020343271 + 0.58108476366842510 + 7.7543355398340452E-002 + 1.7388641777451141E-002 + 0.29707825098842378 + 0.38183422980812054 + 0.73884564959254817 + 0.67197842873743596 + 0.73599078841893473 + 0.99906414263992005 + 0.23719295414896990 + 0.29814696950393227 + 0.13291026961452701 + 0.58601680058732342 + 0.62154676634001405 + 0.25451339892073932 + 0.31260041196477673 + 9.7069412538008493E-002 + 0.50224763634274794 + 0.35619915554918080 + 0.23935156209466157 + 0.37336625771253651 + 1.2369698223213987E-002 + 0.34136559720325721 + 0.11017712492900600 + 0.74912064943983836 + 0.18070536750009580 + 0.49286434862546002 + 0.57986229638101960 + 0.98886002639123660 + 0.48226954608717776 + 0.69492656006587694 + 0.42707052418293756 + 0.33448971911181857 + 0.53908362255230813 + 0.75801351081264556 + 0.10915606885155427 + 0.21830299867344749 + 0.46813105023782242 + 0.39963358763756318 + 0.21433097842775339 + 0.52259855604177119 + 0.72075984093170575 + 0.50389637622084038 + 2.4373607876761127E-002 + 0.74425736913556406 + 0.46410665733550260 + 0.86585005715718211 + 0.28039344394222354 + 0.79624103219791564 + 0.15979878109718371 + 0.82072429653535295 + 8.6086495272265751E-002 + 0.70607868155129694 + 0.69337763768394467 + 2.1409348732930766E-002 + 0.88112760963274539 + 0.70721370676366035 + 0.91622662881070482 + 9.8614301866138732E-002 + 9.9281624680585878E-002 + 0.82472945885024984 + 5.1907069686841112E-002 + 0.84016769331044117 + 0.20102834979289241 + 0.16807787853937128 + 0.62876845789692126 + 0.99543089289756281 + 0.13494449856878177 + 0.35657143296177551 + 0.85555030185167169 + 0.20953289380073770 + 0.41657234820349487 + 0.91444812552706267 + 0.82654108856169728 + 0.92520408165309220 + 0.84504246589307286 + 0.37163925677521803 + 0.23218198459457540 + 0.60265004614421258 + 0.31387514993012289 + 0.41000329509889966 + 0.20527825677785927 + 0.98950365583741728 + 0.97283544247425624 + 0.73479550677496519 + 0.68875514001355143 + 0.95439256147139417 + 0.14216467179825898 + 4.3646151288456281E-002 + 0.42907170457118582 + 0.54868393131082982 + 0.27996658066949820 + 0.62004954089318431 + 0.68822117766336177 + 0.63095603117775667 + 0.27300518652981154 + 3.7281943031523213E-002 + 9.3959564720353939E-002 + 9.7169381291188017E-002 + 0.20633383159733043 + 0.67349556126177035 + 0.95821715134439245 + 0.92729407144461717 + 0.96671286063364903 + 0.66699092661017900 + 0.17740484432721004 + 4.4641688239241972E-002 + 0.64183374293180862 + 0.74168293360887461 + 3.9225728304700169E-002 + 0.89398416877949671 + 0.79520191889594116 + 0.54128317655580460 + 0.14402585464293693 + 0.20376813083881018 + 0.82505465637876796 + 0.63889554341918142 + 4.3467667240172148E-002 + 0.37088191979264096 + 0.96925518104082897 + 0.54220632535502489 + 0.97441976706182842 + 0.20385822206242921 + 0.37041552642137177 + 0.23492261365965561 + 4.3951053138179930E-002 + 0.12196236410233396 + 0.21740406042093596 + 0.80051545678352909 + 0.14381300568658162 + 0.72332072569340511 + 0.23572331665182489 + 0.44817553259780851 + 0.91616068740353640 + 0.10631561038890069 + 0.47916994476991803 + 0.48940447134899046 + 0.25479999529605024 + 0.88054897486578199 + 0.50909308440123624 + 0.55078336384144677 + 0.65523378672363108 + 0.29857621442787163 + 0.68784189038595045 + 0.83727214028596819 + 0.52409818150328036 + 0.36215662978765195 + 0.53395519406301162 + 0.28479322326557011 + 0.43781313941258304 + 0.61745902873723324 + 0.56316216922065365 + 0.53604193467990058 + 0.53438894391144132 + 0.80372400540051814 + 0.68022860258893303 + 0.55957512601506210 + 4.0534357922563657E-002 + 0.25441822157545957 + 0.82842845775856588 + 6.7044515733972077E-002 + 0.19964019745497907 + 0.46142423624182882 + 0.92460624211602038 + 0.84206872091855800 + 0.55937217407144502 + 0.62363003002528217 + 0.28150732082043817 + 0.41771500116203342 + 0.23251509984273611 + 0.54081739077343727 + 0.45141020334454041 + 0.32482625161830470 + 0.95410615754259354 + 0.24123251178570015 + 0.50821615087825478 + 0.15316740108570670 + 0.11205657700804750 + 0.39324624893320603 + 0.24856377911824978 + 0.20343745892967036 + 0.74472311381824596 + 0.40624323949095498 + 0.21404384330451620 + 0.38590679063393907 + 0.62884322579906282 + 0.29749345136205108 + 0.23387309019370406 + 0.89078383955289553 + 0.13558361493640092 + 0.48658804174344539 + 0.62737194564095589 + 0.78598225154866697 + 0.13137624163984540 + 0.58196012880768677 + 0.10018062919492365 + 0.14579136748324828 + 9.3305828765455345E-002 + 0.42693437972586779 + 0.16272711176750576 + 0.62265360319290863 + 0.40930881593081736 + 0.41091205064884662 + 0.82340461008796595 + 0.68701474294215714 + 0.93040886766721798 + 0.55248226680274470 + 0.60774571909526642 + 6.8810131601534152E-002 + 0.66020312901976297 + 7.3430509311354086E-002 + 0.25163218169350188 + 0.46899198341295545 + 0.41394940277212200 + 0.90919958122827893 + 0.21159826169762752 + 0.42380637387735476 + 0.45415712968395994 + 0.86560859659824985 + 0.54308418754210308 + 0.22768108114869534 + 0.63250210466362589 + 0.27603846678341171 + 8.7062595874559889E-002 + 0.84879094420920964 + 0.58808073063696042 + 0.78143208320231139 + 0.83978092577298469 + 0.45729783032871296 + 0.76173542435132191 + 0.17923980016979613 + 0.20103480137447605 + 0.93840830661349273 + 0.34711230919593916 + 0.58116366914373430 + 0.67699509330761387 + 0.29991363046530495 + 0.20923001835816279 + 9.4750828680822963E-002 + 0.23385249724359625 + 0.18997745559904544 + 0.54884521152075294 + 0.98965268505164872 + 0.83461958062056851 + 0.14622633500608373 + 0.98342229836803696 + 0.26111904008478248 + 8.3766894533169989E-002 + 0.26203549105587598 + 0.92050292818672474 + 0.12683696399781397 + 0.24038715562976165 + 0.79859373597746597 + 0.96855039688318456 + 0.92726143335753619 + 2.3000738910852192E-003 + 0.45250532143043642 + 0.28392395256364367 + 0.81347148322686280 + 2.6315498313508101E-002 + 0.79015433579546723 + 0.82125078165874044 + 0.29534291913652311 + 1.0330770504083375E-002 + 0.33648617676590575 + 0.78416281904277696 + 0.98798999860693471 + 0.80836934645357061 + 0.72253137488112174 + 0.80318806866686998 + 0.77159811022642444 + 0.61098275132179225 + 0.71318518273869103 + 0.82905619305262235 + 0.43927283573212250 + 0.91953182742218331 + 0.11989538298481506 + 0.35586690373822805 + 1.1183235533493985E-003 + 0.67342906618577558 + 7.2930855770394487E-002 + 0.75464020085112438 + 0.98386680411371330 + 0.77289896350504961 + 0.56275930457820778 + 0.51851366712667613 + 0.95504408477258806 + 0.85896574770908884 + 5.0489964233619844E-003 + 0.38869435627021431 + 8.2106162304569352E-003 + 7.4217648476267328E-002 + 0.71430827948955056 + 8.9488183561449830E-002 + 0.62715051038177094 + 0.34592413808423927 + 0.59324287364264805 + 0.60394280170542558 + 0.85355694015056116 + 0.74001415268891080 + 0.76706069755032757 + 0.82784832438217038 + 0.60858969865472190 + 0.98926852290765765 + 0.12724946149202054 + 0.95740681396613070 + 0.93885642671968839 + 0.30145730919106484 + 0.21078063365247601 + 0.21385139243814777 + 0.90224746667208322 + 0.82503164439856036 + 5.3814738656463135E-002 + 0.62543823810650423 + 0.83095986728138982 + 0.33169061153166624 + 0.81679911959487583 + 0.59425653215748753 + 0.78802643162518748 + 0.75990166426681682 + 0.32420649227556098 + 0.98686132380390390 + 0.98626672091111089 + 0.97791442860325617 + 0.78544875788069746 + 0.77419814985276503 + 0.78117220767950002 + 0.92994647466491998 + 0.21593818392123865 + 0.50239618473163006 + 0.15039641948255422 + 0.65776713738440407 + 0.33477708520495497 + 0.85625491258544884 + 0.36094993645332707 + 0.90993949942234664 + 0.34720649086859012 + 0.51404136294097214 + 0.79449456932379903 + 0.54251461546341417 + 0.98975088419386736 + 0.49885170704877524 + 4.4391629771411090E-002 + 0.94017216845187690 + 0.25151323682252524 + 0.49020425528447831 + 0.61440920982823144 + 0.46432078562274626 + 0.93425385111629566 + 0.54733002316946866 + 0.80123157826903579 + 0.13549204673158854 + 0.80100736915681026 + 0.42093770357993776 + 0.83049545121010127 + 0.57467921324095883 + 0.72135185311027072 + 0.14115987240230510 + 0.17994318854805513 + 0.59278210419826749 + 0.93969513145815853 + 0.93539545173686633 + 0.87699610068994005 + 0.30150166861178107 + 0.97399049562242013 + 0.35477529095203764 + 0.58610203938174266 + 0.40644477256361355 + 0.71793056595648608 + 0.66477589590931174 + 0.97927429696548529 + 0.36499696765057976 + 0.52802936784698673 + 0.52193872997570168 + 0.65870406107171320 + 0.61820421766104161 + 0.90040398303971614 + 0.29803533235775248 + 0.43464180099291738 + 0.87943860239172977 + 0.55864292921723191 + 0.13078376121440627 + 0.32540694310914731 + 0.34218271962835800 + 0.85482213249557404 + 0.94370033620028693 + 0.47511272032083696 + 0.47345305476497757 + 0.30221722705385190 + 0.90788085689764131 + 0.73088894292231998 + 0.17160524082518336 + 0.49465372870198721 + 0.96147695954616808 + 0.88148268494196458 + 0.55764885791823815 + 8.9901851454712300E-002 + 0.65913871676668379 + 0.72138163168975211 + 0.12502379216927295 + 0.71709960280769991 + 0.18777107318550890 + 0.10248155941998149 + 0.98617977224360587 + 0.68567655766027968 + 0.17195690648704343 + 0.43362203783085818 + 0.61442574093912583 + 0.22795440666493150 + 0.45159776477714786 + 0.35506644719964342 + 0.22091367714392263 + 0.19869581487634491 + 0.39823284736400311 + 0.48020328462351358 + 3.9444513547934434E-002 + 0.12587307223420652 + 0.44774786276711254 + 2.2021640108484064E-002 + 0.56493299073228442 + 0.73226615157794228 + 7.1254567266961999E-002 + 0.31994815977919089 + 0.31708865283838250 + 0.36656558366404113 + 0.16644798526758109 + 0.93486265063832974 + 0.71011988102743473 + 0.54926645936444274 + 0.58605107884734053 + 2.7772399667945535E-002 + 0.75909909283966215 + 0.92398273313319024 + 0.94833569515493821 + 0.97384004056465656 + 0.39397350717486646 + 0.27285621224040568 + 0.95653472169096787 + 0.70328398407865222 + 0.87180350219484026 + 6.6432874578069345E-002 + 0.69072941477110916 + 0.73734474714848730 + 0.80319825618801843 + 0.42729616541454263 + 5.4919787710240087E-002 + 0.15450456340634489 + 0.74034100276525905 + 0.96935078189734725 + 0.90443234384611060 + 0.62283710241925050 + 0.46785604484035659 + 0.46286005241418948 + 0.31773798027739275 + 0.63765611427091784 + 0.80328051712626092 + 0.19525323066950762 + 0.74756144304188865 + 0.92027981114775770 + 0.23242189664972557 + 0.56477479140523101 + 0.16691241907533438 + 0.39313171095178490 + 0.83634321388461430 + 0.85824982606269629 + 0.83048557891225272 + 0.20967159121899215 + 0.50465997873369872 + 0.69439147801213608 + 0.89473894925435360 + 0.15885310306500600 + 0.18406558912944249 + 0.44741975100717823 + 0.24417447469403086 + 0.23280846840193803 + 0.94881850753792207 + 0.93461161448051300 + 0.98519339601865497 + 0.38559516159581975 + 0.39606912367768388 + 3.1572368911323423E-002 + 0.59106852644135799 + 4.9805119209725746E-002 + 6.7688840788029836E-002 + 0.85560432841582923 + 0.65653631149139713 + 0.44057114606664527 + 4.3629071715606926E-002 + 0.85186666330847416 + 8.8802471360835966E-003 + 0.98174099879466681 + 2.5836412163702249E-002 + 0.52300842691343874 + 0.64894410785534262 + 0.56494700296770617 + 0.78858968385953077 + 3.7948233691199817E-002 + 0.20123976558154411 + 0.34807867093762468 + 0.81695343131666576 + 0.43351293926762935 + 0.84384262048552827 + 4.2577172366605964E-002 + 0.85361888738422920 + 0.95788831084358605 + 0.79807765581852763 + 0.91327714868102916 + 0.85284136018775314 + 0.55569980891232262 + 0.49083490210034242 + 0.43150479510447681 + 0.70729472554743467 + 0.86498769409025300 + 0.28848072947738501 + 0.16175396436556611 + 0.25688428814351738 + 0.34400203933857298 + 0.86244684696218243 + 0.64051358750307230 + 0.92926672202570515 + 0.22008928936727301 + 0.92435618627948912 + 0.73972259057434897 + 0.73416767574040165 + 0.76481953984293227 + 0.10591075720374121 + 0.97739621257275289 + 0.70589339623840175 + 0.52087884406080320 + 0.62461198422844078 + 0.28044763214094104 + 0.97068743889651898 + 0.53039819641475816 + 0.52483948157134819 + 0.31710449859349765 + 0.61502373095249752 + 0.48691376936032427 + 0.31241425478234319 + 0.47538384794357214 + 0.19715054098147888 + 0.88328283504554506 + 0.41738258089505820 + 0.49131169213141135 + 0.23547550930690520 + 0.69041567038755147 + 9.4628745392359548E-002 + 0.45154820637795723 + 0.25110076290679473 + 0.50124555005024973 + 0.56332580935005439 + 0.85765721095451042 + 0.59394939796868229 + 0.82372824154009194 + 0.12752887845827843 + 0.76413749367498696 + 0.90069522975852934 + 0.43607909818563684 + 0.96611608037018826 + 0.76994630144758958 + 0.13937315847451615 + 0.83244975735589222 + 0.66534691176449101 + 0.73242656767334324 + 0.86430061446515793 + 4.2056980144323575E-002 + 0.47900168812325816 + 0.20884876281339260 + 0.67552678318658366 + 0.74538444831178197 + 0.16351980201258698 + 0.97782578979865775 + 0.70761116689389780 + 0.92787506925407115 + 0.31626157696342716 + 0.29271893560768447 + 0.22480754433519579 + 0.41979954655704077 + 0.41116196821139184 + 0.27439289669445444 + 0.46883878934423961 + 0.11048974599079386 + 0.84023055802836666 + 0.33963728790632786 + 0.33246132254972238 + 0.30830043066972834 + 0.61320074002953362 + 0.83682518188176047 + 0.20590560052593787 + 0.17592274953600651 + 8.6346862720994011E-002 + 0.53554714921266466 + 0.38131617737101919 + 0.80932337908682328 + 0.13750965464620180 + 0.88575174572093474 + 0.12909987218158037 + 0.83251873127817788 + 0.24842627494136238 + 0.59683076582325967 + 0.48883168275802191 + 0.26625902271628021 + 0.40682078748693229 + 4.2754145616687111E-002 + 1.5114071010209784E-002 + 0.53437721266600491 + 0.71543972709202208 + 0.11686018319695179 + 0.21189493264208892 + 0.46511154443140512 + 0.12719825429922338 + 0.15157980707422425 + 0.43520946435180718 + 0.64153069898346970 + 0.37847826275947938 + 0.34488022400942242 + 1.9218596094404461E-002 + 0.55378127201960936 + 0.46399180569913767 + 0.53279711094453930 + 0.34199054770159520 + 0.60928719568182288 + 5.1042739039672824E-003 + 0.11368173665286108 + 0.99593573631802101 + 0.25335611153120041 + 0.16593627654569687 + 0.65036738048539178 + 0.91844469229673109 + 0.36588645515485752 + 0.42575656951619223 + 2.5948008105135756E-002 + 0.96553752554667938 + 0.10453811537968249 + 0.24916182732853542 + 0.13315022009238220 + 0.69797803367777789 + 0.83052361278405584 + 0.98353783475080903 + 0.46303658422554861 + 0.76821323829073052 + 0.20502108939150432 + 0.47510410940603265 + 0.95107594096149484 + 0.71316678992811688 + 0.89444924307004214 + 0.58122364358867173 + 0.63452067971237724 + 0.19035939388105660 + 0.60941755714207346 + 0.14823703378512221 + 0.56958247344090651 + 0.74383718236159879 + 0.53582182822951552 + 8.8118712314528835E-002 + 0.78735186159694592 + 0.10682429744509747 + 0.34889746129283239 + 0.27316917982397371 + 0.55020936546745247 + 0.19875822849681057 + 0.32727051053365130 + 0.10423212591144448 + 0.31047609908745599 + 0.27039021172527811 + 0.55134565163007210 + 0.98024063073614442 + 0.93304599528342891 + 5.7322413622490132E-002 + 9.8757603947650097E-002 + 0.10199126743695430 + 0.86290047098718148 + 0.74772555208704361 + 0.33560674163423343 + 7.0401022012212877E-002 + 0.31742669012715297 + 0.67553427505066566 + 0.63814134176603332 + 7.9834348807100497E-002 + 1.2513608927477549E-002 + 0.49244428138305452 + 1.2885830355461536E-002 + 0.78574658950779153 + 0.81226540440050599 + 0.61111607966359216 + 0.83461606111027464 + 0.98460264355779259 + 0.56666672379289196 + 3.2891494475933314E-002 + 0.81526318772061046 + 0.74199714281300899 + 0.32318822348783272 + 0.26285999044327824 + 9.5929780332578929E-002 + 0.34220067007473176 + 1.9889802059953610E-002 + 0.78488176387158504 + 0.83129726012966998 + 0.11245932704061801 + 0.34448869023827910 + 0.25034910384552944 + 0.26776978458689626 + 0.80637984526830309 + 0.10776415004085393 + 0.74385243239834153 + 0.54217682357920438 + 0.26669251756461776 + 0.54915979130428383 + 0.12673814774015213 + 0.21949275531181556 + 7.1380720677826304E-002 + 0.71886143071972342 + 0.53028125570763152 + 0.96570017853737866 + 0.83442533023564991 + 5.3762733444703059E-002 + 4.9926535455153243E-002 + 0.84241172350688487 + 0.54902115351462655 + 0.32045901028729418 + 0.97950703602923284 + 0.72804286298322296 + 0.58162277554273345 + 0.36128911712554768 + 0.94084764263085674 + 0.20746205049784194 + 0.49491212202195101 + 9.9624782102264220E-002 + 0.84587432233623616 + 0.75936085307670709 + 0.47998934585895725 + 0.30116224349299969 + 0.30309134003009319 + 0.68274399405911623 + 0.23239154807731666 + 0.82776191812662958 + 0.12872698483300127 + 0.26795550598453133 + 0.75973958764646810 + 0.19596665348190356 + 0.58401906663879544 + 0.58321919930093813 + 0.47019635167153950 + 0.70969636950427883 + 3.6987763860924616E-002 + 0.52712512496501063 + 0.86525578925498081 + 0.98764182352961427 + 0.32921808042189582 + 0.52332690527862979 + 0.70111792721126420 + 0.28244241559992389 + 3.3281502203404756E-002 + 0.44487603775949580 + 0.18870694862308923 + 0.65327822013202663 + 0.86304215202964940 + 0.69406828766550888 + 0.29018680399104468 + 0.20219921884479319 + 3.9787057060440389E-002 + 0.95461058087433770 + 0.44452620693538591 + 0.83873898049369799 + 0.86473192718546699 + 0.31256996801438319 + 0.36029421237918413 + 0.57620203302522555 + 0.99345519679777539 + 0.32074721641011905 + 0.37235741323139848 + 0.76115650217565900 + 4.3422824443165808E-002 + 0.62205970424373191 + 0.10669865052003757 + 0.56339179824833607 + 0.84008286585745040 + 0.78754420609820031 + 0.65426888804310579 + 0.63180694043031238 + 0.27379358059283021 + 0.52376982437886355 + 6.5628737063754272E-003 + 0.64381119211424931 + 0.53105016953987771 + 0.39386857664553077 + 0.64703913418516734 + 1.1410243733678271E-002 + 0.24476978596169019 + 0.49633517217042922 + 0.48278468524535256 + 9.1042016025070893E-002 + 0.23093119090698622 + 0.47154985607143018 + 0.83803143217707188 + 0.28971577849473817 + 0.65480573910208406 + 0.92414692946266896 + 0.75968153955727757 + 0.37535282600894249 + 0.24570338052825846 + 0.81945735509187045 + 7.4884898638600816E-002 + 0.37484470171379414 + 0.77891273362762092 + 2.5869671853797627E-002 + 0.93857063044766420 + 9.5879165930575994E-002 + 3.7519810481470017E-003 + 0.59141653656473636 + 0.13623968337627090 + 0.27304589165438742 + 0.66854607975901814 + 0.92761332260016260 + 0.27199437101797130 + 0.85100955192947225 + 0.95673207295768137 + 0.26699168962499797 + 0.51735911015909153 + 0.39926348765866493 + 0.62833487265679722 + 0.61030784341147637 + 0.45351774287358282 + 0.39071599079986186 + 0.19600530855928255 + 0.66411151984994632 + 0.18022452258902888 + 0.61703693681279503 + 0.84114228390776802 + 0.81330048042515202 + 0.47916226599670608 + 9.0463484149008622E-002 + 5.6510441728043759E-002 + 0.79256901137416591 + 0.67690113722304801 + 0.70870980115194016 + 0.59410263206372704 + 0.67276664667762631 + 0.94685030780890855 + 0.18017878875817672 + 0.49846175377805224 + 1.0900501685927821E-002 + 0.59778308466137986 + 0.66404830524820468 + 0.92313745314454820 + 0.31609444824582056 + 0.57297586549800528 + 0.24276090521697569 + 2.5336238987033965E-002 + 0.21722352990184746 + 0.34444630009062038 + 0.14813575696799930 + 0.29470476428820547 + 0.82647206348174862 + 0.86397162628984603 + 0.87253762249390476 + 0.99437831408630828 + 0.96611436633802228 + 0.86535980027614912 + 0.69833716481997854 + 4.3079072220191250E-002 + 0.50765982176574553 + 0.80666539779437585 + 5.2176252401643808E-002 + 0.75671390607858768 + 0.21419749381046316 + 0.70426965121627205 + 0.20699836748160649 + 0.21348292857078022 + 0.14277546900442317 + 0.93722724172188165 + 0.77110485349003710 + 0.14104322419489890 + 0.45596034255804341 + 0.17004193950362989 + 0.88899914280643344 + 2.2196643937938632E-002 + 0.93835843600527014 + 0.25706680177818697 + 0.77307646204329927 + 0.71809822519050215 + 0.72993867820703429 + 0.68004235380489320 + 0.42387308547439417 + 0.72245210178429886 + 0.34701690304286359 + 0.12182489990669154 + 0.50910218461131151 + 0.80371985724276485 + 0.84350836630640558 + 0.64527143750703431 + 0.22311490663459210 + 0.67078404720487939 + 0.99213295191400874 + 0.91995804396966108 + 0.33238785754262778 + 0.46942009324286715 + 0.37553893107884662 + 0.61018540998448856 + 0.46097967881933499 + 0.33339569286177095 + 0.57300834014620960 + 0.63104708963166445 + 0.91731549321855610 + 0.50681402887710902 + 0.47047084346093015 + 2.5512286327135314E-002 + 0.99612308445772513 + 0.74275349309682781 + 0.47591020530570560 + 0.40181828773070549 + 0.78958817331287179 + 4.7109378789517820E-002 + 7.6896431363802975E-002 + 0.52024921083216213 + 0.10480716966488046 + 0.42315609462054837 + 5.4088876400001595E-002 + 0.50191886444818934 + 0.57294879761911588 + 0.61273083454976174 + 0.30781987367103980 + 0.29015147800931018 + 2.1794091398266602E-002 + 0.63088344450125078 + 0.79661323318752508 + 0.17043623105301720 + 0.41032335358113770 + 0.92030312356238753 + 0.87396824733972167 + 0.74899509338264281 + 0.31838721599348929 + 0.86054367420573641 + 0.84019508502745310 + 0.83760455345749918 + 4.0671404166424452E-002 + 8.7429545787472307E-002 + 0.62751823707785803 + 0.86310241290249579 + 5.6408304683458255E-002 + 0.64132246071872601 + 5.7409004448881973E-002 + 0.45600569783792011 + 0.83987595170225404 + 0.91812015949614789 + 0.66968416911208450 + 0.10216675578983825 + 0.39572187272063175 + 0.92676645819906867 + 0.54303841682618881 + 0.39691687027067601 + 0.26140082137998988 + 0.88851342077088091 + 0.39595075046580419 + 0.33741583893651139 + 0.86202860286726590 + 0.43769682316377612 + 0.50440613242815147 + 0.76225606266665125 + 0.96186039885528274 + 0.43798261467920696 + 0.37437952559121612 + 0.96408342797511537 + 6.3187664110252229E-002 + 0.16024309193573316 + 0.18450941891428840 + 0.47306309311417749 + 0.35900644215697497 + 0.61470913305812047 + 0.10474165984858530 + 0.95917068078074053 + 0.88940297818810876 + 0.36746756311191220 + 0.62620203727022172 + 5.2412185597020766E-002 + 0.85704224052903655 + 5.1788849909250700E-002 + 0.93533060539331103 + 0.77906133022465696 + 0.82492165062861744 + 0.61374629857715846 + 0.68768843507729471 + 5.0853126173038277E-002 + 0.17810889780727379 + 0.29695389645936032 + 2.8190332292901132E-002 + 1.3266090115848783E-002 + 0.65968091915802063 + 0.55206255062179110 + 0.62410948926586585 + 6.5703440779245170E-002 + 0.78994459416097129 + 0.39692255819054978 + 0.21402161817816179 + 0.57677106604464967 + 0.83319482350367835 + 0.17790186849569523 + 0.73078871483060226 + 0.22665571786620475 + 0.74671159152102717 + 0.86438106596336439 + 0.95413637335751744 + 0.14620227943607134 + 5.7936809913204712E-002 + 0.98535172252024594 + 2.0865933557963956E-002 + 0.65052522272778290 + 0.83708978486930263 + 0.60027810148141469 + 0.85099195813480932 + 0.85443785633967906 + 0.73517901754262738 + 0.97714779185145417 + 0.26830391876777426 + 0.57658970543655030 + 0.73134209808441497 + 0.69279360310381577 + 0.22293560047807759 + 0.22272825397147145 + 0.88005286031377139 + 0.96564411627953461 + 0.87145362842802143 + 0.56230611494263272 + 0.93014355526995374 + 0.90883360596208718 + 0.45265482883941033 + 7.5890123196877823E-003 + 0.13891899600998414 + 0.19395797792807912 + 0.40822572570344562 + 0.91984262678367656 + 0.41035009880865658 + 0.21031624559043749 + 0.18100789306921783 + 0.79352208861650197 + 0.54701620377705495 + 0.43573916585428663 + 0.45475459895084569 + 0.56263253074866171 + 0.61249511196383466 + 1.7521048876169232E-002 + 0.34828113003582928 + 0.52769822167224234 + 9.2073402614058608E-002 + 0.67385854686032332 + 6.2394989126019595E-002 + 0.67562353970063427 + 0.47211798936650595 + 9.8665257294559439E-002 + 0.43082834190490971 + 0.14249350583460796 + 0.50526362688148296 + 0.38193182809615323 + 0.15678567120378872 + 0.84734656947749087 + 0.63450811669369500 + 0.69169034334883506 + 0.98033573532169171 + 0.32444405697311041 + 0.37046074085349545 + 0.89749076441217213 + 0.92944504213263102 + 0.30065964450140470 + 0.82602067292728165 + 0.82831342461735602 + 0.82782600616961588 + 0.35433425353576808 + 0.89621590808965834 + 0.53228509433131066 + 0.20190963812276053 + 0.67000526616971356 + 0.93890699885969653 + 0.10520203411017093 + 0.62848355019055191 + 0.33034544583888703 + 7.8076170394218991E-003 + 0.72417687182973012 + 0.98800541160080257 + 0.74327516345928402 + 0.92590555866150481 + 0.60352349719456910 + 0.15680962659125086 + 4.9434958757949232E-002 + 0.55686051658955904 + 0.97810570859908452 + 0.12872754429692890 + 0.95547914162817804 + 0.95573437656426208 + 0.12967304813757963 + 0.45951519961713316 + 0.68151881426853933 + 0.46566941554450381 + 0.62813314240559137 + 0.54948969763289313 + 0.77136885714444148 + 0.80877759415522377 + 0.65958877927092274 + 0.89058500654762440 + 0.90982465711328686 + 0.61749100212857400 + 0.54754063509364670 + 0.35713498382768094 + 2.4839537655822141E-002 + 0.44473647181794362 + 0.48318333042530526 + 0.15052230596724669 + 0.98645625807854387 + 0.16983403901664573 + 0.24743079769734777 + 0.75834503525907948 + 0.39212420611110943 + 0.64782771756848589 + 0.41577658200279899 + 0.15879365042174598 + 0.94978611458295603 + 0.66463121868276431 + 0.84009097217167650 + 0.90795928852065444 + 0.52724828450050865 + 0.38418444434347165 + 0.16407355312341210 + 8.8496510745926571E-002 + 0.50958525975331526 + 7.0102045493356968E-002 + 0.35537675549345948 + 0.93728436913066560 + 0.26853869097094929 + 5.9162881009825696E-002 + 0.46807856276806703 + 0.34526803158516728 + 0.77813731816475240 + 0.70936884855344218 + 0.36739517626832097 + 0.36261806234035987 diff --git a/tests/TERAPI-PIMD/scrdir0001.ref b/tests/TERAPI-PIMD/scrdir0001.ref new file mode 100644 index 00000000..e69de29b diff --git a/tests/TERAPI-PIMD/scrdir0002.ref b/tests/TERAPI-PIMD/scrdir0002.ref new file mode 100644 index 00000000..e69de29b diff --git a/tests/TERAPI-PIMD/scrdir0003.ref b/tests/TERAPI-PIMD/scrdir0003.ref new file mode 100644 index 00000000..e69de29b diff --git a/tests/TERAPI-PIMD/scrdir0004.ref b/tests/TERAPI-PIMD/scrdir0004.ref new file mode 100644 index 00000000..e69de29b diff --git a/tests/TERAPI-PIMD/tc_server.cpp b/tests/TERAPI-PIMD/tc_server.cpp new file mode 100644 index 00000000..725cd176 --- /dev/null +++ b/tests/TERAPI-PIMD/tc_server.cpp @@ -0,0 +1,61 @@ +#include +#include + +#include "../tc_mpi_api.h" + +using namespace std; + +int main(int argc, char* argv[]) +{ + char *serverName = NULL; + + // Due to a bug in hydra_nameserver, it crashes + // when multiple TC servers call `MPI_Unpublish_name()` + // Hence, we want to allow invoking without this parameter, + // in which case TC server will just print the port to stdin, + // where it could be grepped and passed via file to ABIN, + // and it will never call MPI_Publish_name/MPI_Unpublish_name + // NOTE: This behaviour is different from real TC, + // which has default serverName and will always try to publish it. + if (argc > 2) { + printf("Only one cmdline argument supported, , but you provided more!"); + throw std::runtime_error("Incorrect invocation"); + } + + if (argc == 2) { + serverName = new char[1024]; + strcpy(serverName, argv[1]); + } + + TCServerMock tc = TCServerMock(serverName); + if (serverName) { + delete[] serverName; + } + + tc.initializeCommunication(); + + tc.receiveNumAtoms(); + tc.receiveAtomTypes(); + + int loop_counter = 0; + int MAX_LOOP_COUNT = 100; + // Will go through this loop until MPI client gives an exit signal. + while (true) { + + int status = tc.receive(); + if (status == MPI_TAG_EXIT) { + break; + } + + tc.send(); + + // This is just a precaution, we don't want endless loop! + loop_counter++; + if (loop_counter > MAX_LOOP_COUNT) { + printf("Maximum number of steps exceeded!\n"); + return(1); + } + } + + return(0); +} diff --git a/tests/TERAPI-PIMD/temper.dat.ref b/tests/TERAPI-PIMD/temper.dat.ref new file mode 100644 index 00000000..727c8ac4 --- /dev/null +++ b/tests/TERAPI-PIMD/temper.dat.ref @@ -0,0 +1,2 @@ + # Time[fs] Temperature T-Average Conserved_quantity_of_thermostat + 0.24 0.42 0.42 diff --git a/tests/TERAPI-PIMD/test.sh b/tests/TERAPI-PIMD/test.sh new file mode 100755 index 00000000..4cdabed3 --- /dev/null +++ b/tests/TERAPI-PIMD/test.sh @@ -0,0 +1,61 @@ +#/bin/bash +set -euo pipefail +# Useful for debugging +#set -x + +ABINEXE=$1 +source ../test_tc_server_utils.sh + +set_default_vars +set_mpich_vars +# If $1 = "clean"; exit early. +if ! clean_output_files $1; then + exit 0 +fi + +N_TERA_SERVERS=$(egrep --only-matching 'nteraservers\s*=\s*[0-9]' $ABININ | egrep -o [0-9]) + +# Exit early for OpenMPI build. +check_for_openmpi + +# Compiled the fake TC server +$MPICXX $TCSRC -Wall -o $TCEXE + +# NOTE: We very intentionally do NOT launch +# hydra_nameserver in this test since it cannot handle +# multiple TC servers due to a bug in MPI_Unpublish_name +# https://github.com/pmodels/mpich/issues/5058 +# +# Therefore, we pass the port_name to ABIN via files, see below. +#TC_SERVER_NAME="tcserver.$$" +#launch_hydra_nameserver $MPICH_HYDRA +#hostname=$HOSTNAME +#MPIRUN="$MPIRUN -nameserver $hostname -n 1" + +MPIRUN="$MPIRUN -n 1" + +ABIN_CMD="$ABINEXE -i $ABININ -x $ABINGEOM" # -M $TC_SERVER_NAME" + +let NUM_JOBS=N_TERA_SERVERS+1 +declare -A job_pids +for ((itera=1;itera<=N_TERA_SERVERS;itera++)) { + #$MPIRUN ./$TCEXE $TC_SERVER_NAME.$itera > $TCOUT.$itera 2>&1 & + $MPIRUN ./$TCEXE > $TCOUT.$itera 2>&1 & + job_pids[$itera]=$! +} +sleep 1 +# Grep port names from TC output, pass to ABIN via a file. +for ((itera=1;itera<=N_TERA_SERVERS;itera++)) { + grep 'port name' $TCOUT.$itera | awk -F"port name: " '{print $2;exit}' > $TC_PORT_FILE.$itera +} + +$MPIRUN $ABIN_CMD > $ABINOUT 2>&1 & +job_pids[$NUM_JOBS]=$! + +function cleanup { + kill -9 ${job_pids[@]} > /dev/null 2>&1 || true + exit 0 +} + +trap cleanup INT ABRT TERM EXIT +check_running_processes ${job_pids[@]} diff --git a/tests/TERAPI-PIMD/velocities.xyz.ref b/tests/TERAPI-PIMD/velocities.xyz.ref new file mode 100644 index 00000000..2eda254e --- /dev/null +++ b/tests/TERAPI-PIMD/velocities.xyz.ref @@ -0,0 +1,14 @@ + 3 + Time step: 1 +O 0.0000000000E+00 -0.9563564009E-05 0.0000000000E+00 +H -0.5851553569E-04 0.7589655783E-04 0.0000000000E+00 +H 0.5851553569E-04 0.7589655783E-04 0.0000000000E+00 +O 0.0000000000E+00 -0.1314779241E-04 0.0000000000E+00 +H -0.8043976155E-04 0.1043410371E-03 0.0000000000E+00 +H 0.8043976155E-04 0.1043410371E-03 0.0000000000E+00 +O 0.0000000000E+00 -0.1434197286E-04 0.0000000000E+00 +H -0.8774274119E-04 0.1138180673E-03 0.0000000000E+00 +H 0.8774274119E-04 0.1138180673E-03 0.0000000000E+00 +O 0.0000000000E+00 -0.1314779241E-04 0.0000000000E+00 +H -0.8043976155E-04 0.1043410371E-03 0.0000000000E+00 +H 0.8043976155E-04 0.1043410371E-03 0.0000000000E+00 diff --git a/tests/TERAPI/README b/tests/TERAPI/README deleted file mode 100644 index 48a8cc71..00000000 --- a/tests/TERAPI/README +++ /dev/null @@ -1,2 +0,0 @@ -File tera-mpiapi.cpp has to be compiled separately with the same MPI compiler as ABIN. -Currently does not work... diff --git a/tests/TERAPI/charges.dat.ref b/tests/TERAPI/charges.dat.ref new file mode 100644 index 00000000..577f91c9 --- /dev/null +++ b/tests/TERAPI/charges.dat.ref @@ -0,0 +1,4 @@ + # Atomic Mulliken charges from current electronic state + # Time_step Bead_index O H H + 0 1 -1.0000 -2.0000 -3.0000 + 1 1 -1.0000 -2.0000 -3.0000 diff --git a/tests/TERAPI/dipoles.dat.ref b/tests/TERAPI/dipoles.dat.ref new file mode 100644 index 00000000..50dadf46 --- /dev/null +++ b/tests/TERAPI/dipoles.dat.ref @@ -0,0 +1,3 @@ + # Time |D| Dx Dy Dz + 0 1 0.3742E-01 -0.1000E-01 -0.2000E-01 -0.3000E-01 + 1 1 0.3742E-01 -0.1000E-01 -0.2000E-01 -0.3000E-01 diff --git a/tests/TERAPI/energies.dat.ref b/tests/TERAPI/energies.dat.ref new file mode 100644 index 00000000..493bdca2 --- /dev/null +++ b/tests/TERAPI/energies.dat.ref @@ -0,0 +1,2 @@ + # Time[fs] E-potential E-kinetic E-Total E-Total-Avg + 0.97 0.5551215316E-03 0.2371941974E-03 0.7923157290E-03 0.7923157290E-03 diff --git a/tests/TERAPI/forces.xyz.ref b/tests/TERAPI/forces.xyz.ref new file mode 100644 index 00000000..4c71fdc8 --- /dev/null +++ b/tests/TERAPI/forces.xyz.ref @@ -0,0 +1,5 @@ + 3 +net force: 0.00000E+00 0.00000E+00 0.00000E+00 torque force: 0.00000E+00 0.00000E+00 0.00000E+00 +O -0.0000000000E+00 -0.2300722731E-01 -0.0000000000E+00 +H -0.7955237226E-02 0.1150361365E-01 -0.0000000000E+00 +H 0.7955237226E-02 0.1150361365E-01 -0.0000000000E+00 diff --git a/tests/TERAPI/input.in b/tests/TERAPI/input.in index aeed52fc..b4f1a7b8 100644 --- a/tests/TERAPI/input.in +++ b/tests/TERAPI/input.in @@ -1,29 +1,28 @@ -This is a sample input file for ABIN -NOTE: order of input sections matters!! +NOTE: The reference files were actually created with +pot='mmwater' +TCServerMock uses the same qTIP4P potential, +so now we are sure that all data are passed correctly. +(Except for charges and dipoles, which are currently randomly +assigned in tc_mpi_api.cpp) &general -pot='_tera_' !not done yet,options should be g09,orca,tera,turbo,molpro,nab,harm,morse,guillot,2dho +!pot='mmwater' +pot='_tera_' watpot=1 -ipimd=0, !classical simulation 0, quantum simulation 1 -nwalk=1, !number of random walkers -imini=0, !equilibration period,!not properly implemented yet -istage=0, !staging transformation (1), without staging (0) -nstep=5, -dt=40., !number of steps and timestep -irandom=131313, !random seed -nabin=50, ! what is the gap between ab initio calculations? +ipimd=0, +nstep=1, +dt=40., +irandom=13131313, -nwrite=1, ! how often some output should be printed (estimators etc.) -nwritex=1, ! how often should we print coordinates? -nrest=1, ! how often we print restart files? -irest=0, ! should we restart from restart.xyz? (ignoring mini.dat), NOT working yet!! - -natom=3, !number of atoms +nwrite=1, +nwritef=1, +nwritev=1, +nwritex=1, +nrest=1, +idebug=3 / &nhcopt -inose=1, ! Thermostating: Nose-Hoover 1, microcanonical 0,GLE 2, LE 3 -temp=100.15, ! initial temperature for Maxwell-Boltzmann sampling [au] -nrespnose=8 +inose=0, +temp=0.0d0 / - diff --git a/tests/TERAPI/movie.xyz.ref b/tests/TERAPI/movie.xyz.ref new file mode 100644 index 00000000..74f66fcf --- /dev/null +++ b/tests/TERAPI/movie.xyz.ref @@ -0,0 +1,5 @@ + 3 +Time step: 1 Sim. Time [au] 40.00 +O 0.00000000E+00 0.11759214E+00 0.00000000E+00 +H 0.75449565E+00 -0.46876320E+00 0.00000000E+00 +H -0.75449565E+00 -0.46876320E+00 0.00000000E+00 diff --git a/tests/TERAPI/restart.xyz.ref b/tests/TERAPI/restart.xyz.ref new file mode 100644 index 00000000..d39bd2fc --- /dev/null +++ b/tests/TERAPI/restart.xyz.ref @@ -0,0 +1,1295 @@ + 1 40.000000000000000 + Cartesian Coordinates [au] + 0.0000000000000000 0.22221693683724300 0.0000000000000000 + 1.4257901548862351 -0.88583407860420382 0.0000000000000000 + -1.4257901548862351 -0.88583407860420382 0.0000000000000000 + Cartesian Velocities [au] + 0.0000000000000000 -3.5046285493045846E-005 0.0000000000000000 + -2.0490214720593456E-004 2.7812773889049629E-004 0.0000000000000000 + 2.0490214720593456E-004 2.7812773889049629E-004 0.0000000000000000 + Cumulative averages of various estimators + 5.2709821651373865E-005 + 0.0000000000000000 0.0000000000000000 + 7.9231572901965783E-004 +PRNG STATE (OPTIONAL) + 595103133 10 + 1 -1.1749474095392607 + 0.49629292000981806 + 0.46025861761134124 + 0.50251434707008613 + 0.19799855572176028 + 0.74317218699772170 + 5.8075842586035265E-002 + 0.27483363561092844 + 0.27719491285128939 + 0.48728943011535364 + 0.46801484136926064 + 0.88903226614713660 + 0.30735767922567803 + 9.9795745428384208E-002 + 6.7650980371311675E-002 + 0.98915190010278309 + 0.73792833387672374 + 4.0845246124789014E-002 + 0.82098919188374353 + 0.18700871449268774 + 0.56585360562937836 + 0.51927011799128664 + 0.69643920914248980 + 0.27634987133468769 + 0.69242326506696728 + 0.91100346773719565 + 0.12426805639334404 + 0.32165353907126715 + 0.62755818230270322 + 0.20285297115891510 + 0.50012849908370427 + 0.96528830685633693 + 0.39059528651803888 + 0.17433940511736878 + 0.47796380040087527 + 0.72702865518551008 + 0.24874863952765836 + 0.18383747726886313 + 0.92088133249984239 + 0.77113521596477241 + 0.47437965445285712 + 0.35130879751342547 + 0.20921294354998210 + 9.1374779842865905E-002 + 0.45186765734940693 + 0.39446020684599148 + 0.95716829603551545 + 0.87867833359246816 + 0.18555385917002098 + 0.80494863829938623 + 0.49589478880382387 + 4.7012530352265713E-002 + 0.98360638479730511 + 0.78896443148071427 + 0.47133181076921105 + 0.95542030148846990 + 0.22086832429566883 + 0.70605708650068877 + 0.46775882110550882 + 0.27314069065646507 + 0.81516711500871608 + 0.15044684215293458 + 0.88950536312020745 + 0.97394254611016606 + 0.85968018508344457 + 0.33301454477990688 + 0.53470345563750854 + 3.2741238003257678E-002 + 0.79340124117738853 + 6.8909814826419336E-002 + 0.44230400378664925 + 0.57034834634476539 + 0.70397596781939953 + 0.55076343247115744 + 5.1053705188852661E-002 + 0.58211370496187698 + 0.48159410839389594 + 0.90804116563111492 + 2.2367772917075257E-002 + 0.67557588472600827 + 0.88921451415548347 + 0.22390347494851781 + 0.57021288317992358 + 0.54176710102172976 + 0.14734870667805211 + 8.2660977967744742E-002 + 0.44563228441164782 + 0.35025334188112112 + 0.33003761522191155 + 0.19750126970936677 + 0.64965277034081126 + 0.75770326930313914 + 5.7049954076624942E-002 + 0.60589031047008390 + 0.16227326127837927 + 6.6519674582050214E-002 + 0.15168121130731649 + 4.8750680956111125E-003 + 0.48918163044671559 + 0.10650604919490192 + 0.48904627178482585 + 0.80901801758646386 + 7.1311012125914175E-002 + 0.55418150844065650 + 0.83438336051749928 + 0.99842587984902664 + 0.64940069478122453 + 6.3054804090619143E-002 + 0.57675469542413182 + 4.9812360626813046E-002 + 0.17272020649907915 + 0.80532406784800514 + 8.5617945919896243E-002 + 0.80418573083733236 + 0.88996023478180675 + 0.77586548514162601 + 0.45048214944494447 + 9.5693150066765043E-002 + 0.22961908735589986 + 0.40548063998852157 + 0.19054339047147195 + 0.73117401118166470 + 9.6009895921724819E-003 + 0.20263952293048249 + 0.79223910414447474 + 5.0685423566193322E-002 + 0.31043009335787275 + 0.46917293070848487 + 0.11351314271628610 + 0.19485346938860459 + 0.70366790666560775 + 0.54243607547062211 + 0.35584089742587111 + 0.93636699788146416 + 0.43156241726620337 + 0.29115796587234044 + 0.83966017964082695 + 0.39510852236282190 + 0.98199268024954378 + 0.87199612884721844 + 0.31106278617873073 + 0.66203481978063294 + 0.23628623649304359 + 0.95803214049719898 + 0.65635758421441537 + 0.79935779178625666 + 0.67263540020343271 + 0.58108476366842510 + 7.7543355398340452E-002 + 1.7388641777451141E-002 + 0.29707825098842378 + 0.38183422980812054 + 0.73884564959254817 + 0.67197842873743596 + 0.73599078841893473 + 0.99906414263992005 + 0.23719295414896990 + 0.29814696950393227 + 0.13291026961452701 + 0.58601680058732342 + 0.62154676634001405 + 0.25451339892073932 + 0.31260041196477673 + 9.7069412538008493E-002 + 0.50224763634274794 + 0.35619915554918080 + 0.23935156209466157 + 0.37336625771253651 + 1.2369698223213987E-002 + 0.34136559720325721 + 0.11017712492900600 + 0.74912064943983836 + 0.18070536750009580 + 0.49286434862546002 + 0.57986229638101960 + 0.98886002639123660 + 0.48226954608717776 + 0.69492656006587694 + 0.42707052418293756 + 0.33448971911181857 + 0.53908362255230813 + 0.75801351081264556 + 0.10915606885155427 + 0.21830299867344749 + 0.46813105023782242 + 0.39963358763756318 + 0.21433097842775339 + 0.52259855604177119 + 0.72075984093170575 + 0.50389637622084038 + 2.4373607876761127E-002 + 0.74425736913556406 + 0.46410665733550260 + 0.86585005715718211 + 0.28039344394222354 + 0.79624103219791564 + 0.15979878109718371 + 0.82072429653535295 + 8.6086495272265751E-002 + 0.70607868155129694 + 0.69337763768394467 + 2.1409348732930766E-002 + 0.88112760963274539 + 0.70721370676366035 + 0.91622662881070482 + 9.8614301866138732E-002 + 9.9281624680585878E-002 + 0.82472945885024984 + 5.1907069686841112E-002 + 0.84016769331044117 + 0.20102834979289241 + 0.16807787853937128 + 0.62876845789692126 + 0.99543089289756281 + 0.13494449856878177 + 0.35657143296177551 + 0.85555030185167169 + 0.20953289380073770 + 0.41657234820349487 + 0.91444812552706267 + 0.82654108856169728 + 0.92520408165309220 + 0.84504246589307286 + 0.37163925677521803 + 0.23218198459457540 + 0.60265004614421258 + 0.31387514993012289 + 0.41000329509889966 + 0.20527825677785927 + 0.98950365583741728 + 0.97283544247425624 + 0.73479550677496519 + 0.68875514001355143 + 0.95439256147139417 + 0.14216467179825898 + 4.3646151288456281E-002 + 0.42907170457118582 + 0.54868393131082982 + 0.27996658066949820 + 0.62004954089318431 + 0.68822117766336177 + 0.63095603117775667 + 0.27300518652981154 + 3.7281943031523213E-002 + 9.3959564720353939E-002 + 9.7169381291188017E-002 + 0.20633383159733043 + 0.67349556126177035 + 0.95821715134439245 + 0.92729407144461717 + 0.96671286063364903 + 0.66699092661017900 + 0.17740484432721004 + 4.4641688239241972E-002 + 0.64183374293180862 + 0.74168293360887461 + 3.9225728304700169E-002 + 0.89398416877949671 + 0.79520191889594116 + 0.54128317655580460 + 0.14402585464293693 + 0.20376813083881018 + 0.82505465637876796 + 0.63889554341918142 + 4.3467667240172148E-002 + 0.37088191979264096 + 0.96925518104082897 + 0.54220632535502489 + 0.97441976706182842 + 0.20385822206242921 + 0.37041552642137177 + 0.23492261365965561 + 4.3951053138179930E-002 + 0.12196236410233396 + 0.21740406042093596 + 0.80051545678352909 + 0.14381300568658162 + 0.72332072569340511 + 0.23572331665182489 + 0.44817553259780851 + 0.91616068740353640 + 0.10631561038890069 + 0.47916994476991803 + 0.48940447134899046 + 0.25479999529605024 + 0.88054897486578199 + 0.50909308440123624 + 0.55078336384144677 + 0.65523378672363108 + 0.29857621442787163 + 0.68784189038595045 + 0.83727214028596819 + 0.52409818150328036 + 0.36215662978765195 + 0.53395519406301162 + 0.28479322326557011 + 0.43781313941258304 + 0.61745902873723324 + 0.56316216922065365 + 0.53604193467990058 + 0.53438894391144132 + 0.80372400540051814 + 0.68022860258893303 + 0.55957512601506210 + 4.0534357922563657E-002 + 0.25441822157545957 + 0.82842845775856588 + 6.7044515733972077E-002 + 0.19964019745497907 + 0.46142423624182882 + 0.92460624211602038 + 0.84206872091855800 + 0.55937217407144502 + 0.62363003002528217 + 0.28150732082043817 + 0.41771500116203342 + 0.23251509984273611 + 0.54081739077343727 + 0.45141020334454041 + 0.32482625161830470 + 0.95410615754259354 + 0.24123251178570015 + 0.50821615087825478 + 0.15316740108570670 + 0.11205657700804750 + 0.39324624893320603 + 0.24856377911824978 + 0.20343745892967036 + 0.74472311381824596 + 0.40624323949095498 + 0.21404384330451620 + 0.38590679063393907 + 0.62884322579906282 + 0.29749345136205108 + 0.23387309019370406 + 0.89078383955289553 + 0.13558361493640092 + 0.48658804174344539 + 0.62737194564095589 + 0.78598225154866697 + 0.13137624163984540 + 0.58196012880768677 + 0.10018062919492365 + 0.14579136748324828 + 9.3305828765455345E-002 + 0.42693437972586779 + 0.16272711176750576 + 0.62265360319290863 + 0.40930881593081736 + 0.41091205064884662 + 0.82340461008796595 + 0.68701474294215714 + 0.93040886766721798 + 0.55248226680274470 + 0.60774571909526642 + 6.8810131601534152E-002 + 0.66020312901976297 + 7.3430509311354086E-002 + 0.25163218169350188 + 0.46899198341295545 + 0.41394940277212200 + 0.90919958122827893 + 0.21159826169762752 + 0.42380637387735476 + 0.45415712968395994 + 0.86560859659824985 + 0.54308418754210308 + 0.22768108114869534 + 0.63250210466362589 + 0.27603846678341171 + 8.7062595874559889E-002 + 0.84879094420920964 + 0.58808073063696042 + 0.78143208320231139 + 0.83978092577298469 + 0.45729783032871296 + 0.76173542435132191 + 0.17923980016979613 + 0.20103480137447605 + 0.93840830661349273 + 0.34711230919593916 + 0.58116366914373430 + 0.67699509330761387 + 0.29991363046530495 + 0.20923001835816279 + 9.4750828680822963E-002 + 0.23385249724359625 + 0.18997745559904544 + 0.54884521152075294 + 0.98965268505164872 + 0.83461958062056851 + 0.14622633500608373 + 0.98342229836803696 + 0.26111904008478248 + 8.3766894533169989E-002 + 0.26203549105587598 + 0.92050292818672474 + 0.12683696399781397 + 0.24038715562976165 + 0.79859373597746597 + 0.96855039688318456 + 0.92726143335753619 + 2.3000738910852192E-003 + 0.45250532143043642 + 0.28392395256364367 + 0.81347148322686280 + 2.6315498313508101E-002 + 0.79015433579546723 + 0.82125078165874044 + 0.29534291913652311 + 1.0330770504083375E-002 + 0.33648617676590575 + 0.78416281904277696 + 0.98798999860693471 + 0.80836934645357061 + 0.72253137488112174 + 0.80318806866686998 + 0.77159811022642444 + 0.61098275132179225 + 0.71318518273869103 + 0.82905619305262235 + 0.43927283573212250 + 0.91953182742218331 + 0.11989538298481506 + 0.35586690373822805 + 1.1183235533493985E-003 + 0.67342906618577558 + 7.2930855770394487E-002 + 0.75464020085112438 + 0.98386680411371330 + 0.77289896350504961 + 0.56275930457820778 + 0.51851366712667613 + 0.95504408477258806 + 0.85896574770908884 + 5.0489964233619844E-003 + 0.38869435627021431 + 8.2106162304569352E-003 + 7.4217648476267328E-002 + 0.71430827948955056 + 8.9488183561449830E-002 + 0.62715051038177094 + 0.34592413808423927 + 0.59324287364264805 + 0.60394280170542558 + 0.85355694015056116 + 0.74001415268891080 + 0.76706069755032757 + 0.82784832438217038 + 0.60858969865472190 + 0.98926852290765765 + 0.12724946149202054 + 0.95740681396613070 + 0.93885642671968839 + 0.30145730919106484 + 0.21078063365247601 + 0.21385139243814777 + 0.90224746667208322 + 0.82503164439856036 + 5.3814738656463135E-002 + 0.62543823810650423 + 0.83095986728138982 + 0.33169061153166624 + 0.81679911959487583 + 0.59425653215748753 + 0.78802643162518748 + 0.75990166426681682 + 0.32420649227556098 + 0.98686132380390390 + 0.98626672091111089 + 0.97791442860325617 + 0.78544875788069746 + 0.77419814985276503 + 0.78117220767950002 + 0.92994647466491998 + 0.21593818392123865 + 0.50239618473163006 + 0.15039641948255422 + 0.65776713738440407 + 0.33477708520495497 + 0.85625491258544884 + 0.36094993645332707 + 0.90993949942234664 + 0.34720649086859012 + 0.51404136294097214 + 0.79449456932379903 + 0.54251461546341417 + 0.98975088419386736 + 0.49885170704877524 + 4.4391629771411090E-002 + 0.94017216845187690 + 0.25151323682252524 + 0.49020425528447831 + 0.61440920982823144 + 0.46432078562274626 + 0.93425385111629566 + 0.54733002316946866 + 0.80123157826903579 + 0.13549204673158854 + 0.80100736915681026 + 0.42093770357993776 + 0.83049545121010127 + 0.57467921324095883 + 0.72135185311027072 + 0.14115987240230510 + 0.17994318854805513 + 0.59278210419826749 + 0.93969513145815853 + 0.93539545173686633 + 0.87699610068994005 + 0.30150166861178107 + 0.97399049562242013 + 0.35477529095203764 + 0.58610203938174266 + 0.40644477256361355 + 0.71793056595648608 + 0.66477589590931174 + 0.97927429696548529 + 0.36499696765057976 + 0.52802936784698673 + 0.52193872997570168 + 0.65870406107171320 + 0.61820421766104161 + 0.90040398303971614 + 0.29803533235775248 + 0.43464180099291738 + 0.87943860239172977 + 0.55864292921723191 + 0.13078376121440627 + 0.32540694310914731 + 0.34218271962835800 + 0.85482213249557404 + 0.94370033620028693 + 0.47511272032083696 + 0.47345305476497757 + 0.30221722705385190 + 0.90788085689764131 + 0.73088894292231998 + 0.17160524082518336 + 0.49465372870198721 + 0.96147695954616808 + 0.88148268494196458 + 0.55764885791823815 + 8.9901851454712300E-002 + 0.65913871676668379 + 0.72138163168975211 + 0.12502379216927295 + 0.71709960280769991 + 0.18777107318550890 + 0.10248155941998149 + 0.98617977224360587 + 0.68567655766027968 + 0.17195690648704343 + 0.43362203783085818 + 0.61442574093912583 + 0.22795440666493150 + 0.45159776477714786 + 0.35506644719964342 + 0.22091367714392263 + 0.19869581487634491 + 0.39823284736400311 + 0.48020328462351358 + 3.9444513547934434E-002 + 0.12587307223420652 + 0.44774786276711254 + 2.2021640108484064E-002 + 0.56493299073228442 + 0.73226615157794228 + 7.1254567266961999E-002 + 0.31994815977919089 + 0.31708865283838250 + 0.36656558366404113 + 0.16644798526758109 + 0.93486265063832974 + 0.71011988102743473 + 0.54926645936444274 + 0.58605107884734053 + 2.7772399667945535E-002 + 0.75909909283966215 + 0.92398273313319024 + 0.94833569515493821 + 0.97384004056465656 + 0.39397350717486646 + 0.27285621224040568 + 0.95653472169096787 + 0.70328398407865222 + 0.87180350219484026 + 6.6432874578069345E-002 + 0.69072941477110916 + 0.73734474714848730 + 0.80319825618801843 + 0.42729616541454263 + 5.4919787710240087E-002 + 0.15450456340634489 + 0.74034100276525905 + 0.96935078189734725 + 0.90443234384611060 + 0.62283710241925050 + 0.46785604484035659 + 0.46286005241418948 + 0.31773798027739275 + 0.63765611427091784 + 0.80328051712626092 + 0.19525323066950762 + 0.74756144304188865 + 0.92027981114775770 + 0.23242189664972557 + 0.56477479140523101 + 0.16691241907533438 + 0.39313171095178490 + 0.83634321388461430 + 0.85824982606269629 + 0.83048557891225272 + 0.20967159121899215 + 0.50465997873369872 + 0.69439147801213608 + 0.89473894925435360 + 0.15885310306500600 + 0.18406558912944249 + 0.44741975100717823 + 0.24417447469403086 + 0.23280846840193803 + 0.94881850753792207 + 0.93461161448051300 + 0.98519339601865497 + 0.38559516159581975 + 0.39606912367768388 + 3.1572368911323423E-002 + 0.59106852644135799 + 4.9805119209725746E-002 + 6.7688840788029836E-002 + 0.85560432841582923 + 0.65653631149139713 + 0.44057114606664527 + 4.3629071715606926E-002 + 0.85186666330847416 + 8.8802471360835966E-003 + 0.98174099879466681 + 2.5836412163702249E-002 + 0.52300842691343874 + 0.64894410785534262 + 0.56494700296770617 + 0.78858968385953077 + 3.7948233691199817E-002 + 0.20123976558154411 + 0.34807867093762468 + 0.81695343131666576 + 0.43351293926762935 + 0.84384262048552827 + 4.2577172366605964E-002 + 0.85361888738422920 + 0.95788831084358605 + 0.79807765581852763 + 0.91327714868102916 + 0.85284136018775314 + 0.55569980891232262 + 0.49083490210034242 + 0.43150479510447681 + 0.70729472554743467 + 0.86498769409025300 + 0.28848072947738501 + 0.16175396436556611 + 0.25688428814351738 + 0.34400203933857298 + 0.86244684696218243 + 0.64051358750307230 + 0.92926672202570515 + 0.22008928936727301 + 0.92435618627948912 + 0.73972259057434897 + 0.73416767574040165 + 0.76481953984293227 + 0.10591075720374121 + 0.97739621257275289 + 0.70589339623840175 + 0.52087884406080320 + 0.62461198422844078 + 0.28044763214094104 + 0.97068743889651898 + 0.53039819641475816 + 0.52483948157134819 + 0.31710449859349765 + 0.61502373095249752 + 0.48691376936032427 + 0.31241425478234319 + 0.47538384794357214 + 0.19715054098147888 + 0.88328283504554506 + 0.41738258089505820 + 0.49131169213141135 + 0.23547550930690520 + 0.69041567038755147 + 9.4628745392359548E-002 + 0.45154820637795723 + 0.25110076290679473 + 0.50124555005024973 + 0.56332580935005439 + 0.85765721095451042 + 0.59394939796868229 + 0.82372824154009194 + 0.12752887845827843 + 0.76413749367498696 + 0.90069522975852934 + 0.43607909818563684 + 0.96611608037018826 + 0.76994630144758958 + 0.13937315847451615 + 0.83244975735589222 + 0.66534691176449101 + 0.73242656767334324 + 0.86430061446515793 + 4.2056980144323575E-002 + 0.47900168812325816 + 0.20884876281339260 + 0.67552678318658366 + 0.74538444831178197 + 0.16351980201258698 + 0.97782578979865775 + 0.70761116689389780 + 0.92787506925407115 + 0.31626157696342716 + 0.29271893560768447 + 0.22480754433519579 + 0.41979954655704077 + 0.41116196821139184 + 0.27439289669445444 + 0.46883878934423961 + 0.11048974599079386 + 0.84023055802836666 + 0.33963728790632786 + 0.33246132254972238 + 0.30830043066972834 + 0.61320074002953362 + 0.83682518188176047 + 0.20590560052593787 + 0.17592274953600651 + 8.6346862720994011E-002 + 0.53554714921266466 + 0.38131617737101919 + 0.80932337908682328 + 0.13750965464620180 + 0.88575174572093474 + 0.12909987218158037 + 0.83251873127817788 + 0.24842627494136238 + 0.59683076582325967 + 0.48883168275802191 + 0.26625902271628021 + 0.40682078748693229 + 4.2754145616687111E-002 + 1.5114071010209784E-002 + 0.53437721266600491 + 0.71543972709202208 + 0.11686018319695179 + 0.21189493264208892 + 0.46511154443140512 + 0.12719825429922338 + 0.15157980707422425 + 0.43520946435180718 + 0.64153069898346970 + 0.37847826275947938 + 0.34488022400942242 + 1.9218596094404461E-002 + 0.55378127201960936 + 0.46399180569913767 + 0.53279711094453930 + 0.34199054770159520 + 0.60928719568182288 + 5.1042739039672824E-003 + 0.11368173665286108 + 0.99593573631802101 + 0.25335611153120041 + 0.16593627654569687 + 0.65036738048539178 + 0.91844469229673109 + 0.36588645515485752 + 0.42575656951619223 + 2.5948008105135756E-002 + 0.96553752554667938 + 0.10453811537968249 + 0.24916182732853542 + 0.13315022009238220 + 0.69797803367777789 + 0.83052361278405584 + 0.98353783475080903 + 0.46303658422554861 + 0.76821323829073052 + 0.20502108939150432 + 0.47510410940603265 + 0.95107594096149484 + 0.71316678992811688 + 0.89444924307004214 + 0.58122364358867173 + 0.63452067971237724 + 0.19035939388105660 + 0.60941755714207346 + 0.14823703378512221 + 0.56958247344090651 + 0.74383718236159879 + 0.53582182822951552 + 8.8118712314528835E-002 + 0.78735186159694592 + 0.10682429744509747 + 0.34889746129283239 + 0.27316917982397371 + 0.55020936546745247 + 0.19875822849681057 + 0.32727051053365130 + 0.10423212591144448 + 0.31047609908745599 + 0.27039021172527811 + 0.55134565163007210 + 0.98024063073614442 + 0.93304599528342891 + 5.7322413622490132E-002 + 9.8757603947650097E-002 + 0.10199126743695430 + 0.86290047098718148 + 0.74772555208704361 + 0.33560674163423343 + 7.0401022012212877E-002 + 0.31742669012715297 + 0.67553427505066566 + 0.63814134176603332 + 7.9834348807100497E-002 + 1.2513608927477549E-002 + 0.49244428138305452 + 1.2885830355461536E-002 + 0.78574658950779153 + 0.81226540440050599 + 0.61111607966359216 + 0.83461606111027464 + 0.98460264355779259 + 0.56666672379289196 + 3.2891494475933314E-002 + 0.81526318772061046 + 0.74199714281300899 + 0.32318822348783272 + 0.26285999044327824 + 9.5929780332578929E-002 + 0.34220067007473176 + 1.9889802059953610E-002 + 0.78488176387158504 + 0.83129726012966998 + 0.11245932704061801 + 0.34448869023827910 + 0.25034910384552944 + 0.26776978458689626 + 0.80637984526830309 + 0.10776415004085393 + 0.74385243239834153 + 0.54217682357920438 + 0.26669251756461776 + 0.54915979130428383 + 0.12673814774015213 + 0.21949275531181556 + 7.1380720677826304E-002 + 0.71886143071972342 + 0.53028125570763152 + 0.96570017853737866 + 0.83442533023564991 + 5.3762733444703059E-002 + 4.9926535455153243E-002 + 0.84241172350688487 + 0.54902115351462655 + 0.32045901028729418 + 0.97950703602923284 + 0.72804286298322296 + 0.58162277554273345 + 0.36128911712554768 + 0.94084764263085674 + 0.20746205049784194 + 0.49491212202195101 + 9.9624782102264220E-002 + 0.84587432233623616 + 0.75936085307670709 + 0.47998934585895725 + 0.30116224349299969 + 0.30309134003009319 + 0.68274399405911623 + 0.23239154807731666 + 0.82776191812662958 + 0.12872698483300127 + 0.26795550598453133 + 0.75973958764646810 + 0.19596665348190356 + 0.58401906663879544 + 0.58321919930093813 + 0.47019635167153950 + 0.70969636950427883 + 3.6987763860924616E-002 + 0.52712512496501063 + 0.86525578925498081 + 0.98764182352961427 + 0.32921808042189582 + 0.52332690527862979 + 0.70111792721126420 + 0.28244241559992389 + 3.3281502203404756E-002 + 0.44487603775949580 + 0.18870694862308923 + 0.65327822013202663 + 0.86304215202964940 + 0.69406828766550888 + 0.29018680399104468 + 0.20219921884479319 + 3.9787057060440389E-002 + 0.95461058087433770 + 0.44452620693538591 + 0.83873898049369799 + 0.86473192718546699 + 0.31256996801438319 + 0.36029421237918413 + 0.57620203302522555 + 0.99345519679777539 + 0.32074721641011905 + 0.37235741323139848 + 0.76115650217565900 + 4.3422824443165808E-002 + 0.62205970424373191 + 0.10669865052003757 + 0.56339179824833607 + 0.84008286585745040 + 0.78754420609820031 + 0.65426888804310579 + 0.63180694043031238 + 0.27379358059283021 + 0.52376982437886355 + 6.5628737063754272E-003 + 0.64381119211424931 + 0.53105016953987771 + 0.39386857664553077 + 0.64703913418516734 + 1.1410243733678271E-002 + 0.24476978596169019 + 0.49633517217042922 + 0.48278468524535256 + 9.1042016025070893E-002 + 0.23093119090698622 + 0.47154985607143018 + 0.83803143217707188 + 0.28971577849473817 + 0.65480573910208406 + 0.92414692946266896 + 0.75968153955727757 + 0.37535282600894249 + 0.24570338052825846 + 0.81945735509187045 + 7.4884898638600816E-002 + 0.37484470171379414 + 0.77891273362762092 + 2.5869671853797627E-002 + 0.93857063044766420 + 9.5879165930575994E-002 + 3.7519810481470017E-003 + 0.59141653656473636 + 0.13623968337627090 + 0.27304589165438742 + 0.66854607975901814 + 0.92761332260016260 + 0.27199437101797130 + 0.85100955192947225 + 0.95673207295768137 + 0.26699168962499797 + 0.51735911015909153 + 0.39926348765866493 + 0.62833487265679722 + 0.61030784341147637 + 0.45351774287358282 + 0.39071599079986186 + 0.19600530855928255 + 0.66411151984994632 + 0.18022452258902888 + 0.61703693681279503 + 0.84114228390776802 + 0.81330048042515202 + 0.47916226599670608 + 9.0463484149008622E-002 + 5.6510441728043759E-002 + 0.79256901137416591 + 0.67690113722304801 + 0.70870980115194016 + 0.59410263206372704 + 0.67276664667762631 + 0.94685030780890855 + 0.18017878875817672 + 0.49846175377805224 + 1.0900501685927821E-002 + 0.59778308466137986 + 0.66404830524820468 + 0.92313745314454820 + 0.31609444824582056 + 0.57297586549800528 + 0.24276090521697569 + 2.5336238987033965E-002 + 0.21722352990184746 + 0.34444630009062038 + 0.14813575696799930 + 0.29470476428820547 + 0.82647206348174862 + 0.86397162628984603 + 0.87253762249390476 + 0.99437831408630828 + 0.96611436633802228 + 0.86535980027614912 + 0.69833716481997854 + 4.3079072220191250E-002 + 0.50765982176574553 + 0.80666539779437585 + 5.2176252401643808E-002 + 0.75671390607858768 + 0.21419749381046316 + 0.70426965121627205 + 0.20699836748160649 + 0.21348292857078022 + 0.14277546900442317 + 0.93722724172188165 + 0.77110485349003710 + 0.14104322419489890 + 0.45596034255804341 + 0.17004193950362989 + 0.88899914280643344 + 2.2196643937938632E-002 + 0.93835843600527014 + 0.25706680177818697 + 0.77307646204329927 + 0.71809822519050215 + 0.72993867820703429 + 0.68004235380489320 + 0.42387308547439417 + 0.72245210178429886 + 0.34701690304286359 + 0.12182489990669154 + 0.50910218461131151 + 0.80371985724276485 + 0.84350836630640558 + 0.64527143750703431 + 0.22311490663459210 + 0.67078404720487939 + 0.99213295191400874 + 0.91995804396966108 + 0.33238785754262778 + 0.46942009324286715 + 0.37553893107884662 + 0.61018540998448856 + 0.46097967881933499 + 0.33339569286177095 + 0.57300834014620960 + 0.63104708963166445 + 0.91731549321855610 + 0.50681402887710902 + 0.47047084346093015 + 2.5512286327135314E-002 + 0.99612308445772513 + 0.74275349309682781 + 0.47591020530570560 + 0.40181828773070549 + 0.78958817331287179 + 4.7109378789517820E-002 + 7.6896431363802975E-002 + 0.52024921083216213 + 0.10480716966488046 + 0.42315609462054837 + 5.4088876400001595E-002 + 0.50191886444818934 + 0.57294879761911588 + 0.61273083454976174 + 0.30781987367103980 + 0.29015147800931018 + 2.1794091398266602E-002 + 0.63088344450125078 + 0.79661323318752508 + 0.17043623105301720 + 0.41032335358113770 + 0.92030312356238753 + 0.87396824733972167 + 0.74899509338264281 + 0.31838721599348929 + 0.86054367420573641 + 0.84019508502745310 + 0.83760455345749918 + 4.0671404166424452E-002 + 8.7429545787472307E-002 + 0.62751823707785803 + 0.86310241290249579 + 5.6408304683458255E-002 + 0.64132246071872601 + 5.7409004448881973E-002 + 0.45600569783792011 + 0.83987595170225404 + 0.91812015949614789 + 0.66968416911208450 + 0.10216675578983825 + 0.39572187272063175 + 0.92676645819906867 + 0.54303841682618881 + 0.39691687027067601 + 0.26140082137998988 + 0.88851342077088091 + 0.39595075046580419 + 0.33741583893651139 + 0.86202860286726590 + 0.43769682316377612 + 0.50440613242815147 + 0.76225606266665125 + 0.96186039885528274 + 0.43798261467920696 + 0.37437952559121612 + 0.96408342797511537 + 6.3187664110252229E-002 + 0.16024309193573316 + 0.18450941891428840 + 0.47306309311417749 + 0.35900644215697497 + 0.61470913305812047 + 0.10474165984858530 + 0.95917068078074053 + 0.88940297818810876 + 0.36746756311191220 + 0.62620203727022172 + 5.2412185597020766E-002 + 0.85704224052903655 + 5.1788849909250700E-002 + 0.93533060539331103 + 0.77906133022465696 + 0.82492165062861744 + 0.61374629857715846 + 0.68768843507729471 + 5.0853126173038277E-002 + 0.17810889780727379 + 0.29695389645936032 + 2.8190332292901132E-002 + 1.3266090115848783E-002 + 0.65968091915802063 + 0.55206255062179110 + 0.62410948926586585 + 6.5703440779245170E-002 + 0.78994459416097129 + 0.39692255819054978 + 0.21402161817816179 + 0.57677106604464967 + 0.83319482350367835 + 0.17790186849569523 + 0.73078871483060226 + 0.22665571786620475 + 0.74671159152102717 + 0.86438106596336439 + 0.95413637335751744 + 0.14620227943607134 + 5.7936809913204712E-002 + 0.98535172252024594 + 2.0865933557963956E-002 + 0.65052522272778290 + 0.83708978486930263 + 0.60027810148141469 + 0.85099195813480932 + 0.85443785633967906 + 0.73517901754262738 + 0.97714779185145417 + 0.26830391876777426 + 0.57658970543655030 + 0.73134209808441497 + 0.69279360310381577 + 0.22293560047807759 + 0.22272825397147145 + 0.88005286031377139 + 0.96564411627953461 + 0.87145362842802143 + 0.56230611494263272 + 0.93014355526995374 + 0.90883360596208718 + 0.45265482883941033 + 7.5890123196877823E-003 + 0.13891899600998414 + 0.19395797792807912 + 0.40822572570344562 + 0.91984262678367656 + 0.41035009880865658 + 0.21031624559043749 + 0.18100789306921783 + 0.79352208861650197 + 0.54701620377705495 + 0.43573916585428663 + 0.45475459895084569 + 0.56263253074866171 + 0.61249511196383466 + 1.7521048876169232E-002 + 0.34828113003582928 + 0.52769822167224234 + 9.2073402614058608E-002 + 0.67385854686032332 + 6.2394989126019595E-002 + 0.67562353970063427 + 0.47211798936650595 + 9.8665257294559439E-002 + 0.43082834190490971 + 0.14249350583460796 + 0.50526362688148296 + 0.38193182809615323 + 0.15678567120378872 + 0.84734656947749087 + 0.63450811669369500 + 0.69169034334883506 + 0.98033573532169171 + 0.32444405697311041 + 0.37046074085349545 + 0.89749076441217213 + 0.92944504213263102 + 0.30065964450140470 + 0.82602067292728165 + 0.82831342461735602 + 0.82782600616961588 + 0.35433425353576808 + 0.89621590808965834 + 0.53228509433131066 + 0.20190963812276053 + 0.67000526616971356 + 0.93890699885969653 + 0.10520203411017093 + 0.62848355019055191 + 0.33034544583888703 + 7.8076170394218991E-003 + 0.72417687182973012 + 0.98800541160080257 + 0.74327516345928402 + 0.92590555866150481 + 0.60352349719456910 + 0.15680962659125086 + 4.9434958757949232E-002 + 0.55686051658955904 + 0.97810570859908452 + 0.12872754429692890 + 0.95547914162817804 + 0.95573437656426208 + 0.12967304813757963 + 0.45951519961713316 + 0.68151881426853933 + 0.46566941554450381 + 0.62813314240559137 + 0.54948969763289313 + 0.77136885714444148 + 0.80877759415522377 + 0.65958877927092274 + 0.89058500654762440 + 0.90982465711328686 + 0.61749100212857400 + 0.54754063509364670 + 0.35713498382768094 + 2.4839537655822141E-002 + 0.44473647181794362 + 0.48318333042530526 + 0.15052230596724669 + 0.98645625807854387 + 0.16983403901664573 + 0.24743079769734777 + 0.75834503525907948 + 0.39212420611110943 + 0.64782771756848589 + 0.41577658200279899 + 0.15879365042174598 + 0.94978611458295603 + 0.66463121868276431 + 0.84009097217167650 + 0.90795928852065444 + 0.52724828450050865 + 0.38418444434347165 + 0.16407355312341210 + 8.8496510745926571E-002 + 0.50958525975331526 + 7.0102045493356968E-002 + 0.35537675549345948 + 0.93728436913066560 + 0.26853869097094929 + 5.9162881009825696E-002 + 0.46807856276806703 + 0.34526803158516728 + 0.77813731816475240 + 0.70936884855344218 + 0.36739517626832097 + 0.36261806234035987 diff --git a/tests/TERAPI/scrdir0001.ref b/tests/TERAPI/scrdir0001.ref new file mode 100644 index 00000000..e69de29b diff --git a/tests/TERAPI/tc_server.cpp b/tests/TERAPI/tc_server.cpp new file mode 100644 index 00000000..d30080c4 --- /dev/null +++ b/tests/TERAPI/tc_server.cpp @@ -0,0 +1,78 @@ +#include +#include + +#include "../tc_mpi_api.h" + +using namespace std; + +int main(int argc, char* argv[]) +{ + char *serverName = NULL; + + // Due to a bug in hydra_nameserver, it crashes + // when multiple TC servers call `MPI_Unpublish_name()` + // Hence, we want to allow invoking without this parameter, + // in which case TC server will just print the port to stdin, + // where it could be grepped and passed via file to ABIN, + // and it will never call MPI_Publish_name/MPI_Unpublish_name + // NOTE: This behaviour is different from real TC, + // which has default serverName and will always try to publish it. + if (argc > 2) { + printf("Only one cmdline argument supported, , but you provided more!"); + throw std::runtime_error("Incorrect invocation"); + } + + if (argc == 2) { + serverName = new char[1024]; + strcpy(serverName, argv[1]); + } + + TCServerMock tc = TCServerMock(serverName); + + if (serverName) { + delete[] serverName; + } + + tc.initializeCommunication(); + + tc.receiveNumAtoms(); + tc.receiveAtomTypes(); + + int loop_counter = 0; + int MAX_LOOP_COUNT = 100; + // Will go through this loop until MPI client gives an exit signal. + while (true) { + + int status = tc.receiveBeginLoop(); + if (status == MPI_TAG_EXIT) { + break; + } + + tc.receiveAtomTypesAndScrdir(); + tc.receiveCoordinates(); + + // Energies and gradients from qTIP4PF potential + double energy = tc.getWaterGradients(); + + int MPI_SCF_DIE = 0; + tc.sendSCFEnergy(energy, MPI_SCF_DIE); + + tc.sendQMCharges(); + + tc.sendQMDipoleMoment(); + + // NOTE: In the real TC interface, gradients are sent + // conditionally only if they are requested. + // But we always request them in ABIN (see tc.receive). + tc.sendQMGradients(); + + // This is just a precaution, we don't want endless loop! + loop_counter++; + if (loop_counter > MAX_LOOP_COUNT) { + printf("Maximum number of steps exceeded!\n"); + return(1); + } + } + + return(0); +} diff --git a/tests/TERAPI/temper.dat.ref b/tests/TERAPI/temper.dat.ref new file mode 100644 index 00000000..6072cb5e --- /dev/null +++ b/tests/TERAPI/temper.dat.ref @@ -0,0 +1,2 @@ + # Time[fs] Temperature T-Average Conserved_quantity_of_thermostat + 0.97 16.64 16.64 diff --git a/tests/TERAPI/tera-mpiapi.cpp b/tests/TERAPI/tera-mpiapi.cpp deleted file mode 100644 index a8171284..00000000 --- a/tests/TERAPI/tera-mpiapi.cpp +++ /dev/null @@ -1,223 +0,0 @@ -/* int.cpp */ -/***********************************************************************/ -/* */ -/* This file is part of the TeraChem software suite */ -/* Copyright PetaChem, LLC 2008- */ -/* No portion may be distributed, modified, used, or divulged without */ -/* express written permission from the copyright holders. */ -/* */ -/***********************************************************************/ -#include "mpi.h" -#include -#include -#define MAX_DATA 200000 -#include - -using namespace std; - -int main(int argc, char* argv[]) -{ - - int UseMPI=1; - int MPI_SCF_DIE=0; - char terachem_port_name[1024]; - strcpy(terachem_port_name,"terachem_port"); - - // DH: added initializations - int MMAtom; - int Dx, Dy, Dz; - int Dxmm, Dymm, Dzmm; - int totNumAtoms; - int MMtotNumAtoms; - int Mode; - - MPI_Comm amber_client; - MPI_Status mpi_status; - char port_name[MPI_MAX_PORT_NAME]; - char buftype[MAX_DATA]; - char bufchar[100*128]; - char tmpbuf[1024]; - double bufcoords[MAX_DATA]; - double bufchrgs[MAX_DATA]; - int bufints[MAX_DATA]; - int mpi_tag = 0; - - // Determine terachem MPI named port name. If already defined through - // the CLI, we will respect that. Alternatively, it may be provided - // as the second undashed option argument. Default was set above - // (terachem_port) - if ((argc > 2) && (strcmp(terachem_port_name,"terachem_port")==0)) { - printf("Assigning ID to terachem_port_name: %s \n", argv[2]); - strcpy(terachem_port_name,argv[2]); - } - - int size_mpi, again_mpi; - int number = 1; - MPI_Init(0, NULL); - MPI_Comm_size(MPI_COMM_WORLD, &size_mpi); - - // MPI_INFO_NULL are the implementation defaults. - MPI_Open_port(MPI_INFO_NULL, port_name); - // establishes a port at which the server may be contacted. - printf("\nTerachem server available at port_name: %s \n",port_name); - printf("Will publish this port_name to '%s'. \n", terachem_port_name); - - // Publish the port name - MPI_Publish_name(terachem_port_name, MPI_INFO_NULL, port_name); - - printf("Waiting to accept MPI communication from AMBER client. \n"); - MPI_Comm_accept( port_name, MPI_INFO_NULL, 0, MPI_COMM_SELF, &amber_client ); // accept communication from AMBER client. - printf("MPI communication accepted. \n"); - - int loop = 1; - int loop_counter = 0; -// Will go through this loop until MPI gives an exit signal. -// If !UseMPI, only go through the loop once - while (loop) { - printf("\nReceiving new QM data. \n"); - // Read in num QM atoms, QM atom types, QM atom coordinates - MPI_Recv( bufints, MAX_DATA, MPI_INT, MPI_ANY_SOURCE, MPI_ANY_TAG, amber_client, &mpi_status ); - printf("\nReceived new QM data. \n"); - if(mpi_status.MPI_TAG == 0) { - printf("Got a 0 tag from client. Freeing and finalizing. \n"); - MPI_Comm_free( &amber_client ); - MPI_Unpublish_name(terachem_port_name, MPI_INFO_NULL, port_name); - MPI_Close_port(port_name); - MPI_Finalize(); - loop = 0; - // DH: This exit is potentially dangerous, - // because statements after while loop are not executed. - // This should probably be "break" - break; - } - if(mpi_status.MPI_TAG == 1) { - printf("Got a 1 tag from client --> Energy mode. \n"); - Mode = 0; - } - if(mpi_status.MPI_TAG == 2) { - printf("Got a 2 tag from client --> Gradient mode. \n"); - Mode = 1; - } - - // atom types - these are stored with two characters each - // in the buftype character array - MPI_Recv( buftype, MAX_DATA, MPI_CHAR, MPI_ANY_SOURCE, MPI_ANY_TAG, - amber_client, &mpi_status ); - // Recieve QM coordinates from AMBER - //printf("Receiving QM coordinates: \n"); - MPI_Recv( bufcoords, MAX_DATA, MPI_DOUBLE, MPI_ANY_SOURCE, - MPI_ANY_TAG, amber_client, &mpi_status ); - // QMMM STARTUP - - // QMMM not yet implemented in ABIN - bool QMMM = false; - MMtotNumAtoms = 0; - MMAtom = 0; - if(QMMM) { - printf("Receiving new MM data. \n"); - // Receive number of MM point charges - MPI_Recv( bufints, MAX_DATA, MPI_INT, MPI_ANY_SOURCE, MPI_ANY_TAG, - amber_client, &mpi_status ); - MMtotNumAtoms = bufints[0]; - printf("Number of MM point charges: %d\n", MMtotNumAtoms); - // Receive new MM charges - MPI_Recv( bufchrgs, MAX_DATA, MPI_DOUBLE, MPI_ANY_SOURCE, MPI_ANY_TAG, - amber_client, &mpi_status ); - double MMcharge = 0.0; - // Recieve new coordinates from ABIN - // printf("\n****** MM coordinates ******\n"); - MPI_Recv( bufcoords, MAX_DATA, MPI_DOUBLE, MPI_ANY_SOURCE, - MPI_ANY_TAG, amber_client, &mpi_status ); - } - - // Born not implemented in ABIN - bool RunBorn = false; - if(RunBorn) { - printf("Receiving Born radii. \n"); - double born_radii[totNumAtoms + MMtotNumAtoms]; - MPI_Recv( bufcoords, MAX_DATA, MPI_DOUBLE, MPI_ANY_SOURCE, MPI_ANY_TAG, amber_client, &mpi_status ); - for(int atom = 0; atom < totNumAtoms + MMtotNumAtoms; atom ++) { - born_radii[atom] = bufcoords[atom]; - printf("Born radii for atom %d: %lf \n", atom+1, born_radii[atom]); - } - } - - - double SCFEnergy; - - // We need to implement this in ABIN. - bool MPI_SCF_DIE = false; - - SCFEnergy = 1.0+loop_counter; - bufcoords[0] = SCFEnergy; - if(MPI_SCF_DIE) { - mpi_tag = 1; // send the error message - printf("SCF did not converge. Setting MPI_TAG = 1. \n"); - MPI_Send( bufcoords, 1, MPI_DOUBLE, 0, mpi_tag, amber_client ); - MPI_SCF_DIE = 0; // reset the flag for the next loop - mpi_tag = 0; - } else { - printf("Sending QM energy, QM population charges, and dipoles (QM, MM and total) via MPI. \n"); - printf("QM energy = %.8f \n" , SCFEnergy); - // Send the energy - MPI_Send( bufcoords, 1, MPI_DOUBLE, 0, mpi_tag, amber_client ); - // Compute the population charges - for(int atom = 0; atom < totNumAtoms; atom ++) { - bufcoords[atom] = -1-atom; - } - MPI_Send( bufcoords, totNumAtoms, MPI_DOUBLE, 0, mpi_tag, amber_client ); - // QM dipole moment - Dx = -0.01 + loop_counter; - Dy = -0.02 + loop_counter; - Dz = -0.03 + loop_counter; - bufcoords[0] = Dx; - bufcoords[1] = Dy; - bufcoords[2] = Dz; - bufcoords[3] = sqrt(Dx*Dx + Dy*Dy + Dz*Dz); - //printf (" DIPOLE: %lf %lf %lf %lf \n", bufcoords[0], bufcoords[1], bufcoords[2], bufcoords[3] ); - MPI_Send( bufcoords, 4, MPI_DOUBLE, 0, mpi_tag, amber_client ); - // MM dipole moment - Dxmm = -0.01 + loop_counter; - Dymm = -0.02 + loop_counter; - Dzmm = -0.03 + loop_counter; - bufcoords[0] = Dxmm; - bufcoords[1] = Dymm; - bufcoords[2] = Dzmm; - bufcoords[3] = sqrt(Dxmm*Dxmm + Dymm*Dymm + Dzmm*Dzmm); - //printf (" DIPOLE: %lf %lf %lf %lf \n", bufcoords[0], bufcoords[1], bufcoords[2], bufcoords[3] ); - MPI_Send( bufcoords, 4, MPI_DOUBLE, 0, mpi_tag, amber_client ); - // Total dipole moment - Dx += Dxmm; - Dy += Dymm; - Dz += Dzmm; - bufcoords[0] = Dx; - bufcoords[1] = Dy; - bufcoords[2] = Dz; - bufcoords[3] = sqrt(Dx*Dx + Dy*Dy + Dz*Dz); - //printf (" DIPOLE: %lf %lf %lf %lf \n", bufcoords[0], bufcoords[1], bufcoords[2], bufcoords[3] ); - MPI_Send( bufcoords, 4, MPI_DOUBLE, 0, mpi_tag, amber_client ); - } - if(UseMPI && Mode == 1) { - printf("Sending gradients via MPI. \n"); - for(int i = 0; i < (totNumAtoms); i ++) { - bufcoords[3*i] = 0.001+0.0001*(3*i); - bufcoords[3*i+1] = 0.001+0.0001*(3*i+1); - bufcoords[3*i+2] = 0.001+0.0001*(3*i+2); - } - for(int i = 0; i < (MMtotNumAtoms); i ++) { - bufcoords[3*totNumAtoms + 3*i ] = 0.0; - bufcoords[3*totNumAtoms + 3*i+1] = 0.0; - bufcoords[3*totNumAtoms + 3*i+2] = 0.0; - } - MPI_Send( bufcoords, 3*totNumAtoms + 3*MMtotNumAtoms, - MPI_DOUBLE, 0, mpi_tag, amber_client ); - } - - loop_counter++; - if(!UseMPI) - loop = 0; - } // end while loop for dynamics via MPI - - return(0); -} - diff --git a/tests/TERAPI/test.sh b/tests/TERAPI/test.sh index ff108429..cae7150e 100755 --- a/tests/TERAPI/test.sh +++ b/tests/TERAPI/test.sh @@ -1,50 +1,46 @@ #/bin/bash -#module load mpich2/1.4.1p1 -#MPIRUN=mpiexec +set -euo pipefail +# Useful for debugging +#set -x + +ABINEXE=$1 +source ../test_tc_server_utils.sh + +set_default_vars +set_mpich_vars +# If $1 = "clean"; exit early. +if ! clean_output_files $1; then + exit 0 +fi -MPIRUN=$MPI_PATH/bin/mpirun +# Exit early for OpenMPI build. +check_for_openmpi -rm -f restart.xyz movie.xyz -if [[ "$1" = "clean" ]];then - rm -f output terapi.out temper.dat energies.dat - exit 0 -fi +# Compile the fake TC server. +# TCSRC end TCEXE is defined in ../test_tc_server_utils.sh. +$MPICXX $TCSRC -Wall -o $TCEXE +launch_hydra_nameserver $MPICH_HYDRA +hostname=$HOSTNAME +MPIRUN="$MPIRUN -nameserver $hostname -n 1" -#DHHack: -if [[ -z $1 ]];then - ABINEXE=../../abin -else - ABINEXE=$1 -fi +TC_PORT="tcport.$$" +ABIN_CMD="$ABINEXE -i $ABININ -x $ABINGEOM -M $TC_PORT" +TC_CMD="./$TCEXE $TC_PORT.1" -$MPIRUN -np 1 ./tera-mpiapi > terapi.out & -# Get PID of the last process -terapid=$! +$MPIRUN $TC_CMD > $TCOUT 2>&1 & +tcpid=$! -sleep 2 -# Ugly workaround because MPI_Lookup does not work -grep port_name: terapi.out | awk '{print $6}' > port.txt -$MPIRUN -np 1 $ABINEXE > output & +$MPIRUN $ABIN_CMD > $ABINOUT 2>&1 & abinpid=$! -while true;do - sleep 1 - if ! `ps|grep -q $terapid` && ! `ps|grep -q $abinpid` ;then - echo "Both ABIN and TeraChem stopped." - break - fi - if ! `ps|grep -q $terapid` ;then - echo "Terachem died. Killing ABIN." - kill -9 $abinpid - break - fi - if ! `ps|grep -q $abinpid` ;then - echo "ABIN died. Killing TeraChem." - echo $terapid - kill -9 $terapid - break - fi -done +function cleanup { + kill -9 $tcpid $abinpid > /dev/null 2>&1 || true + #kill -9 $tcpid $abinpid $hydrapid > /dev/null 2>&1 || true + exit 0 +} + +trap cleanup INT ABRT TERM EXIT +check_running_processes $abinpid $tcpid diff --git a/tests/TERAPI/velocities.xyz.ref b/tests/TERAPI/velocities.xyz.ref new file mode 100644 index 00000000..a5c8fcf6 --- /dev/null +++ b/tests/TERAPI/velocities.xyz.ref @@ -0,0 +1,5 @@ + 3 + Time step: 1 +O 0.0000000000E+00 -0.3504628549E-04 0.0000000000E+00 +H -0.2049021472E-03 0.2781277389E-03 0.0000000000E+00 +H 0.2049021472E-03 0.2781277389E-03 0.0000000000E+00 diff --git a/tests/WITHOUT_MPI/ERROR.ref b/tests/WITHOUT_MPI/ERROR.ref new file mode 100644 index 00000000..ab32ffa2 --- /dev/null +++ b/tests/WITHOUT_MPI/ERROR.ref @@ -0,0 +1,2 @@ + FATAL ERROR encountered in subroutine: initialize_terachem_interface + Check standard output for further information. diff --git a/tests/WITHOUT_MPI/input.in b/tests/WITHOUT_MPI/input.in new file mode 100644 index 00000000..7fae46ff --- /dev/null +++ b/tests/WITHOUT_MPI/input.in @@ -0,0 +1,20 @@ +&general +pot='_tera_' +watpot=1 +ipimd=0, +nstep=1, +dt=40., +irandom=13131313, + +nwrite=1, +nwritef=1, +nwritev=1, +nwritex=1, +nrest=1, +idebug=3 +/ + +&nhcopt +inose=1, +temp=100.15, +/ diff --git a/tests/WITHOUT_MPI/mini.xyz b/tests/WITHOUT_MPI/mini.xyz new file mode 100644 index 00000000..099f6aa0 --- /dev/null +++ b/tests/WITHOUT_MPI/mini.xyz @@ -0,0 +1,4 @@ + 2 + + H 0.000000 0.000000 0.000000 + H 0.000000 0.000000 3.000000 diff --git a/tests/tc_mpi_api.cpp b/tests/tc_mpi_api.cpp new file mode 100644 index 00000000..fd6e5112 --- /dev/null +++ b/tests/tc_mpi_api.cpp @@ -0,0 +1,408 @@ +#include +#include +#include "tc_mpi_api.h" + +using namespace std; + +TCServerMock::TCServerMock(char *serverName) { + + tcServerName = NULL; + gradients = coordinates = NULL; + atomTypes = NULL; + if (serverName) { + tcServerName = new char[strlen(serverName)+1]; + strcpy(tcServerName, serverName); + } + + // Initialize MPI in the constructor + MPI_Init(0, NULL); + + // Check that we're only running one MPI process + // i.e. that we've been invoked by `mpirun -n 1` + int commSize; + MPI_Comm_size(MPI_COMM_WORLD, &commSize); + printf("MPI_Comm_size = %d\n", commSize); + if (commSize != 1) { + printf("ERROR: Comm_size != 1!\n"); + printf("Please execute this program with 'mpirun -n 1'\n"); + throw std::runtime_error("Incorrect mpirun invocation"); + } +} + + +TCServerMock::~TCServerMock(void) { + printf("Freeing and finalizing MPI.\n"); + MPI_Comm_free(&abin_client); + if (mpiPortName) { + MPI_Close_port(mpiPortName); + } + MPI_Finalize(); + if (atomTypes) { + delete[] atomTypes; + } + if (gradients) { + delete[] gradients; + } + if (coordinates) { + delete[] coordinates; + } +} + + +void TCServerMock::checkRecvCount(MPI_Status *mpiStatus, + MPI_Datatype datatype, + int expected_count) { + int recvCount; + MPI_Get_count(mpiStatus, datatype, &recvCount); + if (recvCount != expected_count) { + printf("Unexpected received count\n"); + printf("Expected %d, got %d\n", expected_count, recvCount); + throw std::runtime_error("Unexpected received count"); + } +} + + +// When we get an error tag from client we exit immediately. +void TCServerMock::checkRecvTag(MPI_Status &mpiStatus) { + if (mpiStatus.MPI_TAG == MPI_TAG_ERROR) { + throw std::runtime_error("Client sent an error tag."); + } +} + + +void TCServerMock::printMPIError(int error_code) { + int *resultLen = NULL; + int new_error_code = MPI_Error_string(error_code, bufchars, resultLen); + if (new_error_code == MPI_SUCCESS) { + printf("%s\n", bufchars); + } +} + + +MPI_Comm* TCServerMock::getABINCommunicator() { + return &abin_client; +} + + +// Publish server name, but only if tcServerName was passed to constructor. +void TCServerMock::publishServerName(char *serverName, char *portName) { + if (!serverName) { + printf("Server name not specified, nothing to publish\n"); + printf("Pass the port name manually.\n"); + return; + } + MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_RETURN); + + // In seconds + double sleepTime = 0.2; + double maxTime = 50; + double elapsedTime = 0; + + // Retry if hydra_nameserver isn't ready. + int ierr = MPI_Publish_name(serverName, MPI_INFO_NULL, portName); + while (ierr != MPI_SUCCESS) { + if (elapsedTime > maxTime) { + printMPIError(ierr); + throw std::runtime_error("could not publish server name"); + } + sleep(sleepTime); + elapsedTime += sleepTime; + ierr = MPI_Publish_name(serverName, MPI_INFO_NULL, portName); + } + + printf("Port published under server name '%s'\n", tcServerName); + + ierr = MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_ARE_FATAL); + if (ierr != MPI_SUCCESS) { + printMPIError(ierr); + throw std::runtime_error("could not set error handler"); + } +} + + +void TCServerMock::initializeCommunication() { + // Establishes a port at which the server may be contacted. + // MPI_INFO_NULL are the implementation defaults. + MPI_Open_port(MPI_INFO_NULL, mpiPortName); + + printf("Fake TeraChem server available at port name: %s\n", mpiPortName); + fflush(stdout); + + publishServerName(tcServerName, mpiPortName); + + printf("Waiting to accept MPI communication from ABIN client.\n"); + fflush(stdout); + MPI_Comm_accept(mpiPortName, MPI_INFO_NULL, 0, MPI_COMM_SELF, &abin_client); + printf("MPI communication accepted.\n"); + + // It's important to unpublish the port_name early, otherwise + // we could get conflicts when other server tried to use the same name. + // WARNING: If more then one tc_server is running, + // concurrent calls MPI_Unpublish_name are crashing the hydra_nameserver. + // https://github.com/pmodels/mpich/issues/5058 + if (tcServerName) { + MPI_Unpublish_name(tcServerName, MPI_INFO_NULL, mpiPortName); + delete[] tcServerName; + } +} + + +// This is called only once at the beginning. +int TCServerMock::receiveNumAtoms() { + printf("Receiving number of atoms...\n"); + fflush(stdout); + int recvCount = 1; + MPI_Recv(bufints, recvCount, MPI_INT, MPI_ANY_SOURCE, MPI_ANY_TAG, abin_client, &mpiStatus); + checkRecvTag(mpiStatus); + checkRecvCount(&mpiStatus, MPI_INT, recvCount); + totNumAtoms = bufints[0]; + if (totNumAtoms < 1) { + printf("ERROR: Invalid number of atoms. Expected positive number, got: %d", totNumAtoms); + throw std::runtime_error("Invalid number of atoms"); + } + printf("totNumAtoms=%d\n", totNumAtoms); + + // Allocate buffers for gradients and coordinates + gradients = new double[totNumAtoms * 3]; + coordinates = new double[totNumAtoms * 3]; + return totNumAtoms; +} + + +char* TCServerMock::receiveAtomTypes() { + printf("Receiving atom types...\n"); + fflush(stdout); + int recvCount = totNumAtoms * 2; + MPI_Recv(bufchars, recvCount, MPI_CHAR, MPI_ANY_SOURCE, MPI_ANY_TAG, abin_client, &mpiStatus); + checkRecvTag(mpiStatus); + checkRecvCount(&mpiStatus, MPI_CHAR, recvCount); + + atomTypes = new char[totNumAtoms * 2 + 1]; + strncpy(atomTypes, bufchars, totNumAtoms * 2); + atomTypes[totNumAtoms * 2] = '\0'; + puts(atomTypes); + return atomTypes; +} + + +void TCServerMock::receiveAtomTypesAndScrdir() { + printf("Receiving atom types and scrdir...\n"); + MPI_Recv(bufchars, MAX_DATA, MPI_CHAR, MPI_ANY_SOURCE, MPI_ANY_TAG, abin_client, &mpiStatus); + checkRecvTag(mpiStatus); + puts(bufchars); + char *atoms = new char[2*totNumAtoms + 1]; + strncpy(atoms, bufchars, totNumAtoms * 2); + atoms[totNumAtoms * 2] = '\0'; + if (strcmp(atomTypes, atoms) != 0) { + printf("ERROR: expected '%s', got '%s'\n", atomTypes, atoms); + throw std::runtime_error("invalid atom types"); + } + delete[] atoms; + + char scrdir[1024]; + parseScrDir(bufchars, scrdir); + validateScrDir(scrdir); + printf("Using scratch directory %s\n", scrdir); +} + +void TCServerMock::validateScrDir(char *scrdir) { + unsigned int beadIdx; + int ret = sscanf(scrdir, "scrdir%4u", &beadIdx); + if (ret == 0 || ret == EOF) { + throw std::runtime_error("invalid scrdir"); + } + printf("Bead index: %u\n", beadIdx); + // To make sure all scrdirs were passed correctly, + // we create empty files with their names so + // that they can be compared with the reference. + std::ofstream output(scrdir); +} + +void TCServerMock::parseScrDir(char *buffer, char *scrdir) { + // DH VARIABLE SCRATCH DIRECTORY, useful for Path Integral MD + // Done a bit awkwardly to preserve AMBER interface + // The format in bufchars needs to be: AtomTypes++scrdir++ + // i.e. "C H H H H ++scrdir01++ + // Code copied directly from terachem/amber.cpp + char delim = '+'; + if (buffer[totNumAtoms*2] == delim && buffer[totNumAtoms*2+1] == delim) { + int i = totNumAtoms * 2 + 2; + int j = 0; + while (bufchars[i] != delim) { + scrdir[j] = bufchars[i]; + i++;j++; + } + scrdir[j] = '\0'; + } +} + +void TCServerMock::receiveCoordinates() { + // Receive QM coordinates from ABIN + printf("Receiving QM coordinates...\n"); + int recvCount = totNumAtoms * 3; + MPI_Recv(bufdoubles, recvCount, MPI_DOUBLE, MPI_ANY_SOURCE, + MPI_ANY_TAG, abin_client, &mpiStatus); + checkRecvTag(mpiStatus); + checkRecvCount(&mpiStatus, MPI_DOUBLE, recvCount); + for (int i = 0; i < totNumAtoms * 3; i++) { + coordinates[i] = bufdoubles[i]; + printf("%g ", bufdoubles[i]); + } + printf("\n"); +} + + +// Receive number of QM atoms, QM atom types, QM atom coordinates. +// This is called repeatedly in the MD loop. +int TCServerMock::receiveBeginLoop() { + printf("\nReceiving new QM data.\n"); + fflush(stdout); + int recvCount = 1; + MPI_Recv(bufints, recvCount, MPI_INT, MPI_ANY_SOURCE, MPI_ANY_TAG, abin_client, &mpiStatus); + checkRecvTag(mpiStatus); + // When ABIN sends it's exit tag, + // we want to allow it to send zero data. + //checkRecvCount(&mpiStatus, MPI_INT, recvCount); + int tag = mpiStatus.MPI_TAG; + + if (tag == MPI_TAG_EXIT) { + printf("Got an EXIT tag from client. \n"); + return tag; + } else if (tag != MPI_TAG_GRADIENT) { + printf("ERROR: Expected mpi_tag=%d, got %d", MPI_TAG_GRADIENT, tag); + throw std::runtime_error("Invalid MPI TAG received"); + } + + if (totNumAtoms != bufints[0]) { + printf("ERROR: Unexpected number of atoms.\n"); + printf("Expected %d, got %d\n", totNumAtoms, bufints[0]); + throw std::runtime_error("Invalid number of atoms received"); + } + return tag; +} + + +// This is what we expect from ABIN every MD iteration +// for classical MD and PIMD. +// For Surface Hopping see receive_sh() +int TCServerMock::receive() { + int tag = receiveBeginLoop(); + if (tag == MPI_TAG_EXIT) { + return tag; + } + receiveAtomTypesAndScrdir(); + receiveCoordinates(); + return tag; +} + + +void TCServerMock::sendSCFEnergy(double energy, int MPI_SCF_DIE) { + bufdoubles[0] = energy; + if (MPI_SCF_DIE) { + printf("SCF did not converge. Setting MPI_TAG = %d.\n", MPI_TAG_SCF_DIED); + MPI_Send(bufdoubles, 1, MPI_DOUBLE, 0, MPI_TAG_SCF_DIED, abin_client); + MPI_SCF_DIE = 0; // reset the flag for the next loop + } + + printf("QM energy = %.8f\n" , energy); + MPI_Send(bufdoubles, 1, MPI_DOUBLE, 0, MPI_TAG_OK, abin_client); +} + + +// Compute fake Mulliken charges. +void TCServerMock::computeFakeQMCharges(double *charges) { + for (int atom = 0; atom < totNumAtoms; atom++) { + charges[atom] = -1 - atom; + } +} + + +void TCServerMock::sendQMCharges() { + // TODO: We could move this computation elsewhere + // and accept charges as inputs here. + computeFakeQMCharges(bufdoubles); + MPI_Send(bufdoubles, totNumAtoms, MPI_DOUBLE, 0, MPI_TAG_OK, abin_client); +} + + +// Compute fake dipole moments. +void TCServerMock::computeFakeQMDipoleMoment(double &Dx, double &Dy, double &Dz, double &DTotal) { + Dx = -0.01; + Dy = -0.02; + Dz = -0.03; + DTotal = sqrt(Dx*Dx + Dy*Dy + Dz*Dz); + printf("QM DIPOLE: %lf %lf %lf %lf\n", Dx, Dy, Dz, DTotal); +} + + +void TCServerMock::sendQMDipoleMoment() { + computeFakeQMDipoleMoment(bufdoubles[0], bufdoubles[1], bufdoubles[2], bufdoubles[3]); + MPI_Send(bufdoubles, 4, MPI_DOUBLE, 0, MPI_TAG_OK, abin_client); +} + + +void TCServerMock::sendQMGradients() { + printf("Sending gradients via MPI. \n"); + for(int i = 0; i < totNumAtoms; i++) { + bufdoubles[3*i] = gradients[3*i]; + bufdoubles[3*i+1] = gradients[3*i+1]; + bufdoubles[3*i+2] = gradients[3*i+2]; + printf("%lf %lf %lf\n", bufdoubles[3*i], bufdoubles[3*i+1], bufdoubles[3*i+2]); + } + MPI_Send(bufdoubles, 3 * totNumAtoms, MPI_DOUBLE, 0, MPI_TAG_OK, abin_client); +} + + +void TCServerMock::send() { + printf("Sending QM energy, QM population charges, QM dipoles and QN gradients via MPI.\n"); + + double energy = getWaterGradients(); + int MPI_SCF_DIE = 0; + + sendSCFEnergy(energy, MPI_SCF_DIE); + + sendQMCharges(); + + sendQMDipoleMoment(); + + // NOTE: In the real TC interface, gradients are sent + // conditionally only if they are requested. + // But we always request them in ABIN (see tc.receive). + sendQMGradients(); +} + + +// Gradients stored internally for now, +// returns energy in atomic units. +double TCServerMock::getWaterGradients() { + // conversion constants + const double ANG = 1.889726132873; + const double AUTOKCAL = 627.50946943; + const double FORCE_FAC = 1 / ANG / AUTOKCAL; + + double *converted_coords = new double[3 * totNumAtoms]; + + if (totNumAtoms % 3 != 0) { + throw std::runtime_error("Number of atoms not divisible by 3"); + } + int nwater = totNumAtoms / 3; + + double E = 0.0; + for (int i = 0; i < totNumAtoms * 3; i++) { + converted_coords[i] = coordinates[i] / ANG; + gradients[i] = 0; + } + + h2o::qtip4pf water_potential; + E = water_potential(nwater, coordinates, gradients); + + // Convert energy and gradients to atomic units + E /= AUTOKCAL; + for (int i = 0; i < totNumAtoms * 3; i++) { + gradients[i] *= FORCE_FAC; + } + + delete[] converted_coords; + + return E; +} diff --git a/tests/tc_mpi_api.h b/tests/tc_mpi_api.h new file mode 100644 index 00000000..b6a53e2b --- /dev/null +++ b/tests/tc_mpi_api.h @@ -0,0 +1,82 @@ +#include +#include +#include +#include +#include "mpi.h" + +#include "../water_potentials/qtip4pf.h" + +#define MAX_DATA 200 + +#define MPI_TAG_EXIT 0 +#define MPI_TAG_GRADIENT 2 +#define MPI_TAG_ERROR 13 + +// This is sent to ABIN for normal MPI_Send +// it is confusing that this is not symmetric, i.e. +// 0 from TC means OK, while 0 from ABIN means exit. +#define MPI_TAG_OK 0 + +#define MPI_TAG_SCF_DIED 1 + +class TCServerMock { + public: + TCServerMock(char *); + ~TCServerMock(void); + + void initializeCommunication(); + + int receiveNumAtoms(); + char* receiveAtomTypes(); + void receiveAtomTypesAndScrdir(); + void receiveCoordinates(); + + int receiveBeginLoop(); + // This combines all expected receive + // methods in their order. But the tc_server + // implementation can also call them individually + // for better granurality. + int receive(); + + // Using qTIP4PF, same as in ABIN and used in all other tests. + // Returns potential energy. + double getWaterGradients(); + + void send(); + void sendSCFEnergy(double, int); + void sendQMCharges(); + void sendQMDipoleMoment(); + void sendQMGradients(); + + MPI_Comm* getABINCommunicator(); + + private: + // This one will be published via MPI_Publish + // ABIN will be looking for this one via hydra_nameserver + char *tcServerName; + // This is the name of the actual port from MPI_Open_port() + char mpiPortName[MPI_MAX_PORT_NAME]; + // Buffers for MPI_Recv and MPI_Send calls + char bufchars[MAX_DATA]; + double bufdoubles[MAX_DATA]; + int bufints[MAX_DATA]; + + double *gradients; + double *coordinates; + + MPI_Comm abin_client; + MPI_Status mpiStatus; + + int totNumAtoms; + char *atomTypes; + + void computeFakeQMDipoleMoment(double&, double&, double&, double&); + void computeFakeQMCharges(double*); + + void publishServerName(char*, char*); + void printMPIError(int); + void parseScrDir(char*, char*); + void validateScrDir(char*); + void checkRecvCount(MPI_Status*, MPI_Datatype, int); + void checkRecvTag(MPI_Status&); +}; diff --git a/tests/test.sh b/tests/test.sh index 2def0931..e0eb0add 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -117,8 +117,10 @@ restart_sh.bin restart_sh.bin.old restart_sh.bin.?? restart.xyz.old restart.xyz. # Run all tests if [[ $TESTS = "all" ]];then # TODO: Re-enable GLE and PIGLE tests! - folders=(CMD SH_EULER SH_RK4 SH_BUTCHER SH_RK4_PHASE LZ_SS LZ_ST LZ_ENE PIMD SHAKE HARMON MINI QMMM \ - ANALYZE_EXT WATER_FAIL) + folders=(CMD SH_EULER SH_RK4 SH_BUTCHER SH_RK4_PHASE \ + LZ_SS LZ_ST LZ_ENE \ + PIMD SHAKE HARMON MINI QMMM \ + ANALYZE_EXT CMDLINE WATER_FAIL) let index=${#folders[@]}+1 # TODO: Split this test, test OpenMP separately @@ -129,9 +131,18 @@ if [[ $TESTS = "all" ]];then if [[ $MPI = "TRUE" ]];then let index=${#folders[@]}+1 folders[index]=REMD - # TODO: Test MPI interface with TC + let index++ + folders[index]=TERAPI + let index++ + folders[index]=TERAPI-PIMD + let index++ + folders[index]=TERAPI-PIMD-PARALLEL + let index++ + folders[index]=TERAPI-FAILS # TODO: Test SH-MPI interface with TC - # folders[index]=TERAPI # does not yet work + else + let index=${#folders[@]}+1 + folders[index]=WITHOUT_MPI fi if [[ $CP2K = "TRUE" ]];then diff --git a/tests/test_tc_server_utils.sh b/tests/test_tc_server_utils.sh new file mode 100644 index 00000000..1c6d72fa --- /dev/null +++ b/tests/test_tc_server_utils.sh @@ -0,0 +1,153 @@ +#!/bin/bash + +# Various utility function that are shared by test.sh scripts +# in tests for TeraChem MPI interface (TERAPI*/) + +# This file is meant to be sourced, NOT executed! + +set -euo pipefail + +export TC_PORT_FILE=port.txt + +hydrapid= +launch_hydra_nameserver() { + # Make sure hydra_nameserver is running + # NOTE: Currently we will always restart the server + # to workaround the existing bug in it. + # https://github.com/pmodels/mpich/issues/5058 + CMD=$1 + hydra=$(ps -C hydra_nameserver -o pid= || true) + if [[ -n ${hydra} ]];then + kill_processes $hydra + fi + #if [[ -z ${hydra-} ]];then + #echo "Launching hydra nameserver for MPI_Lookup" + #fi + $CMD & + hydrapid=$! + # Sometime tests fail connecting to the nameserver, + # let's try to give it some time. + sleep 0.2 +} + +check_for_openmpi() { + if [[ -f "${MPI_PATH-}/bin/orterun" ]];then + # TeraChem is compiled with MPICH so there's no + # point in trying to make this work with OpenMPI. + # We'll skip this test by faking it was successfull. + # Here are some pointers if we ever want to make it work: + # https://techdiagnosys.blogspot.com/2016/12/openmpi-working-nameserver-publish.html + # https://www.open-mpi.org/doc/v4.1/man1/ompi-server.1.php + # https://www.open-mpi.org/doc/v4.1/man1/mpirun.1.php#sect6 (search for ompi-server) + echo "Skipping this test with OpenMPI build" + for f in `ls *ref`;do + cp $f `basename $f .ref` + done + exit 1 + fi +} + +set_mpich_vars() { + if [[ -z ${MPI_PATH-} ]];then + export MPIRUN=mpirun + export MPICXX=mpicxx + export MPICH_HYDRA=hydra_nameserver + else + export MPIRUN=$MPI_PATH/bin/mpirun + export MPICXX=$MPI_PATH/bin/mpicxx + export MPICH_HYDRA=$MPI_PATH/bin/hydra_nameserver + fi +} + +set_default_vars() { + export ABINOUT=abin.out + export ABININ=input.in + export ABINGEOM=mini.xyz + export TCSRC="../tc_mpi_api.cpp ../../water_potentials/qtip4pf.cpp tc_server.cpp" + export TCEXE=tc_server + export TCOUT=tc.out +} + +clean_output_files() { + local return_code=0 + if [[ -z "${1-}" ]];then + echo "Incorrect invocation of the clean_output_files function!" + return 1 + fi + if [[ "${1-}" = "clean" ]];then + return_code=1 + fi + # WARNING: This shift is important, since by default + # the first parameter is the path to ABIN binary, + # we do not want to delete that! + shift + # Remove predefined files + any additional arguments that were passed. + rm -f $* + rm -f *dat *diff + rm -f restart.xyz velocities.xyz forces.xyz movie.xyz restart.xyz.old + rm -f $TCEXE $TCOUT* $ABINOUT $TC_PORT_FILE.* ERROR scrdir000? + return $return_code +} + + +# Sillently kill all processes whose PIDs +# are passed as parameters. Note that some of +# them could have already ended sucessfully. +kill_processes() { + kill -9 $* > /dev/null 2>&1 || true +} + +# Not that it is hard in general to know +# whether we ended successfully or not, +# so we always return 0. Validation is then +# always done on the output files. +cleanup() { + kill_processeses $* + exit 0 +} + +# Helper function for building a regex expression +join_by() { + local IFS="$1" + shift + echo "$*" +} + +# This function accepts PIDs of ABIN and all TC servers +# end periodically checks whether they are still running. +# If only some of them stopped, it kills the rest. +check_running_processes() { + # The MPI interface is prone to deadlocks, where + # both server and client are waiting on MPI_Recv. + # We need to kill both processes if that happens. + pids="$*" + num_jobs=$# + regex=$(join_by \| $pids) + MAX_ITER=100 + iter=1 + while true;do + running=$(ps -eo pid|grep -E "$regex"|wc -l) + if [[ $running -eq 0 ]];then + #echo "Both ABIN and TeraChem servers stopped" + break + elif [[ $running -lt $num_jobs ]];then + # Give the others time to finish + sleep 1.2 + running=$(ps -eo pid|grep -E "$regex"|wc -l) + if [[ $running -ne 0 ]];then + echo "One of the TC servers or ABIN died. Killing the rest." + cat ${ABINOUT}* ${TCOUT}* + fi + break + fi + + sleep 0.2 + let ++iter + if [[ $iter -gt $MAX_ITER ]];then + echo "Maximum time exceeded." + break + fi + done + # ALWAYS call cleanup + cleanup $pids +}