#!/bin/bash
#PBS -S /bin/bash
#PBS -l select=25:ncpus=24:mpiprocs=24:model=bro
#PBS -l walltime=8:00:00
#PBS -l site=needed=/home4+/nobackupp12+/nobackupp16+/nobackupp13
#PBS -l place=excl
#PBS -N C90
#PBS -m abe
#PBS -r y
#PBS -W group_list=s2387
# loading environment
source /u/dzhang8/gchp-intel.202202.env
cd $PBS_O_WORKDIR

module list	# print loaded modules
set -e		# if a subsequent command fails, treat it as fatal (don't continue)
set -x		# for remainder of script, echo commands to the job's log file

ulimit -c 0
ulimit -l unlimited
ulimit -u 50000
ulimit -v unlimited
ulimit -s unlimited

export MPI_LAUNCH_TIMEOUT=40
export PATH=$PATH:/u/scicon/tools/bin
#export MPI_VERBOSE=1
#export MPI_DISPLAY_SETTINGS=1
export MPI_OMP_NUM_THREADS=1
export FOR_IGNORE_EXCEPTIONS=false
export MPI_COLL_REPRODUCIBLE
unset MPI_MEMMAP_OFF
unset MPI_NUM_MEMORY_REGIONS
export MPI_XPMEM_ENABLED=yes
unset SUPPRESS_XPMEM_TRIM_THRESH
unset PMI_RANK

function last_checkpoint() {
    ls -1 gcchem_internal_checkpoint*.nc4 | tail -n 1
}

function last_checkpoint_date() {
    last_checkpoint | sed 's/gcchem_internal_checkpoint.\(20[12][0-9][0-9][0-9][0123][0-9]\).*/\1/'
}
RESTART_DATE=SIM_START
# Execute simulation
if [ ! -f CONTINUE_SEM ] ; then
    rm -f cap_restart #gcchem* 
    ./runConfig.sh
    touch CONTINUE_SEM
else
    RESTART_FILE=$(last_checkpoint)
    RESTART_DATE=$(last_checkpoint_date)
    echo "$RESTART_DATE 000000" > cap_restart
    sed -i "s/GCHPchem_INTERNAL_RESTART_FILE: .*/GCHPchem_INTERNAL_RESTART_FILE: $RESTART_FILE/g" GCHP.rc
fi

rm -f gcchem_internal_checkpoint
several_tries mpiexec -np 600 ./gchp &> run-C90-HTAPv3-$(date +"%Y%m%d_%H%M").log