diff --git a/support/SubmitScripts/Wheeler.sh b/support/SubmitScripts/Wheeler.sh index 78588bc91fd0..2af58e5d857c 100755 --- a/support/SubmitScripts/Wheeler.sh +++ b/support/SubmitScripts/Wheeler.sh @@ -65,6 +65,39 @@ echo cd ${RUN_DIR} -mpirun -n ${SLURM_NTASKS} \ +checkpoints=0 +current_checkpoint=000000 +if [[ $checkpoints == 0 ]]; then + mpirun -n ${SLURM_NTASKS} \ ${SPECTRE_EXECUTABLE} ++ppn ${CHARM_PPN} +setcpuaffinity \ --input-file ${SPECTRE_INPUT_FILE} + sleep 10s + # If a checkpoint is found add one to checkpoint and sumbit next job + if test -e "${PWD}/SpectreCheckpoint$current_checkpoint"; then + mv spectre.out "${PWD}/SpectreCheckpoint$current_checkpoint" + sed -i "s/^checkpoints=0/checkpoints=1/" Wheeler.sh + sbatch Wheeler.sh + fi +# Section to start from checkpoint +elif [[ $checkpoints -gt 0 && $checkpoints -lt 1000000 ]]; then + mpirun -n ${SLURM_NTASKS} \ + ${SPECTRE_EXECUTABLE} ++ppn ${CHARM_PPN} +setcpuaffinity \ + --input-file ${SPECTRE_INPUT_FILE} \ + +restart SpectreCheckpoint${current_checkpoint} + sleep 10s + # If the next checkpoint was created modify variables and submit next job + printf -v next_checkpoint %06d $checkpoints + if test -e "${PWD}/SpectreCheckpoint$next_checkpoint"; then + # We move the current spectre.out to the checkpoint directory because SLURM + # will overwrite it once the new job is run. + mv spectre.out "${PWD}/SpectreCheckpoint$next_checkpoint" + next_num_of_checkpoints=$(($checkpoints + 1)) + #Updating variables for the next possible checkpoint + sed -i "s/^checkpoints=$checkpoints/"\ +"checkpoints=$next_num_of_checkpoints/" Wheeler.sh + sed -i "s/^current_checkpoint=$current_checkpoint/"\ +"current_checkpoint=$next_checkpoint/" Wheeler.sh + sbatch Wheeler.sh + fi +fi +