train_attack_NN_model.sh

#!/bin/bash
#SBATCH --gres=gpu:1
#SBATCH --job-name=trAttack
#SBATCH --partition=all
#SBATCH --qos=default
#SBATCH --cpus-per-task=3
#SBATCH --mem=64g
#SBATCH --open-mode=append
#SBATCH --signal=B:USR1@120
#SBATCH --requeue
#SBATCH --output=logs/%A_%a.stdout
#SBATCH --error=logs/%A_%a.stderr
#SBATCH --array=0
#SBATCH --exclude=n[1-5,51-55]



SECONDS=0

restart(){
    echo "Calling restart" 
    scontrol requeue $SLURM_JOB_ID
    echo "Scheduled job for restart" 
}

ignore(){
    echo "Ignored SIGTERM" 
}
trap restart USR1
trap ignore TERM

date 



args=()


args+=("train_attack_NN_model.py --zip")



echo "Starting python ${args[${SLURM_ARRAY_TASK_ID}]}"

srun python ${args[${SLURM_ARRAY_TASK_ID}]}

echo "End python ${args[${SLURM_ARRAY_TASK_ID}]}"



DURATION=$SECONDS

echo "End of the program! $(($DURATION / 60)) minutes and $(($DURATION % 60)) seconds elapsed."