Skip to content
Permalink
Browse files

Add scripts for Stampede2 and increase max-depth to 30 levels

  • Loading branch information...
Dhairya Malhotra
Dhairya Malhotra committed Mar 22, 2019
1 parent 67595dd commit 979d503349e509c44ade30924fe4b867c76c16cc
Showing with 63 additions and 1 deletion.
  1. +1 −1 include/pvfmm_common.hpp
  2. +53 −0 scripts/.job.stampede2-skx
  3. +9 −0 scripts/.submit_jobs.sh
@@ -26,7 +26,7 @@
//Verbose
//#define PVFMM_VERBOSE

#define PVFMM_MAX_DEPTH 15
#define PVFMM_MAX_DEPTH 30

#define PVFMM_BC_LEVELS 30

@@ -0,0 +1,53 @@
#!/bin/bash
#SBATCH -J fmm
#SBATCH --export=ALL
#
# Options for Stampede
#SBATCH -p skx-normal
#SBATCH -A TG-DPP130002

module list
pwd
date


eval $nodes_;
eval $cores_;
eval $mpi_proc_;
eval $threads_;
eval $max_time_;
eval $fname_;
eval $args_;

WORK_DIR=${PWD}
cd ${WORK_DIR}

#export KMP_AFFINITY=verbose
#Set mic environment variables
#export OFFLOAD_REPORT=2
export MIC_ENV_PREFIX=MIC
export MIC_KMP_AFFINITY=explicit,granularity=fine,proclist=[1-240:1]
export MIC_OMP_NUM_THREADS=240
export MIC_USE_2MB_BUFFERS=64K

export KMP_AFFINITY=scatter
export I_MPI_HYDRA_BRANCH_COUNT=32

for (( k=0; k<${#nodes[@]}; k++ )) ; do
if [ "${nodes[k]}" == "${NODES}" ] &&
[ "${mpi_proc[k]}" == "${MPI_PROC}" ]; then
export OMP_NUM_THREADS=${threads[k]};

# CPU only
EXEC_=${EXEC};
FNAME=${RESULT_DIR}/$(basename ${EXEC_})_${fname[k]}.out;
if [ -f ${EXEC_} ] && [ ! -s ${FNAME} ] ; then
printf '%*s\n\n' "100" ' ' | tr ' ' "#" | tee -a ${FNAME};
printf "COMMAND: ${EXEC_} ${args[k]}\n" | tee -a ${FNAME};
${TIMEOUT} ${max_time[k]} time ibrun tacc_affinity ${EXEC_} ${args[k]} &> >(tee -a ${FNAME});
printf '\n%*s\n\n' "100" ' ' | tr ' ' "#" | tee -a ${FNAME};
fi;

fi;
done;

@@ -22,6 +22,7 @@ for (( k=0; k<${#nodes[@]}; k++ )) ; do
# Set output filename
case $HOSTNAME in
*titan*) fname[$k]="host_titan";;
*stampede2*) fname[$k]="host_stampede2";;
*stampede*) fname[$k]="host_stampede";;
*ls4*) fname[$k]="host_lonestar";;
*ronaldo*) fname[$k]="host_ronaldo";;
@@ -81,6 +82,14 @@ for (( k=0; k<${#nodes[@]}; k++ )) ; do
-l walltime=${TOTAL_TIME} \
./scripts/.job.titan
;;
*stampede2*) #stampede2.tacc.utexas.edu (Slurm Batch)
if (( ${TOTAL_TIME} > 14400 )); then TOTAL_TIME="14400"; fi
#if (( ${NODES} > 128 )) ; then continue; fi;
sbatch -N${NODES} -n${MPI_PROC} \
-o ${FNAME}.out -e ${FNAME}.err -D ${PWD} \
--time=00:00:${TOTAL_TIME} \
./scripts/.job.stampede2-skx
;;
*stampede*) #stampede.tacc.utexas.edu (Slurm Batch)
if (( ${TOTAL_TIME} > 14400 )); then TOTAL_TIME="14400"; fi
#if (( ${NODES} > 128 )) ; then continue; fi;

0 comments on commit 979d503

Please sign in to comment.
You can’t perform that action at this time.