Add scripts for Stampede2 and increase max-depth to 30 levels

include/pvfmm_common.hpp

@@ -26,7 +26,7 @@
 //Verbose
 //#define PVFMM_VERBOSE
 
-#define PVFMM_MAX_DEPTH 15
+#define PVFMM_MAX_DEPTH 30
 
 #define PVFMM_BC_LEVELS 30
 

scripts/.job.stampede2-skx

@@ -0,0 +1,53 @@
+#!/bin/bash
+#SBATCH -J fmm
+#SBATCH --export=ALL
+#
+# Options for Stampede
+#SBATCH -p skx-normal
+#SBATCH -A TG-DPP130002
+
+module list
+pwd
+date
+
+
+eval    $nodes_;
+eval    $cores_;
+eval $mpi_proc_;
+eval  $threads_;
+eval $max_time_;
+eval    $fname_;
+eval     $args_;
+
+WORK_DIR=${PWD}
+cd ${WORK_DIR}
+
+#export KMP_AFFINITY=verbose
+#Set mic environment variables
+#export OFFLOAD_REPORT=2
+export MIC_ENV_PREFIX=MIC
+export MIC_KMP_AFFINITY=explicit,granularity=fine,proclist=[1-240:1]
+export MIC_OMP_NUM_THREADS=240
+export MIC_USE_2MB_BUFFERS=64K
+
+export KMP_AFFINITY=scatter
+export I_MPI_HYDRA_BRANCH_COUNT=32
+
+for (( k=0; k<${#nodes[@]}; k++ )) ; do
+  if [ "${nodes[k]}" == "${NODES}" ] &&
+     [ "${mpi_proc[k]}" == "${MPI_PROC}" ]; then
+    export OMP_NUM_THREADS=${threads[k]};
+
+    # CPU only
+    EXEC_=${EXEC};
+    FNAME=${RESULT_DIR}/$(basename ${EXEC_})_${fname[k]}.out;
+    if [ -f ${EXEC_} ] && [ ! -s ${FNAME} ] ; then
+      printf '%*s\n\n' "100" ' ' | tr ' ' "#" | tee -a ${FNAME};
+      printf "COMMAND: ${EXEC_} ${args[k]}\n" | tee -a ${FNAME};
+      ${TIMEOUT} ${max_time[k]} time ibrun tacc_affinity ${EXEC_} ${args[k]} &> >(tee -a ${FNAME});
+      printf '\n%*s\n\n' "100" ' ' | tr ' ' "#" | tee -a ${FNAME};
+    fi;
+
+  fi;
+done;
+

scripts/.submit_jobs.sh

@@ -22,6 +22,7 @@ for (( k=0; k<${#nodes[@]}; k++ )) ; do
   # Set output filename
   case $HOSTNAME in
     *titan*) fname[$k]="host_titan";;
+    *stampede2*) fname[$k]="host_stampede2";;
     *stampede*) fname[$k]="host_stampede";;
     *ls4*) fname[$k]="host_lonestar";;
     *ronaldo*) fname[$k]="host_ronaldo";;
@@ -81,6 +82,14 @@ for (( k=0; k<${#nodes[@]}; k++ )) ; do
              -l walltime=${TOTAL_TIME} \
              ./scripts/.job.titan
       ;;
+    *stampede2*) #stampede2.tacc.utexas.edu (Slurm Batch)
+        if (( ${TOTAL_TIME} > 14400 )); then TOTAL_TIME="14400"; fi
+        #if (( ${NODES} > 128 )) ; then continue; fi;
+        sbatch -N${NODES} -n${MPI_PROC} \
+               -o ${FNAME}.out -e ${FNAME}.err -D ${PWD} \
+               --time=00:00:${TOTAL_TIME} \
+               ./scripts/.job.stampede2-skx
+      ;;
     *stampede*) #stampede.tacc.utexas.edu (Slurm Batch)
         if (( ${TOTAL_TIME} > 14400 )); then TOTAL_TIME="14400"; fi
         #if (( ${NODES} > 128 )) ; then continue; fi;