#!/bin/bash
# Copyright 2013-2021 Axel Huebl, Richard Pausch, Rene Widera, Sergei Bastrakov, Klaus Steinger
#
# This file is part of PIConGPU.
#
# PIConGPU is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# PIConGPU is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with PIConGPU.
# If not, see .
#
# PIConGPU batch script for spock's SLURM batch system
# #SBATCH --account=!TBG_nameProject
#SBATCH --partition=!TBG_queue
#SBATCH --time=!TBG_wallTime
# Sets batch job's name
#SBATCH --job-name=!TBG_jobName
#SBATCH --nodes=!TBG_nodes # Nb of nodes
#SBATCH --ntasks=!TBG_tasks # Nb of MPI tasks
#SBATCH --gres=gpu:!TBG_tasks
# #SBATCH --ntasks-per-node=!TBG_devicesPerNode
# #SBATCH --mincpus=!TBG_mpiTasksPerNode
#SBATCH --cpus-per-task=!TBG_coresPerGPU # CPU Cores per MPI process
# #SBATCH --mem=128G # Requested Total Job Memory / Node
#SBATCH --mem=0 # Requested Total Job Memory / Node
#SBATCH --mem-per-gpu=!TBG_memPerDevice
#SBATCH --gpu-bind=closest
#SBATCH --mail-type=!TBG_mailSettings
#SBATCH --mail-user=!TBG_mailAddress
#SBATCH --chdir=!TBG_dstPath
#SBATCH -o pog_%j.out
#SBATCH -e pog_%j.err
# #SBATCH --account=!TBG_nameProject
# #SBATCH --partition=!TBG_queue
# #SBATCH --time=!TBG_wallTime
# Sets batch job's name
# #SBATCH --job-name=!TBG_jobName
# #SBATCH --nodes=!TBG_nodes
# #SBATCH --ntasks=!TBG_tasks
# #SBATCH --ntasks-per-node=!TBG_devicesPerNode
# #SBATCH --mincpus=!TBG_mpiTasksPerNode
# #SBATCH --cpus-per-task=!TBG_coresPerGPU
# #SBATCH --mem-per-gpu=!TBG_memPerDevice
# # #SBATCH --gpus-per-task=1
# #SBATCH --gpu-bind=closest
# #SBATCH --mail-type=!TBG_mailSettings
# #SBATCH --mail-user=!TBG_mailAddress
# #SBATCH --chdir=!TBG_dstPath
## calculations will be performed by tbg ##
.TBG_queue="gpu"
.TBG_jobName="pic_tests"
# settings that can be controlled by environment variables before submit
.TBG_mailSettings=${MY_MAILNOTIFY:-"NONE"}
.TBG_mailAddress=${MY_MAIL:-"d.bertini@gsi.de"}
.TBG_author=${MY_NAME:+--author \"dbertini\"}
.TBG_nameProject=${PROJID:-"PIC"}
.TBG_profile=${PIC_PROFILE:-"/lustre/rz/dbertini/gpu/picongpu.profile"}
# number of available/hosted devices per node in the system
.TBG_numHostedDevicesPerNode=8
# host memory per device
.TBG_memPerDevice=64000000
# number of CPU cores to block per GPU
# we have 16 CPU cores per GPU (64cores/4gpus ~ 16cores)
#.TBG_coresPerGPU=16
.TBG_coresPerGPU=8
# Assign one OpenMP thread per available core per GPU (=task)
export OMP_NUM_THREADS=!TBG_coresPerGPU
## These must be set before running (according to spock quick start guide)
#export MPIR_CVAR_GPU_EAGER_DEVICE_MEM=0
#export MPICH_GPU_SUPPORT_ENABLED=1
# required GPUs per node for the current job
.TBG_devicesPerNode=$(if [ $TBG_tasks -gt $TBG_numHostedDevicesPerNode ] ; then echo $TBG_numHostedDevicesPerNode; else echo $TBG_tasks; fi)
# We only start 1 MPI task per device
.TBG_mpiTasksPerNode="$(( TBG_devicesPerNode * 1 ))"
# use ceil to caculate nodes
.TBG_nodes="$((( TBG_tasks + TBG_devicesPerNode - 1 ) / TBG_devicesPerNode))"
## end calculations ##
echo 'Running program...'
cd !TBG_dstPath
export MODULES_NO_OUTPUT=1
source !TBG_profile
if [ $? -ne 0 ] ; then
echo "Error: PIConGPU environment profile under \"!TBG_profile\" not found!"
exit 1
fi
unset MODULES_NO_OUTPUT
# set user rights to u=rwx;g=r-x;o=---
umask 0027
mkdir simOutput 2> /dev/null
cd simOutput
ln -s ../stdout output
# cuda_memtest is available only on CUDA hardware
# Compilers
export CC=/opt/rocm/llvm/bin/clang
export CXX=/opt/rocm/bin/hipcc
# Main environment variables
export PICHOME=/lustre/rz/dbertini/gpu/picongpu
export PICSRC=$PICHOME
export PIC_EXAMPLES=$PICSRC/share/picongpu/examples
export PATH=$PICSRC:$PATH
export PATH=$PICSRC/bin:$PATH
export PATH=$PICSRC/src/tools/bin:$PATH
export PYTHONPATH=$PICSRC/lib/python:$PYTHONPATH
export ADIOS2_DIR=/opt/adios/2.7.1
export openPMD_DIR=/opt/openPMD-api/0.14.3/
export PNGwriter_DIR=/opt/pngwriter/0.7.0/
export ISAAC_DIR=/opt/isaac/1.5.2/
# output data
export SCRATCH=/lustre/rz/dbertini/gpu/data
export WORKDIR=/lustre/rz/dbertini/gpu
### environment
export PATH=/usr/local/bin:$PATH
export PATH=/opt/rocm/bin:$PATH
# ## picongpu dependencies
export LD_LIBRARY_PATH=/opt/adios/1.13.1/lib:$LD_LIBRARY_PATH
export LD_LIBRARY_PATH=/opt/adios/2.7.1/lib:$LD_LIBRARY_PATH
export LD_LIBRARY_PATH=/opt/boost/1.75.0/lib:$LD_LIBRARY_PATH
export LD_LIBRARY_PATH=/opt/hdf5/1.10.7/lib:$LD_LIBRARY_PATH
export LD_LIBRARY_PATH=/opt/icet/2.9.0/lib:$LD_LIBRARY_PATH
export LD_LIBRARY_PATH=/opt/isaac/1.5.2/lib:$LD_LIBRARY_PATH
export LD_LIBRARY_PATH=/opt/janson/2.9.0/lib:$LD_LIBRARY_PATH
export LD_LIBRARY_PATH=/opt/libsplash/1.7.0/lib:$LD_LIBRARY_PATH
export LD_LIBRARY_PATH=/opt/libpngwriter/0.7.0/lib:$LD_LIBRARY_PATH
export LD_LIBRARY_PATH=/opt/openPMD-api/0.14.3/lib:$LD_LIBRARY_PATH
export LD_LIBRARY_PATH=/opt/rocm/lib:$LD_LIBRARY_PATH
# add necessary rocm-libs part
export LD_LIBRARY_PATH=/opt/rocm/hiprand/lib:/opt/rocm/rocrand/lib:$LD_LIBRARY_PATH
export PATH=/opt/rocm/hiprand/bin:$PATH
# add adios 2
export LD_LIBRARY_PATH=/opt/adios/2.7.1/lib:$LD_LIBRARY_PATH
export PATH=/opt/adios/2.7.1/bin:$PATH
export CPLUS_INCLUDE_PATH=/opt/adios/1.13.1/include:$CPLUS_INCLUDE_PATH
export CPLUS_INCLUDE_PATH=/opt/adios/2.7.1/include:$CPLUS_INCLUDE_PATH
export CPLUS_INCLUDE_PATH=/opt/boost/1.75.0/include:$CPLUS_INCLUDE_PATH
export CPLUS_INCLUDE_PATH=/opt/hdf5/1.10.7/include:$CPLUS_INCLUDE_PATH
export CPLUS_INCLUDE_PATH=/opt/icet/2.9.0/include:$CPLUS_INCLUDE_PATH
export CPLUS_INCLUDE_PATH=/opt/isaac/1.5.2/include:$CPLUS_INCLUDE_PATH
export CPLUS_INCLUDE_PATH=/opt/janson/2.9.0/include:$CPLUS_INCLUDE_PATH
export CPLUS_INCLUDE_PATH=/opt/libsplash/1.7.0/include:$CPLUS_INCLUDE_PATH
export CPLUS_INCLUDE_PATH=/opt/pngwriter/0.7.0/include:$CPLUS_INCLUDE_PATH
export CPLUS_INCLUDE_PATH=/opt/openPMD-api/0.14.3/include:$CPLUS_INCLUDE_PATH
export CPLUS_INCLUDE_PATH=/opt/rocm/include:$CPLUS_INCLUDE_PATH
export PATH=/opt/adios/1.13.1/bin:$PATH
export PATH=/opt/hdf5/1.10.7/bin:$PATH
export PATH=/opt/libsplash/1.7.0/bin:$PATH
export PATH=/opt/openPMD-api/0.14.3/bin:$PATH
## add hip+adios
export LD_LIBRARY_PATH=/opt/rocm/hiprand/lib:/opt/rocm/rocrand/lib:$LD_LIBRARY_PATH
export LD_LIBRARY_PATH=/opt/adios/2.7.1/lib:$LD_LIBRARY_PATH
export PATH=/opt/rocm/hiprand/bin:$PATH
export PATH=/opt/adios/2.7.1/bin:$PATH
## MPI
export OMPI_MCA_io=^ompio
export OMPI_MCA_mpi_leave_pinned=0
# setup openMPI
export PMIX_MCA_gds=^ds21
export OMPI_MCA_io=^ompio
export OMPI_MCA_mpi_leave_pinned=0
export OMPI_MCA_btl_openib_allow_ib=1
export OMPI_MCA_btl_openib_rdma_pipeline_send_length=100000000
export OMPI_MCA_btl_openib_rdma_pipeline_frag_size=100000000
#export OMP_NUM_THREADS=1
cat > !TBG_dstPath/tbg/pic_sub.sh <