#!/bin/bash # Copyright 2013-2021 Axel Huebl, Richard Pausch, Rene Widera, Sergei Bastrakov, Klaus Steinger # # This file is part of PIConGPU. # # PIConGPU is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # PIConGPU is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with PIConGPU. # If not, see . # # PIConGPU batch script for spock's SLURM batch system # #SBATCH --account=!TBG_nameProject #SBATCH --partition=!TBG_queue #SBATCH --time=!TBG_wallTime # Sets batch job's name #SBATCH --job-name=!TBG_jobName #SBATCH --nodes=!TBG_nodes # Nb of nodes #SBATCH --ntasks=!TBG_tasks # Nb of MPI tasks #SBATCH --gres=gpu:!TBG_tasks # #SBATCH --ntasks-per-node=!TBG_devicesPerNode # #SBATCH --mincpus=!TBG_mpiTasksPerNode #SBATCH --cpus-per-task=!TBG_coresPerGPU # CPU Cores per MPI process # #SBATCH --mem=128G # Requested Total Job Memory / Node #SBATCH --mem=0 # Requested Total Job Memory / Node #SBATCH --mem-per-gpu=!TBG_memPerDevice #SBATCH --gpu-bind=closest #SBATCH --mail-type=!TBG_mailSettings #SBATCH --mail-user=!TBG_mailAddress #SBATCH --chdir=!TBG_dstPath #SBATCH -o pog_%j.out #SBATCH -e pog_%j.err # #SBATCH --account=!TBG_nameProject # #SBATCH --partition=!TBG_queue # #SBATCH --time=!TBG_wallTime # Sets batch job's name # #SBATCH --job-name=!TBG_jobName # #SBATCH --nodes=!TBG_nodes # #SBATCH --ntasks=!TBG_tasks # #SBATCH --ntasks-per-node=!TBG_devicesPerNode # #SBATCH --mincpus=!TBG_mpiTasksPerNode # #SBATCH --cpus-per-task=!TBG_coresPerGPU # #SBATCH --mem-per-gpu=!TBG_memPerDevice # # #SBATCH --gpus-per-task=1 # #SBATCH --gpu-bind=closest # #SBATCH --mail-type=!TBG_mailSettings # #SBATCH --mail-user=!TBG_mailAddress # #SBATCH --chdir=!TBG_dstPath ## calculations will be performed by tbg ## .TBG_queue="gpu" .TBG_jobName="pic_tests" # settings that can be controlled by environment variables before submit .TBG_mailSettings=${MY_MAILNOTIFY:-"NONE"} .TBG_mailAddress=${MY_MAIL:-"d.bertini@gsi.de"} .TBG_author=${MY_NAME:+--author \"dbertini\"} .TBG_nameProject=${PROJID:-"PIC"} .TBG_profile=${PIC_PROFILE:-"/lustre/rz/dbertini/gpu/picongpu.profile"} # number of available/hosted devices per node in the system .TBG_numHostedDevicesPerNode=8 # host memory per device .TBG_memPerDevice=64000000 # number of CPU cores to block per GPU # we have 16 CPU cores per GPU (64cores/4gpus ~ 16cores) #.TBG_coresPerGPU=16 .TBG_coresPerGPU=8 # Assign one OpenMP thread per available core per GPU (=task) export OMP_NUM_THREADS=!TBG_coresPerGPU ## These must be set before running (according to spock quick start guide) #export MPIR_CVAR_GPU_EAGER_DEVICE_MEM=0 #export MPICH_GPU_SUPPORT_ENABLED=1 # required GPUs per node for the current job .TBG_devicesPerNode=$(if [ $TBG_tasks -gt $TBG_numHostedDevicesPerNode ] ; then echo $TBG_numHostedDevicesPerNode; else echo $TBG_tasks; fi) # We only start 1 MPI task per device .TBG_mpiTasksPerNode="$(( TBG_devicesPerNode * 1 ))" # use ceil to caculate nodes .TBG_nodes="$((( TBG_tasks + TBG_devicesPerNode - 1 ) / TBG_devicesPerNode))" ## end calculations ## echo 'Running program...' cd !TBG_dstPath export MODULES_NO_OUTPUT=1 source !TBG_profile if [ $? -ne 0 ] ; then echo "Error: PIConGPU environment profile under \"!TBG_profile\" not found!" exit 1 fi unset MODULES_NO_OUTPUT # set user rights to u=rwx;g=r-x;o=--- umask 0027 mkdir simOutput 2> /dev/null cd simOutput ln -s ../stdout output # cuda_memtest is available only on CUDA hardware # Compilers export CC=/opt/rocm/llvm/bin/clang export CXX=/opt/rocm/bin/hipcc # Main environment variables export PICHOME=/lustre/rz/dbertini/gpu/picongpu export PICSRC=$PICHOME export PIC_EXAMPLES=$PICSRC/share/picongpu/examples export PATH=$PICSRC:$PATH export PATH=$PICSRC/bin:$PATH export PATH=$PICSRC/src/tools/bin:$PATH export PYTHONPATH=$PICSRC/lib/python:$PYTHONPATH export ADIOS2_DIR=/opt/adios/2.7.1 export openPMD_DIR=/opt/openPMD-api/0.14.3/ export PNGwriter_DIR=/opt/pngwriter/0.7.0/ export ISAAC_DIR=/opt/isaac/1.5.2/ # output data export SCRATCH=/lustre/rz/dbertini/gpu/data export WORKDIR=/lustre/rz/dbertini/gpu ### environment export PATH=/usr/local/bin:$PATH export PATH=/opt/rocm/bin:$PATH # ## picongpu dependencies export LD_LIBRARY_PATH=/opt/adios/1.13.1/lib:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=/opt/adios/2.7.1/lib:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=/opt/boost/1.75.0/lib:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=/opt/hdf5/1.10.7/lib:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=/opt/icet/2.9.0/lib:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=/opt/isaac/1.5.2/lib:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=/opt/janson/2.9.0/lib:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=/opt/libsplash/1.7.0/lib:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=/opt/libpngwriter/0.7.0/lib:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=/opt/openPMD-api/0.14.3/lib:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=/opt/rocm/lib:$LD_LIBRARY_PATH # add necessary rocm-libs part export LD_LIBRARY_PATH=/opt/rocm/hiprand/lib:/opt/rocm/rocrand/lib:$LD_LIBRARY_PATH export PATH=/opt/rocm/hiprand/bin:$PATH # add adios 2 export LD_LIBRARY_PATH=/opt/adios/2.7.1/lib:$LD_LIBRARY_PATH export PATH=/opt/adios/2.7.1/bin:$PATH export CPLUS_INCLUDE_PATH=/opt/adios/1.13.1/include:$CPLUS_INCLUDE_PATH export CPLUS_INCLUDE_PATH=/opt/adios/2.7.1/include:$CPLUS_INCLUDE_PATH export CPLUS_INCLUDE_PATH=/opt/boost/1.75.0/include:$CPLUS_INCLUDE_PATH export CPLUS_INCLUDE_PATH=/opt/hdf5/1.10.7/include:$CPLUS_INCLUDE_PATH export CPLUS_INCLUDE_PATH=/opt/icet/2.9.0/include:$CPLUS_INCLUDE_PATH export CPLUS_INCLUDE_PATH=/opt/isaac/1.5.2/include:$CPLUS_INCLUDE_PATH export CPLUS_INCLUDE_PATH=/opt/janson/2.9.0/include:$CPLUS_INCLUDE_PATH export CPLUS_INCLUDE_PATH=/opt/libsplash/1.7.0/include:$CPLUS_INCLUDE_PATH export CPLUS_INCLUDE_PATH=/opt/pngwriter/0.7.0/include:$CPLUS_INCLUDE_PATH export CPLUS_INCLUDE_PATH=/opt/openPMD-api/0.14.3/include:$CPLUS_INCLUDE_PATH export CPLUS_INCLUDE_PATH=/opt/rocm/include:$CPLUS_INCLUDE_PATH export PATH=/opt/adios/1.13.1/bin:$PATH export PATH=/opt/hdf5/1.10.7/bin:$PATH export PATH=/opt/libsplash/1.7.0/bin:$PATH export PATH=/opt/openPMD-api/0.14.3/bin:$PATH ## add hip+adios export LD_LIBRARY_PATH=/opt/rocm/hiprand/lib:/opt/rocm/rocrand/lib:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=/opt/adios/2.7.1/lib:$LD_LIBRARY_PATH export PATH=/opt/rocm/hiprand/bin:$PATH export PATH=/opt/adios/2.7.1/bin:$PATH ## MPI export OMPI_MCA_io=^ompio export OMPI_MCA_mpi_leave_pinned=0 # setup openMPI export PMIX_MCA_gds=^ds21 export OMPI_MCA_io=^ompio export OMPI_MCA_mpi_leave_pinned=0 export OMPI_MCA_btl_openib_allow_ib=1 export OMPI_MCA_btl_openib_rdma_pipeline_send_length=100000000 export OMPI_MCA_btl_openib_rdma_pipeline_frag_size=100000000 #export OMP_NUM_THREADS=1 cat > !TBG_dstPath/tbg/pic_sub.sh <