Caltech central cluster

Sample sbatch script

Here is a sbatch script template for setting up simulations using multiple CPUs on Caltech central cluster.

#!/bin/bash
#SBATCH --ntasks=32
#SBATCH --job-name=[YOUR_JOB_NAME]
#SBATCH --time=2:00:00

module purge
module load julia/1.9.3 cuda/11.8 ucx/1.14.1_cuda-11.8 openmpi/4.1.5_cuda-11.8 hdf5/1.12.2-ompi415 nsight-systems/2023.2.1

CA_PATH=[YOUR_CLIMAATMOS_PATH]
CA_EXAMPLE=$CA_PATH'examples/'
DRIVER=$CA_EXAMPLE'hybrid/driver.jl'
CONFIG_FILE=[YOUR_CONFIG_FILE]

export OPENBLAS_NUM_THREADS=1
export JULIA_NVTX_CALLBACKS=gc
export OMPI_MCA_opal_warn_on_missing_libcuda=0
export JULIA_MAX_NUM_PRECOMPILE_FILES=100
export JULIA_LOAD_PATH="${JULIA_LOAD_PATH}:${CA_PATH}.buildkite"
export CLIMACORE_DISTRIBUTED="MPI"

julia -e 'using Pkg; Pkg.add("MPIPreferences"); using MPIPreferences; use_system_binary()'
julia --project=$CA_EXAMPLE -e 'using Pkg; Pkg.instantiate()'
julia --project=$CA_EXAMPLE -e 'using Pkg; Pkg.precompile()'
julia --project=$CA_EXAMPLE -e 'using Pkg; Pkg.status()'

srun julia --project=$CA_EXAMPLE $DRIVER --config_file $CONFIG_FILE