From dafe5d817ad281031d3aaf09f0fe378592608ad0 Mon Sep 17 00:00:00 2001 From: Daniel Vickers Date: Fri, 31 Oct 2025 10:23:13 -0400 Subject: [PATCH 1/5] This configuration works with multiple MPI ranks, but it quite slow --- toolchain/modules | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/toolchain/modules b/toolchain/modules index 27783e9407..30aab6fce3 100644 --- a/toolchain/modules +++ b/toolchain/modules @@ -88,3 +88,13 @@ n-gpu CC=nvc CXX=nvc++ FC=nvfortran san CSCS Santis san-all cmake python san-gpu nvhpc cuda cray-mpich + +h hipergator +h-gpu nvhpc/25.9 +h-gpu CUDA_HOME="/apps/compilers/cuda/12.8.1" +h-all HPC_OMPI_DIR="/apps/mpi/cuda/12.8.1/nvhpc/25.3/openmpi/5.0.7" +h-all HPC_OMPI_BIN="/apps/mpi/cuda/12.8.1/nvhpc/25.3/openmpi/5.0.7/bin" +h-all OMPI_MCA_pml=ob1 OMPI_MCA_coll_hcoll_enable=0 +h-gpu PATH="/apps/mpi/cuda/12.8.1/nvhpc/25.3/openmpi/5.0.7/bin:/apps/compilers/cuda/12.8.1/bin:${PATH}" +h-gpu LD_LIBRARY_PATH="/apps/compilers/cuda/12.8.1/lib64:${LD_LIBRARY_PATH}" +h-gpu MFC_CUDA_CC=100 NVHPC_CUDA_HOME=$CUDA_HOME \ No newline at end of file From 3c79d48068c9f726ebb323e3a740090d2efd2ec5 Mon Sep 17 00:00:00 2001 From: Daniel Vickers Date: Fri, 31 Oct 2025 10:31:25 -0400 Subject: [PATCH 2/5] make file to no longer require using -b --- toolchain/templates/hipergator.mako | 61 +++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 toolchain/templates/hipergator.mako diff --git a/toolchain/templates/hipergator.mako b/toolchain/templates/hipergator.mako new file mode 100644 index 0000000000..3de36fea5b --- /dev/null +++ b/toolchain/templates/hipergator.mako @@ -0,0 +1,61 @@ +#!/usr/bin/env bash + +<%namespace name="helpers" file="helpers.mako"/> + +% if engine == 'batch': +#SBATCH --nodes=${nodes} +#SBATCH --ntasks-per-node=${tasks_per_node} +#SBATCH --job-name="${name}" +#SBATCH --output="${name}.out" +#SBATCH --time=${walltime} +#SBATCH --cpus-per-task=7 +% if gpu: +#SBATCH --gpus-per-task=1 +#SBATCH --gpu-bind=closest +% endif +% if account: +#SBATCH --account=${account} +% endif +% if partition: +#SBATCH --partition=${partition} +% else: +#SBATCH --partition=hpg-b200 +% endif +% if quality_of_service: +#SBATCH --qos=${quality_of_service} +% endif +% if email: +#SBATCH --mail-user=${email} +#SBATCH --mail-type="BEGIN, END, FAIL" +% endif +% endif + +${helpers.template_prologue()} + +ok ":) Loading modules:\n" +cd "${MFC_ROOT_DIR}" +% if engine == 'batch': +. ./mfc.sh load -c h -m ${'g' if gpu else 'c'} +% endif +cd - > /dev/null +echo + + +% for target in targets: + ${helpers.run_prologue(target)} + + % if not mpi: + (set -x; ${profiler} "${target.get_install_binpath(case)}") + % else: + (set -x; ${profiler} \ + mpirun -np ${nodes*tasks_per_node} \ + --bind-to none \ + "${target.get_install_binpath(case)}") + % endif + + ${helpers.run_epilogue(target)} + + echo +% endfor + +${helpers.template_epilogue()} From 361018fa2630338a35bd96ea29ae33d18c4e50a5 Mon Sep 17 00:00:00 2001 From: Daniel Vickers Date: Fri, 31 Oct 2025 10:39:59 -0400 Subject: [PATCH 3/5] One last module update --- toolchain/modules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/toolchain/modules b/toolchain/modules index 30aab6fce3..7ab9115c33 100644 --- a/toolchain/modules +++ b/toolchain/modules @@ -97,4 +97,4 @@ h-all HPC_OMPI_BIN="/apps/mpi/cuda/12.8.1/nvhpc/25.3/openmpi/5.0.7/bin" h-all OMPI_MCA_pml=ob1 OMPI_MCA_coll_hcoll_enable=0 h-gpu PATH="/apps/mpi/cuda/12.8.1/nvhpc/25.3/openmpi/5.0.7/bin:/apps/compilers/cuda/12.8.1/bin:${PATH}" h-gpu LD_LIBRARY_PATH="/apps/compilers/cuda/12.8.1/lib64:${LD_LIBRARY_PATH}" -h-gpu MFC_CUDA_CC=100 NVHPC_CUDA_HOME=$CUDA_HOME \ No newline at end of file +h-gpu MFC_CUDA_CC=100 NVHPC_CUDA_HOME="/apps/compilers/cuda/12.8.1" \ No newline at end of file From 5af77ba4d8e0e9cf0c02840b06108295176e8891 Mon Sep 17 00:00:00 2001 From: Daniel Vickers Date: Fri, 31 Oct 2025 11:16:26 -0400 Subject: [PATCH 4/5] We don't need to set as many cuda parameters --- toolchain/modules | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/toolchain/modules b/toolchain/modules index 7ab9115c33..89436a40a1 100644 --- a/toolchain/modules +++ b/toolchain/modules @@ -95,6 +95,5 @@ h-gpu CUDA_HOME="/apps/compilers/cuda/12.8.1" h-all HPC_OMPI_DIR="/apps/mpi/cuda/12.8.1/nvhpc/25.3/openmpi/5.0.7" h-all HPC_OMPI_BIN="/apps/mpi/cuda/12.8.1/nvhpc/25.3/openmpi/5.0.7/bin" h-all OMPI_MCA_pml=ob1 OMPI_MCA_coll_hcoll_enable=0 -h-gpu PATH="/apps/mpi/cuda/12.8.1/nvhpc/25.3/openmpi/5.0.7/bin:/apps/compilers/cuda/12.8.1/bin:${PATH}" -h-gpu LD_LIBRARY_PATH="/apps/compilers/cuda/12.8.1/lib64:${LD_LIBRARY_PATH}" +h-gpu PATH="/apps/mpi/cuda/12.8.1/nvhpc/25.3/openmpi/5.0.7/bin:${PATH}" h-gpu MFC_CUDA_CC=100 NVHPC_CUDA_HOME="/apps/compilers/cuda/12.8.1" \ No newline at end of file From 7e6c44c467eb60045a55f2a59c8d9a51a06b168b Mon Sep 17 00:00:00 2001 From: Ben Wilfong <48168887+wilfonba@users.noreply.github.com> Date: Fri, 31 Oct 2025 11:49:09 -0400 Subject: [PATCH 5/5] spelling --- benchmarks/5eq_rk3_weno3_hllc/case.py | 2 +- examples/2D_axisym_shockwatercavity/case.py | 2 +- examples/3D_shockdroplet/case.py | 2 +- examples/3D_shockdroplet_muscl/case.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/benchmarks/5eq_rk3_weno3_hllc/case.py b/benchmarks/5eq_rk3_weno3_hllc/case.py index 7f5e03ec4c..e3f196d02c 100644 --- a/benchmarks/5eq_rk3_weno3_hllc/case.py +++ b/benchmarks/5eq_rk3_weno3_hllc/case.py @@ -81,7 +81,7 @@ c_w = math.sqrt(gamw * (p0w + piw) / rho0w) # Shock Mach number of interest. Note that the post-shock properties can be defined in terms of either -# Min or psOp0a. Just comment/uncomment appropriatelly +# Min or psOp0a. Just comment/uncomment appropriately Min = 2.4 ## Pos to pre shock ratios - AIR diff --git a/examples/2D_axisym_shockwatercavity/case.py b/examples/2D_axisym_shockwatercavity/case.py index 0a5d0d517a..46c9e5eb7f 100644 --- a/examples/2D_axisym_shockwatercavity/case.py +++ b/examples/2D_axisym_shockwatercavity/case.py @@ -59,7 +59,7 @@ c_w = math.sqrt(gamw * (p0w + piw) / rho0w) # Shock Mach number of interest. Note that the post-shock properties can be defined in terms of either -# Min or psOp0a. Just comment/uncomment appropriatelly +# Min or psOp0a. Just comment/uncomment appropriately Min = 2.146 ## Pos to pre shock ratios - AIR diff --git a/examples/3D_shockdroplet/case.py b/examples/3D_shockdroplet/case.py index 9063b2d495..b8ba2c2f46 100644 --- a/examples/3D_shockdroplet/case.py +++ b/examples/3D_shockdroplet/case.py @@ -59,7 +59,7 @@ c_w = math.sqrt(gamw * (p0w + piw) / rho0w) # Shock Mach number of interest. Note that the post-shock properties can be defined in terms of either -# Min or psOp0a. Just comment/uncomment appropriatelly +# Min or psOp0a. Just comment/uncomment appropriately Min = 2.4 ## Pos to pre shock ratios - AIR diff --git a/examples/3D_shockdroplet_muscl/case.py b/examples/3D_shockdroplet_muscl/case.py index e2476ed4c4..e8c4069caf 100644 --- a/examples/3D_shockdroplet_muscl/case.py +++ b/examples/3D_shockdroplet_muscl/case.py @@ -59,7 +59,7 @@ c_w = math.sqrt(gamw * (p0w + piw) / rho0w) # Shock Mach number of interest. Note that the post-shock properties can be defined in terms of either -# Min or psOp0a. Just comment/uncomment appropriatelly +# Min or psOp0a. Just comment/uncomment appropriately Min = 2.4 ## Pos to pre shock ratios - AIR