diff --git a/src/simulation/m_bubbles.fpp b/src/simulation/m_bubbles.fpp index 2bec7e1609..14c6ab472c 100644 --- a/src/simulation/m_bubbles.fpp +++ b/src/simulation/m_bubbles.fpp @@ -4,7 +4,7 @@ #:include 'macros.fpp' -!> @brief This module contains the procedures shared by the ensemble-averaged and volume-averaged bubble models. +!> @brief This module contains the procedures shared by ensemble-averaged and volume-averaged bubble models. module m_bubbles use m_derived_types !< Definitions of the derived types diff --git a/toolchain/templates/bridges2.mako b/toolchain/templates/bridges2.mako index 4536551943..e64b637c1f 100644 --- a/toolchain/templates/bridges2.mako +++ b/toolchain/templates/bridges2.mako @@ -14,7 +14,7 @@ % if account: #SBATCH --account="${account}" % endif -% if gpu: +% if is_gpu_active: #SBATCH --gpu-bind=verbose,closest #SBATCH --gres=gpu:v100-16:${tasks_per_node} % endif @@ -31,7 +31,7 @@ ${helpers.template_prologue()} ok ":) Loading modules:\n" cd "${MFC_ROOT_DIR}" -. ./mfc.sh load -c b -m ${'g' if gpu else 'c'} +. ./mfc.sh load -c b -m ${'g' if is_gpu_active else 'c'} cd - > /dev/null echo diff --git a/toolchain/templates/carpenter-cray.mako b/toolchain/templates/carpenter-cray.mako index bfaad0b427..44c2a5e974 100644 --- a/toolchain/templates/carpenter-cray.mako +++ b/toolchain/templates/carpenter-cray.mako @@ -25,7 +25,7 @@ ${helpers.template_prologue()} ok ":) Loading modules:\n" cd "${MFC_ROOT_DIR}" -. ./mfc.sh load -c cc -m ${'g' if gpu else 'c'} +. ./mfc.sh load -c cc -m ${'g' if is_gpu_active else 'c'} cd - > /dev/null echo diff --git a/toolchain/templates/carpenter.mako b/toolchain/templates/carpenter.mako index a652a887f7..242340d0a1 100644 --- a/toolchain/templates/carpenter.mako +++ b/toolchain/templates/carpenter.mako @@ -25,7 +25,7 @@ ${helpers.template_prologue()} ok ":) Loading modules:\n" cd "${MFC_ROOT_DIR}" -. ./mfc.sh load -c c -m ${'g' if gpu else 'c'} +. ./mfc.sh load -c c -m ${'g' if is_gpu_active else 'c'} cd - > /dev/null echo diff --git a/toolchain/templates/default.mako b/toolchain/templates/default.mako index b1cdaf81e0..15ba06dd72 100644 --- a/toolchain/templates/default.mako +++ b/toolchain/templates/default.mako @@ -2,6 +2,7 @@ import os from mako.exceptions import RuntimeException %> + % if os.name == 'nt': @echo off % else: @@ -48,7 +49,7 @@ if engine == 'batch': (set -x; ${profiler} \ jsrun --nrs ${tasks_per_node*nodes} \ --cpu_per_rs 1 \ - --gpu_per_rs ${1 if gpu else 0} \ + --gpu_per_rs ${1 if is_gpu_active else 0} \ --tasks_per_rs 1 \ "${target.get_install_binpath(case)}") elif [ "$binary" == "srun" ]; then diff --git a/toolchain/templates/delta.mako b/toolchain/templates/delta.mako index 694f22c457..a2489e7345 100644 --- a/toolchain/templates/delta.mako +++ b/toolchain/templates/delta.mako @@ -14,7 +14,7 @@ % if account: #SBATCH --account="${account}" % endif -% if gpu: +% if gpu != gpuConfigOptions.NONE.value: #SBATCH --gpus-per-node=${tasks_per_node} #SBATCH --mem=208G #SBATCH --gpu-bind=closest @@ -32,7 +32,7 @@ ${helpers.template_prologue()} ok ":) Loading modules:\n" cd "${MFC_ROOT_DIR}" -. ./mfc.sh load -c d -m ${'g' if gpu else 'c'} +. ./mfc.sh load -c d -m ${'g' if gpu != gpuConfigOptions.NONE.value else 'c'} cd - > /dev/null echo diff --git a/toolchain/templates/deltaai.mako b/toolchain/templates/deltaai.mako index 8492aa7a06..d098d201e1 100644 --- a/toolchain/templates/deltaai.mako +++ b/toolchain/templates/deltaai.mako @@ -14,7 +14,7 @@ % if account: #SBATCH --account="${account}" % endif -% if gpu: +% if is_gpu_active: #SBATCH --gpus-per-node=${tasks_per_node} #SBATCH --mem=208G #SBATCH --gpu-bind=closest @@ -32,7 +32,7 @@ ${helpers.template_prologue()} ok ":) Loading modules:\n" cd "${MFC_ROOT_DIR}" -. ./mfc.sh load -c dai -m ${'g' if gpu else 'c'} +. ./mfc.sh load -c dai -m ${'g' if is_gpu_active else 'c'} cd - > /dev/null echo diff --git a/toolchain/templates/frontier.mako b/toolchain/templates/frontier.mako index b6d4a9a887..e10398c191 100644 --- a/toolchain/templates/frontier.mako +++ b/toolchain/templates/frontier.mako @@ -10,7 +10,7 @@ #SBATCH --time=${walltime} #SBATCH --cpus-per-task=7 #SBATCH -C nvme -% if gpu: +% if is_gpu_active: #SBATCH --gpus-per-task=1 #SBATCH --gpu-bind=closest % endif @@ -34,12 +34,12 @@ ${helpers.template_prologue()} ok ":) Loading modules:\n" cd "${MFC_ROOT_DIR}" % if engine == 'batch': -. ./mfc.sh load -c f -m ${'g' if gpu else 'c'} +. ./mfc.sh load -c f -m ${'g' if is_gpu_active else 'c'} % endif cd - > /dev/null echo -% if gpu: +% if is_gpu_active: export MPICH_GPU_SUPPORT_ENABLED=1 % else: export MPICH_GPU_SUPPORT_ENABLED=0 @@ -66,7 +66,7 @@ ulimit -s unlimited % if engine == 'interactive': --unbuffered --nodes ${nodes} --ntasks-per-node ${tasks_per_node} \ --cpus-per-task 7 \ - % if gpu: + % if is_gpu_active: --gpus-per-task 1 --gpu-bind closest \ % endif ${profiler} "${target.get_install_binpath(case)}") diff --git a/toolchain/templates/hipergator.mako b/toolchain/templates/hipergator.mako index 3de36fea5b..3ed8e54529 100644 --- a/toolchain/templates/hipergator.mako +++ b/toolchain/templates/hipergator.mako @@ -9,7 +9,7 @@ #SBATCH --output="${name}.out" #SBATCH --time=${walltime} #SBATCH --cpus-per-task=7 -% if gpu: +% if is_gpu_active: #SBATCH --gpus-per-task=1 #SBATCH --gpu-bind=closest % endif @@ -35,7 +35,7 @@ ${helpers.template_prologue()} ok ":) Loading modules:\n" cd "${MFC_ROOT_DIR}" % if engine == 'batch': -. ./mfc.sh load -c h -m ${'g' if gpu else 'c'} +. ./mfc.sh load -c h -m ${'g' if is_gpu_active else 'c'} % endif cd - > /dev/null echo diff --git a/toolchain/templates/include/helpers.mako b/toolchain/templates/include/helpers.mako index a4086ee381..0e57d63b3d 100644 --- a/toolchain/templates/include/helpers.mako +++ b/toolchain/templates/include/helpers.mako @@ -1,4 +1,9 @@ -<%! import os %> +<%! +import os +from mfc.state import gpuConfigOptions +%> + +<% is_gpu_active = (gpu != gpuConfigOptions.NONE.value) %> <%def name="template_prologue()"> % if os.name != 'nt': diff --git a/toolchain/templates/nautilus.mako b/toolchain/templates/nautilus.mako index 5dfbdfdd84..00c8cd190d 100644 --- a/toolchain/templates/nautilus.mako +++ b/toolchain/templates/nautilus.mako @@ -14,7 +14,7 @@ % if account: #SBATCH --account="${account}" % endif -% if gpu: +% if is_gpu_active: #SBATCH --gpu-bind=verbose,closest #SBATCH --gres=gpu:v100-16:${tasks_per_node} % endif @@ -31,7 +31,7 @@ ${helpers.template_prologue()} ok ":) Loading modules:\n" cd "${MFC_ROOT_DIR}" -. ./mfc.sh load -c n -m ${'g' if gpu else 'c'} +. ./mfc.sh load -c n -m ${'g' if is_gpu_active else 'c'} cd - > /dev/null echo diff --git a/toolchain/templates/oscar.mako b/toolchain/templates/oscar.mako index 158a217cc1..e0f99a4e97 100644 --- a/toolchain/templates/oscar.mako +++ b/toolchain/templates/oscar.mako @@ -14,7 +14,7 @@ % if account: #SBATCH --account="${account}" % endif -% if gpu: +% if is_gpu_active: #SBATCH --gpus-per-node=${tasks_per_node} #SBATCH --mem=64G #SBATCH --gpu-bind=closest @@ -32,7 +32,7 @@ ${helpers.template_prologue()} ok ":) Loading modules:\n" cd "${MFC_ROOTDIR}" -. ./mfc.sh load -c o -m ${'g' if gpu else 'c'} +. ./mfc.sh load -c o -m ${'g' if is_gpu_active else 'c'} cd - > /dev/null echo diff --git a/toolchain/templates/phoenix-bench.mako b/toolchain/templates/phoenix-bench.mako index e699da3e4c..aee7db5445 100644 --- a/toolchain/templates/phoenix-bench.mako +++ b/toolchain/templates/phoenix-bench.mako @@ -17,7 +17,7 @@ % if quality_of_service: #SBATCH --qos=${quality_of_service} % endif -% if gpu: +% if is_gpu_active: #SBATCH --gres=gpu:V100:${tasks_per_node} #SBATCH --mem-per-gpu=16G\ % endif @@ -31,7 +31,7 @@ ${helpers.template_prologue()} ok ":) Loading modules:\n" cd "${MFC_ROOT_DIR}" -. ./mfc.sh load -c p -m ${'g' if gpu else 'c'} +. ./mfc.sh load -c p -m ${'g' if is_gpu_active else 'c'} cd - > /dev/null echo diff --git a/toolchain/templates/phoenix.mako b/toolchain/templates/phoenix.mako index e5e139905f..a550140a45 100644 --- a/toolchain/templates/phoenix.mako +++ b/toolchain/templates/phoenix.mako @@ -17,7 +17,7 @@ % if quality_of_service: #SBATCH --qos=${quality_of_service} % endif -% if gpu: +% if is_gpu_active: #SBATCH --gres=gpu:V100:${tasks_per_node} #SBATCH --mem-per-gpu=16G\ % endif @@ -31,7 +31,7 @@ ${helpers.template_prologue()} ok ":) Loading modules:\n" cd "${MFC_ROOT_DIR}" -. ./mfc.sh load -c p -m ${'g' if gpu else 'c'} +. ./mfc.sh load -c p -m ${'g' if is_gpu_active else 'c'} cd - > /dev/null echo diff --git a/toolchain/templates/santis.mako b/toolchain/templates/santis.mako index cb4b330625..a1b41ab2ef 100644 --- a/toolchain/templates/santis.mako +++ b/toolchain/templates/santis.mako @@ -59,7 +59,7 @@ ${helpers.template_prologue()} ok ":) Loading modules:\n" cd "${MFC_ROOT_DIR}" % if engine == 'batch': -. ./mfc.sh load -c san -m ${'g' if gpu else 'c'} +. ./mfc.sh load -c san -m ${'g' if is_gpu_active else 'c'} % endif cd - > /dev/null echo @@ -74,7 +74,7 @@ echo --ntasks=${nodes*tasks_per_node} \ --cpus-per-task 72 \ --cpu-bind=none \ - % if gpu: + % if is_gpu_active: --gpus-per-task 1 \ % endif --wait 200 --bcast=/tmp/${target.name} \ diff --git a/toolchain/templates/summit.mako b/toolchain/templates/summit.mako index 1ca902b2b1..a1e2ad24d4 100644 --- a/toolchain/templates/summit.mako +++ b/toolchain/templates/summit.mako @@ -2,6 +2,7 @@ <%namespace name="helpers" file="helpers.mako"/> + % if engine == 'batch': #BSUB -J {{{name}}} #BSUB -nnodes {{{nodes}}} @@ -16,7 +17,7 @@ ${helpers.template_prologue()} ok ":) Loading modules:\n" cd "${MFC_ROOT_DIR}" -. ./mfc.sh load -c s -m ${'g' if gpu else 'c'} +. ./mfc.sh load -c s -m ${'g' if is_gpu_active else 'c'} cd - > /dev/null echo