Add OpenMP support for Nvidia hardware

.github/workflows/bench.yml

@@ -36,21 +36,40 @@ jobs:
             labels: gt
             flag: p
             device: cpu
+            interface: none
             build_script: ""
           - cluster: phoenix
             name: Georgia Tech | Phoenix (NVHPC)
             group: phoenix
             labels: gt
             flag: p
             device: gpu
+            interface: acc
             build_script: ""
+          - cluster: phoenix
+            name: Georgia Tech | Phoenix (NVHPC)
+            group: phoenix
+            labels: gt
+            flag: p
+            device: gpu
+            interface: omp
+            build_script: ""
+          - cluster: frontier
+            name: Oak Ridge | Frontier (CCE)
+            group: phoenix
+            labels: frontier
+            flag: f
+            device: gpu
+            interface: acc
+            build_script: "bash .github/workflows/frontier/build.sh gpu acc bench"
           - cluster: frontier
             name: Oak Ridge | Frontier (CCE)
             group: phoenix
             labels: frontier
             flag: f
             device: gpu
-            build_script: "bash .github/workflows/frontier/build.sh gpu bench"
+            interface: omp
+            build_script: "bash .github/workflows/frontier/build.sh gpu omp bench"
     runs-on:
       group: ${{ matrix.group }}
       labels: ${{ matrix.labels }}
@@ -80,29 +99,29 @@ jobs:
 
       - name: Bench (Master v. PR)
         run: |
-          (cd pr     && bash .github/workflows/${{ matrix.cluster }}/submit-bench.sh .github/workflows/${{ matrix.cluster }}/bench.sh ${{ matrix.device }}) &
-          (cd master && bash .github/workflows/${{ matrix.cluster }}/submit-bench.sh .github/workflows/${{ matrix.cluster }}/bench.sh ${{ matrix.device }}) &
+          (cd pr     && bash .github/workflows/${{ matrix.cluster }}/submit-bench.sh .github/workflows/${{ matrix.cluster }}/bench.sh ${{ matrix.device }} ${{ matrix.interface }}) &
+          (cd master && bash .github/workflows/${{ matrix.cluster }}/submit-bench.sh .github/workflows/${{ matrix.cluster }}/bench.sh ${{ matrix.device }} ${{ matrix.interface }}) &
           wait %1 && wait %2
 
       - name: Generate & Post Comment
         run: |
           (cd pr && . ./mfc.sh load -c ${{ matrix.flag }} -m g)
-          (cd pr && ./mfc.sh bench_diff ../master/bench-${{ matrix.device }}.yaml ../pr/bench-${{ matrix.device }}.yaml)
+          (cd pr && ./mfc.sh bench_diff ../master/bench-${{ matrix.device }}-${{ matrix.interface }}.yaml ../pr/bench-${{ matrix.device }}-${{ matrix.interface }}.yaml)
 
       - name: Print Logs
         if:   always()
         run: |
-          cat pr/bench-${{ matrix.device }}.* 2>/dev/null || true
-          cat master/bench-${{ matrix.device }}.* 2>/dev/null || true
+          cat pr/bench-${{ matrix.device }}-${{ matrix.interface }}.* 2>/dev/null || true
+          cat master/bench-${{ matrix.device }}-${{ matrix.interface }}.* 2>/dev/null || true
 
       # All other runners (non-Phoenix) just run without special env
       - name: Archive Logs (Frontier)
         if: always() && matrix.cluster != 'phoenix'
         uses: actions/upload-artifact@v4
         with:
-          name: ${{ matrix.cluster }}-${{ matrix.device }}
+          name: ${{ matrix.cluster }}-${{ matrix.device }}-${{ matrix.interface }}
           path: |
-            pr/bench-${{ matrix.device }}.*
+            pr/bench-${{ matrix.device }}-${{ matrix.interface }}.*
             pr/build/benchmarks/*
-            master/bench-${{ matrix.device }}.*
+            master/bench-${{ matrix.device }}-${{ matrix.interface }}.*
             master/build/benchmarks/*

.github/workflows/frontier/bench.sh

@@ -1,12 +1,18 @@
 #!/bin/bash
 
 n_ranks=12
-
+device_opts=""
 if [ "$job_device" = "gpu" ]; then
     gpus=$(rocm-smi --showid | awk '{print $1}' | grep -Eo '[0-9]+' | uniq | tr '\n' ' ')
     n_ranks=$(echo "$gpus" | wc -w)         # number of GPUs on node
     gpu_ids=$(echo "$gpus" | tr ' ' '\n' | tr '\n' ' ' | sed 's/ $//')  # GPU IDs from rocm-smi
-    device_opts="--gpu -g $gpu_ids"
+    device_opts+="--gpu"
+    if [ "$job_interface" = "acc" ]; then
+        device_opts+=" acc"
+    elif [ "$job_interface" = "omp" ]; then
+        device_opts+=" mp"
+    fi
+    device_opts+=" -g $gpu_ids"
 fi
 
 if [ "$job_device" = "gpu" ]; then

.github/workflows/frontier/build.sh

@@ -1,13 +1,21 @@
 #!/bin/bash
 
+job_device=$1
+job_interface=$2
+run_bench=$3
 build_opts=""
-if [ "$1" = "gpu" ]; then
-    build_opts="--gpu"
+if [ "$job_device" = "gpu" ]; then
+  build_opts+="--gpu"
+  if [ "$job_interface" = "acc" ]; then
+      build_opts+=" acc"
+  elif [ "$job_interface" = "omp" ]; then
+      build_opts+=" mp"
+  fi
 fi
 
 . ./mfc.sh load -c f -m g
 
-if [ "$2" == "bench" ]; then
+if [ "$run_bench" == "bench" ]; then
     for dir in benchmarks/*/; do
         dirname=$(basename "$dir")
         ./mfc.sh run "$dir/case.py" --case-optimization -j 8 --dry-run $build_opts

.github/workflows/frontier/submit-bench.sh

@@ -24,7 +24,7 @@ else
 fi
 
 
-job_slug="`basename "$1" | sed 's/\.sh$//' | sed 's/[^a-zA-Z0-9]/-/g'`-$2"
+job_slug="`basename "$1" | sed 's/\.sh$//' | sed 's/[^a-zA-Z0-9]/-/g'`-$2-$3"
 
 sbatch <<EOT
 #!/bin/bash
@@ -45,6 +45,7 @@ echo "Running in $(pwd):"
 
 job_slug="$job_slug"
 job_device="$2"
+job_interface="$3"
 
 . ./mfc.sh load -c f -m g
 

.github/workflows/frontier/submit.sh

@@ -25,7 +25,7 @@ else
 fi
 
 
-job_slug="`basename "$1" | sed 's/\.sh$//' | sed 's/[^a-zA-Z0-9]/-/g'`-$2"
+job_slug="`basename "$1" | sed 's/\.sh$//' | sed 's/[^a-zA-Z0-9]/-/g'`-$2-$3"
 
 sbatch <<EOT
 #!/bin/bash
@@ -46,6 +46,7 @@ echo "Running in $(pwd):"
 
 job_slug="$job_slug"
 job_device="$2"
+job_interface="$3"
 
 . ./mfc.sh load -c f -m g
 

.github/workflows/frontier/test.sh

@@ -3,8 +3,18 @@
 gpus=`rocm-smi --showid | awk '{print $1}' | grep -Eo '[0-9]+' | uniq | tr '\n' ' '`
 ngpus=`echo "$gpus" | tr -d '[:space:]' | wc -c`
 
+device_opts=""
 if [ "$job_device" = "gpu" ]; then
-    ./mfc.sh test -a --rdma-mpi --max-attempts 3 -j $ngpus -- -c frontier
+    device_opts+="--gpu"
+    if [ "$job_interface" = "acc" ]; then
+        device_opts+=" acc"
+    elif [ "$job_interface" = "omp" ]; then
+        device_opts+=" mp"
+    fi
+fi
+
+if [ "$job_device" = "gpu" ]; then
+    ./mfc.sh test -a --rdma-mpi --max-attempts 3 -j $ngpus $device_opts -- -c frontier
 else
     ./mfc.sh test -a --max-attempts 3 -j 32 -- -c frontier
 fi

.github/workflows/lint-source.yml

@@ -30,7 +30,7 @@ jobs:
 
     - name: Looking for raw directives
       run: |
-        ! grep -iR '!\$acc\|!\$omp' --exclude="parallel_macros.fpp" --exclude="syscheck.fpp" ./src/*
+        ! grep -iR '!\$acc\|!\$omp' --exclude="parallel_macros.fpp" --exclude="acc_macros.fpp" --exclude="omp_macros.fpp" --exclude="shared_parallel_macros.fpp" --exclude="syscheck.fpp" ./src/*
 
     - name: No double precision intrinsics
       run: |

.github/workflows/phoenix/bench.sh

@@ -2,10 +2,18 @@
 
 n_ranks=12
 
+echo "My interface is:" $job_interface
+device_opts=""
 if [ "$job_device" = "gpu" ]; then
     n_ranks=$(nvidia-smi -L | wc -l)        # number of GPUs on node
     gpu_ids=$(seq -s ' ' 0 $(($n_ranks-1))) # 0,1,2,...,gpu_count-1
-    device_opts="--gpu -g $gpu_ids"
+    device_opts+="--gpu"
+    if [ "$job_interface" = "acc" ]; then
+        device_opts+=" acc"
+    elif [ "$job_interface" = "omp" ]; then
+        device_opts+=" mp"
+    fi
+    device_opts+=" -g $gpu_ids"
 fi
 
 tmpbuild=/storage/scratch1/6/sbryngelson3/mytmp_build

.github/workflows/phoenix/submit-bench.sh

@@ -3,7 +3,7 @@
 set -e
 
 usage() {
-    echo "Usage: $0 [script.sh] [cpu|gpu]"
+    echo "Usage: $0 [script.sh] [cpu|gpu] [none|acc|omp]"
 }
 
 if [ ! -z "$1" ]; then
@@ -34,7 +34,7 @@ else
     exit 1
 fi
 
-job_slug="`basename "$1" | sed 's/\.sh$//' | sed 's/[^a-zA-Z0-9]/-/g'`-$2"
+job_slug="`basename "$1" | sed 's/\.sh$//' | sed 's/[^a-zA-Z0-9]/-/g'`-$2-$3"
 
 sbatch <<EOT
 #!/bin/bash
@@ -55,6 +55,7 @@ echo "Running in $(pwd):"
 
 job_slug="$job_slug"
 job_device="$2"
+job_interface="$3"
 
 . ./mfc.sh load -c p -m $2
 

.github/workflows/phoenix/submit.sh

@@ -3,7 +3,7 @@
 set -e
 
 usage() {
-    echo "Usage: $0 [script.sh] [cpu|gpu]"
+    echo "Usage: $0 [script.sh] [cpu|gpu] [none|acc|omp]"
 }
 
 if [ ! -z "$1" ]; then
@@ -34,7 +34,7 @@ else
     exit 1
 fi
 
-job_slug="`basename "$1" | sed 's/\.sh$//' | sed 's/[^a-zA-Z0-9]/-/g'`-$2"
+job_slug="`basename "$1" | sed 's/\.sh$//' | sed 's/[^a-zA-Z0-9]/-/g'`-$2-$3"
 
 sbatch <<EOT
 #!/bin/bash
@@ -55,10 +55,10 @@ echo "Running in $(pwd):"
 
 job_slug="$job_slug"
 job_device="$2"
+job_interface="$3"
 
 . ./mfc.sh load -c p -m $2
 
 $sbatch_script_contents
 
-EOT
-
+EOT

.github/workflows/phoenix/test.sh

@@ -3,6 +3,11 @@
 build_opts=""
 if [ "$job_device" = "gpu" ]; then
     build_opts="--gpu"
+    if [ "$job_interface" = "omp" ]; then
+      build_opts+=" mp"
+    elif [ "$job_interface" = "acc" ]; then
+      build_opts+=" acc"
+    fi
 fi
 
 ./mfc.sh test --dry-run -j 8 $build_opts

.github/workflows/test.yml

@@ -96,8 +96,20 @@ jobs:
     timeout-minutes: 1400
     strategy:
       matrix:
-        device: ['cpu', 'gpu']
+        device: ['gpu']
+        interface: ['acc', 'omp']
         lbl: ['gt', 'frontier']
+        include:
+          - device: 'cpu'
+            interface: 'none'
+            lbl: 'gt'
+          - device: 'cpu'
+            interface: 'none'
+            lbl: 'frontier'
+        exclude:
+          - device: 'gpu'
+            interface: 'omp'
+            lbl: 'frontier'
     runs-on:
       group:  phoenix
       labels: ${{ matrix.lbl }}
@@ -111,23 +123,23 @@ jobs:
 
       - name: Build & Test
         if:   matrix.lbl == 'gt'
-        run:  bash .github/workflows/phoenix/submit.sh .github/workflows/phoenix/test.sh ${{ matrix.device }}
+        run:  bash .github/workflows/phoenix/submit.sh .github/workflows/phoenix/test.sh ${{ matrix.device }} ${{ matrix.interface }}
 
       - name: Build
         if:   matrix.lbl == 'frontier'
-        run:  bash .github/workflows/frontier/build.sh ${{ matrix.device }}
+        run:  bash .github/workflows/frontier/build.sh ${{ matrix.device }} ${{ matrix.interface }}
 
       - name: Test
         if:   matrix.lbl == 'frontier'
-        run:  bash .github/workflows/frontier/submit.sh .github/workflows/frontier/test.sh ${{matrix.device}}
+        run:  bash .github/workflows/frontier/submit.sh .github/workflows/frontier/test.sh ${{matrix.device}} ${{ matrix.interface }}
 
       - name: Print Logs
         if:   always()
-        run:  cat test-${{ matrix.device }}.out
+        run:  cat test-${{ matrix.device }}-${{ matrix.interface }}.out
 
       - name: Archive Logs
         uses: actions/upload-artifact@v4
         if:   matrix.lbl == 'frontier'
         with:
-          name: logs-${{ strategy.job-index }}-${{ matrix.device }}
-          path: test-${{ matrix.device }}.out
+          name: logs-${{ strategy.job-index }}-${{ matrix.device }}-${{ matrix.interface }}
+          path: test-${{ matrix.device }}-${{ matrix.interface }}.out