Merge pull request #822 from danielpeter/devel

various small updates; updates testing; adds CUDA12 support; fixes inlining problem w/ Cray compilers
SPECFEM · Jun 2, 2023 · 525daa9 · 525daa9
2 parents bb3519f + 93d1560
commit 525daa9
Show file tree

Hide file tree

Showing 25 changed files with 671 additions and 341 deletions.
diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml
@@ -72,7 +72,7 @@ jobs:
           # tests/ directory
           RUN_CHECKS=1
         elif [ "$directory" == .azure-pipelines ]; then
-          # .travis/ directory
+          # azure directory
           RUN_CHECKS=1
         fi
       done <<< "$DIFF"
@@ -84,7 +84,8 @@ jobs:
     displayName: 'Run checks'
 
 - job: compilation_default
-  displayName: 'Compilation Default GCC 9'
+  # ubuntu-latest: ubuntu-22.04 w/ GCC 11
+  displayName: 'Compilation Default GCC'
   steps:
   - template: .azure-pipelines/install-template.yml
     parameters:
@@ -95,14 +96,14 @@ jobs:
       CUDA: false
       BUILD: true
 
-- job: compilation_default_gcc7
-  displayName: 'Compilation Default GCC 7'
+- job: compilation_default_gcc9
+  displayName: 'Compilation Default GCC 9'
   pool:
-    vmImage: 'ubuntu-18.04'
+    vmImage: 'ubuntu-20.04'
   variables:
-    CC: gcc-7
-    CXX: g++-7
-    FC: gfortran-7
+    CC: gcc-9
+    CXX: g++-9
+    FC: gfortran-9
   steps:
   - template: .azure-pipelines/install-template.yml
     parameters:
@@ -131,27 +132,33 @@ jobs:
       CUDA: false
       BUILD: true
 
-- job: compilation_CUDA10
-  displayName: 'Compilation CUDA 10'
+- job: compilation_CUDA11_gcc9
+  displayName: 'Compilation CUDA 11 GCC 9'
   pool:
-    vmImage: 'ubuntu-18.04'
+    vmImage: 'ubuntu-20.04'
   variables:
-    CC: gcc-7
-    CXX: g++-7
-    FC: gfortran-7
+    CC: gcc-9
+    CXX: g++-9
+    FC: gfortran-9
   steps:
   - template: .azure-pipelines/install-template.yml
     parameters:
       CUDA: true
-      CUDA_V: '10.2'
+      CUDA_V: '11.4'
   - template: .azure-pipelines/configure-template.yml
     parameters:
       TESTFLAGS: '--enable-vectorization --with-cuda=cuda10'
       CUDA: true
       BUILD: true
 
-- job: compilation_CUDA11
-  displayName: 'Compilation CUDA 11'
+- job: compilation_CUDA11_gcc10
+  displayName: 'Compilation CUDA 11 GCC 10'
+  pool:
+    vmImage: 'ubuntu-20.04'
+  variables:
+    CC: gcc-10
+    CXX: g++-10
+    FC: gfortran-10
   steps:
   - template: .azure-pipelines/install-template.yml
     parameters:
@@ -163,6 +170,38 @@ jobs:
       CUDA: true
       BUILD: true
 
+- job: compilation_CUDA12
+  displayName: 'Compilation CUDA 12 GCC 10'
+  pool:
+    vmImage: 'ubuntu-20.04'
+  variables:
+    CC: gcc-10
+    CXX: g++-10
+    FC: gfortran-10
+  steps:
+  - template: .azure-pipelines/install-template.yml
+    parameters:
+      CUDA: true
+      CUDA_V: '12.1'
+  - template: .azure-pipelines/configure-template.yml
+    parameters:
+      TESTFLAGS: '--enable-vectorization --with-cuda=cuda11'
+      CUDA: true
+      BUILD: true
+
+- job: compilation_CUDA12_latest
+  displayName: 'Compilation CUDA 12'
+  steps:
+  - template: .azure-pipelines/install-template.yml
+    parameters:
+      CUDA: true
+      CUDA_V: '12.1'
+  - template: .azure-pipelines/configure-template.yml
+    parameters:
+      TESTFLAGS: '--enable-vectorization --with-cuda=cuda12'
+      CUDA: true
+      BUILD: true
+
 - job: test_example_1
   displayName: 'Test example 1 - regional_Greece_small'
   dependsOn: compilation_default

diff --git a/.azure-pipelines/install-template.yml b/.azure-pipelines/install-template.yml
@@ -4,8 +4,10 @@
 # software setup on VM nodes
 # ubuntu-18.04:
 # https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu1804-README.md
-# ubuntu-20.04 "ubuntu-latest":
+# ubuntu-20.04:
 # https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-README.md
+# ubuntu-22.04 "ubuntu-latest":
+# https://github.com/actions/runner-images/blob/main/images/linux/Ubuntu2204-Readme.md
 #
 parameters:
 - name: CUDA
@@ -20,10 +22,27 @@ steps:
     # fortran/openMPI compiler
     echo "CC: ${CC} CXX: ${CXX} FC: ${FC}"
     # updates repository
+    echo; echo `uname -a`; lsb_release -a; echo
     sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 6B05F25D762E3157
     sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 78BD65473CB3BD13
     sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 762E3157
-    if [ "${FC}" == "gfortran-10" ]; then
+    if [ "${FC}" == "gfortran-9" ]; then
+      echo "gfortran: gfortran-9 update"
+      # updating gfortran version
+      sudo add-apt-repository ppa:ubuntu-toolchain-r/test
+      sudo apt-get update
+      sudo apt-get install -y --reinstall gcc-9 g++-9 gfortran-9
+      # updates alternatives
+      echo
+      update-alternatives --query gfortran
+      echo
+      sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 100
+      sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-9 100
+      sudo update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-9 100
+      echo
+      update-alternatives --query gfortran
+      echo
+    elif [ "${FC}" == "gfortran-10" ]; then
       echo "gfortran: gfortran-10 update"
       # updating gfortran version
       sudo add-apt-repository ppa:ubuntu-toolchain-r/test
@@ -94,9 +113,14 @@ steps:
       ## distribution xenial: from ubuntu 16.04
       #UBUNTU_VERSION=ubuntu1604
       ## distribution bionic: from ubuntu 18.04
-      UBUNTU_VERSION=ubuntu1804
+      #UBUNTU_VERSION=ubuntu1804
       ## distribution focal: from ubuntu 20.04
       #UBUNTU_VERSION=ubuntu2004
+      ## distribution jammy: from ubuntu 22.04
+      #UBUNTU_VERSION=ubuntu2204
+
+      # default
+      UBUNTU_VERSION=ubuntu2004
 
       # CUDA_VERSION - specifies CUDA toolkit version
       echo "CUDA version: $CUDA_V"
@@ -110,10 +134,20 @@ steps:
       elif [ "$CUDA_V" == "10.2" ]; then
         ## bionic
         CUDA_VERSION=10.2.89-1
+      elif [ "$CUDA_V" == "11.4" ]; then
+        ## focal
+        CUDA_VERSION=11.4.0-1
+      elif [ "$CUDA_V" == "12.1" ]; then
+        ## focal
+        CUDA_VERSION=12.1.1-1
       else
-        # note: on azure VM nodes with ubuntu-latest, default gcc version is 9.3;
+        # note: - on azure VM nodes with ubuntu 18.04, default gcc version is 9.3;
+        #       needs at least CUDA version 10.x
+        #       - on azure VM nodes with ubuntu 20.04, default gcc version is 10.3;
         #       needs at least CUDA version 11.x
-        CUDA_VERSION=11.4.0-1
+        #       - on azure VM nodes with ubuntu-latest (22.04), default gcc version is 11.3;
+        #       needs at least CUDA version 11.7
+        CUDA_VERSION=12.1.1-1
       fi
       echo
 
@@ -151,7 +185,12 @@ steps:
       echo
 
       # gets repo
-      if [ "${CUDA_VERSION}" == "11.4.0-1" ]; then
+      if [ "${CUDA_VERSION}" == "10.2.89-1" ]; then
+        # gets packages
+        INSTALLER=cuda-repo-${UBUNTU_VERSION}_${CUDA_VERSION}_${CUDA_ARCH}.deb
+        wget http://developer.download.nvidia.com/compute/cuda/repos/${UBUNTU_VERSION}/${CUDA_OS}/${INSTALLER}
+        sudo dpkg -i ${INSTALLER}
+      elif [ "${CUDA_VERSION}" == "11.4.0-1" ]; then
         # new CUDA version 11.4 has no cuda-repo-** file, following instructions from website,
         # see: https://developer.nvidia.com/cuda-downloads?target_os=Linux&target_arch=x86_64&Distribution=Ubuntu&target_version=18.04&target_type=deb_network
         wget https://developer.download.nvidia.com/compute/cuda/repos/${UBUNTU_VERSION}/${CUDA_OS}/cuda-${UBUNTU_VERSION}.pin
@@ -160,10 +199,12 @@ steps:
         # adds repo
         sudo add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/${UBUNTU_VERSION}/${CUDA_OS}/ /"
       else
-        # gets packages
-        INSTALLER=cuda-repo-${UBUNTU_VERSION}_${CUDA_VERSION}_${CUDA_ARCH}.deb
-        wget http://developer.download.nvidia.com/compute/cuda/repos/${UBUNTU_VERSION}/${CUDA_OS}/${INSTALLER}
-        sudo dpkg -i ${INSTALLER}
+        # new versions
+        wget https://developer.download.nvidia.com/compute/cuda/repos/${UBUNTU_VERSION}/${CUDA_OS}/cuda-${UBUNTU_VERSION}.pin
+        sudo mv cuda-${UBUNTU_VERSION}.pin /etc/apt/preferences.d/cuda-repository-pin-600
+        echo
+        # adds repo
+        sudo add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/${UBUNTU_VERSION}/${CUDA_OS}/ /"
       fi
       #echo
 

diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
 
     - name: Check for changes
       id: diff
@@ -29,9 +29,20 @@ jobs:
           export DIFF=$( git diff --name-only ${{ github.event.before }} $GITHUB_SHA )
           echo "  diff between ${{ github.event.before }} and $GITHUB_SHA"
         fi
-        echo "$DIFF"
+        echo "***"; echo "$DIFF"; echo "***"
         # Escape newlines (replace \n with %0A)
-        echo "::set-output name=diff::$( echo "$DIFF" | sed ':a;N;$!ba;s/\n/%0A/g' )"
+        # deprecated:
+        #echo "::set-output name=diff::$( echo "$DIFF" | sed ':a;N;$!ba;s/\n/%0A/g' )"
+        # new:
+        # replace new line with %0A - will result finding only one file with a very long name...
+        #echo "diff=$( echo "$DIFF" | sed ':a;N;$!ba;s/\n/%0A/g' )" >> $GITHUB_OUTPUT
+        # doesn't work...
+        #echo "diff=\"$DIFF\"" >> "$GITHUB_OUTPUT"
+        # new multi-line format:
+        # (https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#multiline-strings)
+        echo "diff<<EOF" >> $GITHUB_OUTPUT
+        echo "$DIFF" >> $GITHUB_OUTPUT
+        echo "EOF" >> $GITHUB_OUTPUT
 
     - name: Output changes
       run: echo "${{ steps.diff.outputs.diff }}"
@@ -75,7 +86,7 @@ jobs:
     needs: changesCheck
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
 
     - name: Install dependencies
       run: |
@@ -128,7 +139,7 @@ jobs:
         os: [ubuntu-latest,ubuntu-20.04]
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
 
     - name: Install packages
       run: ./.github/scripts/run_install.sh
@@ -147,11 +158,11 @@ jobs:
     needs: changesCheck
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
 
     - name: Cache Intel oneapi packages
       id: cache-intel-oneapi
-      uses: actions/cache@v2
+      uses: actions/cache@v3
       with:
         path: /opt/intel/oneapi
         key: install-${{ runner.os }}-all
@@ -224,7 +235,7 @@ jobs:
     needs: [linuxCheck]
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
 
     - name: Install packages
       run: ./.github/scripts/run_install.sh
@@ -242,7 +253,7 @@ jobs:
      needs: [linuxCheck]
 
      steps:
-     - uses: actions/checkout@v2
+     - uses: actions/checkout@v3
 
      - name: Install packages
        run: ./.github/scripts/run_install.sh
@@ -267,7 +278,7 @@ jobs:
     needs: [linuxCheck]
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
 
     - name: Install packages
       run: ./.github/scripts/run_install.sh
@@ -291,7 +302,7 @@ jobs:
     needs: [linuxCheck]
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
 
     - name: Install packages
       run: ./.github/scripts/run_install.sh
@@ -315,7 +326,7 @@ jobs:
     needs: [linuxCheck]
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
 
     - name: Install packages
       run: ./.github/scripts/run_install.sh
@@ -340,7 +351,7 @@ jobs:
     needs: [linuxCheck]
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
 
     - name: Install packages
       run: ./.github/scripts/run_install.sh
@@ -364,7 +375,7 @@ jobs:
     needs: [linuxCheck]
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
 
     - name: Install packages
       run: ./.github/scripts/run_install.sh