Merge pull request #55 from luraess/master

Refactor to run with latest Julia GPU and MPI stack

.github/workflows/ci.yml

@@ -20,13 +20,13 @@ jobs:
       fail-fast: false
       matrix:
         version:
-          - '1.7' # Minimum required Julia version (due to dependency of AMDGPU.jl)
+          # - '1.7' # Skipping this version because of AMDGPU deps compat issue (rocBLAS_jll)
           - '1'   # Latest stable 1.x release of Julia
-          # - 'nightly'
+          - 'nightly'
         os:
           - ubuntu-latest
           - macOS-latest
-          - windows-latest
+          # - windows-latest
         arch:
           - x64
     steps:
@@ -51,22 +51,22 @@ jobs:
       - uses: codecov/codecov-action@v2
         with:
           files: lcov.info
-  docs:
-    name: Documentation
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: julia-actions/setup-julia@v1
-        with:
-          version: '1'
-      - uses: julia-actions/julia-buildpkg@v1
-      - uses: julia-actions/julia-docdeploy@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }}
-      - run: |
-          julia --project=docs -e '
-            using Documenter: DocMeta, doctest
-            using ImplicitGlobalGrid
-            DocMeta.setdocmeta!(ImplicitGlobalGrid, :DocTestSetup, :(using ImplicitGlobalGrid); recursive=true)
-            doctest(ImplicitGlobalGrid)'
+  # docs:
+  #   name: Documentation
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - uses: actions/checkout@v2
+  #     - uses: julia-actions/setup-julia@v1
+  #       with:
+  #         version: '1'
+  #     - uses: julia-actions/julia-buildpkg@v1
+  #     - uses: julia-actions/julia-docdeploy@v1
+  #       env:
+  #         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  #         DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }}
+  #     - run: |
+  #         julia --project=docs -e '
+  #           using Documenter: DocMeta, doctest
+  #           using ImplicitGlobalGrid
+  #           DocMeta.setdocmeta!(ImplicitGlobalGrid, :DocTestSetup, :(using ImplicitGlobalGrid); recursive=true)
+  #           doctest(ImplicitGlobalGrid)'

Project.toml

@@ -1,14 +1,14 @@
-authors = ["Samuel Omlin", "Ludovic Räss", "Ivan Utkin"]
+authors = ["Samuel Omlin", "Ludovic Raess", "Ivan Utkin"]
 name = "ImplicitGlobalGrid"
 uuid = "4d7a3746-15be-11ea-1130-334b0c4f5fa0"
-version = "0.12.0"
+version = "0.13.0"
 
 [compat]
-AMDGPU = "0.3.7"
-CUDA = "1, ~3.1, ~3.2, ~3.3, ~3.7.1, ~3.8, ~3.9, ~3.10, ~3.11, ~3.12, 4"
+AMDGPU = "0.5"
+CUDA = "1, ~3.1, ~3.2, ~3.3, ~3.7.1, ~3.8, ~3.9, ~3.10, ~3.11, ~3.12, ~3.13, 4"
 LoopVectorization = "0.12"
-MPI = "0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19"
-julia = "1.7"
+MPI = "0.20"
+julia = "1.9"
 
 [deps]
 AMDGPU = "21141c5a-9bdb-4563-92ae-f87d6854732e"

docs/Project.toml

@@ -1,4 +1,3 @@
 [deps]
-ImplicitGlobalGrid = "d35fcfd7-7af4-4c67-b1aa-d78070614af4"
 DocExtensions = "cbdad009-89f1-4e05-85a0-06b07b50707d"
 Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"

src/init_global_grid.jl

@@ -86,7 +86,7 @@ function init_global_grid(nx::Integer, ny::Integer, nz::Integer; dimx::Integer=0
     comm_cart = MPI.Cart_create(comm, dims, periods, reorder);
     me        = MPI.Comm_rank(comm_cart);
     coords    = MPI.Cart_coords(comm_cart);
-    neighbors = fill(MPI.MPI_PROC_NULL, NNEIGHBORS_PER_DIM, NDIMS_MPI);
+    neighbors = fill(MPI.PROC_NULL, NNEIGHBORS_PER_DIM, NDIMS_MPI);
     for i = 1:NDIMS_MPI
         neighbors[:,i] .= MPI.Cart_shift(comm_cart, i-1, disp);
     end

src/select_device.jl

@@ -20,14 +20,14 @@ function select_device()
             nb_devices = length(CUDA.devices())
         elseif amdgpu_enabled()
             @assert AMDGPU.functional()
-            nb_devices = length(AMDGPU.get_agents(:gpu))
+            nb_devices = length(AMDGPU.devices())
         end
-        comm_l = MPI.Comm_split_type(comm(), MPI.MPI_COMM_TYPE_SHARED, me())
+        comm_l = MPI.Comm_split_type(comm(), MPI.COMM_TYPE_SHARED, me())
         if (MPI.Comm_size(comm_l) > nb_devices) error("More processes have been launched per node than there are GPUs available."); end
         me_l      = MPI.Comm_rank(comm_l)
         device_id = amdgpu_enabled() ? me_l+1 : me_l
         if     cuda_enabled()   CUDA.device!(device_id)
-        elseif amdgpu_enabled() AMDGPU.device!(device_id)
+        elseif amdgpu_enabled() AMDGPU.device_id!(device_id)
         end
         return device_id
     else

src/shared.jl

@@ -102,7 +102,7 @@ amdgpuaware_MPI()                      = global_grid().amdgpuaware_MPI
 amdgpuaware_MPI(dim::Integer)          = global_grid().amdgpuaware_MPI[dim]
 loopvectorization()                    = global_grid().loopvectorization
 loopvectorization(dim::Integer)        = global_grid().loopvectorization[dim]
-has_neighbor(n::Integer, dim::Integer) = neighbor(n, dim) != MPI.MPI_PROC_NULL
+has_neighbor(n::Integer, dim::Integer) = neighbor(n, dim) != MPI.PROC_NULL
 any_array(fields::GGArray...)          = any([is_array(A) for A in fields])
 any_cuarray(fields::GGArray...)        = any([is_cuarray(A) for A in fields])
 any_rocarray(fields::GGArray...)       = any([is_rocarray(A) for A in fields])
@@ -125,5 +125,5 @@ end
 ## AMDGPU functions
 
 function register(::Type{<:ROCArray},buf::Array{T}) where T <: GGNumber
-    return unsafe_wrap(ROCArray,pointer(buf),size(buf)), pointer(buf);
+    return unsafe_wrap(ROCArray, pointer(buf), size(buf))
 end