Refactor pcms

.github/workflows/clang-tidy.yml

@@ -63,7 +63,7 @@ jobs:
         repo-path: 'kokkos/kokkos'
         repo-ref: '4.6.01'
         cache: true
-        options: '-DCMAKE_CXX_STANDARD=17
+        options: '-DCMAKE_CXX_STANDARD=20
                   -DBUILD_SHARED_LIBS=OFF
                   -DKokkos_ENABLE_SERIAL=ON
                   -DKokkos_ENABLE_OPENMP=OFF
@@ -78,7 +78,7 @@ jobs:
         repo-path: 'kokkos/kokkos-kernels'
         repo-ref: '4.6.01'
         cache: true
-        options: '-DCMAKE_CXX_STANDARD=17
+        options: '-DCMAKE_CXX_STANDARD=20
                   -DBUILD_SHARED_LIBS=OFF
                   -DKokkos_DIR=${{ runner.temp }}/build-kokkos-openmpi/install/lib/cmake/Kokkos'
 

.github/workflows/cmake-test.yml

@@ -25,6 +25,23 @@ jobs:
         compiler: [g++]
         language: ['cpp']
         python_api: [OFF, ON]
+        meshfields: [ON]
+        petsc: [ON]
+        include:
+          - build_type: Release
+            memory_test: OFF
+            compiler: g++
+            language: 'cpp'
+            python_api: OFF
+            meshfields: OFF
+            petsc: ON
+          - build_type: Release
+            memory_test: OFF
+            compiler: g++
+            language: 'cpp'
+            python_api: OFF
+            meshfields: ON
+            petsc: OFF
         exclude:
           - build_type: Release
             memory_test: ON
@@ -68,7 +85,7 @@ jobs:
         repo-ref: '4.6.01'
         cache: true
         cache-suffix: ${{ matrix.python_api == 'ON' && '-shared' || '' }}
-        options: '-DCMAKE_CXX_STANDARD=17
+        options: '-DCMAKE_CXX_STANDARD=20
                   -DBUILD_SHARED_LIBS=${{ matrix.python_api }}
                   -DKokkos_ENABLE_SERIAL=ON
                   -DKokkos_ENABLE_OPENMP=OFF
@@ -84,7 +101,7 @@ jobs:
         repo-ref: '4.6.01'
         cache: true
         cache-suffix: ${{ matrix.python_api == 'ON' && '-shared' || '' }}
-        options: '-DCMAKE_CXX_STANDARD=17
+        options: '-DCMAKE_CXX_STANDARD=20
                   -DBUILD_SHARED_LIBS=${{ matrix.python_api }}
                   -DKokkos_DIR=${{ runner.temp }}/build-kokkos/install/lib/cmake/Kokkos'
 
@@ -106,6 +123,7 @@ jobs:
                   -DKokkos_DIR=${{ runner.temp }}/build-kokkos/install/lib/cmake/Kokkos'
 
     - name: build meshFields
+      if: matrix.meshfields == 'ON'
       uses: ./.github/actions/install-repo
       with:
         repo-name: 'meshFields'
@@ -156,9 +174,14 @@ jobs:
     - name: Set LD_LIBRARY_PATH for shared libraries
       if: matrix.python_api == 'ON'
       run: |
-        echo "LD_LIBRARY_PATH=${{ runner.temp }}/build-kokkos/install/lib:${{ runner.temp }}/build-omega_h/install/lib:${{ runner.temp }}/build-meshFields/install/lib:${{ runner.temp }}/build-redev/install/lib:${{ runner.temp }}/build-ADIOS2/install/lib:${{ runner.temp }}/build-perfstubs/install/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV
+        meshfields_lib=""
+        if [ "${{ matrix.meshfields }}" = "ON" ]; then
+          meshfields_lib="${{ runner.temp }}/build-meshFields/install/lib:"
+        fi
+        echo "LD_LIBRARY_PATH=${{ runner.temp }}/build-kokkos/install/lib:${{ runner.temp }}/build-omega_h/install/lib:${meshfields_lib}${{ runner.temp }}/build-redev/install/lib:${{ runner.temp }}/build-ADIOS2/install/lib:${{ runner.temp }}/build-perfstubs/install/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV
 
     - name: clone petsc
+      if: matrix.petsc == 'ON'
       id: clone-petsc
       run: |
         cd ${{ runner.temp }}
@@ -167,14 +190,15 @@ jobs:
         echo "petsc-commit-hash=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
 
     - name: Cache PETSc
+      if: matrix.petsc == 'ON'
       id: cache-petsc
       uses: actions/cache@v3
       with:
         path: ${{ runner.temp }}/petsc
         key: build-petsc-${{ steps.clone-petsc.outputs.petsc-commit-hash }}-${{ matrix.python_api }}
 
     - name: build petsc
-      if: steps.cache-petsc.outputs.cache-hit != 'true'
+      if: matrix.petsc == 'ON' && steps.cache-petsc.outputs.cache-hit != 'true'
       run: |
         cd ${{ runner.temp }}/petsc
         ./configure \
@@ -186,6 +210,15 @@ jobs:
           --download-fblaslapack
         make all check
 
+    - name: Set PCMS PETSc options
+      run: |
+        echo "PCMS_ENABLE_PETSC=-DPCMS_ENABLE_PETSC=${{ matrix.petsc }}" >> $GITHUB_ENV
+        if [ "${{ matrix.petsc }}" = "ON" ]; then
+          echo "PCMS_PETSC_OPTIONS=-DPETSC_LINK_STATIC=ON -DPETSC_DIR=${{ runner.temp }}/petsc -DPETSC_ARCH=ubuntu-kokkos" >> $GITHUB_ENV
+        else
+          echo "PCMS_PETSC_OPTIONS=" >> $GITHUB_ENV
+        fi
+
     - name: checkout pcms_testcases
       uses: actions/checkout@v3
       with:
@@ -195,6 +228,15 @@ jobs:
     - name: Install fftw3
       run: sudo apt-get install -yq libfftw3-dev pkg-config
 
+    - name: Set PCMS MeshFields options
+      run: |
+        echo "PCMS_ENABLE_MESHFIELDS=-DPCMS_ENABLE_MESHFIELDS=${{ matrix.meshfields }}" >> $GITHUB_ENV
+        if [ "${{ matrix.meshfields }}" = "ON" ]; then
+          echo "PCMS_MESHFIELDS_DIR=-Dmeshfields_DIR=${{ runner.temp }}/build-meshFields/install/lib/cmake/meshfields" >> $GITHUB_ENV
+        else
+          echo "PCMS_MESHFIELDS_DIR=" >> $GITHUB_ENV
+        fi
+
     - name: build pcms
       uses: ./.github/actions/install-repo
       with:
@@ -207,15 +249,14 @@ jobs:
                   -DCMAKE_CXX_COMPILER=`which mpicxx`
                   -DCMAKE_Fortran_COMPILER=`which mpifort`
                   -DMPIEXEC_PREFLAGS="--oversubscribe"
-                  -DPCMS_ENABLE_PETSC=ON
-                  -DPETSC_LINK_STATIC=ON
-                  -DPETSC_DIR=${{ runner.temp }}/petsc
-                  -DPETSC_ARCH=ubuntu-kokkos
+                  ${{ env.PCMS_ENABLE_MESHFIELDS }}
+                  ${{ env.PCMS_ENABLE_PETSC }}
+                  ${{ env.PCMS_PETSC_OPTIONS }}
                   -DPCMS_TIMEOUT=10
                   -DPCMS_ENABLE_Python=${{ matrix.python_api }}
                   -DCatch2_DIR=${{ runner.temp }}/build-Catch2/install/lib/cmake/Catch2
                   -DOmega_h_DIR=${{ runner.temp }}/build-omega_h/install/lib/cmake/Omega_h
-                  -Dmeshfields_DIR=${{ runner.temp }}/build-meshFields/install/lib/cmake/meshfields
+                  ${{ env.PCMS_MESHFIELDS_DIR }}
                   -Dredev_DIR=${{ runner.temp }}/build-redev/install/lib/cmake/redev
                   -DMPIEXEC_EXECUTABLE=`which mpirun`
                   -DADIOS2_DIR=${{ runner.temp }}/build-ADIOS2/install/lib/cmake/adios2
@@ -263,8 +304,6 @@ jobs:
           -B ${{github.workspace}}/examples/external-usage-example/build \
           -S ${{github.workspace}}/examples/external-usage-example/ \
           -Dpcms_DIR=${{ runner.temp }}/build-pcms/install/lib/cmake/pcms \
-          -DPETSC_LINK_STATIC=ON \
-          -DPETSC_DIR=${{ runner.temp }}/petsc \
-          -DPETSC_ARCH=ubuntu-kokkos \
+          ${{ matrix.petsc == 'ON' && format('-DPETSC_LINK_STATIC=ON -DPETSC_DIR={0}/petsc -DPETSC_ARCH=ubuntu-kokkos', runner.temp) || '' }} \
           --debug-output
         cmake --build ${{github.workspace}}/examples/external-usage-example/build

.github/workflows/perlmutter/install.sh

@@ -45,7 +45,7 @@ build-pcms/install
 # cmake -S kokkos-kernels -B build-kokkos-kernels \
 #   -DCMAKE_INSTALL_PREFIX=$PWD/build-kokkos-kernels/install \
 #   -DCMAKE_CXX_COMPILER=CC \
-#   -DCMAKE_CXX_STANDARD=17
+#   -DCMAKE_CXX_STANDARD=20
 # cmake --build build-kokkos-kernels -j24 --target install
 
 
@@ -131,4 +131,4 @@ cmake -S pcms -B build-pcms \
   -DPCMS_TIMEOUT=100 \
   -DCMAKE_CXX_STANDARD=20 \
   -DPCMS_TEST_DATA_DIR=$PWD/pcms_testcases
-cmake --build build-pcms -j8
+cmake --build build-pcms -j8

README.md

@@ -32,7 +32,7 @@ module load cuda/12.1.1-zxa4msk
 git clone --branch 4.6.01 --depth 1 git@github.com:kokkos/kokkos.git
 cmake -S kokkos -B build-kokkos \
   -DCMAKE_INSTALL_PREFIX=build-kokkos/install \
-  -DCMAKE_CXX_STANDARD=17 \
+  -DCMAKE_CXX_STANDARD=20 \
   -DKokkos_ENABLE_SERIAL=ON \
   -DKokkos_ENABLE_OPENMP=OFF \
   -DKokkos_ENABLE_CUDA=OFF \
@@ -44,7 +44,7 @@ cmake --build build-kokkos --target install
 git clone --branch 4.6.01 --depth 1 git@github.com:kokkos/kokkos-kernels.git
 cmake -S kokkos-kernels -B build-kokkos-kernels \
   -DCMAKE_INSTALL_PREFIX=build-kokkos-kernels/install \
-  -DCMAKE_CXX_STANDARD=17 \
+  -DCMAKE_CXX_STANDARD=20 \
   -DKokkos_ROOT=$PWD/build-kokkos/install/lib64/cmake \
   -DBUILD_SHARED_LIBS=OFF
 cmake --build build-kokkos-kernels --target install
@@ -97,7 +97,7 @@ cmake --build build-redev --target install
 git clone --branch 4.6.01 --depth 1 git@github.com:kokkos/kokkos.git
 cmake -S kokkos -B build-kokkos \
   -DCMAKE_INSTALL_PREFIX=build-kokkos/install \
-  -DCMAKE_CXX_STANDARD=17 \
+  -DCMAKE_CXX_STANDARD=20 \
   -DCMAKE_BUILD_TYPE="Release" \
   -DCMAKE_CXX_COMPILER=$PWD/kokkos/bin/nvcc_wrapper \
   -DKokkos_ARCH_AMPERE80=ON \
@@ -112,7 +112,7 @@ cmake --build build-kokkos --target install
 git clone --branch 4.6.01 --depth 1 git@github.com:kokkos/kokkos-kernels.git
 cmake -S kokkos-kernels -B build-kokkos-kernels \
   -DCMAKE_INSTALL_PREFIX=build-kokkos-kernels/install \
-  -DCMAKE_CXX_STANDARD=17 \
+  -DCMAKE_CXX_STANDARD=20 \
   -DKokkos_ROOT=$PWD/build-kokkos/install/lib64/cmake \
   -DBUILD_SHARED_LIBS=off
 cmake --build build-kokkos-kernels --target install
@@ -248,9 +248,19 @@ If work in SCOREC system, you can directly use the spack-scorec.yaml which conta
    Then add the binded python module to your `PYTHONPATH` environment variable. You can find the install prefix with `find $(spack location -i pcms) -name "*.so" | grep -i pcms`.
    ```console
     $ export PYTHONPATH=$<your-path-to-module>:$PYTHONPATH
-    ```
+   ```
 
    Test the python API with `python -c "import pcms; print(pcms.__version__)"`. You should see the version of PCMS printed out without any errors. 
+
+   Point-evaluator construction in Python uses `EvaluationRequest`, for
+   example:
+   ```python
+   request = pcms.EvaluationRequest.from_coordinates(eval_coords)
+   evaluator = source_space.create_point_evaluator(request)
+
+   optimized_request = pcms.EvaluationRequest.from_function_space(target_space)
+   evaluator = source_space.create_point_evaluator(optimized_request)
+   ```
 
    PS: If you need certain config options for the python API, you can specify them in the spack spec. For example, to build the python API with Exodus support, one compatible config is `pcms+python ^omega-h+trilinos ^trilinos@15.0.0:+exodus ^netcdf-c@4.8.1+mpi`. At this moment, omega-h spack package does not have the exodus support, so you need to manually add `args.append("-Omega_h_USE_SEACASExodus:BOOL=ON")` to the `package.py` file of omega-h in spack before installing pcms with spack.
 
@@ -270,4 +280,3 @@ module load clang-format
 module load llvm
 find . -regex '.*\.\(cpp\|cxx\|cc\|c\|hpp\|h\)' -exec clang-format -style=file -i {} \;
 ```
-

config.cmake.in

@@ -30,7 +30,10 @@ if(@PCMS_ENABLE_OMEGA_H@)
 endif()
 
 include("${CMAKE_CURRENT_LIST_DIR}/pcms_utility-targets.cmake")
+include("${CMAKE_CURRENT_LIST_DIR}/pcms_discretization-targets.cmake")
 include("${CMAKE_CURRENT_LIST_DIR}/pcms_localization-targets.cmake")
+include("${CMAKE_CURRENT_LIST_DIR}/pcms_field-targets.cmake")
+include("${CMAKE_CURRENT_LIST_DIR}/pcms_coupler-targets.cmake")
 include("${CMAKE_CURRENT_LIST_DIR}/pcms_core-targets.cmake")
 include("${CMAKE_CURRENT_LIST_DIR}/pcms_transfer-targets.cmake")
 

examples/external-usage-example/main.cpp

@@ -1,6 +1,5 @@
 #include <iostream>
 #include <pcms/utility/print.h>
-#include <pcms/create_field.h>
 #include <pcms/capi/interpolator.h>
 int main()
 {

examples/python-api-example/flow_example.py

@@ -1,7 +1,6 @@
 #!/usr/bin/env python3
 import pcms
 import numpy as np
-import os
 # import matplotlib.pyplot as plt
 
 
@@ -45,23 +44,31 @@ def demonstrate_face_field_transfer(mesh):
         mesh, face_field_values, face_dim
     )
     print(
-        f"Vertex field (MLS): min={vertex_field_values.min():.6f}, "
+        f"Vertex field: min={vertex_field_values.min():.6f}, "
         f"max={vertex_field_values.max():.6f}, "
         f"mean={vertex_field_values.mean():.6f}"
     )
 
-    vertex_layout = pcms.create_lagrange_layout(mesh, 1, 1, pcms.CoordinateSystem.Cartesian)
-    omega_h_field = vertex_layout.create_field()
+    omega_h_factory = pcms.LagrangeFunctionSpace.from_mesh(
+        mesh, 1, 1, pcms.CoordinateSystem.Cartesian
+    )
+    omega_h_field = omega_h_factory.create_field()
     omega_h_field.set_dof_holder_data(vertex_field_values)
 
     divisions = [1000, 500]
     grid = pcms.create_uniform_grid_from_mesh(mesh, divisions)
-    ug_layout = pcms.UniformGridFieldLayout2D(grid, 1, pcms.CoordinateSystem.Cartesian)
-    ug_field = ug_layout.create_field()
+    ug_factory = pcms.LagrangeFunctionSpace.from_uniform_grid(
+        grid, 1, pcms.CoordinateSystem.Cartesian
+    )
+    ug_field = ug_factory.create_field()
 
-    omega_h_field.set_out_of_bounds_mode(pcms.OutOfBoundsMode.FILL, 0.0)
     print(f"Interpolating field from OmegaH mesh to uniform grid...")
-    pcms.interpolate_field(omega_h_field, ug_field)
+    interp = pcms.Interpolator(
+        omega_h_factory,
+        ug_factory,
+        pcms.OutOfBoundsPolicy(pcms.OutOfBoundsMode.FILL, 0.0),
+    )
+    interp.apply(omega_h_field, ug_field)
 
     transferred_data = ug_field.get_dof_holder_data()
     print(f"Transferred field: min={np.min(transferred_data):.6f}, max={np.max(transferred_data):.6f}, mean={np.mean(transferred_data):.6f}")

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "scikit_build_core.build"
 
 [project]
 name = "pcms"
-version = "0.3.0"
+version = "0.4.0"
 
 [tool.scikit-build.cmake.define]
 PCMS_ENABLE_Python = true