3D IGF Solver

.github/workflows/codeql.yml

@@ -30,10 +30,10 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
-      - name: Install Packages (C++)
+      - name: Install Packages
         run: |
           sudo apt-get update
-          sudo apt-get install --yes cmake openmpi-bin libopenmpi-dev libhdf5-openmpi-dev
+          sudo apt-get install --yes cmake openmpi-bin libfftw3-dev libfftw3-mpi-dev libopenmpi-dev libhdf5-openmpi-dev
 
           python -m pip install --upgrade pip
           python -m pip install --upgrade wheel
@@ -46,7 +46,7 @@ jobs:
       - name: Configure (C++)
         if: ${{ matrix.language == 'cpp' }}
         run: |
-          $CMAKE -S . -B build -DImpactX_PYTHON=OFF
+          $CMAKE -S . -B build -DImpactX_FFT=ON -DImpactX_PYTHON=OFF
 
       - name: Initialize CodeQL
         uses: github/codeql-action/init@v3
@@ -58,6 +58,8 @@ jobs:
       - name: Build (py)
         uses: github/codeql-action/autobuild@v3
         if: ${{ matrix.language == 'python' }}
+        env:
+          IMPACTX_FFT: ON
 
       - name: Build (C++)
         if: ${{ matrix.language == 'cpp' }}

.github/workflows/cuda.yml

@@ -47,6 +47,7 @@ jobs:
         cmake -S . -B build            \
           -DCMAKE_VERBOSE_MAKEFILE=ON  \
           -DImpactX_COMPUTE=CUDA       \
+          -DImpactX_FFT=ON             \
           -DImpactX_PYTHON=ON          \
           -DAMReX_CUDA_ARCH=6.0        \
           -DImpactX_PRECISION=SINGLE   \

.github/workflows/dependencies/clang-san-openmpi.sh

@@ -17,6 +17,8 @@ sudo apt-get install -y \
     gnupg               \
     libc++-dev          \
     libc++abi-dev       \
+    libfftw3-dev        \
+    libfftw3-mpi-dev    \
     libhdf5-openmpi-dev \
     libomp-dev          \
     libopenmpi-dev      \

.github/workflows/dependencies/clang-tidy.sh

@@ -16,6 +16,8 @@ sudo apt-get install -y \
     clang-tidy          \
     cmake               \
     gnupg               \
+    libfftw3-dev        \
+    libfftw3-mpi-dev    \
     libhdf5-openmpi-dev \
     libomp-dev          \
     libopenmpi-dev      \

.github/workflows/dependencies/gcc-openmpi.sh

@@ -14,6 +14,8 @@ sudo apt-get install -y \
     ccache              \
     cmake               \
     gnupg               \
+    libfftw3-dev        \
+    libfftw3-mpi-dev    \
     libhdf5-openmpi-dev \
     libopenmpi-dev      \
     ninja-build         \

.github/workflows/dependencies/gcc.sh

@@ -14,6 +14,7 @@ sudo apt-get install -y \
     ccache              \
     cmake               \
     gnupg               \
+    libfftw3-dev        \
     libhdf5-dev         \
     ninja-build         \
     pkg-config          \

.github/workflows/hip.yml

@@ -53,5 +53,6 @@ jobs:
           -DCMAKE_VERBOSE_MAKEFILE=ON \
           -DAMReX_AMD_ARCH=gfx900     \
           -DImpactX_COMPUTE=HIP       \
+          -DImpactX_FFT=ON            \
           -DImpactX_PYTHON=ON
         cmake --build build -j 4

.github/workflows/macos.yml

@@ -25,6 +25,7 @@ jobs:
         brew install adios2
         brew install ccache
         brew install cmake
+        brew install fftw
         brew install hdf5-mpi
         brew install libomp
         brew link --overwrite --force libomp
@@ -67,6 +68,7 @@ jobs:
 
         cmake -S . -B build            \
           -DCMAKE_VERBOSE_MAKEFILE=ON  \
+          -DImpactX_FFT=ON             \
           -DImpactX_PYTHON=ON          \
           -DPython_EXECUTABLE=$(which python3)
         cmake --build build -j 3

.github/workflows/stubs.yml

@@ -58,6 +58,7 @@ jobs:
 
         cmake -S . -B build            \
           -DCMAKE_VERBOSE_MAKEFILE=ON  \
+          -DImpactX_FFT=ON             \
           -DImpactX_PYTHON=ON          \
           -DMPIEXEC_POSTFLAGS="--use-hwthread-cpus"
         cmake --build build -j 4 --target pip_install

.github/workflows/tooling.yml

@@ -44,6 +44,7 @@ jobs:
       run: |
         cmake -S . -B build                  \
           -DpyAMReX_IPO=OFF                  \
+          -DImpactX_FFT=ON                   \
           -DImpactX_PYTHON=ON                \
           -DImpactX_PYTHON_IPO=OFF           \
           -DCMAKE_BUILD_TYPE=RelWithDebInfo  \

.github/workflows/ubuntu.yml

@@ -47,6 +47,7 @@ jobs:
           -DBUILD_SHARED_LIBS=ON       \
           -DCMAKE_INSTALL_PREFIX=/usr  \
           -DCMAKE_VERBOSE_MAKEFILE=ON  \
+          -DImpactX_FFT=ON             \
           -DImpactX_PYTHON=ON          \
           -DMPIEXEC_POSTFLAGS="--use-hwthread-cpus"
         cmake --build build -j 4
@@ -110,6 +111,7 @@ jobs:
           -DCMAKE_BUILD_TYPE=Debug     \
           -DCMAKE_INSTALL_PREFIX=/usr  \
           -DCMAKE_VERBOSE_MAKEFILE=ON  \
+          -DImpactX_FFT=ON             \
           -DImpactX_MPI=OFF            \
           -DImpactX_PYTHON=ON
         cmake --build build -j 4

cmake/dependencies/ABLASTR.cmake

@@ -178,7 +178,7 @@ set(ImpactX_openpmd_src ""
 set(ImpactX_ablastr_repo "https://github.com/ECP-WarpX/WarpX.git"
     CACHE STRING
     "Repository URI to pull and build ABLASTR from if(ImpactX_ablastr_internal)")
-set(ImpactX_ablastr_branch "24.07"
+set(ImpactX_ablastr_branch "b58343a9cf31ad2e2e0c47b2c5f0b15cc122a961"
     CACHE STRING
     "Repository branch for ImpactX_ablastr_repo if(ImpactX_ablastr_internal)")
 

cmake/dependencies/pyAMReX.cmake

@@ -79,7 +79,7 @@ option(ImpactX_pyamrex_internal "Download & build pyAMReX" ON)
 set(ImpactX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git"
     CACHE STRING
     "Repository URI to pull and build pyamrex from if(ImpactX_pyamrex_internal)")
-set(ImpactX_pyamrex_branch "24.07"
+set(ImpactX_pyamrex_branch "ff4643869c63d4ee40a87054b901f61eefcb97a3"
     CACHE STRING
     "Repository branch for ImpactX_pyamrex_repo if(ImpactX_pyamrex_internal)")
 

docs/source/install/hpc/lumi-csc/lumi_cpu_impactx.profile.example

@@ -5,6 +5,7 @@
 module load LUMI/23.09  partition/C
 module load PrgEnv-aocc
 module load buildtools/23.09
+module load cray-fftw/3.3.10.5
 
 # optional: just an additional text editor
 module load nano

docs/source/install/hpc/lumi.rst

@@ -163,15 +163,15 @@ Use the following :ref:`cmake commands <building-cmake>` to compile the applicat
 
          cd $HOME/src/impactx
 
-         cmake --fresh -S . -B build_lumi_gpu -DImpactX_COMPUTE=HIP
+         cmake --fresh -S . -B build_lumi_gpu -DImpactX_COMPUTE=HIP -DImpactX_FFT=ON
          cmake --build build_lumi_gpu -j 16
 
       The ImpactX application executables are now in ``$HOME/src/impactx/build_lumi_gpu/bin/``.
       Additionally, the following commands will install ImpactX as a Python module:
 
       .. code-block:: bash
 
-         cmake --fresh -S . -B build_lumi_gpu_py -DImpactX_COMPUTE=HIP -DImpactX_PYTHON=ON
+         cmake --fresh -S . -B build_lumi_gpu_py -DImpactX_COMPUTE=HIP -DImpactX_FFT=ON -DImpactX_PYTHON=ON
          cmake --build build_lumi_gpu_py -j 16 --target pip_install
 
    .. tab-item:: CPU Nodes
@@ -180,15 +180,15 @@ Use the following :ref:`cmake commands <building-cmake>` to compile the applicat
 
          cd $HOME/src/impactx
 
-         cmake --fresh -S . -B build_lumi_cpu
+         cmake --fresh -S . -B build_lumi_cpu -DImpactX_FFT=ON
          cmake --build build_lumi_cpu -j 16
 
       The ImpactX application executables are now in ``$HOME/src/impactx/build_lumi_cpu/bin/``.
       Additionally, the following commands will install ImpactX as a Python module:
 
       .. code-block:: bash
 
-         cmake --fresh -S . -B build_lumi_cpu_py -DImpactX_PYTHON=ON
+         cmake --fresh -S . -B build_lumi_cpu_py -DImpactX_FFT=ON -DImpactX_PYTHON=ON
          cmake --build build_lumi_cpu_py -j 16 --target pip_install
 
 Now, you can :ref:`submit LUMI compute jobs <running-cpp-lumi>` for ImpactX :ref:`Python scripts <usage-picmi>` (:ref:`example scripts <usage-examples>`).

docs/source/install/hpc/perlmutter.rst

@@ -153,7 +153,7 @@ Use the following :ref:`cmake commands <building-cmake>` to compile the applicat
          cd $HOME/src/impactx
          rm -rf build_pm_gpu
 
-         cmake -S . -B build_pm_gpu -DImpactX_COMPUTE=CUDA
+         cmake -S . -B build_pm_gpu -DImpactX_COMPUTE=CUDA -DImpactX_FFT=ON
          cmake --build build_pm_gpu -j 16
 
       The ImpactX application executables are now in ``$HOME/src/impactx/build_pm_gpu/bin/``.
@@ -164,7 +164,7 @@ Use the following :ref:`cmake commands <building-cmake>` to compile the applicat
          cd $HOME/src/impactx
          rm -rf build_pm_gpu_py
 
-         cmake -S . -B build_pm_gpu_py -DImpactX_COMPUTE=CUDA -DImpactX_APP=OFF -DImpactX_PYTHON=ON
+         cmake -S . -B build_pm_gpu_py -DImpactX_COMPUTE=CUDA -DImpactX_APP=OFF -DImpactX_FFT=ON -DImpactX_PYTHON=ON
          cmake --build build_pm_gpu_py -j 16 --target pip_install
 
    .. tab-item:: CPU Nodes
@@ -174,7 +174,7 @@ Use the following :ref:`cmake commands <building-cmake>` to compile the applicat
          cd $HOME/src/impactx
          rm -rf build_pm_cpu
 
-         cmake -S . -B build_pm_cpu -DImpactX_COMPUTE=OMP
+         cmake -S . -B build_pm_cpu -DImpactX_COMPUTE=OMP -DImpactX_FFT=ON
          cmake --build build_pm_cpu -j 16
 
       The ImpactX application executables are now in ``$HOME/src/impactx/build_pm_cpu/bin/``.
@@ -184,7 +184,7 @@ Use the following :ref:`cmake commands <building-cmake>` to compile the applicat
 
          rm -rf build_pm_cpu_py
 
-         cmake -S . -B build_pm_cpu_py -DImpactX_COMPUTE=OMP -DImpactX_APP=OFF -DImpactX_PYTHON=ON
+         cmake -S . -B build_pm_cpu_py -DImpactX_COMPUTE=OMP -DImpactX_APP=OFF -DImpactX_FFT=ON -DImpactX_PYTHON=ON
          cmake --build build_pm_cpu_py -j 16 --target pip_install
 
 Now, you can :ref:`submit Perlmutter compute jobs <running-cpp-perlmutter>` for ImpactX :ref:`Python scripts <usage-python>` (:ref:`example scripts <usage-examples>`).

docs/source/usage/parameters.rst

@@ -691,6 +691,20 @@ Numerics and algorithms
 * ``algo.space_charge`` (``boolean``, optional, default: ``false``)
     Whether to calculate space charge effects.
 
+* ``algo.poisson_solver`` (``string``, optional, default: ``"multigrid"``)
+    The numerical solver to solve the Poisson equation when calculating space charge effects.
+    Options:
+
+    * ``multigrid``: Poisson's equation is solved using an iterative multigrid (MLMG) solver.
+      See the `AMReX documentation <https://amrex-codes.github.io/amrex/docs_html/LinearSolvers.html#>`__ for details of the MLMG solver.
+
+    * ``fft``: Poisson's equation is solved using an Integrated Green Function method (which requires FFT calculations).
+      See these references for more details `Qiang et al. (2006) <https://doi.org/10.1103/PhysRevSTAB.9.044204>`__ (+ `Erratum <https://doi.org/10.1103/PhysRevSTAB.10.129901>`__).
+      It only works in 3D and it requires the compilation flag ``-DImpactX_FFT=ON``.
+      If mesh refinement is enabled, this solver only works on the coarsest level.
+      On the refined patches, the Poisson equation is solved with the multigrid solver.
+      The boundary conditions are assumed to be open.
+
 * ``algo.mlmg_relative_tolerance`` (``float``, optional, default: ``1.e-7``)
     The relative precision with which the electrostatic space-charge fields should be calculated.
     More specifically, the space-charge fields are computed with an iterative Multi-Level Multi-Grid (MLMG) solver.

docs/source/usage/python.rst

@@ -66,6 +66,21 @@ General
 
       Whether to calculate space charge effects.
 
+   .. py:property:: poisson_solver
+
+      The numerical solver to solve the Poisson equation when calculating space charge effects.
+      Either ``"multigrid"`` (default) or ``"fft"``.
+
+      * ``multigrid``: Poisson's equation is solved using an iterative multigrid (MLMG) solver.
+        See the `AMReX documentation <https://amrex-codes.github.io/amrex/docs_html/LinearSolvers.html#>`__ for details of the MLMG solver.
+
+      * ``fft``: Poisson's equation is solved using an Integrated Green Function method (which requires FFT calculations).
+        See these references for more details `Qiang et al. (2006) <https://doi.org/10.1103/PhysRevSTAB.9.044204>`__ (+ `Erratum <https://doi.org/10.1103/PhysRevSTAB.10.129901>`__).
+        It only works in 3D and it requires the compilation flag ``-DImpactX_FFT=ON``.
+        If mesh refinement is enabled, this solver only works on the coarsest level.
+        On the refined patches, the Poisson equation is solved with the multigrid solver.
+        The boundary conditions are assumed to be open.
+
    .. py:property:: mlmg_relative_tolerance
 
       Default: ``1.e-7``

examples/CMakeLists.txt

@@ -256,23 +256,41 @@ add_impactx_test(cfchannel
 
 # Constant Focusing Channel with Space Charge #################################
 #
-add_impactx_test(cfchannel_spacecharge
-    examples/cfchannel/input_cfchannel_10nC.in
+add_impactx_test(cfchannel_spacecharge_mlmg
+    examples/cfchannel/input_cfchannel_10nC_mlmg.in
       OFF  # ImpactX MPI-parallel
       OFF  # ImpactX Python interface
     examples/cfchannel/analysis_cfchannel_10nC.py
     OFF  # no plot script yet
 )
+if(ImpactX_FFT)
+    add_impactx_test(cfchannel_spacecharge_fft
+        examples/cfchannel/input_cfchannel_10nC_fft.in
+        OFF  # ImpactX MPI-parallel
+        OFF  # ImpactX Python interface
+        examples/cfchannel/analysis_cfchannel_10nC.py
+        OFF  # no plot script yet
+    )
+endif()
 
 # Python: Constant Focusing Channel with Space Charge #########################
 #
-add_impactx_test(cfchannel_spacecharge.py
-    examples/cfchannel/run_cfchannel_10nC.py
+add_impactx_test(cfchannel_spacecharge_mlmg.py
+    examples/cfchannel/run_cfchannel_10nC_mlmg.py
       OFF  # ImpactX MPI-parallel
       ON   # ImpactX Python interface
     examples/cfchannel/analysis_cfchannel_10nC.py
     OFF  # no plot script yet
 )
+if(ImpactX_FFT)
+    add_impactx_test(cfchannel_spacecharge_fft.py
+        examples/cfchannel/run_cfchannel_10nC_fft.py
+        OFF  # ImpactX MPI-parallel
+        ON   # ImpactX Python interface
+        examples/cfchannel/analysis_cfchannel_10nC.py
+        OFF  # no plot script yet
+    )
+endif()
 
 # Python: Constant Focusing Channel ###########################################
 #
@@ -366,23 +384,41 @@ add_impactx_test(multipole
 
 # Expanding Beam Test #########################################################
 #
-add_impactx_test(expanding_beam
-    examples/expanding_beam/input_expanding.in
+add_impactx_test(expanding_beam_mlmg
+    examples/expanding_beam/input_expanding_mlmg.in
       OFF  # ImpactX MPI-parallel
       OFF  # ImpactX Python interface
     examples/expanding_beam/analysis_expanding.py
     OFF  # no plot script yet
 )
+if(ImpactX_FFT)
+    add_impactx_test(expanding_beam_fft
+        examples/expanding_beam/input_expanding_fft.in
+        OFF  # ImpactX MPI-parallel
+        OFF  # ImpactX Python interface
+        examples/expanding_beam/analysis_expanding.py
+        OFF  # no plot script yet
+    )
+endif()
 
 # Python: Expanding Beam Test #################################################
 #
-add_impactx_test(expanding_beam.py
-    examples/expanding_beam/run_expanding.py
+add_impactx_test(expanding_beam_mlmg.py
+    examples/expanding_beam/run_expanding_mlmg.py
       OFF  # ImpactX MPI-parallel
       ON   # ImpactX Python interface
     examples/expanding_beam/analysis_expanding.py
     OFF  # no plot script yet
 )
+if(ImpactX_FFT)
+    add_impactx_test(expanding_beam_fft.py
+        examples/expanding_beam/run_expanding_fft.py
+        OFF  # ImpactX MPI-parallel
+        ON   # ImpactX Python interface
+        examples/expanding_beam/analysis_expanding.py
+        OFF  # no plot script yet
+    )
+endif()
 
 # Python: Chain of Multipoles Test ############################################
 #