CI: Windows + Python (#208)

* CI: Windows + Python

Run Python scripts in Windows CI tests.

* [Draft] Only Py tests

* CMake Python Tests: Multi-Config Hints

Make sure that we can find the extra build config suffix directory
that is appended in multi-config generators.

* Python: Test After Install

Calling `os.add_dll_directory()` injection is too complicated for now.
https://bugs.python.org/issue43173

* Python DLL Loading Note

* Lib Symlink: Allow Static, too

* pyAMReX: .dll support

* Static: Missing Env for Install

* DLL: Debug Loader Logic

* Windows: Add PATH to .dll directories

Python 3.8+ on Windows: DLL search paths for dependent
shared libraries.

Refs.:
- python/cpython#80266
- https://docs.python.org/3.8/library/os.html#os.add_dll_directory

* CI: Debug

* CMake: Update Comments

* CMake: Fix Install

* Win PATH: Add ABLASTR & AMReX DLLs

* MSVC: w/ shared (dll) libs

* CI: Cleanup

* Test Before Install

Make sure we can run tests from the build tree.

.github/workflows/windows.yml

@@ -8,7 +8,7 @@ concurrency:
 
 jobs:
   build_win_msvc:
-    name: MSVC C++17 w/o MPI
+    name: MSVC w/o MPI shared
     runs-on: windows-latest
     if: github.event.pull_request.draft == false
     steps:
@@ -29,30 +29,36 @@ jobs:
         restore-keys: |
           ccache-windows-winmsvc-${{ hashFiles('.github/workflows/windows.yml') }}-
           ccache-windows-winmsvc-
-    - name: Build & Install
+    - name: Build
       run: |
         python3 -m pip install -U pip setuptools wheel pytest
         python3 -m pip install -r requirements.txt
         python3 -m pip install -r examples/requirements.txt
 
         cmake -S . -B build               `
-              -DBUILD_SHARED_LIBS=OFF        `
-              -DCMAKE_BUILD_TYPE=RelWithDebInfo    `
+              -DBUILD_SHARED_LIBS=ON      `
+              -DCMAKE_BUILD_TYPE=RelWithDebInfo  `
               -DCMAKE_VERBOSE_MAKEFILE=ON `
               -DImpactX_COMPUTE=NOACC     `
               -DImpactX_MPI=OFF           `
-              -DImpactX_PYTHON=ON
+              -DImpactX_PYTHON=ON         `
+              -DPython_EXECUTABLE=python3
         if(!$?) { Exit $LASTEXITCODE }
         cmake --build build --config RelWithDebInfo --parallel 2
         if(!$?) { Exit $LASTEXITCODE }
-        cmake --build build --config RelWithDebInfo --target pip_install
+    - name: Test
+      run: |
+        ctest --test-dir build --build-config RelWithDebInfo --output-on-failure -E AMReX
         if(!$?) { Exit $LASTEXITCODE }
-
-        ctest --test-dir build --build-config RelWithDebInfo --output-on-failure -E "(AMReX|py|pytest|analysis|plot)"
+    - name: Install
+      run: |
+        cmake --build build --config RelWithDebInfo --target install
+        if(!$?) { Exit $LASTEXITCODE }
+        cmake --build build --config RelWithDebInfo --target pip_install
         if(!$?) { Exit $LASTEXITCODE }
 
   build_win_clang:
-    name: Clang C++17 w/ OMP w/o MPI
+    name: Clang w/ OMP w/o MPI
     runs-on: windows-2019
     if: github.event.pull_request.draft == false
     steps:
@@ -74,7 +80,7 @@ jobs:
           ccache-windows-winclang-${{ hashFiles('.github/workflows/windows.yml') }}-
           ccache-windows-winclang-
     - uses: seanmiddleditch/gha-setup-ninja@master
-    - name: Build & Install
+    - name: Build
       shell: cmd
       run: |
         python3 -m pip install -U pip setuptools wheel pytest
@@ -86,17 +92,27 @@ jobs:
               -G "Ninja"      ^
               -DCMAKE_C_COMPILER=clang-cl   ^
               -DCMAKE_CXX_COMPILER=clang-cl ^
-              -DBUILD_SHARED_LIBS=OFF           ^
+              -DBUILD_SHARED_LIBS=OFF       ^
               -DCMAKE_BUILD_TYPE=Release    ^
               -DCMAKE_VERBOSE_MAKEFILE=ON   ^
               -DImpactX_COMPUTE=OMP         ^
               -DImpactX_MPI=OFF             ^
-              -DImpactX_PYTHON=ON
+              -DImpactX_PYTHON=ON           ^
+              -DPython_EXECUTABLE=python3
         if errorlevel 1 exit 1
         cmake --build build --config Release --parallel 2
         if errorlevel 1 exit 1
-        cmake --build build --config Release --target pip_install
+    - name: Test
+      shell: cmd
+      run: |
+        call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\vc\Auxiliary\build\vcvarsall.bat" x64
+        ctest --test-dir build --build-config Release --output-on-failure -E AMReX
         if errorlevel 1 exit 1
-
-        ctest --test-dir build --build-config Release --output-on-failure -E "(AMReX|py|pytest|analysis|plot)"
+    - name: Install
+      shell: cmd
+      run: |
+        call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\vc\Auxiliary\build\vcvarsall.bat" x64
+        cmake --build build --config Release --target install
+        if errorlevel 1 exit 1
+        cmake --build build --config Release --target pip_install
         if errorlevel 1 exit 1

CMakeLists.txt

@@ -187,6 +187,21 @@ if(ImpactX_PYTHON)
         PDB_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/impactx
         COMPILE_PDB_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/impactx
     )
+    get_property(isMultiConfig GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG)
+    if(isMultiConfig)
+        foreach(CFG IN LISTS CMAKE_CONFIGURATION_TYPES)
+            string(TOUPPER "${CFG}" CFG_UPPER)
+            set_target_properties(pyImpactX PROPERTIES
+                # build output directories - mainly set to run tests from CMake & IDEs
+                # note: same as above, but for Multi-Config generators
+                ARCHIVE_OUTPUT_DIRECTORY_${CFG_UPPER} ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/impactx
+                LIBRARY_OUTPUT_DIRECTORY_${CFG_UPPER} ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/impactx
+                RUNTIME_OUTPUT_DIRECTORY_${CFG_UPPER} ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/impactx
+                PDB_OUTPUT_DIRECTORY_${CFG_UPPER} ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/impactx
+                COMPILE_PDB_OUTPUT_DIRECTORY_${CFG_UPPER} ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/impactx
+            )
+        endforeach()
+    endif()
     if(EMSCRIPTEN)
         set_target_properties(pyImpactX PROPERTIES
             PREFIX "")
@@ -328,19 +343,17 @@ install(TARGETS ${ImpactX_INSTALL_TARGET_NAMES}
 
 # simplified library alias
 if(ImpactX_LIB)
-    if(WIN32)
-        set(mod_ext "dll")
-    else()
-        set(mod_ext "so")
-    endif()
     if(IS_ABSOLUTE ${CMAKE_INSTALL_LIBDIR})
-        set(ABS_INSTALL_LIB_DIR ${CMAKE_INSTALL_LIBDIR})
+        set(ABS_INSTALL_LIB_DIR "${CMAKE_INSTALL_LIBDIR}")
     else()
-        set(ABS_INSTALL_LIB_DIR ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR})
+        set(ABS_INSTALL_LIB_DIR "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
     endif()
+    # escape spaces for generated cmake_install.cmake file
+    file(TO_CMAKE_PATH "${ABS_INSTALL_LIB_DIR}" ABS_INSTALL_LIB_DIR)
+
     install(CODE "file(CREATE_LINK
         $<TARGET_FILE_NAME:lib>
-        ${ABS_INSTALL_LIB_DIR}/libImpactX.${mod_ext}
+        \"${ABS_INSTALL_LIB_DIR}/libImpactX$<TARGET_FILE_SUFFIX:lib>\"
         COPY_ON_ERROR SYMBOLIC)")
 endif()
 

cmake/ImpactXFunctions.cmake

@@ -124,15 +124,23 @@ macro(impactx_set_default_install_dirs_python)
 endmacro()
 
 
-# function to set the PYTHONPATH on a test
+# function to set the PYTHONPATH and PATH (for .dll files) on a test
 # this avoids that we need to install our python packages to run ctest
 #
 function(impactx_test_set_pythonpath test_name)
     if(WIN32)
         string(REPLACE ";" "\\;" WIN_PYTHONPATH "$ENV{PYTHONPATH}")
+        string(REPLACE ";" "\\;" WIN_PATH "$ENV{PATH}")  # DLLs
         string(REGEX REPLACE "/" "\\\\" WIN_PYTHON_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY})
+        # shared library note:
+        #   For Windows Python 3.8+, this also needs to be injected via
+        #   os.add_dll_directory.
+        #   https://github.com/python/cpython/issues/80266
+        #   https://docs.python.org/3.8/library/os.html#os.add_dll_directory
         set_property(TEST ${test_name}
-            APPEND PROPERTY ENVIRONMENT "PYTHONPATH=${WIN_PYTHON_OUTPUT_DIRECTORY}\;${WIN_PYTHONPATH}"
+            APPEND PROPERTY ENVIRONMENT
+                "PYTHONPATH=${WIN_PYTHON_OUTPUT_DIRECTORY}\;${WIN_PYTHONPATH}"
+                "PATH=$<TARGET_FILE_DIR:lib>\;$<TARGET_FILE_DIR:WarpX::ablastr>\;$<TARGET_FILE_DIR:AMReX::amrex>\;${WIN_PATH}"
         )
     else()
         set_property(TEST ${test_name}
@@ -241,9 +249,9 @@ function(impactx_set_binary_name)
         list(APPEND ImpactX_bin_names app)
     endif()
     if(ImpactX_LIB)
-        list(APPEND ImpactX_bin_names shared)
+        list(APPEND ImpactX_bin_names lib)
     endif()
-    foreach(tgt IN LISTS _ALL_TARGETS)
+    foreach(tgt IN LISTS ImpactX_bin_names)
         set_target_properties(${tgt} PROPERTIES OUTPUT_NAME "impactx")
 
         if(ImpactX_MPI)
@@ -287,15 +295,10 @@ function(impactx_set_binary_name)
         )
     endif()
     if(ImpactX_LIB)
-        if(WIN32)  # TODO: handle static lib extensions
-            set(mod_ext "dll")
-        else()
-            set(mod_ext "so")
-        endif()
         add_custom_command(TARGET lib POST_BUILD
             COMMAND ${CMAKE_COMMAND} -E create_symlink
                 $<TARGET_FILE_NAME:lib>
-                $<TARGET_FILE_DIR:lib>/libimpactx.${mod_ext}
+                $<TARGET_FILE_DIR:lib>/libimpactx$<TARGET_FILE_SUFFIX:lib>
         )
     endif()
 endfunction()

cmake/dependencies/ABLASTR.cmake

@@ -55,7 +55,7 @@ macro(find_ablastr)
             # BETWEEN MULTIPLE PYTHON MODULES
             # TODO this is likely an export/symbol hiding issue that we could
             #      alleviate later on
-            set(BUILD_SHARED_LIBS ON CACHE BOOL "" FORCE)
+            set(AMReX_BUILD_SHARED_LIBS ON CACHE BOOL "Build AMReX shared library" FORCE)
         endif()
 
         if(ImpactX_ablastr_src)

cmake/dependencies/pyAMReX.cmake

@@ -76,7 +76,7 @@ option(ImpactX_pyamrex_internal "Download & build pyAMReX" ON)
 set(ImpactX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git"
     CACHE STRING
     "Repository URI to pull and build pyamrex from if(ImpactX_pyamrex_internal)")
-set(ImpactX_pyamrex_branch "23.01"
+set(ImpactX_pyamrex_branch "f3ba017531e2d1569e63428919632cf908435e56"
     CACHE STRING
     "Repository branch for ImpactX_pyamrex_repo if(ImpactX_pyamrex_internal)")
 

setup.py

@@ -145,7 +145,7 @@ def build_extension(self, ext):
             cmake_args += [
                 "-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{}={}".format(
                     cfg.upper(), os.path.join(extdir, "impactx")
-                )
+                ),
             ]
             if sys.maxsize > 2**32:
                 cmake_args += ["-A", "x64"]

src/python/impactx/__init__.py

@@ -1,3 +1,21 @@
+import os
+
+# Python 3.8+ on Windows: DLL search paths for dependent
+# shared libraries
+# Refs.:
+# - https://github.com/python/cpython/issues/80266
+# - https://docs.python.org/3.8/library/os.html#os.add_dll_directory
+if os.name == "nt":
+    # add anything in the current directory
+    pwd = __file__.rsplit(os.sep, 1)[0] + os.sep
+    os.add_dll_directory(pwd)
+    # add anything in PATH
+    paths = os.environ.get("PATH", "")
+    for p in paths.split(";"):
+        if os.path.exists(p):
+            os.add_dll_directory(p)
+
+# import core bindings to C++
 from . import impactx_pybind as cxx
 from .impactx_pybind import *  # noqa
 from .madx_to_impactx import read_beam, read_lattice  # noqa

tests/python/CMakeLists.txt

@@ -17,5 +17,5 @@ add_test(NAME ${pytest_name}
     WORKING_DIRECTORY ${pytest_rundir}
 )
 
-#   set PYTHONPATH
+#   set PYTHONPATH and PATH (for .dll files)
 impactx_test_set_pythonpath(${pytest_name})