From 4e2ab32892471a9dad68f18dd2210ce4345aefd5 Mon Sep 17 00:00:00 2001
From: Izaak Beekman <zbeekman@gmail.com>
Date: Wed, 23 Mar 2022 14:58:07 -0700
Subject: [PATCH 01/17] Build: Fix paths passed as flag args w/ spaces

Add some quotes to paths passed as compiler flag arguments in CMake
---
 CMakeLists.txt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 2d86a7ca..cebd3fde 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -104,11 +104,11 @@ if(DEFINED CMAKE_BUILD_TYPE AND "${CMAKE_BUILD_TYPE}" MATCHES "[Rr][Ee][Ll]")
   if(CMAKE_Fortran_COMPILER_ID MATCHES "GNU")
     set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fno-working-directory")
   endif()
-  set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fdebug-prefix-map=${CMAKE_SOURCE_DIR}=.")
+  set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fdebug-prefix-map=\"${CMAKE_SOURCE_DIR}\"=.")
   if(CMAKE_C_COMPILER_ID MATCHES "GNU")
     set(CMAKE_C_FLAGS       "${CMAKE_C_FLAGS} -fno-working-directory")
   endif()
-  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fdebug-prefix-map=${CMAKE_SOURCE_DIR}=.")
+  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fdebug-prefix-map=\"${CMAKE_SOURCE_DIR}\"=.")
 endif()
 
 message( STATUS "Building OpenCoarrays version: ${full_git_describe}" )

From cf4f9abdcad06b4541351d83c792f049df34cdea Mon Sep 17 00:00:00 2001
From: Izaak Beekman <zbeekman@gmail.com>
Date: Wed, 23 Mar 2022 14:59:49 -0700
Subject: [PATCH 02/17] Build: Remove genexp in CMake try_compile() flag

This fixes a broken build on windows, and seems like it should not
have been working when using Makefile generators on any platform,
i.e., macOS & other *nix
---
 CMakeLists.txt | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index cebd3fde..a572394b 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -234,7 +234,7 @@ endif()
 
 if(gfortran_compiler)
   set(OLD_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS})
-  set(CMAKE_REQUIRED_FLAGS $<$<COMPILE_LANGUAGE:Fortran>:-fcoarray=single -ffree-form>)
+  set(CMAKE_REQUIRED_FLAGS -fcoarray=single)
 endif()
 include(CheckFortranSourceCompiles)
 CHECK_Fortran_SOURCE_COMPILES("
@@ -243,7 +243,8 @@ CHECK_Fortran_SOURCE_COMPILES("
     integer :: i
     i = this_image()
   end program
-" Check_Simple_Coarray_Fortran_Source_Compiles)
+" Check_Simple_Coarray_Fortran_Source_Compiles
+  SRC_EXT F90)
 if(gfortran_compiler)
   set (CMAKE_REQUIRED_FLAGS ${OLD_REQUIRED_FLAGS})
   unset(OLD_REQUIRED_FLAGS)

From 344ac3010bfaac6a53c48f502f7396555cdae7ed Mon Sep 17 00:00:00 2001
From: Izaak Beekman <zbeekman@gmail.com>
Date: Fri, 25 Mar 2022 18:14:52 -0700
Subject: [PATCH 03/17] CMake will build OpenCoarrays on windows w/ GCC & IMPI

The library now builds with Intel MPI (from Intel OneAPI) and GCC +
GFortran. There are still some path handling quirks and string quoting
things that need to be resolved to get the caf and cafrun wrapper
scripts working reliably. A test program can be built and run by
tweaking/fixing the commands attempted by the caf script. The program
executes correctly and then encounters an error during MPI
finalize/when shutting down.
---
 CMakeLists.txt         | 200 +++++++++++++++++++++++++++--------------
 src/mpi/CMakeLists.txt |  14 +++
 2 files changed, 148 insertions(+), 66 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index a572394b..b8c28e6e 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -204,6 +204,9 @@ endif()
 
 if(CMAKE_BUILD_TYPE MATCHES "Debug|DEBUG|debug")
   add_definitions(-DEXTRA_DEBUG_OUTPUT)
+  set(IMPI_BUILD debug)
+else()
+  set(IMPI_BUILD release)
 endif()
 
   # We have populated CMAKE_Fortran_COMPILER_VERSION if it was missing
@@ -274,33 +277,97 @@ if (C_COMPILER_NAME MATCHES "^[mM][pP][iI]")
   set (MPI_C_COMPILER "${CMAKE_C_COMPILER}")
 endif()
 
-find_package( MPI )
+if(WIN32) # Only support building with GCC & GFortran using Intel MPI (OneAPI)
+# Here we assume Intel ONEAPI and the environment is loaded
+cmake_path(SET MPI_ROOT NORMALIZE "$ENV{I_MPI_ROOT}")
+set (IMPI_INCLUDES "${MPI_ROOT}/include")
+set (IMPI_LIB_DIR "${MPI_ROOT}/lib/${IMPI_BUILD}")
+set (IMPI_DLL_DIR "${MPI_ROOT}/bin/${IMPI_BUILD}")
+set (IMPI_BIN_DIR "${MPI_ROOT}/bin")
+
+find_library(IMPI_LIB
+  "impi.lib"
+  HINTS "${IMPI_LIB_DIR}"
+  DOC "Location of the Intel MPI impi.lib file"
+  REQUIRED
+  NO_DEFAULT_PATH)
+
+set(MPI_impi_LIBRARY "${IMPI_LIB}")
+set(MPI_impicxx_LIBRARY "${IMPI_LIB_DIR}/impicxx.lib")
+
+find_file(IMPI_DLL
+  "impi.dll"
+  HINTS "${IMPI_DLL_DIR}"
+  DOC "Location of the Intel MPI impi.dll file"
+  REQUIRED
+  NO_DEFAULT_PATH)
+
+
+add_library(IMPI::MPI SHARED IMPORTED)
+set_target_properties(IMPI::MPI
+  PROPERTIES
+  IMPORTED_IMPLIB "${IMPI_LIB}"
+  IMPORTED_LOCATION "${IMPI_DLL}")
+target_include_directories(IMPI::MPI
+  INTERFACE "${IMPI_INCLUDES}")
+
+set(MPI_C_INCLUDE_DIRS "${IMPI_INCLUDES}")
+set(MPI_C_INCLUDE_PATH "${IMPI_INCLUDES}")
+set(MPI_C_HEADER_DIR "${IMPI_INCLUDES}")
+set(MPI_Fortran_COMPILER_INCLUDE_DIRS "${IMPI_INCLUDES}")
+set(MPI_Fortran_F77_HEADER_DIR "${IMPI_INCLUDES}")
+set(MPI_Fortran_INCLUDE_PATH "${IMPI_INCLUDES}")
+
+set(MPI_C_LIB_NAMES impi;impicxx)
+set(MPI_C_LIBRARIES "${IMPI_LIB}")
+set(MPI_Fortran_LIB_NAMES impi)
+set(MPI_Fortran_LIBRARIES "${IMPI_LIB}")
+
+set(MPI_Fortran_HAVE_F90_MODULE FALSE)
+set(MPI_Fortran_HAVE_F08_MODULE FALSE)
+
+find_program(MPIEXEC_EXECUTABLE
+  mpiexec
+  HINTS "${IMPI_BIN_DIR}"
+  DOC "Location of Intel MPI implementations mpiexec launcher program"
+  REQUIRED
+  NO_DEFAULT_PATH)
+set(MPIEXEC "${MPIEXEC_EXECUTABLE}")
+set(MPI_EXEC_NUMPROC_FLAG -n)
+set(MPI_EXEC_PREFLAGS -print-all-exitcodes)
+
+set(MPI_C_FOUND TRUE)
+set(MPI_Fortran_FOUND TRUE)
+
+else()
+  find_package( MPI )
 
-if ( (NOT MPI_C_FOUND) OR (NOT MPI_Fortran_FOUND) OR (NOT MPIEXEC))
-  # Get default install location of MPICH from install.sh
-  message(WARNING "Could not find all MPI components!")
-  message(WARNING "
+  if ( (NOT MPI_C_FOUND) OR (NOT MPI_Fortran_FOUND) OR (NOT MPIEXEC_EXECUTABLE))
+    # Get default install location of MPICH from install.sh
+    message(WARNING "Could not find all MPI components!")
+    message(WARNING "
 MPI_C_FOUND        = ${MPI_C_FOUND}
 MPI_Fortran_FOUND  = ${MPI_Fortran_FOUND}
-MPIEXEC = ${MPIEXEC}
+MPIEXEC = ${MPIEXEC_EXECUTABLE}
 ")
-  execute_process( COMMAND "./install.sh" -P mpich
-    WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
-    OUTPUT_VARIABLE DEFAULT_MPICH_INSTALL_LOC
-    OUTPUT_QUIET
-    OUTPUT_STRIP_TRAILING_WHITESPACE
-  )
-  find_program (MY_MPI_EXEC NAMES mpirun mpiexec lamexec srun
-    PATHS "${DEFAULT_MPICH_INSTALL_LOC}" ENV PATH
-    HINTS "${FTN_COMPILER_DIR}" "${C_COMPILER_DIR}"
-    PATH_SUFFIXES bin)
-  set ( MPI_HOME "${MPI_HOME}" "${MY_MPI_EXEC}" "${MY_MPI_EXEC}/.." )
-  find_package( MPI REQUIRED )
+    execute_process( COMMAND "./install.sh" -P mpich
+      WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
+      OUTPUT_VARIABLE DEFAULT_MPICH_INSTALL_LOC
+      OUTPUT_QUIET
+      OUTPUT_STRIP_TRAILING_WHITESPACE
+      )
+    find_program (MY_MPI_EXEC NAMES mpirun mpiexec lamexec srun
+      PATHS "${DEFAULT_MPICH_INSTALL_LOC}" ENV PATH
+      HINTS "${FTN_COMPILER_DIR}" "${C_COMPILER_DIR}"
+      PATH_SUFFIXES bin)
+    set ( MPI_HOME "${MPI_HOME}" "${MY_MPI_EXEC}" "${MY_MPI_EXEC}/.." )
+    find_package( MPI REQUIRED )
+  endif()
 endif()
 list(REMOVE_DUPLICATES MPI_Fortran_INCLUDE_PATH)
 
-# Test for consistent MPI environment
-if (NOT MPIEXEC)
+# Test for consistent MPI environment but not on windows
+if (NOT MPIEXEC_EXECUTABLE)
   message ( ERROR "CMake failed to find `mpiexec` or similar. If building with `./install.sh` please
 report this bug to the OpenCoarrays developers at
 https://github.com/sourceryinstitute/opencoarrays/issues, otherwise point CMake
@@ -308,68 +375,69 @@ to the desired MPI runtime.")
 else()
   add_definitions(-DHAVE_MPI)
 endif()
-
-get_filename_component(MPIEXEC_RELATIVE_LOC "${MPIEXEC}"
-  PROGRAM)
-get_filename_component(MPIEXEC_ABS_LOC "${MPIEXEC_RELATIVE_LOC}"
-  REALPATH)
-get_filename_component(MPIEXEC_DIR "${MPIEXEC_ABS_LOC}"
-  DIRECTORY)
-
-get_filename_component(MPICC_RELATIVE_LOC "${MPI_C_COMPILER}"
-  PROGRAM)
-get_filename_component(MPICC_ABS_LOC "${MPICC_RELATIVE_LOC}"
-  REALPATH)
-get_filename_component(MPICC_DIR "${MPICC_ABS_LOC}"
-  DIRECTORY)
-
-get_filename_component(MPIFC_RELATIVE_LOC "${MPI_Fortran_COMPILER}"
-  PROGRAM)
-get_filename_component(MPIFC_ABS_LOC "${MPIFC_RELATIVE_LOC}"
-  REALPATH)
-get_filename_component(MPIFC_DIR "${MPIFC_ABS_LOC}"
-  DIRECTORY)
-
-if ((MPIEXEC_DIR STREQUAL MPICC_DIR) AND (MPIEXEC_DIR STREQUAL MPIFC_DIR))
-  message ( STATUS "MPI runtime and compile time environments appear to be consistent")
-else()
-  message ( WARNING "MPIEXEC is in \"${MPIEXEC_DIR},\"
+if(NOT WIN32)
+  get_filename_component(MPIEXEC_RELATIVE_LOC "${MPIEXEC}"
+    PROGRAM)
+  get_filename_component(MPIEXEC_ABS_LOC "${MPIEXEC_RELATIVE_LOC}"
+    REALPATH)
+  get_filename_component(MPIEXEC_DIR "${MPIEXEC_ABS_LOC}"
+    DIRECTORY)
+
+  get_filename_component(MPICC_RELATIVE_LOC "${MPI_C_COMPILER}"
+    PROGRAM)
+  get_filename_component(MPICC_ABS_LOC "${MPICC_RELATIVE_LOC}"
+    REALPATH)
+  get_filename_component(MPICC_DIR "${MPICC_ABS_LOC}"
+    DIRECTORY)
+
+  get_filename_component(MPIFC_RELATIVE_LOC "${MPI_Fortran_COMPILER}"
+    PROGRAM)
+  get_filename_component(MPIFC_ABS_LOC "${MPIFC_RELATIVE_LOC}"
+    REALPATH)
+  get_filename_component(MPIFC_DIR "${MPIFC_ABS_LOC}"
+    DIRECTORY)
+
+  if ((MPIEXEC_DIR STREQUAL MPICC_DIR) AND (MPIEXEC_DIR STREQUAL MPIFC_DIR))
+    message ( STATUS "MPI runtime and compile time environments appear to be consistent")
+  else()
+    message ( WARNING "MPIEXEC is in \"${MPIEXEC_DIR},\"
 which differs from the location of MPICC and/or MPIFC which are in
 \"${MPICC_DIR}\" and \"${MPIFC_DIR},\" respectively.
 This is likely indicative of a problem. If building with `./install.sh` please report
 this to the OpenCoarrays developers by filing a new issue at:
 https://github.com/sourceryinstitute/OpenCoarrays/issues/new")
-endif()
+  endif()
 
-#-----------------------------------------------
-# Work around bug #317 present on fedora systems
-#-----------------------------------------------
-if( (MPI_C_LINK_FLAGS MATCHES "noexecstack") OR (MPI_Fortran_LINK_FLAGS MATCHES "noexecstack") )
-  message ( WARNING
-"The `noexecstack` linker flag was found in the MPI_<lang>_LINK_FLAGS variable. This is
+  #-----------------------------------------------
+  # Work around bug #317 present on fedora systems
+  #-----------------------------------------------
+  if( (MPI_C_LINK_FLAGS MATCHES "noexecstack") OR (MPI_Fortran_LINK_FLAGS MATCHES "noexecstack") )
+    message ( WARNING
+      "The `noexecstack` linker flag was found in the MPI_<lang>_LINK_FLAGS variable. This is
 known to cause segmentation faults for some Fortran codes. See, e.g.,
 https://gcc.gnu.org/bugzilla/show_bug.cgi?id=71729 or
 https://github.com/sourceryinstitute/OpenCoarrays/issues/317.
 
 `noexecstack` is being replaced with `execstack`"
-    )
-  string(REPLACE "noexecstack"
-    "execstack" MPI_C_LINK_FLAGS_FIXED ${MPI_C_LINK_FLAGS})
-  string(REPLACE "noexecstack"
-    "execstack" MPI_Fortran_LINK_FLAGS_FIXED ${MPI_Fortran_LINK_FLAGS})
-  set(MPI_C_LINK_FLAGS "${MPI_C_LINK_FLAGS_FIXED}" CACHE STRING
-    "MPI C linking flags" FORCE)
-  set(MPI_Fortran_LINK_FLAGS "${MPI_Fortran_LINK_FLAGS_FIXED}" CACHE STRING
-    "MPI Fortran linking flags" FORCE)
+      )
+    string(REPLACE "noexecstack"
+      "execstack" MPI_C_LINK_FLAGS_FIXED ${MPI_C_LINK_FLAGS})
+    string(REPLACE "noexecstack"
+      "execstack" MPI_Fortran_LINK_FLAGS_FIXED ${MPI_Fortran_LINK_FLAGS})
+    set(MPI_C_LINK_FLAGS "${MPI_C_LINK_FLAGS_FIXED}" CACHE STRING
+      "MPI C linking flags" FORCE)
+    set(MPI_Fortran_LINK_FLAGS "${MPI_Fortran_LINK_FLAGS_FIXED}" CACHE STRING
+      "MPI Fortran linking flags" FORCE)
+  endif()
 endif()
 
 #--------------------------------------------------------
 # Make sure a simple "hello world" C mpi program compiles
 #--------------------------------------------------------
 set(OLD_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS})
-set(CMAKE_REQUIRED_FLAGS ${MPI_C_COMPILE_FLAGS} ${MPI_C_LINK_FLAGS})
+set(CMAKE_REQUIRED_FLAGS ${MPI_C_COMPILE_OPTIONS} ${MPI_C_COMPILE_DEFINITIONS} ${MPI_C_LINK_FLAGS})
 set(OLD_INCLUDES ${CMAKE_REQUIRED_INCLUDES})
-set(CMAKE_REQUIRED_INCLUDES ${MPI_C_INCLUDE_PATH})
+set(CMAKE_REQUIRED_INCLUDES ${MPI_C_INCLUDE_DIRS})
 set(OLD_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES})
 set(CMAKE_REQUIRED_LIBRARIES ${MPI_C_LIBRARIES})
 include (CheckCSourceCompiles)
@@ -410,7 +478,7 @@ endif()
 # Try using mpi.mod first then fall back on includ 'mpif.h'
 #--------------------------------------------------------------
 set(OLD_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS})
-set(CMAKE_REQUIRED_FLAGS "-ffree-form" ${MPI_Fortran_COMPILE_FLAGS} ${MPI_Fortran_LINK_FLAGS})
+set(CMAKE_REQUIRED_FLAGS "-ffree-form" ${MPI_Fortran_COMPILE_OPTIONS} ${MPI_Fortran_COMPILE_DEFINITIONS} ${MPI_Fortran_LINK_FLAGS})
 set(OLD_INCLUDES ${CMAKE_REQUIRED_INCLUDES})
 set(CMAKE_REQUIRED_INCLUDES ${MPI_Fortran_INCLUDE_PATH})
 set(OLD_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES})
@@ -442,7 +510,7 @@ unset(OLD_LIBRARIES)
 # If that failed try using mpif.h
 #--------------------------------
 set(OLD_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS})
-set(CMAKE_REQUIRED_FLAGS "-ffree-form" ${MPI_Fortran_COMPILE_FLAGS} ${MPI_Fortran_LINK_FLAGS})
+set(CMAKE_REQUIRED_FLAGS "-ffree-form" ${MPI_Fortran_COMPILE_OPTIONS} ${MPI_Fortra_COMPILE_DEFINITIONS} ${MPI_Fortran_LINK_FLAGS})
 set(OLD_INCLUDES ${CMAKE_REQUIRED_INCLUDES})
 set(CMAKE_REQUIRED_INCLUDES ${MPI_Fortran_INCLUDE_PATH})
 set(OLD_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES})
diff --git a/src/mpi/CMakeLists.txt b/src/mpi/CMakeLists.txt
index 649774de..2925ac72 100644
--- a/src/mpi/CMakeLists.txt
+++ b/src/mpi/CMakeLists.txt
@@ -46,6 +46,20 @@ target_link_libraries(caf_mpi_static
 set_target_properties(caf_mpi_static
   PROPERTIES
   POSITION_INDEPENDENT_CODE TRUE)
+if(WIN32)
+  target_compile_options(caf_mpi
+    PUBLIC -pthread)
+  target_link_libraries(caf_mpi
+    PRIVATE pthread)
+  set_target_properties(caf_mpi
+    PROPERTIES LINKER_LANGUAGE Fortran)
+  target_compile_options(caf_mpi_static
+    PUBLIC -pthread)
+  target_link_options(caf_mpi_static
+    PUBLIC -pthread)
+  target_link_libraries(caf_mpi_static
+    PRIVATE gfortran)
+endif()
 target_include_directories(caf_mpi PUBLIC
   $<$<COMPILE_LANGUAGE:C>:${MPI_C_INCLUDE_PATH}>
   $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/src>

From 9cac9677c3e12ca623e10c280a2707fbf3ce02b9 Mon Sep 17 00:00:00 2001
From: Izaak Beekman <zbeekman@gmail.com>
Date: Tue, 26 Apr 2022 18:06:34 -0400
Subject: [PATCH 04/17] Fixed typo and made caf.in more robust

---
 src/extensions/caf.in | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/extensions/caf.in b/src/extensions/caf.in
index fbe4204b..63c9b361 100755
--- a/src/extensions/caf.in
+++ b/src/extensions/caf.in
@@ -61,7 +61,7 @@ oca_compiler="@opencoarrays_aware_compiler@" # True for GFortran > 5.0.0
 
 # Compiler used to build OpenCoarrays; runtime must match compiler used during build
 cafc="@Fortran_COMPILER@"
-if [[ "${cafc}" == @*@ ]]; then
+if [[ "${cafc}" == @*@ || -z "${cafc}" ]]; then
   cafc=gfortran
 fi
 caf_mod_dir="@CAF_MODDIR@"    # location of extensions module, needed for non-OCA compilers
@@ -176,7 +176,7 @@ __only_compiling () {
 substitute_lib () {
   # Try to substitute a shared or static library if requested library is missing
   # Some package managers only install dynamic or static libs
-  if ! [[ -f "${1}" ]] ; then
+  if ! [[ -f "${1}" && "${1}" = *.* ]] ; then
       case "${1##*.}" in
           a|lib)
               for suff in so dylib dll ; do
@@ -370,7 +370,7 @@ if "${cafc}" "${compiler_args[@]}" ; then
   exit $?
 else
   return_code=$?
-  echo "Error: comand:" >&2
+  echo "Error: command:" >&2
   echo "   \`${cafc} ${compiler_args[*]}\`" >&2
   echo "failed to compile." >&2
   exit "${return_code}"

From 0abce5ae80000c3012507316d2a0ebd3e25b50dd Mon Sep 17 00:00:00 2001
From: Izaak Beekman <zbeekman@gmail.com>
Date: Tue, 26 Apr 2022 18:07:22 -0400
Subject: [PATCH 05/17] Make cafrun.in more robust & runable on windows

No special changes needed beyond robustness improvements
---
 src/extensions/cafrun.in | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/extensions/cafrun.in b/src/extensions/cafrun.in
index d1f34a2b..bb7b9534 100755
--- a/src/extensions/cafrun.in
+++ b/src/extensions/cafrun.in
@@ -56,25 +56,25 @@ set -o pipefail
 
 caf_version='@CAF_VERSION@'
 CAFRUN="@MPIEXEC@"
-if [[ "${CAFRUN}" == @*@ ]]; then
+if [[ "${CAFRUN}" == @*@ || -z "${CAFRUN}" ]]; then
   CAFRUN=mpiexec
 fi
 have_failed_img=@HAVE_FAILED_IMG@
-if [[ ${have_failed_img} == @*@ ]]; then
+if [[ ${have_failed_img} == @*@  || -z "${have_failed_img}" ]]; then
   have_failed_img=false
 fi
 numproc_flag='@MPIEXEC_NUMPROC_FLAG@'
-if [[ ${numproc_flag} == @*@ ]]; then
+if [[ ${numproc_flag} == @*@  || -z "${numproc_flag}" ]]; then
   numproc_flag='-np'
 fi
 preflags="@MPIEXEC_PREFLAGS@"
 preflags="${preflags//;/ }"
-if [[ "${preflags}" == @*@ ]]; then
+if [[ "${preflags}" == @*@  || -z "${preflags}" ]]; then
   unset preflags
 fi
 postflags="@MPIEXEC_POSTFLAGS@"
 postflags="${postflags//;/ }"
-if [[ "${postflags}" == @*@ ]]; then
+if [[ "${postflags}" == @*@  || -z "${postflags}" ]]; then
   unset postflags
 fi
 #-------------------------

From a65958ca41a2fde614366b6309dead35733f6d77 Mon Sep 17 00:00:00 2001
From: Izaak Beekman <zbeekman@gmail.com>
Date: Tue, 26 Apr 2022 18:11:32 -0400
Subject: [PATCH 06/17] Improve windows support: find pthreads & use in caf

Use CMakes FindThreads feature/module to find pthreads since
they are used in the mpi opencoarrays library. As a result
the caf wrapper script should also link in/build with pthreads
---
 src/extensions/caf.in  |  8 +++++++-
 src/mpi/CMakeLists.txt | 37 +++++++++++++++++++++----------------
 2 files changed, 28 insertions(+), 17 deletions(-)

diff --git a/src/extensions/caf.in b/src/extensions/caf.in
index 63c9b361..9c6ee10f 100755
--- a/src/extensions/caf.in
+++ b/src/extensions/caf.in
@@ -52,7 +52,7 @@ set -o pipefail
 #
 # CAF_VERSION, opencoarrays_aware_compiler, Fortran_COMPILER, CAF_MODDIR,
 # CAF_MPI_Fortran_LINK_FLAGS, CAF_MPI_Fortran_COMPILE_FLAGS,
-# CAF_LIBS, CAF_MPI_LIBS
+# CAF_LIBS, THREADS_LIB, CAF_MPI_LIBS
 #
 
 caf_version='@CAF_VERSION@' # fetched from `git describe` and/or
@@ -87,6 +87,7 @@ mpi_link_flags=(@CAF_MPI_Fortran_LINK_FLAGS@) # e.g. `pkg-config
 # --libs-only-L`
 mpi_compile_flags=(@CAF_MPI_Fortran_COMPILE_FLAGS@)
 caf_libs=(@CAF_LIBS@) # e.g. "libcaf_mpi" "libcaf_extensions",
+threads_lib=@THREADS_LIB@ # pthreads or compatible, needed for windows
 # preferably full paths, but could be
 # combination of -L... and -lcaf_mpi...
 mpi_libs=(@CAF_MPI_LIBS@) # e.g. `pkg-config --libs-only-l` or full paths
@@ -235,6 +236,11 @@ if [[ -n "${caf_libs[*]:-}" ]]; then
     caf_added_libs+=("$(substitute_lib "${prefix%/}/${lib}")")
   done
 fi
+
+if [[ -n "${threads_lib}" ]]; then
+  caf_added_libs+=("${threads_lib}")
+fi
+
 if [[ -n "${mpi_libs[*]:-}" ]]; then
   for lib in "${mpi_libs[@]:-}"; do
     caf_added_libs+=("$(substitute_lib "${lib}")")
diff --git a/src/mpi/CMakeLists.txt b/src/mpi/CMakeLists.txt
index 2925ac72..1527d096 100644
--- a/src/mpi/CMakeLists.txt
+++ b/src/mpi/CMakeLists.txt
@@ -46,20 +46,23 @@ target_link_libraries(caf_mpi_static
 set_target_properties(caf_mpi_static
   PROPERTIES
   POSITION_INDEPENDENT_CODE TRUE)
-if(WIN32)
-  target_compile_options(caf_mpi
-    PUBLIC -pthread)
-  target_link_libraries(caf_mpi
-    PRIVATE pthread)
-  set_target_properties(caf_mpi
-    PROPERTIES LINKER_LANGUAGE Fortran)
-  target_compile_options(caf_mpi_static
-    PUBLIC -pthread)
-  target_link_options(caf_mpi_static
-    PUBLIC -pthread)
-  target_link_libraries(caf_mpi_static
-    PRIVATE gfortran)
-endif()
+
+set(THREADS_PREFER_PTHREAD_FLAG TRUE)
+find_package(Threads REQUIRED)
+
+message(STATUS "Threads found? ${Threads_FOUND}")
+message(STATUS "Thread library to use: ${CMAKE_THREAD_LIBS_INIT}")
+message(STATUS "Found threads library pthread compatible? ${CMAKE_USE_PTHREADS_INIT}")
+
+target_link_libraries(caf_mpi
+  PUBLIC Threads::Threads)
+set_target_properties(caf_mpi
+  PROPERTIES LINKER_LANGUAGE Fortran)
+target_link_libraries(caf_mpi_static
+  PUBLIC Threads::Threads)
+target_link_libraries(caf_mpi_static
+  PRIVATE gfortran)
+
 target_include_directories(caf_mpi PUBLIC
   $<$<COMPILE_LANGUAGE:C>:${MPI_C_INCLUDE_PATH}>
   $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/src>
@@ -305,13 +308,14 @@ set(CMAKE_REQUIRED_INCLUDES ${old_cmake_required_includes})
 #
 # @CAF_VERSION@ @opencoarrays_aware_compiler@ @Fortran_COMPILER@ @CAF_MODDIR@
 # @CAF_MPI_Fortran_LINK_FLAGS@ @CAF_MPI_Fortran_COMPILE_FLAGS@
-# @CAF_LIBS@ @CAF_MPI_LIBS@
+# @CAF_LIBS@ @THREADS_LIB@ @CAF_MPI_LIBS@
 #
 
 set(CAF_VERSION "${full_git_describe}")
 set(Fortran_COMPILER "${CMAKE_Fortran_COMPILER}")
 set(CAF_MODDIR "${CMAKE_INSTALL_INCLUDEDIR}/${mod_dir_tail}")
 set(MOD_DIR_FLAG "${CMAKE_Fortran_MODDIR_FLAG}")
+set(THREADS_LIB "${CMAKE_THREAD_LIBS_INIT}")
 set(CAF_MPI_LIBS "")
 foreach( lib IN LISTS MPI_Fortran_LIBRARIES)
   set(CAF_MPI_LIBS "${CAF_MPI_LIBS} \"${lib}\"")
@@ -330,7 +334,8 @@ string(STRIP "${CAF_MPI_Fortran_COMPILE_FLAGS}" CAF_MPI_Fortran_COMPILE_FLAGS)
 set_target_properties(caf_mpi_static
   PROPERTIES OUTPUT_NAME caf_mpi)
 get_target_property(libcaf_static caf_mpi_static OUTPUT_NAME)
-set(CAF_LIBS "${CMAKE_INSTALL_LIBDIR}/${CMAKE_STATIC_LIBRARY_PREFIX}${libcaf_static}${CMAKE_STATIC_LIBRARY_SUFFIX}")
+set(CAF_LIBS
+  "${CMAKE_INSTALL_LIBDIR}/${CMAKE_STATIC_LIBRARY_PREFIX}${libcaf_static}${CMAKE_STATIC_LIBRARY_SUFFIX}")
 
 configure_file("${CMAKE_SOURCE_DIR}/src/extensions/caf.in" "${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_BINDIR}/caf"
   @ONLY)

From 95f93556a373efd0dfe1cde10f7acbddad714027 Mon Sep 17 00:00:00 2001
From: Izaak Beekman <zbeekman@gmail.com>
Date: Wed, 4 May 2022 18:28:19 -0400
Subject: [PATCH 07/17] Use FindMPI on Windows

The intrinsic FindMPI module seems like it has some bugs or
documentation bugs. I got this working, but it is a bit of black magic
as to how/why it works. It simplifies the code, however, and should
offload some maintanence upstream, so it's worth doing.
---
 CMakeLists.txt | 130 +++++++++++++++++++------------------------------
 1 file changed, 49 insertions(+), 81 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index b8c28e6e..1371cc1a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -278,91 +278,59 @@ if (C_COMPILER_NAME MATCHES "^[mM][pP][iI]")
 endif()
 
 if(WIN32) # Only support building with GCC & GFortran using Intel MPI (OneAPI)
-# Here we assume Intel ONEAPI and the environment is loaded
-cmake_path(SET MPI_ROOT NORMALIZE "$ENV{I_MPI_ROOT}")
-set (IMPI_INCLUDES "${MPI_ROOT}/include")
-set (IMPI_LIB_DIR "${MPI_ROOT}/lib/${IMPI_BUILD}")
-set (IMPI_DLL_DIR "${MPI_ROOT}/bin/${IMPI_BUILD}")
-set (IMPI_BIN_DIR "${MPI_ROOT}/bin")
-
-find_library(IMPI_LIB
-  "impi.lib"
-  HINTS "${IMPI_LIB_DIR}"
-  DOC "Location of the Intel MPI impi.lib file"
-  REQUIRED
-  NO_DEFAULT_PATH)
-
-set(MPI_impi_LIBRARY "${IMPI_LIB}")
-set(MPI_impicxx_LIBRARY "${IMPI_LIB_DIR}/impicxx.lib")
-
-find_file(IMPI_DLL
-  "impi.dll"
-  HINTS "${IMPI_DLL_DIR}"
-  DOC "Location of the Intel MPI impi.dll file"
-  REQUIRED
-  NO_DEFAULT_PATH)
-
-
-add_library(IMPI::MPI SHARED IMPORTED)
-set_target_properties(IMPI::MPI
-  PROPERTIES
-  IMPORTED_IMPLIB "${IMPI_LIB}"
-  IMPORTED_LOCATION "${IMPI_DLL}")
-target_include_directories(IMPI::MPI
-  INTERFACE "${IMPI_INCLUDES}")
-
-set(MPI_C_INCLUDE_DIRS "${IMPI_INCLUDES}")
-set(MPI_C_INCLUDE_PATH "${IMPI_INCLUDES}")
-set(MPI_C_HEADER_DIR "${IMPI_INCLUDES}")
-set(MPI_Fortran_COMPILER_INCLUDE_DIRS "${IMPI_INCLUDES}")
-set(MPI_Fortran_F77_HEADER_DIR "${IMPI_INCLUDES}")
-set(MPI_Fortran_INCLUDE_PATH "${IMPI_INCLUDES}")
-
-set(MPI_C_LIB_NAMES impi;impicxx)
-set(MPI_C_LIBRARIES "${IMPI_LIB}")
-set(MPI_Fortran_LIB_NAMES impi)
-set(MPI_Fortran_LIBRARIES "${IMPI_LIB}")
-
-set(MPI_Fortran_HAVE_F90_MODULE FALSE)
-set(MPI_Fortran_HAVE_F08_MODULE FALSE)
-
-find_program(MPIEXEC_EXECUTABLE
-  mpiexec
-  HINTS "${IMPI_BIN_DIR}"
-  DOC "Location of Intel MPI implementations mpiexec launcher program"
-  REQUIRED
-  NO_DEFAULT_PATH)
-set(MPIEXEC "${MPIEXEC_EXECUTABLE}")
-set(MPI_EXEC_NUMPROC_FLAG -n)
-set(MPI_EXEC_PREFLAGS -print-all-exitcodes)
-
-set(MPI_C_FOUND TRUE)
-set(MPI_Fortran_FOUND TRUE)
-
-else()
-  find_package( MPI )
-
-  if ( (NOT MPI_C_FOUND) OR (NOT MPI_Fortran_FOUND) OR (NOT MPIEXEC_EXECUTABLE))
-    # Get default install location of MPICH from install.sh
-    message(WARNING "Could not find all MPI components!")
-    message(WARNING "
+  # Here we assume Intel ONEAPI and the environment is loaded
+  set( MPI_ASSUME_NO_BUILTIN_MPI TRUE )
+  set( MPI_CXX_SKIP_MPICXX TRUE )
+  cmake_path(SET MPI_ROOT NORMALIZE "$ENV{I_MPI_ROOT}")
+  set (IMPI_LIB_DIR "${MPI_ROOT}/lib/${IMPI_BUILD}")
+  set (IMPI_DLL_DIR "${MPI_ROOT}/bin/${IMPI_BUILD}")
+
+  find_library(IMPI_LIB
+    "impi.lib"
+    HINTS "${IMPI_LIB_DIR}"
+    DOC "Location of the Intel MPI impi.lib file"
+    REQUIRED
+    NO_DEFAULT_PATH)
+
+  find_file(IMPI_DLL
+    "impi.dll"
+    HINTS "${IMPI_DLL_DIR}"
+    DOC "Location of the Intel MPI impi.dll file"
+    REQUIRED
+    NO_DEFAULT_PATH)
+
+
+  set( MPI_C_LIBRARIES ${IMPI_LIB};${IMPI_DLL} CACHE FILEPATH "MPI C libs to link" )
+  set( MPI_C_LIB_NAMES impi CACHE FILEPATH "MPI lib names" )
+  set( MPI_Fortran_LIBRARIES ${IMPI_LIB};${IMPI_DLL} CACHE FILEPATH "MPI Fortran libs to link" )
+  set( MPI_Fortran_LIB_NAMES impi CACHE FILEPATH "MPI Fortran lib names" )
+  set( MPI_impi_LIBRARY ${IMPI_LIB} CACHE FILEPATH "MPI lib to link" )
+
+  set(MPI_Fortran_HAVE_F90_MODULE FALSE)
+  set(MPI_Fortran_HAVE_F08_MODULE FALSE)
+
+endif()
+find_package( MPI )
+
+if ( (NOT MPI_C_FOUND) OR (NOT MPI_Fortran_FOUND) OR (NOT MPIEXEC_EXECUTABLE))
+  # Get default install location of MPICH from install.sh
+  message(WARNING "Could not find all MPI components!")
+  message(WARNING "
 MPI_C_FOUND        = ${MPI_C_FOUND}
 MPI_Fortran_FOUND  = ${MPI_Fortran_FOUND}
 MPIEXEC = ${MPIEXEC_EXECUTABLE}
 ")
-    execute_process( COMMAND "./install.sh" -P mpich
-      WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
-      OUTPUT_VARIABLE DEFAULT_MPICH_INSTALL_LOC
-      OUTPUT_QUIET
-      OUTPUT_STRIP_TRAILING_WHITESPACE
-      )
-    find_program (MY_MPI_EXEC NAMES mpirun mpiexec lamexec srun
-      PATHS "${DEFAULT_MPICH_INSTALL_LOC}" ENV PATH
-      HINTS "${FTN_COMPILER_DIR}" "${C_COMPILER_DIR}"
-      PATH_SUFFIXES bin)
-    set ( MPI_HOME "${MPI_HOME}" "${MY_MPI_EXEC}" "${MY_MPI_EXEC}/.." )
-    find_package( MPI REQUIRED )
-  endif()
+  execute_process( COMMAND "./install.sh" -P mpich
+    WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
+    OUTPUT_VARIABLE DEFAULT_MPICH_INSTALL_LOC
+    OUTPUT_QUIET
+    OUTPUT_STRIP_TRAILING_WHITESPACE)
+  find_program (MY_MPI_EXEC NAMES mpirun mpiexec lamexec srun
+    PATHS "${DEFAULT_MPICH_INSTALL_LOC}" ENV PATH
+    HINTS "${FTN_COMPILER_DIR}" "${C_COMPILER_DIR}"
+    PATH_SUFFIXES bin)
+  set ( MPI_HOME "${MPI_HOME}" "${MY_MPI_EXEC}" "${MY_MPI_EXEC}/.." )
+  find_package( MPI REQUIRED )
 endif()
 list(REMOVE_DUPLICATES MPI_Fortran_INCLUDE_PATH)
 

From 6e81ea4830979d7d658b210a2f2c9ae234d40c88 Mon Sep 17 00:00:00 2001
From: Izaak Beekman <zbeekman@gmail.com>
Date: Thu, 5 May 2022 11:28:15 -0400
Subject: [PATCH 08/17] Bump min. cmake ver. & modernize FindMPI

Some usage of deprecated variables provided in older version of
FindMPI were removed, updating them to the new, correct variables.

The cmake_minimum_required_version was also bumped along with CMake
policies.
---
 CMakeLists.txt           | 36 ++++++++----------------------------
 src/extensions/cafrun.in |  4 ++--
 src/mpi/CMakeLists.txt   | 24 ++++++++++++------------
 3 files changed, 22 insertions(+), 42 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 1371cc1a..dd1615b9 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1,6 +1,6 @@
-cmake_minimum_required(VERSION 3.10)
+cmake_minimum_required(VERSION 3.19)
 
-cmake_policy(VERSION 3.10...3.14)
+cmake_policy(VERSION 3.19...3.23)
 
 if(DEFINED ENV{SOURCE_DATE_EPOCH})
   set(SOURCE_DATE_EPOCH "$ENV{SOURCE_DATE_EPOCH}")
@@ -318,7 +318,7 @@ if ( (NOT MPI_C_FOUND) OR (NOT MPI_Fortran_FOUND) OR (NOT MPIEXEC_EXECUTABLE))
   message(WARNING "
 MPI_C_FOUND        = ${MPI_C_FOUND}
 MPI_Fortran_FOUND  = ${MPI_Fortran_FOUND}
-MPIEXEC = ${MPIEXEC_EXECUTABLE}
+MPIEXEC_EXECUTABLE = ${MPIEXEC_EXECUTABLE}
 ")
   execute_process( COMMAND "./install.sh" -P mpich
     WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
@@ -332,7 +332,7 @@ MPIEXEC = ${MPIEXEC_EXECUTABLE}
   set ( MPI_HOME "${MPI_HOME}" "${MY_MPI_EXEC}" "${MY_MPI_EXEC}/.." )
   find_package( MPI REQUIRED )
 endif()
-list(REMOVE_DUPLICATES MPI_Fortran_INCLUDE_PATH)
+list(REMOVE_DUPLICATES MPI_Fortran_INCLUDE_DIRS)
 
 # Test for consistent MPI environment but not on windows
 if (NOT MPIEXEC_EXECUTABLE)
@@ -344,7 +344,7 @@ else()
   add_definitions(-DHAVE_MPI)
 endif()
 if(NOT WIN32)
-  get_filename_component(MPIEXEC_RELATIVE_LOC "${MPIEXEC}"
+  get_filename_component(MPIEXEC_RELATIVE_LOC "${MPIEXEC_EXECUTABLE}"
     PROGRAM)
   get_filename_component(MPIEXEC_ABS_LOC "${MPIEXEC_RELATIVE_LOC}"
     REALPATH)
@@ -448,7 +448,7 @@ endif()
 set(OLD_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS})
 set(CMAKE_REQUIRED_FLAGS "-ffree-form" ${MPI_Fortran_COMPILE_OPTIONS} ${MPI_Fortran_COMPILE_DEFINITIONS} ${MPI_Fortran_LINK_FLAGS})
 set(OLD_INCLUDES ${CMAKE_REQUIRED_INCLUDES})
-set(CMAKE_REQUIRED_INCLUDES ${MPI_Fortran_INCLUDE_PATH})
+set(CMAKE_REQUIRED_INCLUDES ${MPI_Fortran_INCLUDE_DIRS})
 set(OLD_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES})
 set(CMAKE_REQUIRED_LIBRARIES ${MPI_Fortran_LIBRARIES})
 include (CheckFortranSourceCompiles)
@@ -480,7 +480,7 @@ unset(OLD_LIBRARIES)
 set(OLD_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS})
 set(CMAKE_REQUIRED_FLAGS "-ffree-form" ${MPI_Fortran_COMPILE_OPTIONS} ${MPI_Fortra_COMPILE_DEFINITIONS} ${MPI_Fortran_LINK_FLAGS})
 set(OLD_INCLUDES ${CMAKE_REQUIRED_INCLUDES})
-set(CMAKE_REQUIRED_INCLUDES ${MPI_Fortran_INCLUDE_PATH})
+set(CMAKE_REQUIRED_INCLUDES ${MPI_Fortran_INCLUDE_DIRS})
 set(OLD_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES})
 set(CMAKE_REQUIRED_LIBRARIES ${MPI_Fortran_LIBRARIES})
 include (CheckFortranSourceCompiles)
@@ -524,26 +524,6 @@ else()
     )
 endif()
 
-#----------------
-# Setup MPI flags
-#----------------
-if(NOT CMAKE_C_COMPILE_FLAGS)
-  set(CMAKE_C_COMPILE_FLAGS "")
-endif()
-if(NOT CMAKE_C_LINK_FLAGS)
-  set(CMAKE_C_LINK_FLAGS "")
-endif()
-if(NOT CMAKE_Fortran_COMPILE_FLAGS)
-  set(CMAKE_Fortran_COMPILE_FLAGS "")
-endif()
-if(NOT CMAKE_Fortran_LINK_FLAGS)
-  set(CMAKE_Fortran_LINK_FLAGS "")
-endif()
-set(CMAKE_C_COMPILE_FLAGS "${CMAKE_C_COMPILE_FLAGS} ${MPI_C_COMPILE_FLAGS}")
-set(CMAKE_C_LINK_FLAGS "${CMAKE_C_LINK_FLAGS} ${MPI_C_LINK_FLAGS}")
-set(CMAKE_Fortran_COMPILE_FLAGS "${CMAKE_Fortran_COMPILE_FLAGS} ${MPI_Fortran_COMPILE_FLAGS}")
-set(CMAKE_Fortran_LINK_FLAGS "${CMAKE_Fortran_LINK_FLAGS} ${MPI_Fortran_LINK_FLAGS}")
-
 #---------------------------------------------------
 # Use standardized GNU install directory conventions
 #---------------------------------------------------
@@ -570,7 +550,7 @@ endif()
 
 function(caf_compile_executable target main_depend)
   set(includes "")
-  foreach(includedir ${MPI_Fortran_INCLUDE_PATH})
+  foreach(includedir ${MPI_Fortran_INCLUDE_DIRS})
     list(APPEND includes "-I${includedir}")
   endforeach()
   string(TOUPPER "${CMAKE_BUILD_TYPE}" build_type)
diff --git a/src/extensions/cafrun.in b/src/extensions/cafrun.in
index bb7b9534..0338667d 100755
--- a/src/extensions/cafrun.in
+++ b/src/extensions/cafrun.in
@@ -50,12 +50,12 @@ set -o pipefail
 # Configured variables
 #---------------------
 #
-# CAF_VERSION, MPIEXEC, MPIEXEC_NUMPROC_FLAG, MPIEXEC_PREFLAGS, MPIEXEC_POSTFLAGS,
+# CAF_VERSION, MPIEXEC_EXECUTABLE, MPIEXEC_NUMPROC_FLAG, MPIEXEC_PREFLAGS, MPIEXEC_POSTFLAGS,
 # HAVE_FAILED_IMG
 #
 
 caf_version='@CAF_VERSION@'
-CAFRUN="@MPIEXEC@"
+CAFRUN="@MPIEXEC_EXECUTABLE@"
 if [[ "${CAFRUN}" == @*@ || -z "${CAFRUN}" ]]; then
   CAFRUN=mpiexec
 fi
diff --git a/src/mpi/CMakeLists.txt b/src/mpi/CMakeLists.txt
index 1527d096..18d9730c 100644
--- a/src/mpi/CMakeLists.txt
+++ b/src/mpi/CMakeLists.txt
@@ -31,7 +31,7 @@ target_link_libraries(opencoarrays_mod
 target_link_libraries(opencoarrays_mod
   PUBLIC caf_mpi_static)
 target_include_directories(opencoarrays_mod PUBLIC
-  $<$<COMPILE_LANGUAGE:Fortran>:${MPI_Fortran_INCLUDE_PATH}>
+  $<$<COMPILE_LANGUAGE:Fortran>:${MPI_Fortran_INCLUDE_DIRS}>
   $<BUILD_INTERFACE:${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_INCLUDEDIR}/${mod_dir_tail}>
   $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}/${mod_dir_tail}>)
 
@@ -64,11 +64,11 @@ target_link_libraries(caf_mpi_static
   PRIVATE gfortran)
 
 target_include_directories(caf_mpi PUBLIC
-  $<$<COMPILE_LANGUAGE:C>:${MPI_C_INCLUDE_PATH}>
+  $<$<COMPILE_LANGUAGE:C>:${MPI_C_INCLUDE_DIRS}>
   $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/src>
   $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>)
 target_include_directories(caf_mpi_static PUBLIC
-  $<$<COMPILE_LANGUAGE:C>:${MPI_C_INCLUDE_PATH}>
+  $<$<COMPILE_LANGUAGE:C>:${MPI_C_INCLUDE_DIRS}>
   $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/src>
   $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>)
 target_compile_options(caf_mpi
@@ -140,7 +140,7 @@ cmake_host_system_information(RESULT N_CPU QUERY NUMBER_OF_LOGICAL_CORES)
 set(N_CPU ${N_CPU} PARENT_SCOPE)
 cmake_host_system_information(RESULT HOST_NAME QUERY HOSTNAME)
 set(HOST_NAME ${HOST_NAME} PARENT_SCOPE)
-execute_process(COMMAND ${MPIEXEC} --version
+execute_process(COMMAND ${MPIEXEC_EXECUTABLE} --version
   OUTPUT_VARIABLE mpi_version_out)
 if (mpi_version_out MATCHES "[Oo]pen[ -][Mm][Pp][Ii]")
   message( STATUS "OpenMPI detected")
@@ -217,15 +217,15 @@ set(NEEDED_SYMBOLS MPIX_ERR_PROC_FAILED;MPIX_ERR_REVOKED;MPIX_Comm_failure_ack;M
 
 set(old_cmake_required_includes "${CMAKE_REQUIRED_INCLUDES}")
 if(CMAKE_REQUIRED_INCLUDES)
-  set(CMAKE_REQUIRED_INCLUDES ${CMAKE_REQUIRED_INCLUDES} ${MPI_C_INCLUDE_PATH})
+  set(CMAKE_REQUIRED_INCLUDES ${CMAKE_REQUIRED_INCLUDES} ${MPI_C_INCLUDE_DIRS})
 else()
-  set(CMAKE_REQUIRED_INCLUDES ${MPI_C_INCLUDE_PATH})
+  set(CMAKE_REQUIRED_INCLUDES ${MPI_C_INCLUDE_DIRS})
 endif()
 set(old_cmake_required_flags "${CMAKE_REQUIRED_FLAGS}")
 if(CMAKE_REQUIRED_FLAGS)
-  set(CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS} ${MPI_C_COMPILE_FLAGS} ${MPI_C_LINK_FLAGS})
+  set(CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS} ${MPI_C_COMPILE_OPTIONS} ${MPI_C_COMPILE_DEFINITIONS} ${MPI_C_LINK_FLAGS})
 else()
-  set(CMAKE_REQUIRED_FLAGS ${MPI_C_COMPILE_FLAGS} ${MPI_C_LINK_FLAGS})
+  set(CMAKE_REQUIRED_FLAGS ${MPI_C_COMPILE_OPTIONS} ${MPI_C_COMPILE_DEFINITIONS} ${MPI_C_LINK_FLAGS})
 endif()
 set(old_cmake_required_libraries "${CMAKE_REQUIRED_LIBRARIES}")
 if(CMAKE_REQUIRED_LIBRARIES)
@@ -287,9 +287,9 @@ endif()
 #---------------------------------------------------
 set(old_cmake_required_includes "${CMAKE_REQUIRED_INCLUDES}")
 if(CMAKE_REQUIRED_INCLUDES)
-  set(CMAKE_REQUIRED_INCLUDES ${CMAKE_REQUIRED_INCLUDES} ${MPI_C_INCLUDE_PATH})
+  set(CMAKE_REQUIRED_INCLUDES ${CMAKE_REQUIRED_INCLUDES} ${MPI_C_INCLUDE_DIRS})
 else()
-  set(CMAKE_REQUIRED_INCLUDES ${MPI_C_INCLUDE_PATH})
+  set(CMAKE_REQUIRED_INCLUDES ${MPI_C_INCLUDE_DIRS})
 endif()
 CHECK_INCLUDE_FILES("mpi.h" HAVE_MPI_H)
 CHECK_SYMBOL_EXISTS(I_MPI_VERSION "mpi.h" HAVE_Intel_MPI)
@@ -327,7 +327,7 @@ foreach( lflag IN LISTS MPI_Fortran_LINK_FLAGS)
 endforeach()
 string(STRIP "${CAF_MPI_Fortran_LINK_FLAGS}" CAF_MPI_Fortran_LINK_FLAGS)
 set(CAF_MPI_Fortran_COMPILE_FLAGS "")
-foreach( fcflag IN LISTS MPI_Fortran_COMPILE_FLAGS)
+foreach( fcflag IN LISTS MPI_Fortran_COMPILE_OPTIONS MPI_Fortran_COMPILE_DEFINITIONS)
   set(CAF_MPI_Fortran_COMPILE_FLAGS "${CAF_MPI_Fortran_COMPILE_FLAGS} ${fcflag}" )
 endforeach()
 string(STRIP "${CAF_MPI_Fortran_COMPILE_FLAGS}" CAF_MPI_Fortran_COMPILE_FLAGS)
@@ -343,7 +343,7 @@ configure_file("${CMAKE_SOURCE_DIR}/src/extensions/caf.in" "${CMAKE_BINARY_DIR}/
 
 # List of cafrun.in variables needing configuration:
 #
-# @CAF_VERSION@ @MPIEXEC@ @MPIEXEC_NUMPROC_FLAG@ @MPIEXEC_PREFLAGS@ @MPIEXEC_POSTFLAGS@
+# @CAF_VERSION@ @MPIEXEC_EXECUTABLE@ @MPIEXEC_NUMPROC_FLAG@ @MPIEXEC_PREFLAGS@ @MPIEXEC_POSTFLAGS@
 # @HAVE_FAILED_IMG@
 #
 

From 2ed3c4a8f5ced1c532f35db4f0fac1fa21f434af Mon Sep 17 00:00:00 2001
From: Izaak Beekman <zbeekman@gmail.com>
Date: Thu, 5 May 2022 14:59:05 -0400
Subject: [PATCH 09/17] Fix some minore trailing whitespace & lint errors

---
 src/tests/installation/installation-scripts.sh       | 4 ++--
 src/tests/installation/installation-scripts.sh-usage | 6 +++---
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/tests/installation/installation-scripts.sh b/src/tests/installation/installation-scripts.sh
index e4567ce7..b1c7b76c 100755
--- a/src/tests/installation/installation-scripts.sh
+++ b/src/tests/installation/installation-scripts.sh
@@ -129,11 +129,11 @@ function cleanup_before_exit() {
 }
 trap cleanup_before_exit EXIT # The signal is specified here. Could be SIGINT, SIGTERM etc.
 
-pushd "${OPENCOARRAYS_SRC_DIR}"/src/tests/installation
+pushd "${OPENCOARRAYS_SRC_DIR}"/src/tests/installation || exit 1
 
 # shellcheck source=../../../prerequisites/stack.sh
 source "${OPENCOARRAYS_SRC_DIR}"/prerequisites/stack.sh
 source test-stack.sh
 test_stack
 
-popd
+popd || exit 1
diff --git a/src/tests/installation/installation-scripts.sh-usage b/src/tests/installation/installation-scripts.sh-usage
index bcd5c1d2..ed60999f 100644
--- a/src/tests/installation/installation-scripts.sh-usage
+++ b/src/tests/installation/installation-scripts.sh-usage
@@ -1,11 +1,11 @@
   -b --install-branch [arg]   Install the specified repository development branch.
   -B --list-branches [arg]    List the available branches in the specified package's repository.
-  -c --with-c [arg]           Use specified C compiler. 
-  -C --with-cxx [arg]         Use specified C++ compiler. 
+  -c --with-c [arg]           Use specified C compiler.
+  -C --with-cxx [arg]         Use specified C++ compiler.
   -d --debug                  Enable debug mode.
   -D --print-downloader [arg] Print download program for package specified in argument.
   -e --verbose                Enable verbose mode, print script as it is executed.
-  -f --with-fortran [arg]     Use specified Fortran compiler. 
+  -f --with-fortran [arg]     Use specified Fortran compiler.
   -h --help                   Print this page.
   -i --install-prefix [arg]   Install package in specified path. Default="${OPENCOARRAYS_SRC_DIR}/prerequisites/installations/${package_name:-}/${version_to_build:-}"
   -I --install-version [arg]  Install package version.

From 6b2c49aa867100dd128ee4cdd0e2da90da4f004c Mon Sep 17 00:00:00 2001
From: Izaak Beekman <zbeekman@gmail.com>
Date: Thu, 5 May 2022 15:00:21 -0400
Subject: [PATCH 10/17] Add caf & cafrun batchfile wrapper scripts on windows

These scripts get configured to call the bash version found during
CMake configuration time. Using this version/installation of bash, the
batch files call the corresponding bash caf & cafrun scripts and pass
all arguments through. This *should* allow things like fpm to work
with the caf wrapper scripts, *I beleive* (untested)

CAVEAT: WSL's bash will dump you into your WSL home directory and
there's no obvious work around. If you put Git-Bash's bash.exe (in the
bin subdirectory) first on your path you *should* be in business (I
think). There may be weird edge cases.
---
 src/extensions/caf.bat.in    |  4 ++++
 src/extensions/cafrun.bat.in |  4 ++++
 src/mpi/CMakeLists.txt       | 17 ++++++++++++++++-
 3 files changed, 24 insertions(+), 1 deletion(-)
 create mode 100644 src/extensions/caf.bat.in
 create mode 100644 src/extensions/cafrun.bat.in

diff --git a/src/extensions/caf.bat.in b/src/extensions/caf.bat.in
new file mode 100644
index 00000000..7840959d
--- /dev/null
+++ b/src/extensions/caf.bat.in
@@ -0,0 +1,4 @@
+@REM Batch file wrapper script to call the caf bash script on windows
+@REM %~dp0 is the directory of the batch script. This assumes caf.bat in same dir as caf
+
+"@BASH_EXECUTABLE@" "%~dp0caf" %*
diff --git a/src/extensions/cafrun.bat.in b/src/extensions/cafrun.bat.in
new file mode 100644
index 00000000..8bb219da
--- /dev/null
+++ b/src/extensions/cafrun.bat.in
@@ -0,0 +1,4 @@
+@REM Batch file wrapper script to call the cafrun bash script on windows
+@REM %~dp0 is the directory of the batch script. This assumes cafrun.bat in same dir as cafrun
+
+"@BASH_EXECUTABLE@" "%~dp0cafrun" %*
diff --git a/src/mpi/CMakeLists.txt b/src/mpi/CMakeLists.txt
index 18d9730c..2dcee0ea 100644
--- a/src/mpi/CMakeLists.txt
+++ b/src/mpi/CMakeLists.txt
@@ -356,10 +356,25 @@ endif()
 configure_file("${CMAKE_SOURCE_DIR}/src/extensions/cafrun.in" "${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_BINDIR}/cafrun"
   @ONLY)
 
-install(PROGRAMS "${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_BINDIR}/cafrun"
+install(PROGRAMS
+  "${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_BINDIR}/cafrun"
   "${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_BINDIR}/caf"
   DESTINATION "${CMAKE_INSTALL_FULL_BINDIR}")
 
+if(WIN32)
+  # Add the batch script wrappers
+  configure_file(
+    "${CMAKE_SOURCE_DIR}/src/extensions/caf.bat.in" "${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_BINDIR}/caf.bat"
+    @ONLY)
+  configure_file(
+    "${CMAKE_SOURCE_DIR}/src/extensions/cafrun.bat.in" "${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_BINDIR}/cafrun.bat"
+    @ONLY)
+  install(PROGRAMS
+    "${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_BINDIR}/cafrun.bat"
+    "${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_BINDIR}/caf.bat"
+    DESTINATION "${CMAKE_INSTALL_FULL_BINDIR}")
+endif()
+
 lint_script("${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_BINDIR}" caf)
 check_script_style("${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_BINDIR}/caf")
 lint_script("${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_BINDIR}" cafrun)

From a426966a0e0fc2a305192680a5b31b4a527cc226 Mon Sep 17 00:00:00 2001
From: Izaak Beekman <zbeekman@gmail.com>
Date: Thu, 5 May 2022 15:07:14 -0400
Subject: [PATCH 11/17] Make CMake, project build & test more robust on Windows

If you have Git-Bash's bash.exe in the front of your path,
you can now do everything from within the Intel OneAPI CMD.exe shell.
If you experience issues you should launch git-bash from the OneAPI
CMD.exe shell and it will inherit your environment variables and be
more robust. If CMake finds WSL, you're probably hosed.
---
 CMakeLists.txt                        | 10 +++++++---
 cmake/AddInstallationScriptTest.cmake |  9 ++++++---
 2 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index dd1615b9..eb58eb7a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -548,6 +548,10 @@ if(APPLE)
   find_program(CODESIGN codesign)
 endif()
 
+# We need bash, even on windows, for now at least. A light version like Git-Bash works best,
+# WSL dumps you into your home directory and there's nothing you can do about it.
+find_program(BASH_EXECUTABLE bash
+  REQUIRED)
 function(caf_compile_executable target main_depend)
   set(includes "")
   foreach(includedir ${MPI_Fortran_INCLUDE_DIRS})
@@ -561,7 +565,7 @@ function(caf_compile_executable target main_depend)
     list(APPEND localDefs "-D${d}")
   endforeach()
   add_custom_command(OUTPUT "${target}"
-    COMMAND "${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_BINDIR}/caf"
+    COMMAND "${BASH_EXECUTABLE}" "${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_BINDIR}/caf"
             ${includes} ${localDefs} ${config_Fortran_flags}
             -o "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${target}"
             "${CMAKE_CURRENT_SOURCE_DIR}/${main_depend}"
@@ -720,9 +724,9 @@ function(add_caf_test name num_caf_img test_target)
   endif()
   set(test_parameters -np ${num_caf_img} ${test_parameters})
   if(DEFINED ARGN)
-    add_test(NAME ${name} COMMAND "bash" "${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_BINDIR}/cafrun" ${test_parameters} "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${test_target}" ${ARGN})
+    add_test(NAME ${name} COMMAND "${BASH_EXECUTABLE}" "${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_BINDIR}/cafrun" ${test_parameters} "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${test_target}" ${ARGN})
   else()
-    add_test(NAME ${name} COMMAND "bash" "${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_BINDIR}/cafrun" ${test_parameters} "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${test_target}")
+    add_test(NAME ${name} COMMAND "${BASH_EXECUTABLE}" "${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_BINDIR}/cafrun" ${test_parameters} "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${test_target}")
   endif()
   set_property(TEST ${name} PROPERTY PASS_REGULAR_EXPRESSION "Test passed.")
 endfunction(add_caf_test)
diff --git a/cmake/AddInstallationScriptTest.cmake b/cmake/AddInstallationScriptTest.cmake
index ecf764bc..22464b34 100644
--- a/cmake/AddInstallationScriptTest.cmake
+++ b/cmake/AddInstallationScriptTest.cmake
@@ -8,7 +8,10 @@ macro(add_installation_script_test name path)
   file( COPY "${CMAKE_CURRENT_SOURCE_DIR}/${path}/${name}-usage" DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/${path}"
     FILE_PERMISSIONS OWNER_READ OWNER_WRITE GROUP_READ WORLD_READ
   )
-  add_test(NAME test-${name} COMMAND "${CMAKE_BINARY_DIR}/${path}/${name}")
-  set_property(TEST test-${name} PROPERTY WORKING_DIRECTORY "${CMAKE_BINARY_DIR}/${path}")
-  set_property(TEST test-${name} PROPERTY ENVIRONMENT "OPENCOARRAYS_SRC_DIR=${CMAKE_SOURCE_DIR}")
+  if(EXISTS ${BASH_EXECUTABLE})
+    message(STATUS "Bash executable found: ${BASH_EXECUTABLE}")
+    add_test(NAME test-${name} COMMAND "${BASH_EXECUTABLE}" "${CMAKE_BINARY_DIR}/${path}/${name}")
+    set_property(TEST test-${name} PROPERTY WORKING_DIRECTORY "${CMAKE_BINARY_DIR}/${path}")
+    set_property(TEST test-${name} PROPERTY ENVIRONMENT "OPENCOARRAYS_SRC_DIR=${CMAKE_SOURCE_DIR}")
+  endif()
 endmacro(add_installation_script_test)

From abd921c905f45864884add03afcf84021b8dda5c Mon Sep 17 00:00:00 2001
From: Izaak Beekman <zbeekman@gmail.com>
Date: Thu, 5 May 2022 15:10:40 -0400
Subject: [PATCH 12/17] More FindMPI and try_compile() CMake modernization

A bunch of the variable manipulation should be abstracted as a
function.
---
 CMakeLists.txt         | 18 +++++++++++++++---
 src/mpi/CMakeLists.txt |  3 +++
 2 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index eb58eb7a..4edf1103 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -139,6 +139,9 @@ endif()
 if(NOT CMAKE_REQUIRED_FLAGS)
   set(CMAKE_REQUIRED_FLAGS "")
 endif()
+if(NOT CMAKE_REQUIRED_DEFINITIONS)
+  set(CMAKE_REQUIRED_DEFINITIONS "")
+endif()
 if(NOT CMAKE_REQUIRED_LIBRARIES)
   set(CMAKE_REQUIRED_LIBRARIES "")
 endif()
@@ -403,7 +406,9 @@ endif()
 # Make sure a simple "hello world" C mpi program compiles
 #--------------------------------------------------------
 set(OLD_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS})
-set(CMAKE_REQUIRED_FLAGS ${MPI_C_COMPILE_OPTIONS} ${MPI_C_COMPILE_DEFINITIONS} ${MPI_C_LINK_FLAGS})
+set(CMAKE_REQUIRED_FLAGS ${MPI_C_COMPILE_OPTIONS} ${MPI_C_LINK_FLAGS})
+set(OLD_REQUIRED_DEFINITIONS ${CMAKE_REQUIRED_DEFINITIONS})
+set(CMAKE_REQUIRED_DEFINITIONS ${MPI_C_COMPILE_DEFINITIONS})
 set(OLD_INCLUDES ${CMAKE_REQUIRED_INCLUDES})
 set(CMAKE_REQUIRED_INCLUDES ${MPI_C_INCLUDE_DIRS})
 set(OLD_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES})
@@ -427,6 +432,7 @@ int main(int argc, char** argv) {
 }"
 MPI_C_COMPILES)
 set(CMAKE_REQUIRED_FLAGS ${OLD_REQUIRED_FLAGS})
+set(CMAKE_REQUIRED_DEFINITIONS ${OLD_REQUIRED_DEFINITIONS})
 set(CMAKE_REQUIRED_INCLUDES ${OLD_INCLUDES})
 set(CMAKE_REQUIRED_LIBRARIES ${OLD_LIBRARIES})
 unset(OLD_REQUIRED_FLAGS)
@@ -446,7 +452,9 @@ endif()
 # Try using mpi.mod first then fall back on includ 'mpif.h'
 #--------------------------------------------------------------
 set(OLD_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS})
-set(CMAKE_REQUIRED_FLAGS "-ffree-form" ${MPI_Fortran_COMPILE_OPTIONS} ${MPI_Fortran_COMPILE_DEFINITIONS} ${MPI_Fortran_LINK_FLAGS})
+set(CMAKE_REQUIRED_FLAGS "-ffree-form" ${MPI_Fortran_COMPILE_OPTIONS} ${MPI_Fortran_LINK_FLAGS})
+set(OLD_REQUIRED_DEFINITIONS ${CMAKE_REQUIRED_DEFINITIONS})
+set(CMAKE_REQUIRED_DEFINITIONS ${MPI_Fortran_COMPILE_DEFINITIONS})
 set(OLD_INCLUDES ${CMAKE_REQUIRED_INCLUDES})
 set(CMAKE_REQUIRED_INCLUDES ${MPI_Fortran_INCLUDE_DIRS})
 set(OLD_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES})
@@ -468,6 +476,7 @@ end program
 "
 MPI_Fortran_MODULE_COMPILES)
 set(CMAKE_REQUIRED_FLAGS ${OLD_REQUIRED_FLAGS})
+set(CMAKE_REQUIRED_DEFINITIONS ${OLD_REQUIRED_DEFINITIONS})
 set(CMAKE_REQUIRED_INCLUDES ${OLD_INCLUDES})
 set(CMAKE_REQUIRED_LIBRARIES ${OLD_LIBRARIES})
 unset(OLD_REQUIRED_FLAGS)
@@ -478,7 +487,9 @@ unset(OLD_LIBRARIES)
 # If that failed try using mpif.h
 #--------------------------------
 set(OLD_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS})
-set(CMAKE_REQUIRED_FLAGS "-ffree-form" ${MPI_Fortran_COMPILE_OPTIONS} ${MPI_Fortra_COMPILE_DEFINITIONS} ${MPI_Fortran_LINK_FLAGS})
+set(CMAKE_REQUIRED_FLAGS "-ffree-form" ${MPI_Fortran_COMPILE_OPTIONS} ${MPI_Fortran_LINK_FLAGS})
+set(OLD_REQUIRED_DEFINITIONS ${CMAKE_REQUIRED_DEFINITIONS})
+set(CMAKE_REQUIRED_DEFINITIONS ${MPI_Fortra_COMPILE_DEFINITIONS})
 set(OLD_INCLUDES ${CMAKE_REQUIRED_INCLUDES})
 set(CMAKE_REQUIRED_INCLUDES ${MPI_Fortran_INCLUDE_DIRS})
 set(OLD_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES})
@@ -500,6 +511,7 @@ end program
 "
   MPI_Fortran_INCLUDE_COMPILES)
 set(CMAKE_REQUIRED_FLAGS ${OLD_REQUIRED_FLAGS})
+set(CMAKE_REQUIRED_DEFINITIONS ${OLD_REQUIRED_DEFINITIONS})
 set(CMAKE_REQUIRED_INCLUDES ${OLD_INCLUDES})
 set(CMAKE_REQUIRED_LIBRARIES ${OLD_LIBRARIES})
 unset(OLD_REQUIRED_FLAGS)
diff --git a/src/mpi/CMakeLists.txt b/src/mpi/CMakeLists.txt
index 2dcee0ea..38eb47f9 100644
--- a/src/mpi/CMakeLists.txt
+++ b/src/mpi/CMakeLists.txt
@@ -2,6 +2,9 @@
 if(NOT CMAKE_REQUIRED_FLAGS)
   set(CMAKE_REQUIRED_FLAGS "")
 endif()
+if(NOT CMAKE_REQUIRED_DEFINITIONS)
+  set(CMAKE_REQUIRED_DEFINITIONS "")
+endif()
 if(NOT CMAKE_REQUIRED_LIBRARIES)
   set(CMAKE_REQUIRED_LIBRARIES "")
 endif()

From 084e1aeeddbeacf02b86b5d5979a0f27c539ec00 Mon Sep 17 00:00:00 2001
From: Izaak Beekman <zbeekman@gmail.com>
Date: Thu, 5 May 2022 15:11:35 -0400
Subject: [PATCH 13/17] Make sure we have perl before trying to run style
 tests, especially on windows. Also invoke perl directly so style/linting on
 windows works.

---
 CMakeLists.txt | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 4edf1103..11e42a92 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -621,10 +621,11 @@ endfunction()
 #-----------------------------------------------
 
 find_program(style_pl style.pl "${CMAKE_SOURCE_DIR}/developer-scripts/")
+find_package(Perl)
 function(check_script_style script_full_path)
-  if(style_pl)
+  if(style_pl AND PERL_FOUND)
     add_test(NAME "style:${script_full_path}"
-      COMMAND "${style_pl}" "${script_full_path}")
+      COMMAND "${PERL_EXECUTABLE}" "${style_pl}" "${script_full_path}")
   endif()
 endfunction()
 

From 9ab99abd512e533048276272b78dd551d5a2db89 Mon Sep 17 00:00:00 2001
From: Izaak Beekman <zbeekman@gmail.com>
Date: Thu, 5 May 2022 15:59:02 -0400
Subject: [PATCH 14/17] Update Windows build & installation instructions

---
 INSTALL.md | 69 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 69 insertions(+)

diff --git a/INSTALL.md b/INSTALL.md
index eb3150b9..dc90afad 100644
--- a/INSTALL.md
+++ b/INSTALL.md
@@ -125,6 +125,71 @@ source.
 
 ### Windows ###
 
+Windows installation support has recently been enhanced. The new installation
+method doesn't require Cygwin or WSL and relies only a light weight git/bash
+distribution like Git-Bash, and the Intel OneAPI base distribution as well as
+the Intel OneAPI HPC Toolkit. This allows native and redistributable statically
+linked binaries to be built if you use the appropriate compiler flags. Clients
+will only require the free redistributable Intel MPI runtime.
+
+#### New and Recommended Windows Installation ####
+
+Please follow these instructions, this is the recommended way to install
+OpenCoarrays on Windows.
+
+##### Pre-requisits #####
+
+* [CMake]
+* [Git-Bash]
+* [GCC, GFortran, Unix Make]
+* [Intel OneAPI Base] and [Intel OneAPI HPC toolkit]
+
+##### Installation Procedure #####
+
+1. Ensure all the prerequisites have been installed following the directions of
+   the provider.
+2. Launch the 64-bit Intel OneAPI developer shell. This will either be in the
+   start menu, or you may need to figure out how to launch it manually. See the
+   note below for help.
+3. Ensure that cmake, git-bash, gfortran, gcc, make are available on your default
+   path. The installers should have taken care of this for you.
+5. Launch a Git-Bash bash shell from the OneAPI environment, or ensure Git-Bash's
+   `bash.exe` is first on your path by either:
+     1. Typing `git-bash` from within the windows `CMD.exe` style OneAPI shell, or
+     2. Editing your system wide `%PATH%` variable, or updating it in the local shell
+        so that git-bash's `bin` subdirectory (containing `bash.exe`) is at the front
+        of your `PATH`: `set PATH=C:\path\to\Git-Bash\bin;%PATH%`.
+6. At this point typing `bash --version` should show you you're using Git-Bash, *AND*
+   typing `bash -c pwd` should match the current directory. __`bash` provided with
+   WSL will show the directory as your home directore, something like `/home/username`
+   and this WILL cause problems later__
+7. cd to the extracted OpenCoarrays source directory (obtained by downloading a
+   release archive, or, for advanced useres, cheked out with git etc.)
+8. Create a build directory and change to it with something like:
+   `mkdir build` and `cd build`
+9. Set GCC and GFortran as the default C and Fortran compilers with something like:
+   `set FC=gfortran` and `set CC=gcc`
+10. Check that Intel's MPI is setup correctly by typing `mpifc -show` which should
+    show you how the compiler wrapper script is planning to build MPI programs.
+    (It's fine if this shows `ifort` being used as the compiler instead of `gfortran`
+    at this point.)
+11. Configure Open-Coarrays with CMake and specify the installation directory you would
+    like to use. (If this is not a user-writable location, you will have to open a new
+    shell later as an administrator to perform the installation step.) Run:
+    `cmake -G "Unix Makefiles" -DCMAKE_INSTALL_PREFIX="C:\path\to\desired\install\location" ".\path\to\opencoarrays-source-dir"`
+    The quotes ensure paths with spaces are handled properly. You may omit them if there
+    are no spaces.
+12. When the configuration completes, build with:
+    `cmake --build . -j <ncores>`
+13. Install OpenCoarrays: `cmake --build . -j <ncores> -t install`
+14. Optionally run the test suite. There are a few known failures on Windows.
+    Also, a large numberof firewall warnings will pop up.
+    `cmake --build . -t check`
+
+#### Old Procedure ####
+
+__NOTE: Use this method at your own risk, it is no longer officially supported!__
+
 Windows users may run the [`install.sh`] script inside the Windows Subsystem
 for Linux ([WSL]). The script uses Ubuntu's [APT] package manager to build
 [GCC] 5.4.0, [CMake], and [MPICH].  Windows users who desire a newer version
@@ -448,6 +513,10 @@ file.
 [INSTALL.pdf]: https://md2pdf.herokuapp.com/sourceryinstitute/OpenCoarrays/blob/master/INSTALL.pdf
 [INSTALL.md]: https://github.com/sourceryinstitute/OpenCoarrays/blob/master/INSTALL.md
 [CMake]: https://cmake.org
+[Git-Bash]: https://gitforwindows.org/
+[GCC, GFortran, Unix Make]: http://www.equation.com/servlet/equation.cmd?fa=programminglog
+[Intel OneAPI Base]: https://www.intel.com/content/www/us/en/developer/tools/oneapi/base-toolkit.htm
+[Intel OneAPI HPC toolkit]: https://www.intel.com/content/www/us/en/developer/tools/oneapi/hpc-toolkit.html
 [Sourcery Institute Store]: http://www.sourceryinstitute.org/store/c1/Featured_Products.html
 [VirtualBox]: https://www.virtualbox.org
 [download and installation instructions]: http://www.sourceryinstitute.org/uploads/4/9/9/6/49967347/overview.pdf

From 02ec7924ad9a8d469f263c419b11030b32459068 Mon Sep 17 00:00:00 2001
From: Izaak Beekman <zbeekman@gmail.com>
Date: Thu, 5 May 2022 16:09:34 -0400
Subject: [PATCH 15/17] Update INSTALL.md with the note I forgot

---
 INSTALL.md | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/INSTALL.md b/INSTALL.md
index dc90afad..04e10f0e 100644
--- a/INSTALL.md
+++ b/INSTALL.md
@@ -186,6 +186,20 @@ OpenCoarrays on Windows.
     Also, a large numberof firewall warnings will pop up.
     `cmake --build . -t check`
 
+__NOTE About the Intel OneAPI CMD.exe shell__
+
+If you don't have a "Intel oneAPI command prompt for Intel 64 for Visual Studio 20xx"
+in your start menu you can figure out how to launch an equivalent `CMD.exe` or
+Powershell instance by examining the shortcut Target that it created for me:
+
+```
+%ComSpec% /E:ON /K ""C:\Program Files (x86)\Intel\oneAPI\setvars.bat" intel64 vs2022"
+```
+
+So it looks like you can launch `CMD.exe` and then run the appropriate `setvars.bat`
+script with the `intel64` and `vs20<xx>` arguments where `vs20<xx>` matches the
+appropriate Visual Studio version.
+
 #### Old Procedure ####
 
 __NOTE: Use this method at your own risk, it is no longer officially supported!__

From eef9548ef88f4f6f4089f90f2483123eaa56d646 Mon Sep 17 00:00:00 2001
From: "Izaak \"Zaak\" Beekman" <zbeekman@gmail.com>
Date: Fri, 6 May 2022 22:07:40 -0400
Subject: [PATCH 16/17] Fix typos & style improvements from code review

Thanks to @rouson and @everythingfunctional for the review and great suggestions!

Co-authored-by: Damian Rouson <rouson@lbl.gov>
Co-authored-by: Brad Richardson <everythingfunctional@protonmail.com>
---
 INSTALL.md | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/INSTALL.md b/INSTALL.md
index 04e10f0e..94c1bd91 100644
--- a/INSTALL.md
+++ b/INSTALL.md
@@ -137,7 +137,7 @@ will only require the free redistributable Intel MPI runtime.
 Please follow these instructions, this is the recommended way to install
 OpenCoarrays on Windows.
 
-##### Pre-requisits #####
+##### Pre-requisites #####
 
 * [CMake]
 * [Git-Bash]
@@ -161,10 +161,10 @@ OpenCoarrays on Windows.
         of your `PATH`: `set PATH=C:\path\to\Git-Bash\bin;%PATH%`.
 6. At this point typing `bash --version` should show you you're using Git-Bash, *AND*
    typing `bash -c pwd` should match the current directory. __`bash` provided with
-   WSL will show the directory as your home directore, something like `/home/username`
-   and this WILL cause problems later__
-7. cd to the extracted OpenCoarrays source directory (obtained by downloading a
-   release archive, or, for advanced useres, cheked out with git etc.)
+   WSL will show the directory as your home directory, something like `/home/username`
+   and this *will* cause problems later__
+7. `cd` to the extracted OpenCoarrays source directory (obtained by downloading a
+   release archive, or, for advanced useres, checked out with `git`, etc.)
 8. Create a build directory and change to it with something like:
    `mkdir build` and `cd build`
 9. Set GCC and GFortran as the default C and Fortran compilers with something like:

From e8bc08e75bf091f88906dd50a3d17334448fd934 Mon Sep 17 00:00:00 2001
From: Izaak Beekman <zbeekman@gmail.com>
Date: Fri, 6 May 2022 22:20:28 -0400
Subject: [PATCH 17/17] Add missing comma in INSTALL.md

---
 INSTALL.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/INSTALL.md b/INSTALL.md
index 94c1bd91..55fe4200 100644
--- a/INSTALL.md
+++ b/INSTALL.md
@@ -159,7 +159,7 @@ OpenCoarrays on Windows.
      2. Editing your system wide `%PATH%` variable, or updating it in the local shell
         so that git-bash's `bin` subdirectory (containing `bash.exe`) is at the front
         of your `PATH`: `set PATH=C:\path\to\Git-Bash\bin;%PATH%`.
-6. At this point typing `bash --version` should show you you're using Git-Bash, *AND*
+6. At this point, typing `bash --version` should show you you're using Git-Bash, *AND*
    typing `bash -c pwd` should match the current directory. __`bash` provided with
    WSL will show the directory as your home directory, something like `/home/username`
    and this *will* cause problems later__