Merge pull request #4673 from knelli2/charm_mpi_ci

Use mpi-smp charm on CI

.github/workflows/Tests.yaml

@@ -321,7 +321,6 @@ jobs:
         build_type: [Debug, Release]
         use_pch: [ON]
         use_xsimd: [ON]
-        charm_name: ["Charm 7.0.0"]
         include:
           # Configure ccache sizes. The cache becomes ineffective if it can't
           # hold at least one full build. The sizes for the build configurations
@@ -380,11 +379,15 @@ jobs:
             BUILD_SHARED_LIBS: OFF
             use_xsimd: OFF
             MEMORY_ALLOCATOR: JEMALLOC
-          # Test compatibility with oldest supported CMake version
+          # Test compatibility with oldest supported CMake version and using an
+          # mpi version of charm
           - compiler: clang-13
             use_pch: ON
             build_type: Debug
             CMAKE_EXECUTABLE: /opt/local/cmake/3.18.2/bin/cmake
+            CHARM_ROOT: /work/charm_7_0_0/mpi-linux-x86_64-smp-clang
+            # MPI running tests is a bit slower than multicore
+            TEST_TIMEOUT_FACTOR: 3
 
     container:
       image: sxscollaboration/spectre:ci
@@ -467,6 +470,7 @@ ${{ env.CACHE_KEY_SUFFIX }}"
           BUILD_SHARED_LIBS=${{ matrix.BUILD_SHARED_LIBS }}
           PYTHON_EXECUTABLE=${{ matrix.PYTHON_EXECUTABLE }}
           CMAKE_EXECUTABLE=${{ matrix.CMAKE_EXECUTABLE }}
+          MATRIX_CHARM_ROOT=${{ matrix.CHARM_ROOT }}
           ASAN=${{ matrix.ASAN }}
           MEMORY_ALLOCATOR=${{ matrix.MEMORY_ALLOCATOR }}
           UBSAN_UNDEFINED=${{ matrix.UBSAN_UNDEFINED }}
@@ -482,7 +486,7 @@ ${{ env.CACHE_KEY_SUFFIX }}"
           -D CMAKE_Fortran_COMPILER=${FC}
           -D CMAKE_CXX_FLAGS="${CXXFLAGS} ${{ matrix.EXTRA_CXX_FLAGS }}"
           -D OVERRIDE_ARCH=x86-64
-          -D CHARM_ROOT=${CHARM_ROOT}
+          -D CHARM_ROOT=${MATRIX_CHARM_ROOT:-$CHARM_ROOT}
           -D CMAKE_BUILD_TYPE=${{ matrix.build_type }}
           -D DEBUG_SYMBOLS=OFF
           -D UNIT_TESTS_IN_TEST_EXECUTABLES=OFF

containers/Dockerfile.buildenv

@@ -246,8 +246,8 @@ RUN git clone --single-branch --branch v7.0.0 --depth 1 \
 
 # - Set the environment variable SPECTRE_CONTAINER so we can check if we are
 #   inside a container (0 is true in bash)
-# - Set CHARM_ROOT so it's easy to find the Charm++ directory. Just use GCC
-#   since clang is GCC-compatible
+# - Set CHARM_ROOT so it's easy to find the Charm++ directory. Just use the
+#   multicore build.
 # - Symlink gfortran-9 to gfortran so it we can just specify gfortran in CMake
 # - The singularity containers work better if the locale is set properly
 ENV SPECTRE_CONTAINER 0
@@ -319,10 +319,17 @@ RUN wget https://github.com/Kitware/CMake/releases/download/v3.18.2/cmake-3.18.2
     && make install \
     && cd .. && rm -rf cmake*
 
-# We build both Clang and GCC versions of Charm++ so that all our tests can
-# use the same build environment.
+# Install OpenMPI 4.0.3
+RUN apt-get install -y openmpi-bin
+
+# We build an mpi version of charm (with clang) because many of our production
+# environments have charm built with mpi and we should test that.
 WORKDIR /work/charm_7_0_0
-RUN ./build LIBS multicore-linux-x86_64 clang \
+# This env variable was found to be necessary when building charm with OpenMPI
+# 4.0.3 which ships with Ubuntu 20.04. For more details see this issue:
+# https://bugs.launchpad.net/ubuntu/+source/openmpi/+bug/1941786
+ENV LDFLAGS="-lopen-pal"
+RUN ./build LIBS mpi-linux-x86_64-smp clang \
       ${PARALLEL_MAKE_ARG} -g -O2 --build-shared\
     && rm -rf /work/charm_7_0_0/doc /work/charm_7_0_0/examples
 

docs/Installation/Installation.md

@@ -369,7 +369,7 @@ Follow these steps:
    build directory.
 2. Determine the location of your Charm++ installation. In the Docker container
    it is `/work/charm_7_0_0/multicore-linux-x86_64-gcc` for GCC builds and
-   `/work/charm_7_0_0/multicore-linux-x86_64-clang` for clang builds. For Spack
+   `/work/charm_7_0_0/mpi-linux-x86_64-smp-clang` for clang builds. For Spack
    installations you can determine it with
    `spack location --install-dir charmpp`. We refer to the install directory as
    `CHARM_ROOT` below.