Merge branch '2023.06-software.eessi.io' into 2023.06-software-imagem…

…agick-foss-2022b

.github/workflows/tests.yml

@@ -22,7 +22,7 @@ jobs:
 
     - name: install Python packages
       run: |
-        pip install archspec
+        pip install archspec==0.2.2
 
     - name: test eessi_software_subdir.py script
       run: |

.github/workflows/tests_init.yml

@@ -22,7 +22,7 @@ jobs:
 
     - name: install Python packages
       run: |
-        pip install archspec pytest
+        pip install archspec==0.2.2 pytest
 
     - name: unit tests for eessi_software_subdir_for_host.py script
       run:

check_missing_installations.sh

@@ -18,6 +18,14 @@ easystack=$1
 
 LOCAL_TMPDIR=$(mktemp -d)
 
+# Clone the develop branch of EasyBuild and use that to search for easyconfigs
+git clone -b develop https://github.com/easybuilders/easybuild-easyconfigs.git $LOCAL_TMPDIR/easyconfigs
+export EASYBUILD_ROBOT_PATHS=$LOCAL_TMPDIR/easyconfigs/easybuild/easyconfigs
+
+# All PRs used in EESSI are supposed to be merged, so we can strip out all cases of from-pr
+tmp_easystack=${LOCAL_TMPDIR}/$(basename ${easystack})
+grep -v from-pr ${easystack} > ${tmp_easystack} 
+
 source $TOPDIR/scripts/utils.sh
 
 source $TOPDIR/configure_easybuild
@@ -27,7 +35,7 @@ ${EB:-eb} --show-config
 
 echo ">> Checking for missing installations in ${EASYBUILD_INSTALLPATH}..."
 eb_missing_out=$LOCAL_TMPDIR/eb_missing.out
-${EB:-eb} --easystack ${easystack} --missing 2>&1 | tee ${eb_missing_out}
+${EB:-eb} --easystack ${tmp_easystack} --missing 2>&1 | tee ${eb_missing_out}
 exit_code=${PIPESTATUS[0]}
 
 ok_msg="Command 'eb --missing ...' succeeded, analysing output..."

create_lmodrc.py

@@ -29,7 +29,7 @@
     return content
 end
 
-local function cuda_enabled_load_hook(t)
+local function eessi_cuda_enabled_load_hook(t)
     local frameStk  = require("FrameStk"):singleton()
     local mt        = frameStk:mt()
     local simpleName = string.match(t.modFullName, "(.-)/")
@@ -94,7 +94,35 @@
     end
 end
 
-hook.register("load", cuda_enabled_load_hook)
+local function eessi_openmpi_load_hook(t)
+    -- disable smcuda BTL when loading OpenMPI module for aarch64/neoverse_v1,
+    -- to work around hang/crash due to bug in OpenMPI;
+    -- see https://gitlab.com/eessi/support/-/issues/41
+    local frameStk = require("FrameStk"):singleton()
+    local mt = frameStk:mt()
+    local moduleName = string.match(t.modFullName, "(.-)/")
+    local cpuTarget = os.getenv("EESSI_SOFTWARE_SUBDIR") or ""
+    if (moduleName == "OpenMPI") and (cpuTarget == "aarch64/neoverse_v1") then
+        local msg = "Adding '^smcuda' to $OMPI_MCA_btl to work around bug in OpenMPI"
+        LmodMessage(msg .. " (see https://gitlab.com/eessi/support/-/issues/41)")
+	local ompiMcaBtl = os.getenv("OMPI_MCA_btl")
+	if ompiMcaBtl == nil then
+            setenv("OMPI_MCA_btl", "^smcuda")
+        else
+            setenv("OMPI_MCA_btl", ompiMcaBtl .. ",^smcuda")
+	end
+    end
+end
+
+-- Combine both functions into a single one, as we can only register one function as load hook in lmod
+-- Also: make it non-local, so it can be imported and extended by other lmodrc files if needed
+function eessi_load_hook(t)
+    eessi_cuda_enabled_load_hook(t)
+    eessi_openmpi_load_hook(t)
+end
+
+
+hook.register("load", eessi_load_hook)
 """
 
 def error(msg):

easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml

@@ -1,4 +1,10 @@
 easyconfigs:
   - SciPy-bundle-2023.02-gfbf-2022b.eb
   - GDAL-3.6.2-foss-2022b.eb
-  - ImageMagick-7.1.0-53-GCCcore-12.2.0.eb
+  - waLBerla-6.1-foss-2022b.eb:
+      options:
+        from-pr: 19324
+  - WRF-4.4.1-foss-2022b-dmpar.eb
+  - ImageMagick-7.1.0-53-GCCcore-12.2.0.eb:
+      options:
+        from-pr: 20086

easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml

@@ -15,3 +15,25 @@ easyconfigs:
       options:
         from-pr: 19573
   - scikit-learn-1.3.1-gfbf-2023a.eb
+  - snakemake-8.4.2-foss-2023a.eb:
+      options:
+        from-pr: 19646
+  - LAMMPS-2Aug2023_update2-foss-2023a-kokkos.eb:
+      # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19471
+      # see https://github.com/easybuilders/easybuild-easyblocks/pull/3036
+      options:
+        from-pr: 19471
+        include-easyblocks-from-pr: 3036
+  - matplotlib-3.7.2-gfbf-2023a.eb
+  - PyQt5-5.15.10-GCCcore-12.3.0.eb:
+      # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19554
+      options:
+        from-pr: 19554
+  - Pillow-SIMD-9.5.0-GCCcore-12.3.0.eb:
+      options:
+        # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19996
+        from-pr: 19996
+  - dask-2023.9.2-foss-2023a.eb
+  - OSU-Micro-Benchmarks-7.2-gompi-2023a-CUDA-12.1.1.eb
+  - OSU-Micro-Benchmarks-7.2-gompi-2023b.eb
+  - JupyterNotebook-7.0.2-GCCcore-12.3.0.eb

easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml

@@ -8,3 +8,38 @@ easyconfigs:
   - matplotlib-3.8.2-gfbf-2023b.eb:
       options:
         from-pr: 19552
+  - AOFlagger-3.4.0-foss-2023b.eb:
+      options:
+        from-pr: 19840
+        include-easyblocks-from-pr: 3088
+  - arpack-ng-3.9.0-foss-2023b.eb:
+      options:
+        from-pr: 19840
+        include-easyblocks-from-pr: 3088
+  - Armadillo-12.8.0-foss-2023b.eb:
+      options:
+        from-pr: 19840
+        include-easyblocks-from-pr: 3088
+  - casacore-3.5.0-foss-2023b.eb:
+      options:
+        from-pr: 19840
+        include-easyblocks-from-pr: 3088
+  - IDG-1.2.0-foss-2023b.eb:
+      options:
+        from-pr: 19840
+        include-easyblocks-from-pr: 3088
+  - EveryBeam-0.5.2-foss-2023b.eb:
+      options:
+        from-pr: 19840
+        include-easyblocks-from-pr: 3088
+  - DP3-6.0-foss-2023b.eb:
+      options:
+        from-pr: 19840
+        include-easyblocks-from-pr: 3088
+  - WSClean-3.4-foss-2023b.eb:
+      options:
+        from-pr: 19840
+        include-easyblocks-from-pr: 3088
+  - CDO-2.2.2-gompi-2023b.eb:
+      options:
+        from-pr: 19792

eb_hooks.py

@@ -21,7 +21,7 @@
 
 
 CPU_TARGET_NEOVERSE_V1 = 'aarch64/neoverse_v1'
-CPU_TARGET_AARCH64_GENERIC = 'aarch64/generic' 
+CPU_TARGET_AARCH64_GENERIC = 'aarch64/generic'
 
 EESSI_RPATH_OVERRIDE_ATTR = 'orig_rpath_override_dirs'
 
@@ -160,6 +160,32 @@ def post_prepare_hook(self, *args, **kwargs):
         POST_PREPARE_HOOKS[self.name](self, *args, **kwargs)
 
 
+def parse_hook_casacore_disable_vectorize(ec, eprefix):
+    """
+    Disable 'vectorize' toolchain option for casacore 3.5.0 on aarch64/neoverse_v1
+    Compiling casacore 3.5.0 with GCC 13.2.0 (foss-2023b) gives an error when building for aarch64/neoverse_v1.
+    See also, https://github.com/EESSI/software-layer/pull/479
+    """
+    if ec.name == 'casacore':
+        tcname, tcversion = ec['toolchain']['name'], ec['toolchain']['version']
+        if (
+            LooseVersion(ec.version) == LooseVersion('3.5.0') and
+            tcname == 'foss' and tcversion == '2023b'
+        ):
+            cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR')
+            if cpu_target == CPU_TARGET_NEOVERSE_V1:
+                if not hasattr(ec, 'toolchainopts'):
+                    ec['toolchainopts'] = {}
+                ec['toolchainopts']['vectorize'] = False
+                print_msg("Changed toochainopts for %s: %s", ec.name, ec['toolchainopts'])
+            else:
+                print_msg("Not changing option vectorize for %s on non-neoverse_v1", ec.name)
+        else:
+            print_msg("Not changing option vectorize for %s %s %s", ec.name, ec.version, ec.toolchain)
+    else:
+        raise EasyBuildError("casacore-specific hook triggered for non-casacore easyconfig?!")
+
+
 def parse_hook_cgal_toolchainopts_precise(ec, eprefix):
     """Enable 'precise' rather than 'strict' toolchain option for CGAL on POWER."""
     if ec.name == 'CGAL':
@@ -246,6 +272,23 @@ def parse_hook_ucx_eprefix(ec, eprefix):
         raise EasyBuildError("UCX-specific hook triggered for non-UCX easyconfig?!")
 
 
+def parse_hook_lammps_remove_deps_for_CI_aarch64(ec, *args, **kwargs):
+    """
+    Remove x86_64 specific dependencies for the CI to pass on aarch64
+    """
+    if ec.name == 'LAMMPS' and ec.version in ('2Aug2023_update2',):
+        if os.getenv('EESSI_CPU_FAMILY') == 'aarch64':
+            # ScaFaCoS and tbb are not compatible with aarch64/* CPU targets,
+            # so remove them as dependencies for LAMMPS (they're optional);
+            # see also https://github.com/easybuilders/easybuild-easyconfigs/pull/19164 +
+            # https://github.com/easybuilders/easybuild-easyconfigs/pull/19000;
+            # we need this hook because we check for missing installations for all CPU targets
+            # on an x86_64 VM in GitHub Actions (so condition based on ARCH in LAMMPS easyconfig is always true)
+            ec['dependencies'] = [dep for dep in ec['dependencies'] if dep[0] not in ('ScaFaCoS', 'tbb')]
+    else:
+        raise EasyBuildError("LAMMPS-specific hook triggered for non-LAMMPS easyconfig?!")
+
+
 def pre_configure_hook(self, *args, **kwargs):
     """Main pre-configure hook: trigger custom functions based on software name."""
     if self.name in PRE_CONFIGURE_HOOKS:
@@ -310,32 +353,14 @@ def pre_configure_hook_wrf_aarch64(self, *args, **kwargs):
             if LooseVersion(self.version) <= LooseVersion('3.9.0'):
                     self.cfg.update('preconfigopts', "sed -i 's/%s/%s/g' arch/configure_new.defaults && " % (pattern, repl))
                     print_msg("Using custom preconfigopts for %s: %s", self.name, self.cfg['preconfigopts'])
-                    
+
             if LooseVersion('4.0.0') <= LooseVersion(self.version) <= LooseVersion('4.2.1'):
                     self.cfg.update('preconfigopts', "sed -i 's/%s/%s/g' arch/configure.defaults && " % (pattern, repl))
                     print_msg("Using custom preconfigopts for %s: %s", self.name, self.cfg['preconfigopts'])
     else:
         raise EasyBuildError("WRF-specific hook triggered for non-WRF easyconfig?!")
 
 
-def pre_configure_hook_LAMMPS_aarch64(self, *args, **kwargs):
-    """
-    pre-configure hook for LAMMPS:
-    - set kokkos_arch on Aarch64
-    """
-
-    cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR')
-    if self.name == 'LAMMPS':
-        if self.version == '23Jun2022':
-            if  get_cpu_architecture() == AARCH64:
-                if cpu_target == CPU_TARGET_AARCH64_GENERIC:
-                    self.cfg['kokkos_arch'] = 'ARM80'
-                else:
-                    self.cfg['kokkos_arch'] = 'ARM81'
-    else:
-        raise EasyBuildError("LAMMPS-specific hook triggered for non-LAMMPS easyconfig?!")
-
-
 def pre_configure_hook_atspi2core_filter_ld_library_path(self, *args, **kwargs):
     """
     pre-configure hook for at-spi2-core:
@@ -415,7 +440,7 @@ def pre_test_hook_ignore_failing_tests_netCDF(self, *args, **kwargs):
     """
     cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR')
     if self.name == 'netCDF' and self.version == '4.9.2' and cpu_target == CPU_TARGET_NEOVERSE_V1:
-        self.cfg['testopts'] = "|| echo ignoring failing tests" 
+        self.cfg['testopts'] = "|| echo ignoring failing tests"
 
 def pre_test_hook_increase_max_failed_tests_arm_PyTorch(self, *args, **kwargs):
     """
@@ -580,12 +605,14 @@ def inject_gpu_property(ec):
 
 
 PARSE_HOOKS = {
+    'casacore': parse_hook_casacore_disable_vectorize,
     'CGAL': parse_hook_cgal_toolchainopts_precise,
     'fontconfig': parse_hook_fontconfig_add_fonts,
     'OpenBLAS': parse_hook_openblas_relax_lapack_tests_num_errors,
     'pybind11': parse_hook_pybind11_replace_catch2,
     'Qt5': parse_hook_qt5_check_qtwebengine_disable,
     'UCX': parse_hook_ucx_eprefix,
+    'LAMMPS': parse_hook_lammps_remove_deps_for_CI_aarch64,
 }
 
 POST_PREPARE_HOOKS = {
@@ -597,7 +624,6 @@ def inject_gpu_property(ec):
     'MetaBAT': pre_configure_hook_metabat_filtered_zlib_dep,
     'OpenBLAS': pre_configure_hook_openblas_optarch_generic,
     'WRF': pre_configure_hook_wrf_aarch64,
-    'LAMMPS': pre_configure_hook_LAMMPS_aarch64,
     'at-spi2-core': pre_configure_hook_atspi2core_filter_ld_library_path,
 }