Merge branch '5.0.x' into extension-run

.github/workflows/container_tests_apptainer.yml

@@ -29,10 +29,18 @@ jobs:
     - name: install OS & Python packages
       run: |
         # for building CentOS 7 container images
-        sudo apt-get install rpm
-        sudo apt-get install dnf
+        APT_PKGS="rpm dnf"
         # for modules tool
-        sudo apt-get install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev
+        APT_PKGS+=" lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev"
+
+        # Avoid apt-get update, as we don't really need it,
+        # and it does more harm than good (it's fairly expensive, and it results in flaky test runs)
+        if ! sudo apt-get install $APT_PKGS; then
+          # Try to update cache, then try again to resolve 404s of old packages
+          sudo apt-get update -yqq || true
+          sudo apt-get install $APT_PKGS
+        fi
+
         # fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082
         # needed for Ubuntu 18.04, but not for Ubuntu 20.04, so skipping symlinking if posix.so already exists
         if [ ! -e /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so ] ; then

.github/workflows/unit_tests.yml

@@ -14,8 +14,6 @@ jobs:
     runs-on: ubuntu-20.04
     outputs:
         lmod8: Lmod-8.7.6
-        modulesTcl: modules-tcl-1.147
-        modules3: modules-3.2.10
         modules4: modules-4.1.4
     steps:
       - run: "true"
@@ -29,8 +27,6 @@ jobs:
           # use variables defined by 'setup' job above, see also
           # https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#needs-context
           - ${{needs.setup.outputs.lmod8}}
-          - ${{needs.setup.outputs.modulesTcl}}
-          - ${{needs.setup.outputs.modules3}}
           - ${{needs.setup.outputs.modules4}}
         lc_all: [""]
         include:
@@ -156,11 +152,7 @@ jobs:
           export PYTHONPATH=$PREFIX/lib/python${{matrix.python}}/site-packages:$PYTHONPATH
           eb --version
           # tell EasyBuild which modules tool is available
-          if [[ ${{matrix.modules_tool}} =~ ^modules-tcl- ]]; then
-            export EASYBUILD_MODULES_TOOL=EnvironmentModulesTcl
-          elif [[ ${{matrix.modules_tool}} =~ ^modules-3 ]]; then
-            export EASYBUILD_MODULES_TOOL=EnvironmentModulesC
-          elif [[ ${{matrix.modules_tool}} =~ ^modules-4 ]]; then
+          if [[ ${{matrix.modules_tool}} =~ ^modules-4 ]]; then
             export EASYBUILD_MODULES_TOOL=EnvironmentModules
           else
             export EASYBUILD_MODULES_TOOL=Lmod
@@ -174,7 +166,7 @@ jobs:
               echo "Not testing with '${module_syntax}' as module syntax with '${EASYBUILD_MODULES_TOOL}' as modules tool"
               continue
             fi
-            printf '\n\n=====================> Using $module_syntax module syntax <=====================\n\n'
+            printf "\n\n=====================> Using $module_syntax module syntax <=====================\n\n"
             export EASYBUILD_MODULE_SYNTAX="${module_syntax}"
             export TEST_EASYBUILD_MODULE_SYNTAX="${EASYBUILD_MODULE_SYNTAX}"
 

easybuild/framework/easyblock.py

@@ -41,7 +41,7 @@
 * Davide Vanzo (Vanderbilt University)
 * Caspar van Leeuwen (SURF)
 """
-
+import concurrent
 import copy
 import glob
 import inspect
@@ -52,7 +52,9 @@
 import tempfile
 import time
 import traceback
+from concurrent.futures import ThreadPoolExecutor
 from datetime import datetime
+from textwrap import indent
 
 import easybuild.tools.environment as env
 import easybuild.tools.toolchain as toolchain
@@ -87,7 +89,7 @@
 from easybuild.tools.hooks import MODULE_STEP, MODULE_WRITE, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTITER_STEP
 from easybuild.tools.hooks import POSTPROC_STEP, PREPARE_STEP, READY_STEP, SANITYCHECK_STEP, SOURCE_STEP
 from easybuild.tools.hooks import SINGLE_EXTENSION, TEST_STEP, TESTCASES_STEP, load_hooks, run_hook
-from easybuild.tools.run import RunShellCmdError, check_async_cmd, run_cmd, run_shell_cmd
+from easybuild.tools.run import RunShellCmdError, raise_run_shell_cmd_error, run_shell_cmd
 from easybuild.tools.jenkins import write_to_xml
 from easybuild.tools.module_generator import ModuleGeneratorLua, ModuleGeneratorTcl, module_generator, dependencies_for
 from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version
@@ -952,7 +954,8 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
                     if download_instructions is None:
                         download_instructions = self.cfg['download_instructions']
                     if download_instructions is not None and download_instructions != "":
-                        msg = "\nDownload instructions:\n\n" + download_instructions + '\n'
+                        msg = "\nDownload instructions:\n\n" + indent(download_instructions, '    ') + '\n\n'
+                        msg += "Make the files available in the active source path: %s\n" % ':'.join(source_paths())
                         print_msg(msg, prefix=False, stderr=True)
                         error_msg += "please follow the download instructions above, and make the file available "
                         error_msg += "in the active source path (%s)" % ':'.join(source_paths())
@@ -1818,41 +1821,31 @@ def skip_extensions_parallel(self, exts_filter):
         self.log.experimental("Skipping installed extensions in parallel")
         print_msg("skipping installed extensions (in parallel)", log=self.log)
 
-        async_cmd_info_cache = {}
-        running_checks_ids = []
         installed_exts_ids = []
-        exts_queue = list(enumerate(self.ext_instances[:]))
         checked_exts_cnt = 0
         exts_cnt = len(self.ext_instances)
+        cmds = [resolve_exts_filter_template(exts_filter, ext) for ext in self.ext_instances]
+
+        with ThreadPoolExecutor(max_workers=self.cfg['parallel']) as thread_pool:
 
-        # asynchronously run checks to see whether extensions are already installed
-        while exts_queue or running_checks_ids:
+            # list of command to run asynchronously
+            async_cmds = [thread_pool.submit(run_shell_cmd, cmd, stdin=stdin, hidden=True, fail_on_error=False,
+                                             asynchronous=True, task_id=idx) for (idx, (cmd, stdin)) in enumerate(cmds)]
 
-            # first handle completed checks
-            for idx in running_checks_ids[:]:
+            # process result of commands as they have completed running
+            for done_task in concurrent.futures.as_completed(async_cmds):
+                res = done_task.result()
+                idx = res.task_id
                 ext_name = self.ext_instances[idx].name
-                # don't read any output, just check whether command completed
-                async_cmd_info = check_async_cmd(*async_cmd_info_cache[idx], output_read_size=0, fail_on_error=False)
-                if async_cmd_info['done']:
-                    out, ec = async_cmd_info['output'], async_cmd_info['exit_code']
-                    self.log.info("exts_filter result for %s: exit code %s; output: %s", ext_name, ec, out)
-                    running_checks_ids.remove(idx)
-                    if ec == 0:
-                        print_msg("skipping extension %s" % ext_name, log=self.log)
-                        installed_exts_ids.append(idx)
-
-                    checked_exts_cnt += 1
-                    exts_pbar_label = "skipping installed extensions "
-                    exts_pbar_label += "(%d/%d checked)" % (checked_exts_cnt, exts_cnt)
-                    self.update_exts_progress_bar(exts_pbar_label)
-
-            # start additional checks asynchronously
-            while exts_queue and len(running_checks_ids) < self.cfg['parallel']:
-                idx, ext = exts_queue.pop(0)
-                cmd, stdin = resolve_exts_filter_template(exts_filter, ext)
-                async_cmd_info_cache[idx] = run_cmd(cmd, log_all=False, log_ok=False, simple=False, inp=stdin,
-                                                    regexp=False, trace=False, asynchronous=True)
-                running_checks_ids.append(idx)
+                self.log.info(f"exts_filter result for {ext_name}: exit code {res.exit_code}; output: {res.output}")
+                if res.exit_code == 0:
+                    print_msg(f"skipping extension {ext_name}", log=self.log)
+                    installed_exts_ids.append(idx)
+
+                checked_exts_cnt += 1
+                exts_pbar_label = "skipping installed extensions "
+                exts_pbar_label += "(%d/%d checked)" % (checked_exts_cnt, exts_cnt)
+                self.update_exts_progress_bar(exts_pbar_label)
 
         # compose new list of extensions, skip over the ones that are already installed;
         # note: original order in extensions list should be preserved!
@@ -1965,6 +1958,8 @@ def install_extensions_parallel(self, install=True):
         """
         self.log.info("Installing extensions in parallel...")
 
+        thread_pool = ThreadPoolExecutor(max_workers=self.cfg['parallel'])
+
         running_exts = []
         installed_ext_names = []
 
@@ -2001,16 +1996,23 @@ def update_exts_progress_bar_helper(running_exts, progress_size):
 
             # check for extension installations that have completed
             if running_exts:
-                self.log.info("Checking for completed extension installations (%d running)...", len(running_exts))
+                self.log.info(f"Checking for completed extension installations ({len(running_exts)} running)...")
                 for ext in running_exts[:]:
-                    if self.dry_run or ext.async_cmd_check():
-                        self.log.info("Installation of %s completed!", ext.name)
-                        ext.install_extension_substep("post_install_extension")
-                        running_exts.remove(ext)
-                        installed_ext_names.append(ext.name)
-                        update_exts_progress_bar_helper(running_exts, 1)
+                    if self.dry_run or ext.async_cmd_task.done():
+                        res = ext.async_cmd_task.result()
+                        if res.exit_code == 0:
+                            self.log.info(f"Installation of extension {ext.name} completed!")
+                            # run post-install method for extension from same working dir as installation of extension
+                            cwd = change_dir(res.work_dir)
+                            ext.install_extension_substep("post_install_extension")
+                            change_dir(cwd)
+                            running_exts.remove(ext)
+                            installed_ext_names.append(ext.name)
+                            update_exts_progress_bar_helper(running_exts, 1)
+                        else:
+                            raise_run_shell_cmd_error(res)
                     else:
-                        self.log.debug("Installation of %s is still running...", ext.name)
+                        self.log.debug(f"Installation of extension {ext.name} is still running...")
 
             # try to start as many extension installations as we can, taking into account number of available cores,
             # but only consider first 100 extensions still in the queue
@@ -2077,9 +2079,9 @@ def update_exts_progress_bar_helper(running_exts, progress_size):
                                           rpath_filter_dirs=self.rpath_filter_dirs)
                     if install:
                         ext.install_extension_substep("pre_install_extension")
-                        ext.install_extension_substep("install_extension_async")
+                        ext.async_cmd_task = ext.install_extension_substep("install_extension_async", thread_pool)
                         running_exts.append(ext)
-                        self.log.info("Started installation of extension %s in the background...", ext.name)
+                        self.log.info(f"Started installation of extension {ext.name} in the background...")
                         update_exts_progress_bar_helper(running_exts, 0)
 
             # print progress info after every iteration (unless that info is already shown via progress bar)
@@ -2092,6 +2094,8 @@ def update_exts_progress_bar_helper(running_exts, progress_size):
                     running_ext_names = ', '.join(x.name for x in running_exts[:3]) + ", ..."
                 print_msg(msg % (installed_cnt, exts_cnt, queued_cnt, running_cnt, running_ext_names), log=self.log)
 
+        thread_pool.shutdown()
+
     #
     # MISCELLANEOUS UTILITY FUNCTIONS
     #
@@ -3038,12 +3042,6 @@ def post_install_step(self):
         - run post install commands if any were specified
         """
 
-        self.run_post_install_commands()
-        self.apply_post_install_patches()
-        self.print_post_install_messages()
-
-        self.fix_shebang()
-
         lib_dir = os.path.join(self.installdir, 'lib')
         lib64_dir = os.path.join(self.installdir, 'lib64')
 
@@ -3064,6 +3062,12 @@ def post_install_step(self):
                 # create *relative* 'lib' symlink to 'lib64';
                 symlink('lib64', lib_dir, use_abspath_source=False)
 
+        self.run_post_install_commands()
+        self.apply_post_install_patches()
+        self.print_post_install_messages()
+
+        self.fix_shebang()
+
     def sanity_check_step(self, *args, **kwargs):
         """
         Do a sanity check on the installation

easybuild/framework/easyconfig/easyconfig.py

@@ -121,7 +121,7 @@ def new_ec_method(self, key, *args, **kwargs):
         if key in DEPRECATED_PARAMETERS:
             depr_key = key
             key, ver = DEPRECATED_PARAMETERS[depr_key]
-            _log.deprecated("Easyconfig parameter '%s' is deprecated, use '%s' instead." % (depr_key, key), ver)
+            _log.deprecated("Easyconfig parameter '%s' is deprecated, use '%s' instead" % (depr_key, key), ver)
         if key in REPLACED_PARAMETERS:
             _log.nosupport("Easyconfig parameter '%s' is replaced by '%s'" % (key, REPLACED_PARAMETERS[key]), '2.0')
         return ec_method(self, key, *args, **kwargs)
@@ -179,7 +179,7 @@ def triage_easyconfig_params(variables, ec):
 
     for key in variables:
         # validations are skipped, just set in the config
-        if key in ec:
+        if key in ec or key in DEPRECATED_PARAMETERS.keys():
             ec_params[key] = variables[key]
             _log.debug("setting config option %s: value %s (type: %s)", key, ec_params[key], type(ec_params[key]))
         elif key in REPLACED_PARAMETERS:
@@ -658,7 +658,7 @@ def set_keys(self, params):
         with self.disable_templating():
             for key in sorted(params.keys()):
                 # validations are skipped, just set in the config
-                if key in self._config.keys():
+                if key in self._config.keys() or key in DEPRECATED_PARAMETERS.keys():
                     self[key] = params[key]
                     self.log.info("setting easyconfig parameter %s: value %s (type: %s)",
                                   key, self[key], type(self[key]))
@@ -792,7 +792,7 @@ def local_var_naming(self, local_var_naming_check):
             msg = "Use of %d unknown easyconfig parameters detected %s: %s\n" % (cnt, in_fn, unknown_keys_msg)
             msg += "If these are just local variables please rename them to start with '%s', " % LOCAL_VAR_PREFIX
             msg += "or try using --fix-deprecated-easyconfigs to do this automatically.\nFor more information, see "
-            msg += "https://easybuild.readthedocs.io/en/latest/Easyconfig-files-local-variables.html ."
+            msg += "https://docs.easybuild.io/easyconfig-files-local-variables/ ."
 
             # always log a warning if local variable that don't follow recommended naming scheme are found
             self.log.warning(msg)
@@ -830,7 +830,7 @@ def check_deprecated(self, path):
             depr_maj_ver = int(str(VERSION).split('.')[0]) + 1
             depr_ver = '%s.0' % depr_maj_ver
 
-            more_info_depr_ec = " (see also http://easybuild.readthedocs.org/en/latest/Deprecated-easyconfigs.html)"
+            more_info_depr_ec = " (see also https://docs.easybuild.io/deprecated-easyconfigs)"
 
             self.log.deprecated(depr_msg, depr_ver, more_info=more_info_depr_ec, silent=build_option('silent'))
 

easybuild/framework/easyconfig/templates.py

@@ -97,6 +97,9 @@
     ('cuda_cc_cmake', "List of CUDA compute capabilities suitable for use with $CUDAARCHS in CMake 3.18+"),
     ('cuda_cc_space_sep', "Space-separated list of CUDA compute capabilities"),
     ('cuda_cc_semicolon_sep', "Semicolon-separated list of CUDA compute capabilities"),
+    ('cuda_int_comma_sep', "Comma-separated list of integer CUDA compute capabilities"),
+    ('cuda_int_space_sep', "Space-separated list of integer CUDA compute capabilities"),
+    ('cuda_int_semicolon_sep', "Semicolon-separated list of integer CUDA compute capabilities"),
     ('cuda_sm_comma_sep', "Comma-separated list of sm_* values that correspond with CUDA compute capabilities"),
     ('cuda_sm_space_sep', "Space-separated list of sm_* values that correspond with CUDA compute capabilities"),
 ]
@@ -365,6 +368,10 @@ def template_constant_dict(config, ignore=None, toolchain=None):
         template_values['cuda_cc_space_sep'] = ' '.join(cuda_compute_capabilities)
         template_values['cuda_cc_semicolon_sep'] = ';'.join(cuda_compute_capabilities)
         template_values['cuda_cc_cmake'] = ';'.join(cc.replace('.', '') for cc in cuda_compute_capabilities)
+        int_values = [cc.replace('.', '') for cc in cuda_compute_capabilities]
+        template_values['cuda_int_comma_sep'] = ','.join(int_values)
+        template_values['cuda_int_space_sep'] = ' '.join(int_values)
+        template_values['cuda_int_semicolon_sep'] = ';'.join(int_values)
         sm_values = ['sm_' + cc.replace('.', '') for cc in cuda_compute_capabilities]
         template_values['cuda_sm_comma_sep'] = ','.join(sm_values)
         template_values['cuda_sm_space_sep'] = ' '.join(sm_values)

easybuild/framework/easyconfig/tools.py

@@ -694,7 +694,9 @@ def check_sha256_checksums(ecs, whitelist=None):
             continue
 
         eb_class = get_easyblock_class(ec['easyblock'], name=ec['name'])
-        checksum_issues.extend(eb_class(ec).check_checksums())
+        eb = eb_class(ec)
+        checksum_issues.extend(eb.check_checksums())
+        eb.close_log()
 
     return checksum_issues
 

easybuild/framework/easyconfig/types.py

@@ -31,7 +31,6 @@
 * Caroline De Brouwer (Ghent University)
 * Kenneth Hoste (Ghent University)
 """
-from distutils.util import strtobool
 
 from easybuild.base import fancylogger
 from easybuild.framework.easyconfig.format.format import DEPENDENCY_PARAMETERS
@@ -280,7 +279,14 @@ def to_toolchain_dict(spec):
             res = {'name': spec[0].strip(), 'version': spec[1].strip()}
         # 3-element list
         elif len(spec) == 3:
-            res = {'name': spec[0].strip(), 'version': spec[1].strip(), 'hidden': strtobool(spec[2].strip())}
+            hidden = spec[2].strip().lower()
+            if hidden in {'yes', 'true', 't', 'y', '1', 'on'}:
+                hidden = True
+            elif hidden in {'no', 'false', 'f', 'n', '0', 'off'}:
+                hidden = False
+            else:
+                raise EasyBuildError("Invalid truth value %s", hidden)
+            res = {'name': spec[0].strip(), 'version': spec[1].strip(), 'hidden': hidden}
         else:
             raise EasyBuildError("Can not convert list %s to toolchain dict. Expected 2 or 3 elements", spec)