From 6b779256c85c89a3aa32b357c16efa1ff881f442 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 21 Apr 2021 08:17:29 +0200
Subject: [PATCH 001/175] add support for installing extensions in parallel

---
 easybuild/framework/easyblock.py | 196 +++++++++++++++++++++++++------
 easybuild/framework/extension.py |  51 +++++++-
 2 files changed, 210 insertions(+), 37 deletions(-)

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index 77e0df72a5..4b3777db34 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -1620,6 +1620,166 @@ def skip_extensions(self):
 
         self.ext_instances = res
 
+    def install_extensions(self, install=True, parallel=False):
+        """
+        Install extensions.
+
+        :param install: actually install extensions, don't just prepare environment for installing
+        :param parallel: install extensions in parallel
+
+        """
+        self.log.debug("List of loaded modules: %s", self.modules_tool.list())
+
+        if parallel:
+            self.install_extensions_parallel(install=install)
+        else:
+            self.install_extensions_sequential(install=install)
+
+    def install_extensions_sequential(self, install=True):
+        """
+        Install extensions sequentially.
+
+        :param install: actually install extensions, don't just prepare environment for installing
+        """
+        self.log.info("Installing extensions sequentially...")
+
+        exts_cnt = len(self.ext_instances)
+        for idx, ext in enumerate(self.ext_instances):
+
+            self.log.debug("Starting extension %s" % ext.name)
+
+            # always go back to original work dir to avoid running stuff from a dir that no longer exists
+            change_dir(self.orig_workdir)
+
+            tup = (ext.name, ext.version or '', idx + 1, exts_cnt)
+            print_msg("installing extension %s %s (%d/%d)..." % tup, silent=self.silent, log=self.log)
+
+            if self.dry_run:
+                tup = (ext.name, ext.version, ext.__class__.__name__)
+                msg = "\n* installing extension %s %s using '%s' easyblock\n" % tup
+                self.dry_run_msg(msg)
+
+            self.log.debug("List of loaded modules: %s", self.modules_tool.list())
+
+            # prepare toolchain build environment, but only when not doing a dry run
+            # since in that case the build environment is the same as for the parent
+            if self.dry_run:
+                self.dry_run_msg("defining build environment based on toolchain (options) and dependencies...")
+            else:
+                # don't reload modules for toolchain, there is no need since they will be loaded already;
+                # the (fake) module for the parent software gets loaded before installing extensions
+                ext.toolchain.prepare(onlymod=self.cfg['onlytcmod'], silent=True, loadmod=False,
+                                      rpath_filter_dirs=self.rpath_filter_dirs)
+
+            # real work
+            if install:
+                ext.prerun()
+                txt = ext.run()
+                if txt:
+                    self.module_extra_extensions += txt
+                ext.postrun()
+
+    def install_extensions_parallel(self, install=True):
+        """
+        Install extensions in parallel.
+
+        :param install: actually install extensions, don't just prepare environment for installing
+        """
+        self.log.info("Installing extensions in parallel...")
+
+        running_exts = []
+        installed_ext_names = []
+
+        all_ext_names = [x['name'] for x in self.exts_all]
+        self.log.debug("List of names of all extensions: %s", all_ext_names)
+
+        # take into account that some extensions may be installed already
+        to_install_ext_names = [x.name for x in self.ext_instances]
+        installed_ext_names = [n for n in all_ext_names if n not in to_install_ext_names]
+
+        exts_cnt = len(all_ext_names)
+        exts_queue = self.ext_instances[:]
+
+        iter_id = 0
+        while exts_queue or running_exts:
+
+            iter_id += 1
+
+            # always go back to original work dir to avoid running stuff from a dir that no longer exists
+            change_dir(self.orig_workdir)
+
+            # check for extension installations that have completed
+            if running_exts:
+                self.log.info("Checking for completed extension installations (%d running)...", len(running_exts))
+                for ext in running_exts[:]:
+                    if self.dry_run or ext.async_cmd_check():
+                        self.log.info("Installation of %s completed!", ext.name)
+                        ext.postrun()
+                        running_exts.remove(ext)
+                        installed_ext_names.append(ext.name)
+                    else:
+                        self.log.debug("Installation of %s is still running...", ext.name)
+
+            # print progress info every now and then
+            if iter_id % 1 == 0:
+                msg = "%d out of %d extensions installed (%d queued, %d running: %s)"
+                installed_cnt, queued_cnt, running_cnt = len(installed_ext_names), len(exts_queue), len(running_exts)
+                if running_cnt <= 3:
+                    running_ext_names = ', '.join(x.name for x in running_exts)
+                else:
+                    running_ext_names = ', '.join(x.name for x in running_exts[:3]) + ", ..."
+                print_msg(msg % (installed_cnt, exts_cnt, queued_cnt, running_cnt, running_ext_names), log=self.log)
+
+            # try to start as many extension installations as we can, taking into account number of available cores,
+            # but only consider first 100 extensions still in the queue
+            max_iter = min(100, len(exts_queue))
+
+            for _ in range(max_iter):
+
+                if not (exts_queue and len(running_exts) < self.cfg['parallel']):
+                    break
+
+                # check whether extension at top of the queue is ready to install
+                ext = exts_queue.pop(0)
+
+                pending_deps = [x for x in ext.required_deps if x not in installed_ext_names]
+
+                if self.dry_run:
+                    tup = (ext.name, ext.version, ext.__class__.__name__)
+                    msg = "\n* installing extension %s %s using '%s' easyblock\n" % tup
+                    self.dry_run_msg(msg)
+                    running_exts.append(ext)
+
+                # if some of the required dependencies are not installed yet, requeue this extension
+                elif pending_deps:
+
+                    # make sure all required dependencies are actually going to be installed,
+                    # to avoid getting stuck in an infinite loop!
+                    missing_deps = [x for x in ext.required_deps if x not in all_ext_names]
+                    if missing_deps:
+                        raise EasyBuildError("Missing required dependencies for %s are not going to be installed: %s",
+                                             ext.name, ', '.join(missing_deps))
+                    else:
+                        self.log.info("Required dependencies missing for extension %s (%s), adding it back to queue...",
+                                      ext.name, ', '.join(pending_deps))
+                        # purposely adding extension back in the queue at Nth place rather than at the end,
+                        # since we assume that the required dependencies will be installed soon...
+                        exts_queue.insert(max_iter, ext)
+
+                else:
+                    tup = (ext.name, ext.version or '')
+                    print_msg("starting installation of extension %s %s..." % tup, silent=self.silent, log=self.log)
+
+                    # don't reload modules for toolchain, there is no need since they will be loaded already;
+                    # the (fake) module for the parent software gets loaded before installing extensions
+                    ext.toolchain.prepare(onlymod=self.cfg['onlytcmod'], silent=True, loadmod=False,
+                                          rpath_filter_dirs=self.rpath_filter_dirs)
+                    if install:
+                        ext.prerun()
+                        ext.run(asynchronous=True)
+                        running_exts.append(ext)
+                        self.log.debug("Started installation of extension %s in the background...", ext.name)
+
     #
     # MISCELLANEOUS UTILITY FUNCTIONS
     #
@@ -2318,41 +2478,7 @@ def extensions_step(self, fetch=False, install=True):
         if self.skip:
             self.skip_extensions()
 
-        exts_cnt = len(self.ext_instances)
-        for idx, ext in enumerate(self.ext_instances):
-
-            self.log.debug("Starting extension %s" % ext.name)
-
-            # always go back to original work dir to avoid running stuff from a dir that no longer exists
-            change_dir(self.orig_workdir)
-
-            tup = (ext.name, ext.version or '', idx + 1, exts_cnt)
-            print_msg("installing extension %s %s (%d/%d)..." % tup, silent=self.silent)
-
-            if self.dry_run:
-                tup = (ext.name, ext.version, cls.__name__)
-                msg = "\n* installing extension %s %s using '%s' easyblock\n" % tup
-                self.dry_run_msg(msg)
-
-            self.log.debug("List of loaded modules: %s", self.modules_tool.list())
-
-            # prepare toolchain build environment, but only when not doing a dry run
-            # since in that case the build environment is the same as for the parent
-            if self.dry_run:
-                self.dry_run_msg("defining build environment based on toolchain (options) and dependencies...")
-            else:
-                # don't reload modules for toolchain, there is no need since they will be loaded already;
-                # the (fake) module for the parent software gets loaded before installing extensions
-                ext.toolchain.prepare(onlymod=self.cfg['onlytcmod'], silent=True, loadmod=False,
-                                      rpath_filter_dirs=self.rpath_filter_dirs)
-
-            # real work
-            if install:
-                ext.prerun()
-                txt = ext.run()
-                if txt:
-                    self.module_extra_extensions += txt
-                ext.postrun()
+        self.install_extensions(install=install)
 
         # cleanup (unload fake module, remove fake module dir)
         if fake_mod_data:
diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py
index 569a3bb414..6b70afd966 100644
--- a/easybuild/framework/extension.py
+++ b/easybuild/framework/extension.py
@@ -40,7 +40,7 @@
 from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP, template_constant_dict
 from easybuild.tools.build_log import EasyBuildError, raise_nosupport
 from easybuild.tools.filetools import change_dir
-from easybuild.tools.run import run_cmd
+from easybuild.tools.run import complete_cmd, get_output_from_process, run_cmd
 from easybuild.tools.py2vs3 import string_type
 
 
@@ -138,6 +138,12 @@ def __init__(self, mself, ext, extra_params=None):
                                key, name, version, value)
 
         self.sanity_check_fail_msgs = []
+        self.async_cmd_info = None
+        self.async_cmd_output = None
+        self.async_cmd_check_cnt = None
+        # initial read size should be relatively small,
+        # to avoid hanging for a long time until desired output is available in async_cmd_check
+        self.async_cmd_read_size = 1024
 
     @property
     def name(self):
@@ -159,7 +165,7 @@ def prerun(self):
         """
         pass
 
-    def run(self):
+    def run(self, *args, **kwargs):
         """
         Actual installation of a extension.
         """
@@ -171,6 +177,47 @@ def postrun(self):
         """
         pass
 
+    def async_cmd_start(self, cmd, inp=None):
+        """
+        Start installation asynchronously using specified command.
+        """
+        self.async_cmd_output = ''
+        self.async_cmd_check_cnt = 0
+        self.async_cmd_info = run_cmd(cmd, log_all=True, simple=False, inp=inp, regexp=False, asynchronous=True)
+
+    def async_cmd_check(self):
+        """
+        Check progress of installation command that was started asynchronously.
+
+        :return: True if command completed, False otherwise
+        """
+        if self.async_cmd_info is None:
+            raise EasyBuildError("No installation command running asynchronously for %s", self.name)
+        else:
+            self.log.debug("Checking on installation of extension %s...", self.name)
+            proc = self.async_cmd_info[0]
+            # use small read size, to avoid waiting for a long time until sufficient output is produced
+            self.async_cmd_output += get_output_from_process(proc, read_size=self.async_cmd_read_size)
+            ec = proc.poll()
+            if ec is None:
+                res = False
+                self.async_cmd_check_cnt += 1
+                # increase read size after sufficient checks,
+                # to avoid that installation hangs due to output buffer filling up...
+                if self.async_cmd_check_cnt % 10 == 0 and self.async_cmd_read_size < (1024 ** 2):
+                    self.async_cmd_read_size *= 2
+            else:
+                self.log.debug("Completing installation of extension %s...", self.name)
+                self.async_cmd_output, _ = complete_cmd(*self.async_cmd_info, output=self.async_cmd_output)
+                res = True
+
+        return res
+
+    @property
+    def required_deps(self):
+        """Return list of required dependencies for this extension."""
+        raise NotImplementedError("Don't know how to determine required dependencies for %s" % self.name)
+
     @property
     def toolchain(self):
         """

From 4450fdf2b1cde2fab7c9f7b0eb6bdbf7ba54b0e8 Mon Sep 17 00:00:00 2001
From: Miguel Dias Costa <migueldiascosta@gmail.com>
Date: Tue, 22 Jun 2021 15:46:04 +0800
Subject: [PATCH 002/175] add --review-pr-max and --review-pr-filter options to
 limit easyconfigs shown in multi-diff

---
 easybuild/framework/easyconfig/tools.py | 6 +++++-
 easybuild/main.py                       | 3 ++-
 easybuild/tools/options.py              | 3 +++
 3 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py
index fadbf03f50..ff6882f33a 100644
--- a/easybuild/framework/easyconfig/tools.py
+++ b/easybuild/framework/easyconfig/tools.py
@@ -486,7 +486,7 @@ def find_related_easyconfigs(path, ec):
     return sorted(res)
 
 
-def review_pr(paths=None, pr=None, colored=True, branch='develop', testing=False):
+def review_pr(paths=None, pr=None, colored=True, branch='develop', testing=False, max_ecs=None, filter_ecs=None):
     """
     Print multi-diff overview between specified easyconfigs or PR and specified branch.
     :param pr: pull request number in easybuild-easyconfigs repo to review
@@ -521,6 +521,10 @@ def review_pr(paths=None, pr=None, colored=True, branch='develop', testing=False
             pr_msg = "new PR"
         _log.debug("File in %s %s has these related easyconfigs: %s" % (pr_msg, ec['spec'], files))
         if files:
+            if filter_ecs is not None:
+                files = [x for x in files if filter_ecs.search(x)]
+            if max_ecs is not None:
+                files = files[:max_ecs]
             lines.append(multidiff(ec['spec'], files, colored=colored))
         else:
             lines.extend(['', "(no related easyconfigs found for %s)\n" % os.path.basename(ec['spec'])])
diff --git a/easybuild/main.py b/easybuild/main.py
index 039b400d2f..9b0d8131d3 100644
--- a/easybuild/main.py
+++ b/easybuild/main.py
@@ -261,7 +261,8 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
         merge_pr(options.merge_pr)
 
     elif options.review_pr:
-        print(review_pr(pr=options.review_pr, colored=use_color(options.color), testing=testing))
+        print(review_pr(pr=options.review_pr, colored=use_color(options.color), testing=testing,
+                        max_ecs=options.review_pr_max, filter_ecs=options.review_pr_filter))
 
     elif options.add_pr_labels:
         add_pr_labels(options.add_pr_labels)
diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py
index 2ba011b2f3..22645e3ce8 100644
--- a/easybuild/tools/options.py
+++ b/easybuild/tools/options.py
@@ -690,6 +690,9 @@ def github_options(self):
             'sync-pr-with-develop': ("Sync pull request with current 'develop' branch",
                                      int, 'store', None, {'metavar': 'PR#'}),
             'review-pr': ("Review specified pull request", int, 'store', None, {'metavar': 'PR#'}),
+            'review-pr-filter': ("Regex used to filter out easyconfigs to diff against in --review-pr",
+                                 None, 'regex', None),
+            'review-pr-max': ("Maximum number of easyconfigs to diff against in --review-pr", int, 'store', None),
             'test-report-env-filter': ("Regex used to filter out variables in environment dump of test report",
                                        None, 'regex', None),
             'update-branch-github': ("Update specified branch in GitHub", str, 'store', None),

From 11612082213a58499e0f691f41abc000c7f56e34 Mon Sep 17 00:00:00 2001
From: Alexander Grund <alexander.grund@tu-dresden.de>
Date: Wed, 14 Jul 2021 10:21:10 +0200
Subject: [PATCH 003/175] Print the hook messages only for debug-mode

The output is VERY noisy, especially with parse_hook and module_write hook
The log already has this info and if you want it back you can use --debug
---
 easybuild/tools/hooks.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py
index cb2d72c472..6046e933f1 100644
--- a/easybuild/tools/hooks.py
+++ b/easybuild/tools/hooks.py
@@ -33,6 +33,7 @@
 
 from easybuild.base import fancylogger
 from easybuild.tools.build_log import EasyBuildError, print_msg
+from easybuild.tools.config import build_option
 
 
 _log = fancylogger.getLogger('hooks', fname=False)
@@ -191,7 +192,8 @@ def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None,
 
         if msg is None:
             msg = "Running %s hook..." % label
-        print_msg(msg)
+        if build_option('debug'):
+            print_msg(msg)
 
         _log.info("Running '%s' hook function (arguments: %s)...", hook.__name__, args)
         res = hook(*args)

From 7910ca94c10c28f5aefcee89e3f660d7e59902e8 Mon Sep 17 00:00:00 2001
From: Alexander Grund <alexander.grund@tu-dresden.de>
Date: Wed, 14 Jul 2021 13:36:09 +0200
Subject: [PATCH 004/175] Fix test

---
 test/framework/hooks.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/test/framework/hooks.py b/test/framework/hooks.py
index f51cd0fc91..1e3c44ea5f 100644
--- a/test/framework/hooks.py
+++ b/test/framework/hooks.py
@@ -29,7 +29,7 @@
 """
 import os
 import sys
-from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered
+from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config
 from unittest import TextTestRunner
 
 import easybuild.tools.hooks
@@ -124,6 +124,8 @@ def test_run_hook(self):
 
         hooks = load_hooks(self.test_hooks_pymod)
 
+        init_config(build_options={'debug': True})
+
         self.mock_stdout(True)
         self.mock_stderr(True)
         run_hook('start', hooks)

From dff7ec589dde3b5a2c169189d7b3eb9db2123a4d Mon Sep 17 00:00:00 2001
From: Alexander Grund <alexander.grund@tu-dresden.de>
Date: Wed, 7 Jul 2021 18:11:31 +0200
Subject: [PATCH 005/175] Filter out duplicate paths added to module files

---
 easybuild/framework/easyblock.py    |  6 +--
 easybuild/tools/module_generator.py | 64 ++++++++++++++++++++++++-----
 test/framework/easyblock.py         |  9 +++-
 3 files changed, 65 insertions(+), 14 deletions(-)

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index d8829d8587..4fdaa99323 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -3210,9 +3210,7 @@ def make_module_step(self, fake=False):
         else:
             trace_msg("generating module file @ %s" % self.mod_filepath)
 
-        txt = self.module_generator.MODULE_SHEBANG
-        if txt:
-            txt += '\n'
+        txt = self.module_generator.prepare_module_creation()
 
         if self.modules_header:
             txt += self.modules_header + '\n'
@@ -3226,6 +3224,8 @@ def make_module_step(self, fake=False):
         txt += self.make_module_extra()
         txt += self.make_module_footer()
 
+        self.module_generator.finalize_module_creation()
+
         hook_txt = run_hook(MODULE_WRITE, self.hooks, args=[self, mod_filepath, txt])
         if hook_txt is not None:
             txt = hook_txt
diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py
index 685f158850..8134c8a1f6 100644
--- a/easybuild/tools/module_generator.py
+++ b/easybuild/tools/module_generator.py
@@ -136,17 +136,21 @@ def __init__(self, application, fake=False):
         self.fake_mod_path = tempfile.mkdtemp()
 
         self.modules_tool = modules_tool()
+        self.added_paths = None
+
+    def prepare_module_creation(self):
+        """Prepares creating a module and returns the file header (shebang) if any including the newline"""
+        if self.added_paths is not None:
+            raise EasyBuildError('Module creation already in process. Did you forget to finalize the module?')
+        self.added_paths = set()  # Clear all
+        txt = self.MODULE_SHEBANG
+        if txt:
+            txt += '\n'
+        return txt
 
-    def append_paths(self, key, paths, allow_abs=False, expand_relpaths=True):
-        """
-        Generate append-path statements for the given list of paths.
-
-        :param key: environment variable to append paths to
-        :param paths: list of paths to append
-        :param allow_abs: allow providing of absolute paths
-        :param expand_relpaths: expand relative paths into absolute paths (by prefixing install dir)
-        """
-        return self.update_paths(key, paths, prepend=False, allow_abs=allow_abs, expand_relpaths=expand_relpaths)
+    def finalize_module_creation(self):
+        """Finish creating a module. Must be called when done with the generator"""
+        self.added_paths = None
 
     def create_symlinks(self, mod_symlink_paths, fake=False):
         """Create moduleclass symlink(s) to actual module file."""
@@ -191,6 +195,43 @@ def get_modules_path(self, fake=False, mod_path_suffix=None):
 
         return os.path.join(mod_path, mod_path_suffix)
 
+    def _filter_paths(self, paths):
+        """Filter out already added paths and return the remaining ones"""
+        if self.added_paths is None:
+            raise EasyBuildError('Module creation has not been started. Call prepare_module_creation first!')
+
+        # paths can be a string
+        if isinstance(paths, string_type):
+            if paths in self.added_paths:
+                filtered_paths = None
+            else:
+                self.added_paths.add(paths)
+                filtered_paths = paths
+        else:
+            # Coerce any iterable/generator into a list
+            if not isinstance(paths, list):
+                paths = list(paths)
+            filtered_paths = [x for x in paths if x not in self.added_paths and not self.added_paths.add(x)]
+        if filtered_paths != paths:
+            removed_paths = paths if filtered_paths is None else [x for x in paths if x not in filtered_paths]
+            self.log.warning("Supressed adding the following path(s) to the module as they were already added: %s",
+                             removed_paths)
+        return filtered_paths
+
+    def append_paths(self, key, paths, allow_abs=False, expand_relpaths=True):
+        """
+        Generate append-path statements for the given list of paths.
+
+        :param key: environment variable to append paths to
+        :param paths: list of paths to append
+        :param allow_abs: allow providing of absolute paths
+        :param expand_relpaths: expand relative paths into absolute paths (by prefixing install dir)
+        """
+        paths = self._filter_paths(paths)
+        if not paths:
+            return ''
+        return self.update_paths(key, paths, prepend=False, allow_abs=allow_abs, expand_relpaths=expand_relpaths)
+
     def prepend_paths(self, key, paths, allow_abs=False, expand_relpaths=True):
         """
         Generate prepend-path statements for the given list of paths.
@@ -200,6 +241,9 @@ def prepend_paths(self, key, paths, allow_abs=False, expand_relpaths=True):
         :param allow_abs: allow providing of absolute paths
         :param expand_relpaths: expand relative paths into absolute paths (by prefixing install dir)
         """
+        paths = self._filter_paths(paths)
+        if not paths:
+            return ''
         return self.update_paths(key, paths, prepend=True, allow_abs=allow_abs, expand_relpaths=expand_relpaths)
 
     def _modulerc_check_module_version(self, module_version):
diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py
index 18bfcf07b0..8310fae89e 100644
--- a/test/framework/easyblock.py
+++ b/test/framework/easyblock.py
@@ -1152,7 +1152,7 @@ def test_make_module_step(self):
         # purposely use a 'nasty' description, that includes (unbalanced) special chars: [, ], {, }
         descr = "This {is a}} [fancy]] [[description]]. {{[[TEST}]"
         modextravars = {'PI': '3.1415', 'FOO': 'bar'}
-        modextrapaths = {'PATH': 'pibin', 'CPATH': 'pi/include'}
+        modextrapaths = {'PATH': ('bin', 'pibin'), 'CPATH': 'pi/include'}
         self.contents = '\n'.join([
             'easyblock = "ConfigureMake"',
             'name = "%s"' % name,
@@ -1177,6 +1177,10 @@ def test_make_module_step(self):
         eb.make_builddir()
         eb.prepare_step()
 
+        # Create a dummy file in bin to test if the duplicate entry of modextrapaths is ignored
+        os.mkdir(os.path.join(eb.installdir, 'bin'))
+        write_file(os.path.join(eb.installdir, 'bin', 'dummy_exe'), 'hello')
+
         modpath = os.path.join(eb.make_module_step(), name, version)
         if get_module_syntax() == 'Lua':
             modpath += '.lua'
@@ -1218,6 +1222,9 @@ def test_make_module_step(self):
             else:
                 self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
             self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt))
+            # Check for duplicates
+            num_prepends = len(regex.finditer(txt))
+            self.assertEqual(num_prepends, 1, "Expected exactly 1 %s command in %s" % (regex.pattern, txt))
 
         for (name, ver) in [('GCC', '6.4.0-2.28')]:
             if get_module_syntax() == 'Tcl':

From e026a09ad62cf084997cbaec2b85f3f62a1c66cf Mon Sep 17 00:00:00 2001
From: Alexander Grund <alexander.grund@tu-dresden.de>
Date: Wed, 7 Jul 2021 18:14:29 +0200
Subject: [PATCH 006/175] Use print_warning

---
 easybuild/tools/module_generator.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py
index 8134c8a1f6..ad3e36d4d6 100644
--- a/easybuild/tools/module_generator.py
+++ b/easybuild/tools/module_generator.py
@@ -41,7 +41,7 @@
 from textwrap import wrap
 
 from easybuild.base import fancylogger
-from easybuild.tools.build_log import EasyBuildError
+from easybuild.tools.build_log import EasyBuildError, print_warning
 from easybuild.tools.config import build_option, get_module_syntax, install_path
 from easybuild.tools.filetools import convert_name, mkdir, read_file, remove_file, resolve_path, symlink, write_file
 from easybuild.tools.modules import ROOT_ENV_VAR_NAME_PREFIX, EnvironmentModulesC, Lmod, modules_tool
@@ -214,8 +214,9 @@ def _filter_paths(self, paths):
             filtered_paths = [x for x in paths if x not in self.added_paths and not self.added_paths.add(x)]
         if filtered_paths != paths:
             removed_paths = paths if filtered_paths is None else [x for x in paths if x not in filtered_paths]
-            self.log.warning("Supressed adding the following path(s) to the module as they were already added: %s",
-                             removed_paths)
+            print_warning("Supressed adding the following path(s) to the module as they were already added: %s",
+                          removed_paths,
+                          log=self.log)
         return filtered_paths
 
     def append_paths(self, key, paths, allow_abs=False, expand_relpaths=True):

From a4dd70e8de57ba2dd1b4c0d0abac66fe631223cb Mon Sep 17 00:00:00 2001
From: Alexander Grund <alexander.grund@tu-dresden.de>
Date: Wed, 7 Jul 2021 18:22:18 +0200
Subject: [PATCH 007/175] Use a contextmanager instead

---
 easybuild/framework/easyblock.py    | 27 ++++++++++++---------------
 easybuild/tools/module_generator.py | 23 +++++++++++++++--------
 2 files changed, 27 insertions(+), 23 deletions(-)

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index 4fdaa99323..6c2d92a657 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -3210,21 +3210,18 @@ def make_module_step(self, fake=False):
         else:
             trace_msg("generating module file @ %s" % self.mod_filepath)
 
-        txt = self.module_generator.prepare_module_creation()
-
-        if self.modules_header:
-            txt += self.modules_header + '\n'
-
-        txt += self.make_module_description()
-        txt += self.make_module_group_check()
-        txt += self.make_module_deppaths()
-        txt += self.make_module_dep()
-        txt += self.make_module_extend_modpath()
-        txt += self.make_module_req()
-        txt += self.make_module_extra()
-        txt += self.make_module_footer()
-
-        self.module_generator.finalize_module_creation()
+        with self.module_generator.start_module_creation() as txt:
+            if self.modules_header:
+                txt += self.modules_header + '\n'
+
+            txt += self.make_module_description()
+            txt += self.make_module_group_check()
+            txt += self.make_module_deppaths()
+            txt += self.make_module_dep()
+            txt += self.make_module_extend_modpath()
+            txt += self.make_module_req()
+            txt += self.make_module_extra()
+            txt += self.make_module_footer()
 
         hook_txt = run_hook(MODULE_WRITE, self.hooks, args=[self, mod_filepath, txt])
         if hook_txt is not None:
diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py
index ad3e36d4d6..f4b4634a8f 100644
--- a/easybuild/tools/module_generator.py
+++ b/easybuild/tools/module_generator.py
@@ -37,6 +37,7 @@
 import os
 import re
 import tempfile
+from contextlib import contextmanager
 from distutils.version import LooseVersion
 from textwrap import wrap
 
@@ -138,19 +139,25 @@ def __init__(self, application, fake=False):
         self.modules_tool = modules_tool()
         self.added_paths = None
 
-    def prepare_module_creation(self):
-        """Prepares creating a module and returns the file header (shebang) if any including the newline"""
+    @contextmanager
+    def start_module_creation(self):
+        """
+        Prepares creating a module and returns the file header (shebang) if any including the newline
+
+        Meant to be used in a with statement:
+            with generator.start_module_creation() as txt:
+                # Write txt
+        """
         if self.added_paths is not None:
-            raise EasyBuildError('Module creation already in process. Did you forget to finalize the module?')
+            raise EasyBuildError('Module creation already in process. You cannot create multiple modules at the same time!')
         self.added_paths = set()  # Clear all
         txt = self.MODULE_SHEBANG
         if txt:
             txt += '\n'
-        return txt
-
-    def finalize_module_creation(self):
-        """Finish creating a module. Must be called when done with the generator"""
-        self.added_paths = None
+        try:
+            yield txt
+        finally:
+            self.added_paths = None
 
     def create_symlinks(self, mod_symlink_paths, fake=False):
         """Create moduleclass symlink(s) to actual module file."""

From 75a41952f38d21f379466b35ab733fd1bf21e289 Mon Sep 17 00:00:00 2001
From: Alexander Grund <alexander.grund@tu-dresden.de>
Date: Wed, 7 Jul 2021 18:24:02 +0200
Subject: [PATCH 008/175] Add line break

---
 easybuild/tools/module_generator.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py
index f4b4634a8f..0623ddaf6b 100644
--- a/easybuild/tools/module_generator.py
+++ b/easybuild/tools/module_generator.py
@@ -149,7 +149,8 @@ def start_module_creation(self):
                 # Write txt
         """
         if self.added_paths is not None:
-            raise EasyBuildError('Module creation already in process. You cannot create multiple modules at the same time!')
+            raise EasyBuildError('Module creation already in process. '
+                                 'You cannot create multiple modules at the same time!')
         self.added_paths = set()  # Clear all
         txt = self.MODULE_SHEBANG
         if txt:

From 88851026173bd1403f48507562d9bd0d15f28893 Mon Sep 17 00:00:00 2001
From: Alexander Grund <alexander.grund@tu-dresden.de>
Date: Thu, 8 Jul 2021 11:13:41 +0200
Subject: [PATCH 009/175] Check paths per key not globally

---
 easybuild/tools/module_generator.py | 34 ++++++++++++++++-------------
 1 file changed, 19 insertions(+), 15 deletions(-)

diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py
index 0623ddaf6b..12d6ba4d26 100644
--- a/easybuild/tools/module_generator.py
+++ b/easybuild/tools/module_generator.py
@@ -137,7 +137,7 @@ def __init__(self, application, fake=False):
         self.fake_mod_path = tempfile.mkdtemp()
 
         self.modules_tool = modules_tool()
-        self.added_paths = None
+        self.added_paths_per_key = None
 
     @contextmanager
     def start_module_creation(self):
@@ -148,17 +148,18 @@ def start_module_creation(self):
             with generator.start_module_creation() as txt:
                 # Write txt
         """
-        if self.added_paths is not None:
+        if self.added_paths_per_key is not None:
             raise EasyBuildError('Module creation already in process. '
                                  'You cannot create multiple modules at the same time!')
-        self.added_paths = set()  # Clear all
+        # Mapping of keys/env vars to paths already added
+        self.added_paths_per_key = dict()
         txt = self.MODULE_SHEBANG
         if txt:
             txt += '\n'
         try:
             yield txt
         finally:
-            self.added_paths = None
+            self.added_paths_per_key = None
 
     def create_symlinks(self, mod_symlink_paths, fake=False):
         """Create moduleclass symlink(s) to actual module file."""
@@ -203,27 +204,30 @@ def get_modules_path(self, fake=False, mod_path_suffix=None):
 
         return os.path.join(mod_path, mod_path_suffix)
 
-    def _filter_paths(self, paths):
-        """Filter out already added paths and return the remaining ones"""
-        if self.added_paths is None:
-            raise EasyBuildError('Module creation has not been started. Call prepare_module_creation first!')
+    def _filter_paths(self, key, paths):
+        """Filter out paths already added to key and return the remaining ones"""
+        if self.added_paths_per_key is None:
+            # For compatibility this is only a warning for now and we don't filter any paths
+            print_warning('Module creation has not been started. Call prepare_module_creation first!')
+            return paths
 
+        added_paths = self.added_paths_per_key.setdefault(key, set())
         # paths can be a string
         if isinstance(paths, string_type):
-            if paths in self.added_paths:
+            if paths in added_paths:
                 filtered_paths = None
             else:
-                self.added_paths.add(paths)
+                added_paths.add(paths)
                 filtered_paths = paths
         else:
             # Coerce any iterable/generator into a list
             if not isinstance(paths, list):
                 paths = list(paths)
-            filtered_paths = [x for x in paths if x not in self.added_paths and not self.added_paths.add(x)]
+            filtered_paths = [x for x in paths if x not in added_paths and not added_paths.add(x)]
         if filtered_paths != paths:
             removed_paths = paths if filtered_paths is None else [x for x in paths if x not in filtered_paths]
-            print_warning("Supressed adding the following path(s) to the module as they were already added: %s",
-                          removed_paths,
+            print_warning("Supressed adding the following path(s) to $%s of the module as they were already added: %s",
+                          key, removed_paths,
                           log=self.log)
         return filtered_paths
 
@@ -236,7 +240,7 @@ def append_paths(self, key, paths, allow_abs=False, expand_relpaths=True):
         :param allow_abs: allow providing of absolute paths
         :param expand_relpaths: expand relative paths into absolute paths (by prefixing install dir)
         """
-        paths = self._filter_paths(paths)
+        paths = self._filter_paths(key, paths)
         if not paths:
             return ''
         return self.update_paths(key, paths, prepend=False, allow_abs=allow_abs, expand_relpaths=expand_relpaths)
@@ -250,7 +254,7 @@ def prepend_paths(self, key, paths, allow_abs=False, expand_relpaths=True):
         :param allow_abs: allow providing of absolute paths
         :param expand_relpaths: expand relative paths into absolute paths (by prefixing install dir)
         """
-        paths = self._filter_paths(paths)
+        paths = self._filter_paths(key, paths)
         if not paths:
             return ''
         return self.update_paths(key, paths, prepend=True, allow_abs=allow_abs, expand_relpaths=expand_relpaths)

From eace6817013d006d07aa8ec0a8d79c60ee04f40f Mon Sep 17 00:00:00 2001
From: Alexander Grund <alexander.grund@tu-dresden.de>
Date: Thu, 8 Jul 2021 11:13:49 +0200
Subject: [PATCH 010/175] Fix test

---
 test/framework/easyblock.py | 27 +++++++++++++++------------
 1 file changed, 15 insertions(+), 12 deletions(-)

diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py
index 8310fae89e..9f325a91ab 100644
--- a/test/framework/easyblock.py
+++ b/test/framework/easyblock.py
@@ -1178,7 +1178,7 @@ def test_make_module_step(self):
         eb.prepare_step()
 
         # Create a dummy file in bin to test if the duplicate entry of modextrapaths is ignored
-        os.mkdir(os.path.join(eb.installdir, 'bin'))
+        os.makedirs(os.path.join(eb.installdir, 'bin'))
         write_file(os.path.join(eb.installdir, 'bin', 'dummy_exe'), 'hello')
 
         modpath = os.path.join(eb.make_module_step(), name, version)
@@ -1214,17 +1214,20 @@ def test_make_module_step(self):
                 self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
             self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt))
 
-        for (key, val) in modextrapaths.items():
-            if get_module_syntax() == 'Tcl':
-                regex = re.compile(r'^prepend-path\s+%s\s+\$root/%s$' % (key, val), re.M)
-            elif get_module_syntax() == 'Lua':
-                regex = re.compile(r'^prepend_path\("%s", pathJoin\(root, "%s"\)\)$' % (key, val), re.M)
-            else:
-                self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
-            self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt))
-            # Check for duplicates
-            num_prepends = len(regex.finditer(txt))
-            self.assertEqual(num_prepends, 1, "Expected exactly 1 %s command in %s" % (regex.pattern, txt))
+        for (key, vals) in modextrapaths.items():
+            if isinstance(vals, string_type):
+                vals = [vals]
+            for val in vals:
+                if get_module_syntax() == 'Tcl':
+                    regex = re.compile(r'^prepend-path\s+%s\s+\$root/%s$' % (key, val), re.M)
+                elif get_module_syntax() == 'Lua':
+                    regex = re.compile(r'^prepend_path\("%s", pathJoin\(root, "%s"\)\)$' % (key, val), re.M)
+                else:
+                    self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
+                self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt))
+                # Check for duplicates
+                num_prepends = len(regex.findall(txt))
+                self.assertEqual(num_prepends, 1, "Expected exactly 1 %s command in %s" % (regex.pattern, txt))
 
         for (name, ver) in [('GCC', '6.4.0-2.28')]:
             if get_module_syntax() == 'Tcl':

From cd0a7b3dcc82ba6ea3043fb2b0cf657e59a48684 Mon Sep 17 00:00:00 2001
From: Alexander Grund <alexander.grund@tu-dresden.de>
Date: Fri, 9 Jul 2021 11:28:23 +0200
Subject: [PATCH 011/175] Fix extensions writing to modules

---
 easybuild/framework/easyblock.py    | 3 ++-
 easybuild/tools/module_generator.py | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index 6c2d92a657..35756b8b51 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -2441,7 +2441,8 @@ def extensions_step(self, fetch=False, install=True):
             if install:
                 try:
                     ext.prerun()
-                    txt = ext.run()
+                    with self.module_generator.start_module_creation():
+                        txt = ext.run()
                     if txt:
                         self.module_extra_extensions += txt
                     ext.postrun()
diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py
index 12d6ba4d26..87c0208024 100644
--- a/easybuild/tools/module_generator.py
+++ b/easybuild/tools/module_generator.py
@@ -208,7 +208,7 @@ def _filter_paths(self, key, paths):
         """Filter out paths already added to key and return the remaining ones"""
         if self.added_paths_per_key is None:
             # For compatibility this is only a warning for now and we don't filter any paths
-            print_warning('Module creation has not been started. Call prepare_module_creation first!')
+            print_warning('Module creation has not been started. Call start_module_creation first!')
             return paths
 
         added_paths = self.added_paths_per_key.setdefault(key, set())

From e7a211c95e08e9952b84f994da586876ff8d9b84 Mon Sep 17 00:00:00 2001
From: Alexander Grund <alexander.grund@tu-dresden.de>
Date: Fri, 16 Jul 2021 12:43:29 +0200
Subject: [PATCH 012/175] Fix adding root path (empty string)

---
 easybuild/tools/module_generator.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py
index 87c0208024..59444b273e 100644
--- a/easybuild/tools/module_generator.py
+++ b/easybuild/tools/module_generator.py
@@ -229,6 +229,8 @@ def _filter_paths(self, key, paths):
             print_warning("Supressed adding the following path(s) to $%s of the module as they were already added: %s",
                           key, removed_paths,
                           log=self.log)
+            if not filtered_paths:
+                filtered_paths = None
         return filtered_paths
 
     def append_paths(self, key, paths, allow_abs=False, expand_relpaths=True):
@@ -241,7 +243,7 @@ def append_paths(self, key, paths, allow_abs=False, expand_relpaths=True):
         :param expand_relpaths: expand relative paths into absolute paths (by prefixing install dir)
         """
         paths = self._filter_paths(key, paths)
-        if not paths:
+        if paths is None:
             return ''
         return self.update_paths(key, paths, prepend=False, allow_abs=allow_abs, expand_relpaths=expand_relpaths)
 
@@ -255,7 +257,7 @@ def prepend_paths(self, key, paths, allow_abs=False, expand_relpaths=True):
         :param expand_relpaths: expand relative paths into absolute paths (by prefixing install dir)
         """
         paths = self._filter_paths(key, paths)
-        if not paths:
+        if paths is None:
             return ''
         return self.update_paths(key, paths, prepend=True, allow_abs=allow_abs, expand_relpaths=expand_relpaths)
 

From 3048a4971bb349b531c574b1b9bade685bf6f966 Mon Sep 17 00:00:00 2001
From: Alexander Grund <alexander.grund@tu-dresden.de>
Date: Fri, 16 Jul 2021 12:43:38 +0200
Subject: [PATCH 013/175] Update tests to use filtering module generator

---
 test/framework/easyblock.py        | 21 +++++++----
 test/framework/module_generator.py | 56 +++++++++++++++++-------------
 2 files changed, 46 insertions(+), 31 deletions(-)

diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py
index 9f325a91ab..7393a35c0d 100644
--- a/test/framework/easyblock.py
+++ b/test/framework/easyblock.py
@@ -435,7 +435,8 @@ def test_make_module_req(self):
         # this is not a path that should be picked up
         os.mkdir(os.path.join(eb.installdir, 'CPATH'))
 
-        guess = eb.make_module_req()
+        with eb.module_generator.start_module_creation():
+            guess = eb.make_module_req()
 
         if get_module_syntax() == 'Tcl':
             self.assertTrue(re.search(r"^prepend-path\s+CLASSPATH\s+\$root/bla.jar$", guess, re.M))
@@ -462,7 +463,8 @@ def test_make_module_req(self):
 
         # check that bin is only added to PATH if there are files in there
         write_file(os.path.join(eb.installdir, 'bin', 'test'), 'test')
-        guess = eb.make_module_req()
+        with eb.module_generator.start_module_creation():
+            guess = eb.make_module_req()
         if get_module_syntax() == 'Tcl':
             self.assertTrue(re.search(r"^prepend-path\s+PATH\s+\$root/bin$", guess, re.M))
             self.assertFalse(re.search(r"^prepend-path\s+PATH\s+\$root/sbin$", guess, re.M))
@@ -481,7 +483,8 @@ def test_make_module_req(self):
             self.assertFalse('prepend_path("CMAKE_LIBRARY_PATH", pathJoin(root, "lib64"))' in guess)
         # -- With files
         write_file(os.path.join(eb.installdir, 'lib64', 'libfoo.so'), 'test')
-        guess = eb.make_module_req()
+        with eb.module_generator.start_module_creation():
+            guess = eb.make_module_req()
         if get_module_syntax() == 'Tcl':
             self.assertTrue(re.search(r"^prepend-path\s+CMAKE_LIBRARY_PATH\s+\$root/lib64$", guess, re.M))
         elif get_module_syntax() == 'Lua':
@@ -490,7 +493,8 @@ def test_make_module_req(self):
         write_file(os.path.join(eb.installdir, 'lib', 'libfoo.so'), 'test')
         shutil.rmtree(os.path.join(eb.installdir, 'lib64'))
         os.symlink('lib', os.path.join(eb.installdir, 'lib64'))
-        guess = eb.make_module_req()
+        with eb.module_generator.start_module_creation():
+            guess = eb.make_module_req()
         if get_module_syntax() == 'Tcl':
             self.assertFalse(re.search(r"^prepend-path\s+CMAKE_LIBRARY_PATH\s+\$root/lib64$", guess, re.M))
         elif get_module_syntax() == 'Lua':
@@ -509,7 +513,8 @@ def test_make_module_req(self):
 
         # check for behavior when a string value is used as dict value by make_module_req_guesses
         eb.make_module_req_guess = lambda: {'PATH': 'bin'}
-        txt = eb.make_module_req()
+        with eb.module_generator.start_module_creation():
+            txt = eb.make_module_req()
         if get_module_syntax() == 'Tcl':
             self.assertTrue(re.match(r"^\nprepend-path\s+PATH\s+\$root/bin\n$", txt, re.M))
         elif get_module_syntax() == 'Lua':
@@ -520,7 +525,8 @@ def test_make_module_req(self):
         # check for correct behaviour if empty string is specified as one of the values
         # prepend-path statements should be included for both the 'bin' subdir and the install root
         eb.make_module_req_guess = lambda: {'PATH': ['bin', '']}
-        txt = eb.make_module_req()
+        with eb.module_generator.start_module_creation():
+            txt = eb.make_module_req()
         if get_module_syntax() == 'Tcl':
             self.assertTrue(re.search(r"\nprepend-path\s+PATH\s+\$root/bin\n", txt, re.M))
             self.assertTrue(re.search(r"\nprepend-path\s+PATH\s+\$root\n", txt, re.M))
@@ -535,7 +541,8 @@ def test_make_module_req(self):
         for path in ['pathA', 'pathB', 'pathC']:
             os.mkdir(os.path.join(eb.installdir, 'lib', path))
             write_file(os.path.join(eb.installdir, 'lib', path, 'libfoo.so'), 'test')
-        txt = eb.make_module_req()
+        with eb.module_generator.start_module_creation():
+            txt = eb.make_module_req()
         if get_module_syntax() == 'Tcl':
             self.assertTrue(re.search(r"\nprepend-path\s+LD_LIBRARY_PATH\s+\$root/lib/pathC\n" +
                                       r"prepend-path\s+LD_LIBRARY_PATH\s+\$root/lib/pathA\n" +
diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py
index 517245e741..4bfa3c3344 100644
--- a/test/framework/module_generator.py
+++ b/test/framework/module_generator.py
@@ -670,6 +670,10 @@ def test_swap(self):
     def test_append_paths(self):
         """Test generating append-paths statements."""
         # test append_paths
+        def append_paths(*args, **kwargs):
+            """Wrap this into start_module_creation which need to be called prior to append_paths"""
+            with self.modgen.start_module_creation():
+                return self.modgen.append_paths(*args, **kwargs)
 
         if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl:
             expected = ''.join([
@@ -678,17 +682,17 @@ def test_append_paths(self):
                 "append-path\tkey\t\t$root\n",
             ])
             paths = ['path1', 'path2', '']
-            self.assertEqual(expected, self.modgen.append_paths("key", paths))
+            self.assertEqual(expected, append_paths("key", paths))
             # 2nd call should still give same result, no side-effects like manipulating passed list 'paths'!
-            self.assertEqual(expected, self.modgen.append_paths("key", paths))
+            self.assertEqual(expected, append_paths("key", paths))
 
             expected = "append-path\tbar\t\t$root/foo\n"
-            self.assertEqual(expected, self.modgen.append_paths("bar", "foo"))
+            self.assertEqual(expected, append_paths("bar", "foo"))
 
-            res = self.modgen.append_paths("key", ["/abs/path"], allow_abs=True)
+            res = append_paths("key", ["/abs/path"], allow_abs=True)
             self.assertEqual("append-path\tkey\t\t/abs/path\n", res)
 
-            res = self.modgen.append_paths('key', ['1234@example.com'], expand_relpaths=False)
+            res = append_paths('key', ['1234@example.com'], expand_relpaths=False)
             self.assertEqual("append-path\tkey\t\t1234@example.com\n", res)
 
         else:
@@ -698,22 +702,22 @@ def test_append_paths(self):
                 'append_path("key", root)\n',
             ])
             paths = ['path1', 'path2', '']
-            self.assertEqual(expected, self.modgen.append_paths("key", paths))
+            self.assertEqual(expected, append_paths("key", paths))
             # 2nd call should still give same result, no side-effects like manipulating passed list 'paths'!
-            self.assertEqual(expected, self.modgen.append_paths("key", paths))
+            self.assertEqual(expected, append_paths("key", paths))
 
             expected = 'append_path("bar", pathJoin(root, "foo"))\n'
-            self.assertEqual(expected, self.modgen.append_paths("bar", "foo"))
+            self.assertEqual(expected, append_paths("bar", "foo"))
 
             expected = 'append_path("key", "/abs/path")\n'
-            self.assertEqual(expected, self.modgen.append_paths("key", ["/abs/path"], allow_abs=True))
+            self.assertEqual(expected, append_paths("key", ["/abs/path"], allow_abs=True))
 
-            res = self.modgen.append_paths('key', ['1234@example.com'], expand_relpaths=False)
+            res = append_paths('key', ['1234@example.com'], expand_relpaths=False)
             self.assertEqual('append_path("key", "1234@example.com")\n', res)
 
         self.assertErrorRegex(EasyBuildError, "Absolute path %s/foo passed to update_paths "
-                                              "which only expects relative paths." % self.modgen.app.installdir,
-                              self.modgen.append_paths, "key2", ["bar", "%s/foo" % self.modgen.app.installdir])
+                                            "which only expects relative paths." % self.modgen.app.installdir,
+                            append_paths, "key2", ["bar", "%s/foo" % self.modgen.app.installdir])
 
     def test_module_extensions(self):
         """test the extensions() for extensions"""
@@ -745,6 +749,10 @@ def test_module_extensions(self):
     def test_prepend_paths(self):
         """Test generating prepend-paths statements."""
         # test prepend_paths
+        def prepend_paths(*args, **kwargs):
+            """Wrap this into start_module_creation which need to be called prior to append_paths"""
+            with self.modgen.start_module_creation():
+                return self.modgen.prepend_paths(*args, **kwargs)
 
         if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl:
             expected = ''.join([
@@ -753,17 +761,17 @@ def test_prepend_paths(self):
                 "prepend-path\tkey\t\t$root\n",
             ])
             paths = ['path1', 'path2', '']
-            self.assertEqual(expected, self.modgen.prepend_paths("key", paths))
+            self.assertEqual(expected, prepend_paths("key", paths))
             # 2nd call should still give same result, no side-effects like manipulating passed list 'paths'!
-            self.assertEqual(expected, self.modgen.prepend_paths("key", paths))
+            self.assertEqual(expected, prepend_paths("key", paths))
 
             expected = "prepend-path\tbar\t\t$root/foo\n"
-            self.assertEqual(expected, self.modgen.prepend_paths("bar", "foo"))
+            self.assertEqual(expected, prepend_paths("bar", "foo"))
 
-            res = self.modgen.prepend_paths("key", ["/abs/path"], allow_abs=True)
+            res = prepend_paths("key", ["/abs/path"], allow_abs=True)
             self.assertEqual("prepend-path\tkey\t\t/abs/path\n", res)
 
-            res = self.modgen.prepend_paths('key', ['1234@example.com'], expand_relpaths=False)
+            res = prepend_paths('key', ['1234@example.com'], expand_relpaths=False)
             self.assertEqual("prepend-path\tkey\t\t1234@example.com\n", res)
 
         else:
@@ -773,22 +781,22 @@ def test_prepend_paths(self):
                 'prepend_path("key", root)\n',
             ])
             paths = ['path1', 'path2', '']
-            self.assertEqual(expected, self.modgen.prepend_paths("key", paths))
+            self.assertEqual(expected, prepend_paths("key", paths))
             # 2nd call should still give same result, no side-effects like manipulating passed list 'paths'!
-            self.assertEqual(expected, self.modgen.prepend_paths("key", paths))
+            self.assertEqual(expected, prepend_paths("key", paths))
 
             expected = 'prepend_path("bar", pathJoin(root, "foo"))\n'
-            self.assertEqual(expected, self.modgen.prepend_paths("bar", "foo"))
+            self.assertEqual(expected, prepend_paths("bar", "foo"))
 
             expected = 'prepend_path("key", "/abs/path")\n'
-            self.assertEqual(expected, self.modgen.prepend_paths("key", ["/abs/path"], allow_abs=True))
+            self.assertEqual(expected, prepend_paths("key", ["/abs/path"], allow_abs=True))
 
-            res = self.modgen.prepend_paths('key', ['1234@example.com'], expand_relpaths=False)
+            res = prepend_paths('key', ['1234@example.com'], expand_relpaths=False)
             self.assertEqual('prepend_path("key", "1234@example.com")\n', res)
 
         self.assertErrorRegex(EasyBuildError, "Absolute path %s/foo passed to update_paths "
-                                              "which only expects relative paths." % self.modgen.app.installdir,
-                              self.modgen.prepend_paths, "key2", ["bar", "%s/foo" % self.modgen.app.installdir])
+                                            "which only expects relative paths." % self.modgen.app.installdir,
+                            prepend_paths, "key2", ["bar", "%s/foo" % self.modgen.app.installdir])
 
     def test_det_user_modpath(self):
         """Test for generic det_user_modpath method."""

From 146773e7b16e02b4b46fdc32b9221ae2da9e56cb Mon Sep 17 00:00:00 2001
From: Alexander Grund <alexander.grund@tu-dresden.de>
Date: Fri, 16 Jul 2021 12:46:32 +0200
Subject: [PATCH 014/175] Fix formatting

---
 test/framework/module_generator.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py
index 4bfa3c3344..c046180883 100644
--- a/test/framework/module_generator.py
+++ b/test/framework/module_generator.py
@@ -716,8 +716,8 @@ def append_paths(*args, **kwargs):
             self.assertEqual('append_path("key", "1234@example.com")\n', res)
 
         self.assertErrorRegex(EasyBuildError, "Absolute path %s/foo passed to update_paths "
-                                            "which only expects relative paths." % self.modgen.app.installdir,
-                            append_paths, "key2", ["bar", "%s/foo" % self.modgen.app.installdir])
+                                              "which only expects relative paths." % self.modgen.app.installdir,
+                              append_paths, "key2", ["bar", "%s/foo" % self.modgen.app.installdir])
 
     def test_module_extensions(self):
         """test the extensions() for extensions"""
@@ -795,8 +795,8 @@ def prepend_paths(*args, **kwargs):
             self.assertEqual('prepend_path("key", "1234@example.com")\n', res)
 
         self.assertErrorRegex(EasyBuildError, "Absolute path %s/foo passed to update_paths "
-                                            "which only expects relative paths." % self.modgen.app.installdir,
-                            prepend_paths, "key2", ["bar", "%s/foo" % self.modgen.app.installdir])
+                                              "which only expects relative paths." % self.modgen.app.installdir,
+                              prepend_paths, "key2", ["bar", "%s/foo" % self.modgen.app.installdir])
 
     def test_det_user_modpath(self):
         """Test for generic det_user_modpath method."""

From 3d673307d9863b6aba16ed0f6e8e2c8d9975c16e Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Tue, 7 Sep 2021 09:02:35 +0200
Subject: [PATCH 015/175] bump version to 4.4.3dev

---
 easybuild/tools/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py
index 70aa973100..2216e1f42d 100644
--- a/easybuild/tools/version.py
+++ b/easybuild/tools/version.py
@@ -43,7 +43,7 @@
 # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like
 #   UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0'
 # This causes problems further up the dependency chain...
-VERSION = LooseVersion('4.4.2')
+VERSION = LooseVersion('4.4.3.dev0')
 UNKNOWN = 'UNKNOWN'
 
 

From df39e6e3fb61b728c00bedde9401da9ee87ee747 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B8rgen=20Nordmoen?= <j.h.nordmoen@usit.uio.no>
Date: Wed, 8 Sep 2021 08:58:06 +0200
Subject: [PATCH 016/175] Add progressbar to installation procedure

Utilize [`tqdm`](https://github.com/tqdm/tqdm) to track progress when
installing EasyBuilds. This require a few changes to allow `print_msg`
and the progressbar to interact.
---
 easybuild/framework/easyblock.py | 57 ++++++++++++++++++++++----------
 easybuild/main.py                | 18 +++++++---
 easybuild/tools/build_log.py     |  6 +++-
 requirements.txt                 |  2 ++
 4 files changed, 60 insertions(+), 23 deletions(-)

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index ee6d2eae49..43a4286b97 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -212,6 +212,9 @@ def __init__(self, ec):
         self.postmsg = ''  # allow a post message to be set, which can be shown as last output
         self.current_step = None
 
+        # Create empty progress bar
+        self.progressbar = None
+
         # list of loaded modules
         self.loaded_modules = []
 
@@ -300,6 +303,13 @@ def close_log(self):
         self.log.info("Closing log for application name %s version %s" % (self.name, self.version))
         fancylogger.logToFile(self.logfile, enable=False)
 
+    def set_progressbar(self, progressbar):
+        """
+        Set progress bar, the progress bar is needed when writing messages so
+        that the progress counter is always at the bottom
+        """
+        self.progressbar = progressbar
+
     #
     # DRY RUN UTILITIES
     #
@@ -318,7 +328,7 @@ def dry_run_msg(self, msg, *args):
         """Print dry run message."""
         if args:
             msg = msg % args
-        dry_run_msg(msg, silent=self.silent)
+        dry_run_msg(msg, silent=self.silent, progressbar=self.progressbar)
 
     #
     # FETCH UTILITY FUNCTIONS
@@ -1637,7 +1647,8 @@ def skip_extensions(self):
                 self.log.debug("exit code: %s, stdout/err: %s", ec, cmdstdouterr)
                 res.append(ext_inst)
             else:
-                print_msg("skipping extension %s" % ext_inst.name, silent=self.silent, log=self.log)
+                print_msg("skipping extension %s" % ext_inst.name, silent=self.silent, log=self.log,
+                          progressbar=self.progressbar)
 
         self.ext_instances = res
 
@@ -1741,7 +1752,8 @@ def handle_iterate_opts(self):
                     self.log.debug("Found list for %s: %s", opt, self.iter_opts[opt])
 
             if self.iter_opts:
-                print_msg("starting iteration #%s ..." % self.iter_idx, log=self.log, silent=self.silent)
+                print_msg("starting iteration #%s ..." % self.iter_idx, log=self.log, silent=self.silent,
+                          progressbar=self.progressbar)
                 self.log.info("Current iteration index: %s", self.iter_idx)
 
             # pop first element from all iterative easyconfig parameters as next value to use
@@ -1882,7 +1894,8 @@ def check_readiness_step(self):
                 hidden = LooseVersion(self.modules_tool.version) < LooseVersion('7.0.0')
 
             self.mod_file_backup = back_up_file(self.mod_filepath, hidden=hidden, strip_fn=strip_fn)
-            print_msg("backup of existing module file stored at %s" % self.mod_file_backup, log=self.log)
+            print_msg("backup of existing module file stored at %s" % self.mod_file_backup, log=self.log,
+                      progressbar=self.progressbar)
 
         # check if main install needs to be skipped
         # - if a current module can be found, skip is ok
@@ -2418,7 +2431,7 @@ def extensions_step(self, fetch=False, install=True):
             change_dir(self.orig_workdir)
 
             tup = (ext.name, ext.version or '', idx + 1, exts_cnt)
-            print_msg("installing extension %s %s (%d/%d)..." % tup, silent=self.silent)
+            print_msg("installing extension %s %s (%d/%d)..." % tup, silent=self.silent, progressbar=self.progressbar)
             start_time = datetime.now()
 
             if self.dry_run:
@@ -2450,9 +2463,11 @@ def extensions_step(self, fetch=False, install=True):
                     if not self.dry_run:
                         ext_duration = datetime.now() - start_time
                         if ext_duration.total_seconds() >= 1:
-                            print_msg("\t... (took %s)", time2str(ext_duration), log=self.log, silent=self.silent)
+                            print_msg("\t... (took %s)", time2str(ext_duration), log=self.log, silent=self.silent,
+                                      progressbar=self.progressbar)
                         elif self.logdebug or build_option('trace'):
-                            print_msg("\t... (took < 1 sec)", log=self.log, silent=self.silent)
+                            print_msg("\t... (took < 1 sec)", log=self.log, silent=self.silent,
+                                      progressbar=self.progressbar)
 
         # cleanup (unload fake module, remove fake module dir)
         if fake_mod_data:
@@ -3250,7 +3265,7 @@ def make_module_step(self, fake=False):
                 else:
                     diff_msg += 'no differences found'
                 self.log.info(diff_msg)
-                print_msg(diff_msg, log=self.log)
+                print_msg(diff_msg, log=self.log, progressbar=self.progressbar)
 
             self.invalidate_module_caches(modpath)
 
@@ -3569,7 +3584,8 @@ def run_all_steps(self, run_test_cases):
 
         steps = self.get_steps(run_test_cases=run_test_cases, iteration_count=self.det_iter_cnt())
 
-        print_msg("building and installing %s..." % self.full_mod_name, log=self.log, silent=self.silent)
+        print_msg("building and installing %s..." % self.full_mod_name, log=self.log, silent=self.silent,
+                  progressbar=self.progressbar)
         trace_msg("installation prefix: %s" % self.installdir)
 
         ignore_locks = build_option('ignore_locks')
@@ -3589,12 +3605,12 @@ def run_all_steps(self, run_test_cases):
         try:
             for (step_name, descr, step_methods, skippable) in steps:
                 if self.skip_step(step_name, skippable):
-                    print_msg("%s [skipped]" % descr, log=self.log, silent=self.silent)
+                    print_msg("%s [skipped]" % descr, log=self.log, silent=self.silent, progressbar=self.progressbar)
                 else:
                     if self.dry_run:
                         self.dry_run_msg("%s... [DRY RUN]\n", descr)
                     else:
-                        print_msg("%s..." % descr, log=self.log, silent=self.silent)
+                        print_msg("%s..." % descr, log=self.log, silent=self.silent, progressbar=self.progressbar)
                     self.current_step = step_name
                     start_time = datetime.now()
                     try:
@@ -3603,9 +3619,11 @@ def run_all_steps(self, run_test_cases):
                         if not self.dry_run:
                             step_duration = datetime.now() - start_time
                             if step_duration.total_seconds() >= 1:
-                                print_msg("... (took %s)", time2str(step_duration), log=self.log, silent=self.silent)
+                                print_msg("... (took %s)", time2str(step_duration), log=self.log, silent=self.silent,
+                                          progressbar=self.progressbar)
                             elif self.logdebug or build_option('trace'):
-                                print_msg("... (took < 1 sec)", log=self.log, silent=self.silent)
+                                print_msg("... (took < 1 sec)", log=self.log, silent=self.silent,
+                                          progressbar=self.progressbar)
 
         except StopException:
             pass
@@ -3631,7 +3649,7 @@ def print_dry_run_note(loc, silent=True):
     dry_run_msg(msg, silent=silent)
 
 
-def build_and_install_one(ecdict, init_env):
+def build_and_install_one(ecdict, init_env, progressbar=None):
     """
     Build the software
     :param ecdict: dictionary contaning parsed easyconfig + metadata
@@ -3649,7 +3667,7 @@ def build_and_install_one(ecdict, init_env):
 
     if dry_run:
         dry_run_msg('', silent=silent)
-    print_msg("processing EasyBuild easyconfig %s" % spec, log=_log, silent=silent)
+    print_msg("processing EasyBuild easyconfig %s" % spec, log=_log, silent=silent, progressbar=progressbar)
 
     if dry_run:
         # print note on interpreting dry run output (argument is reference to location of dry run messages)
@@ -3674,6 +3692,7 @@ def build_and_install_one(ecdict, init_env):
     try:
         app_class = get_easyblock_class(easyblock, name=name)
         app = app_class(ecdict['ec'])
+        app.set_progressbar(progressbar)
         _log.info("Obtained application instance of for %s (easyblock: %s)" % (name, easyblock))
     except EasyBuildError as err:
         print_error("Failed to get application instance for %s (easyblock: %s): %s" % (name, easyblock, err.msg),
@@ -3856,7 +3875,8 @@ def ensure_writable_log_dir(log_dir):
         application_log = app.logfile
 
     req_time = time2str(end_timestamp - start_timestamp)
-    print_msg("%s: Installation %s %s (took %s)" % (summary, ended, succ, req_time), log=_log, silent=silent)
+    print_msg("%s: Installation %s %s (took %s)" % (summary, ended, succ, req_time), log=_log, silent=silent,
+              progressbar=progressbar)
 
     # check for errors
     if run.errors_found_in_log > 0:
@@ -3864,7 +3884,7 @@ def ensure_writable_log_dir(log_dir):
                      "build logs, please verify the build.", run.errors_found_in_log)
 
     if app.postmsg:
-        print_msg("\nWARNING: %s\n" % app.postmsg, log=_log, silent=silent)
+        print_msg("\nWARNING: %s\n" % app.postmsg, log=_log, silent=silent, progressbar=progressbar)
 
     if dry_run:
         # print note on interpreting dry run output (argument is reference to location of dry run messages)
@@ -3878,7 +3898,8 @@ def ensure_writable_log_dir(log_dir):
     if application_log:
         # there may be multiple log files, or the file name may be different due to zipping
         logs = glob.glob('%s*' % application_log)
-        print_msg("Results of the build can be found in the log file(s) %s" % ', '.join(logs), log=_log, silent=silent)
+        print_msg("Results of the build can be found in the log file(s) %s" % ', '.join(logs), log=_log, silent=silent,
+                  progressbar=progressbar)
 
     del app
 
diff --git a/easybuild/main.py b/easybuild/main.py
index 748c068376..1da4362597 100644
--- a/easybuild/main.py
+++ b/easybuild/main.py
@@ -39,6 +39,7 @@
 import os
 import stat
 import sys
+import tqdm
 import traceback
 
 # IMPORTANT this has to be the first easybuild import as it customises the logging
@@ -98,7 +99,7 @@ def find_easyconfigs_by_specs(build_specs, robot_path, try_to_generate, testing=
     return [(ec_file, generated)]
 
 
-def build_and_install_software(ecs, init_session_state, exit_on_failure=True):
+def build_and_install_software(ecs, init_session_state, exit_on_failure=True, progress=None):
     """
     Build and install software for all provided parsed easyconfig files.
 
@@ -113,9 +114,11 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True):
 
     res = []
     for ec in ecs:
+        if progress:
+            progress.set_description("Installing %s" % ec['short_mod_name'])
         ec_res = {}
         try:
-            (ec_res['success'], app_log, err) = build_and_install_one(ec, init_env)
+            (ec_res['success'], app_log, err) = build_and_install_one(ec, init_env, progressbar=progress)
             ec_res['log_file'] = app_log
             if not ec_res['success']:
                 ec_res['err'] = EasyBuildError(err)
@@ -153,6 +156,8 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True):
                 raise EasyBuildError(test_msg)
 
         res.append((ec, ec_res))
+        if progress:
+            progress.update()
 
     return res
 
@@ -520,8 +525,13 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
     # build software, will exit when errors occurs (except when testing)
     if not testing or (testing and do_build):
         exit_on_failure = not (options.dump_test_report or options.upload_test_report)
-
-        ecs_with_res = build_and_install_software(ordered_ecs, init_session_state, exit_on_failure=exit_on_failure)
+        # Create progressbar around software to install
+        progress_bar = tqdm.tqdm(total=len(ordered_ecs), desc="EasyBuild",
+                                 leave=False, unit='EB')
+        ecs_with_res = build_and_install_software(
+            ordered_ecs, init_session_state, exit_on_failure=exit_on_failure,
+            progress=progress_bar)
+        progress_bar.close()
     else:
         ecs_with_res = [(ec, {}) for ec in ordered_ecs]
 
diff --git a/easybuild/tools/build_log.py b/easybuild/tools/build_log.py
index 2cf97c5f2d..2242a78dea 100644
--- a/easybuild/tools/build_log.py
+++ b/easybuild/tools/build_log.py
@@ -258,6 +258,7 @@ def print_msg(msg, *args, **kwargs):
     prefix = kwargs.pop('prefix', True)
     newline = kwargs.pop('newline', True)
     stderr = kwargs.pop('stderr', False)
+    pbar = kwargs.pop('progressbar', None)
     if kwargs:
         raise EasyBuildError("Unknown named arguments passed to print_msg: %s", kwargs)
 
@@ -272,6 +273,8 @@ def print_msg(msg, *args, **kwargs):
 
         if stderr:
             sys.stderr.write(msg)
+        elif pbar:
+            pbar.write(msg, end='')
         else:
             sys.stdout.write(msg)
 
@@ -304,6 +307,7 @@ def dry_run_msg(msg, *args, **kwargs):
         msg = msg % args
 
     silent = kwargs.pop('silent', False)
+    pbar = kwargs.pop('progressbar', None)
     if kwargs:
         raise EasyBuildError("Unknown named arguments passed to dry_run_msg: %s", kwargs)
 
@@ -311,7 +315,7 @@ def dry_run_msg(msg, *args, **kwargs):
         if dry_run_var is not None:
             msg = dry_run_var[0].sub(dry_run_var[1], msg)
 
-    print_msg(msg, silent=silent, prefix=False)
+    print_msg(msg, silent=silent, prefix=False, progressbar=pbar)
 
 
 def dry_run_warning(msg, *args, **kwargs):
diff --git a/requirements.txt b/requirements.txt
index e63085eb51..77d393c398 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -62,3 +62,5 @@ archspec; python_version >= '2.7'
 # cryptography 3.4.0 no longer supports Python 2.7
 cryptography==3.3.2; python_version == '2.7'
 cryptography; python_version >= '3.5'
+
+tqdm; python_version >= '2.7'

From 671e7faceda523c0a7511268bb363c15cb572d34 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B8rgen=20Nordmoen?= <j.h.nordmoen@usit.uio.no>
Date: Wed, 8 Sep 2021 09:06:58 +0200
Subject: [PATCH 017/175] Moved initialization of easyblock progressbar

Moved initialization outside try block and added logging for easier
future debugging
---
 easybuild/framework/easyblock.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index 43a4286b97..9bdd64ed68 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -3692,12 +3692,15 @@ def build_and_install_one(ecdict, init_env, progressbar=None):
     try:
         app_class = get_easyblock_class(easyblock, name=name)
         app = app_class(ecdict['ec'])
-        app.set_progressbar(progressbar)
         _log.info("Obtained application instance of for %s (easyblock: %s)" % (name, easyblock))
     except EasyBuildError as err:
         print_error("Failed to get application instance for %s (easyblock: %s): %s" % (name, easyblock, err.msg),
                     silent=silent)
 
+    # Setup progressbar
+    if progressbar:
+        app.set_progressbar(progressbar)
+        _log.info("Updated progressbar instance for easyblock %s" % easyblock)
     # application settings
     stop = build_option('stop')
     if stop is not None:

From 9f5fff0b41c4027a3dea6d03ed1b079ecc8d3155 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B8rgen=20Nordmoen?= <j.h.nordmoen@usit.uio.no>
Date: Thu, 9 Sep 2021 08:52:42 +0200
Subject: [PATCH 018/175] Moved to `rich` library and more fine grained ticks

---
 easybuild/framework/easyblock.py | 64 ++++++++++++++++----------------
 easybuild/main.py                | 29 +++++++++------
 easybuild/tools/build_log.py     |  6 +--
 requirements.txt                 |  2 +-
 4 files changed, 53 insertions(+), 48 deletions(-)

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index 9bdd64ed68..8f4c4a713b 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -214,6 +214,7 @@ def __init__(self, ec):
 
         # Create empty progress bar
         self.progressbar = None
+        self.pbar_task = None
 
         # list of loaded modules
         self.loaded_modules = []
@@ -303,12 +304,20 @@ def close_log(self):
         self.log.info("Closing log for application name %s version %s" % (self.name, self.version))
         fancylogger.logToFile(self.logfile, enable=False)
 
-    def set_progressbar(self, progressbar):
+    def set_progressbar(self, progressbar, task_id):
         """
         Set progress bar, the progress bar is needed when writing messages so
         that the progress counter is always at the bottom
         """
         self.progressbar = progressbar
+        self.pbar_task = task_id
+
+    def advance_progress(self, tick=1.0):
+        """
+        Advance the progress bar forward with `tick`
+        """
+        if self.progressbar and self.pbar_task is not None:
+            self.progressbar.advance(self.pbar_task, tick)
 
     #
     # DRY RUN UTILITIES
@@ -328,7 +337,7 @@ def dry_run_msg(self, msg, *args):
         """Print dry run message."""
         if args:
             msg = msg % args
-        dry_run_msg(msg, silent=self.silent, progressbar=self.progressbar)
+        dry_run_msg(msg, silent=self.silent)
 
     #
     # FETCH UTILITY FUNCTIONS
@@ -1647,8 +1656,7 @@ def skip_extensions(self):
                 self.log.debug("exit code: %s, stdout/err: %s", ec, cmdstdouterr)
                 res.append(ext_inst)
             else:
-                print_msg("skipping extension %s" % ext_inst.name, silent=self.silent, log=self.log,
-                          progressbar=self.progressbar)
+                print_msg("skipping extension %s" % ext_inst.name, silent=self.silent, log=self.log)
 
         self.ext_instances = res
 
@@ -1752,8 +1760,7 @@ def handle_iterate_opts(self):
                     self.log.debug("Found list for %s: %s", opt, self.iter_opts[opt])
 
             if self.iter_opts:
-                print_msg("starting iteration #%s ..." % self.iter_idx, log=self.log, silent=self.silent,
-                          progressbar=self.progressbar)
+                print_msg("starting iteration #%s ..." % self.iter_idx, log=self.log, silent=self.silent)
                 self.log.info("Current iteration index: %s", self.iter_idx)
 
             # pop first element from all iterative easyconfig parameters as next value to use
@@ -1894,8 +1901,7 @@ def check_readiness_step(self):
                 hidden = LooseVersion(self.modules_tool.version) < LooseVersion('7.0.0')
 
             self.mod_file_backup = back_up_file(self.mod_filepath, hidden=hidden, strip_fn=strip_fn)
-            print_msg("backup of existing module file stored at %s" % self.mod_file_backup, log=self.log,
-                      progressbar=self.progressbar)
+            print_msg("backup of existing module file stored at %s" % self.mod_file_backup, log=self.log)
 
         # check if main install needs to be skipped
         # - if a current module can be found, skip is ok
@@ -2431,7 +2437,7 @@ def extensions_step(self, fetch=False, install=True):
             change_dir(self.orig_workdir)
 
             tup = (ext.name, ext.version or '', idx + 1, exts_cnt)
-            print_msg("installing extension %s %s (%d/%d)..." % tup, silent=self.silent, progressbar=self.progressbar)
+            print_msg("installing extension %s %s (%d/%d)..." % tup, silent=self.silent)
             start_time = datetime.now()
 
             if self.dry_run:
@@ -2463,11 +2469,9 @@ def extensions_step(self, fetch=False, install=True):
                     if not self.dry_run:
                         ext_duration = datetime.now() - start_time
                         if ext_duration.total_seconds() >= 1:
-                            print_msg("\t... (took %s)", time2str(ext_duration), log=self.log, silent=self.silent,
-                                      progressbar=self.progressbar)
+                            print_msg("\t... (took %s)", time2str(ext_duration), log=self.log, silent=self.silent)
                         elif self.logdebug or build_option('trace'):
-                            print_msg("\t... (took < 1 sec)", log=self.log, silent=self.silent,
-                                      progressbar=self.progressbar)
+                            print_msg("\t... (took < 1 sec)", log=self.log, silent=self.silent)
 
         # cleanup (unload fake module, remove fake module dir)
         if fake_mod_data:
@@ -3265,7 +3269,7 @@ def make_module_step(self, fake=False):
                 else:
                     diff_msg += 'no differences found'
                 self.log.info(diff_msg)
-                print_msg(diff_msg, log=self.log, progressbar=self.progressbar)
+                print_msg(diff_msg, log=self.log)
 
             self.invalidate_module_caches(modpath)
 
@@ -3583,9 +3587,10 @@ def run_all_steps(self, run_test_cases):
             return True
 
         steps = self.get_steps(run_test_cases=run_test_cases, iteration_count=self.det_iter_cnt())
+        # Calculate progress bar tick
+        tick = 1.0 / float(len(steps))
 
-        print_msg("building and installing %s..." % self.full_mod_name, log=self.log, silent=self.silent,
-                  progressbar=self.progressbar)
+        print_msg("building and installing %s..." % self.full_mod_name, log=self.log, silent=self.silent)
         trace_msg("installation prefix: %s" % self.installdir)
 
         ignore_locks = build_option('ignore_locks')
@@ -3605,12 +3610,12 @@ def run_all_steps(self, run_test_cases):
         try:
             for (step_name, descr, step_methods, skippable) in steps:
                 if self.skip_step(step_name, skippable):
-                    print_msg("%s [skipped]" % descr, log=self.log, silent=self.silent, progressbar=self.progressbar)
+                    print_msg("%s [skipped]" % descr, log=self.log, silent=self.silent)
                 else:
                     if self.dry_run:
                         self.dry_run_msg("%s... [DRY RUN]\n", descr)
                     else:
-                        print_msg("%s..." % descr, log=self.log, silent=self.silent, progressbar=self.progressbar)
+                        print_msg("%s..." % descr, log=self.log, silent=self.silent)
                     self.current_step = step_name
                     start_time = datetime.now()
                     try:
@@ -3619,11 +3624,10 @@ def run_all_steps(self, run_test_cases):
                         if not self.dry_run:
                             step_duration = datetime.now() - start_time
                             if step_duration.total_seconds() >= 1:
-                                print_msg("... (took %s)", time2str(step_duration), log=self.log, silent=self.silent,
-                                          progressbar=self.progressbar)
+                                print_msg("... (took %s)", time2str(step_duration), log=self.log, silent=self.silent)
                             elif self.logdebug or build_option('trace'):
-                                print_msg("... (took < 1 sec)", log=self.log, silent=self.silent,
-                                          progressbar=self.progressbar)
+                                print_msg("... (took < 1 sec)", log=self.log, silent=self.silent)
+                self.advance_progress(tick)
 
         except StopException:
             pass
@@ -3649,7 +3653,7 @@ def print_dry_run_note(loc, silent=True):
     dry_run_msg(msg, silent=silent)
 
 
-def build_and_install_one(ecdict, init_env, progressbar=None):
+def build_and_install_one(ecdict, init_env, progressbar=None, task_id=None):
     """
     Build the software
     :param ecdict: dictionary contaning parsed easyconfig + metadata
@@ -3667,7 +3671,7 @@ def build_and_install_one(ecdict, init_env, progressbar=None):
 
     if dry_run:
         dry_run_msg('', silent=silent)
-    print_msg("processing EasyBuild easyconfig %s" % spec, log=_log, silent=silent, progressbar=progressbar)
+    print_msg("processing EasyBuild easyconfig %s" % spec, log=_log, silent=silent)
 
     if dry_run:
         # print note on interpreting dry run output (argument is reference to location of dry run messages)
@@ -3698,8 +3702,8 @@ def build_and_install_one(ecdict, init_env, progressbar=None):
                     silent=silent)
 
     # Setup progressbar
-    if progressbar:
-        app.set_progressbar(progressbar)
+    if progressbar and task_id is not None:
+        app.set_progressbar(progressbar, task_id)
         _log.info("Updated progressbar instance for easyblock %s" % easyblock)
     # application settings
     stop = build_option('stop')
@@ -3878,8 +3882,7 @@ def ensure_writable_log_dir(log_dir):
         application_log = app.logfile
 
     req_time = time2str(end_timestamp - start_timestamp)
-    print_msg("%s: Installation %s %s (took %s)" % (summary, ended, succ, req_time), log=_log, silent=silent,
-              progressbar=progressbar)
+    print_msg("%s: Installation %s %s (took %s)" % (summary, ended, succ, req_time), log=_log, silent=silent)
 
     # check for errors
     if run.errors_found_in_log > 0:
@@ -3887,7 +3890,7 @@ def ensure_writable_log_dir(log_dir):
                      "build logs, please verify the build.", run.errors_found_in_log)
 
     if app.postmsg:
-        print_msg("\nWARNING: %s\n" % app.postmsg, log=_log, silent=silent, progressbar=progressbar)
+        print_msg("\nWARNING: %s\n" % app.postmsg, log=_log, silent=silent)
 
     if dry_run:
         # print note on interpreting dry run output (argument is reference to location of dry run messages)
@@ -3901,8 +3904,7 @@ def ensure_writable_log_dir(log_dir):
     if application_log:
         # there may be multiple log files, or the file name may be different due to zipping
         logs = glob.glob('%s*' % application_log)
-        print_msg("Results of the build can be found in the log file(s) %s" % ', '.join(logs), log=_log, silent=silent,
-                  progressbar=progressbar)
+        print_msg("Results of the build can be found in the log file(s) %s" % ', '.join(logs), log=_log, silent=silent)
 
     del app
 
diff --git a/easybuild/main.py b/easybuild/main.py
index 1da4362597..987819fb0d 100644
--- a/easybuild/main.py
+++ b/easybuild/main.py
@@ -39,7 +39,6 @@
 import os
 import stat
 import sys
-import tqdm
 import traceback
 
 # IMPORTANT this has to be the first easybuild import as it customises the logging
@@ -74,6 +73,7 @@
 from easybuild.tools.parallelbuild import submit_jobs
 from easybuild.tools.repository.repository import init_repository
 from easybuild.tools.testing import create_test_report, overall_test_report, regtest, session_state
+from rich.progress import Progress, TextColumn, BarColumn, TimeElapsedColumn
 
 _log = None
 
@@ -112,13 +112,17 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True, pr
     # e.g. via easyconfig.handle_allowed_system_deps
     init_env = copy.deepcopy(os.environ)
 
+    # Initialize progress bar with overall installation task
+    if progress:
+        task_id = progress.add_task("", total=len(ecs))
     res = []
     for ec in ecs:
         if progress:
-            progress.set_description("Installing %s" % ec['short_mod_name'])
+            progress.update(task_id, description=ec['short_mod_name'])
         ec_res = {}
         try:
-            (ec_res['success'], app_log, err) = build_and_install_one(ec, init_env, progressbar=progress)
+            (ec_res['success'], app_log, err) = build_and_install_one(ec, init_env, progressbar=progress,
+                                                                      task_id=task_id)
             ec_res['log_file'] = app_log
             if not ec_res['success']:
                 ec_res['err'] = EasyBuildError(err)
@@ -156,8 +160,6 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True, pr
                 raise EasyBuildError(test_msg)
 
         res.append((ec, ec_res))
-        if progress:
-            progress.update()
 
     return res
 
@@ -526,12 +528,17 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
     if not testing or (testing and do_build):
         exit_on_failure = not (options.dump_test_report or options.upload_test_report)
         # Create progressbar around software to install
-        progress_bar = tqdm.tqdm(total=len(ordered_ecs), desc="EasyBuild",
-                                 leave=False, unit='EB')
-        ecs_with_res = build_and_install_software(
-            ordered_ecs, init_session_state, exit_on_failure=exit_on_failure,
-            progress=progress_bar)
-        progress_bar.close()
+        progress_bar = Progress(
+            TextColumn("[bold blue]Installing {task.description} ({task.completed:.0f}/{task.total})"),
+            BarColumn(),
+            "[progress.percentage]{task.percentage:>3.1f}%",
+            "•",
+            TimeElapsedColumn()
+        )
+        with progress_bar:
+            ecs_with_res = build_and_install_software(
+                ordered_ecs, init_session_state, exit_on_failure=exit_on_failure,
+                progress=progress_bar)
     else:
         ecs_with_res = [(ec, {}) for ec in ordered_ecs]
 
diff --git a/easybuild/tools/build_log.py b/easybuild/tools/build_log.py
index 2242a78dea..2cf97c5f2d 100644
--- a/easybuild/tools/build_log.py
+++ b/easybuild/tools/build_log.py
@@ -258,7 +258,6 @@ def print_msg(msg, *args, **kwargs):
     prefix = kwargs.pop('prefix', True)
     newline = kwargs.pop('newline', True)
     stderr = kwargs.pop('stderr', False)
-    pbar = kwargs.pop('progressbar', None)
     if kwargs:
         raise EasyBuildError("Unknown named arguments passed to print_msg: %s", kwargs)
 
@@ -273,8 +272,6 @@ def print_msg(msg, *args, **kwargs):
 
         if stderr:
             sys.stderr.write(msg)
-        elif pbar:
-            pbar.write(msg, end='')
         else:
             sys.stdout.write(msg)
 
@@ -307,7 +304,6 @@ def dry_run_msg(msg, *args, **kwargs):
         msg = msg % args
 
     silent = kwargs.pop('silent', False)
-    pbar = kwargs.pop('progressbar', None)
     if kwargs:
         raise EasyBuildError("Unknown named arguments passed to dry_run_msg: %s", kwargs)
 
@@ -315,7 +311,7 @@ def dry_run_msg(msg, *args, **kwargs):
         if dry_run_var is not None:
             msg = dry_run_var[0].sub(dry_run_var[1], msg)
 
-    print_msg(msg, silent=silent, prefix=False, progressbar=pbar)
+    print_msg(msg, silent=silent, prefix=False)
 
 
 def dry_run_warning(msg, *args, **kwargs):
diff --git a/requirements.txt b/requirements.txt
index 77d393c398..3f82c41b26 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -63,4 +63,4 @@ archspec; python_version >= '2.7'
 cryptography==3.3.2; python_version == '2.7'
 cryptography; python_version >= '3.5'
 
-tqdm; python_version >= '2.7'
+rich; python_version >= '2.7'

From 9b0a12d99aad17a6a093fb671e6c3affcc7dbd10 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Sat, 11 Sep 2021 14:05:43 +0200
Subject: [PATCH 019/175] use Rich as optional dependency for showing progress
 bar

---
 easybuild/framework/easyblock.py | 21 +++++++------
 easybuild/main.py                | 54 +++++++++++++++++++++-----------
 2 files changed, 46 insertions(+), 29 deletions(-)

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index 8f4c4a713b..58bed6afde 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -213,7 +213,7 @@ def __init__(self, ec):
         self.current_step = None
 
         # Create empty progress bar
-        self.progressbar = None
+        self.progress_bar = None
         self.pbar_task = None
 
         # list of loaded modules
@@ -304,20 +304,20 @@ def close_log(self):
         self.log.info("Closing log for application name %s version %s" % (self.name, self.version))
         fancylogger.logToFile(self.logfile, enable=False)
 
-    def set_progressbar(self, progressbar, task_id):
+    def set_progress_bar(self, progress_bar, task_id):
         """
         Set progress bar, the progress bar is needed when writing messages so
         that the progress counter is always at the bottom
         """
-        self.progressbar = progressbar
+        self.progress_bar = progress_bar
         self.pbar_task = task_id
 
     def advance_progress(self, tick=1.0):
         """
         Advance the progress bar forward with `tick`
         """
-        if self.progressbar and self.pbar_task is not None:
-            self.progressbar.advance(self.pbar_task, tick)
+        if self.progress_bar and self.pbar_task is not None:
+            self.progress_bar.advance(self.pbar_task, tick)
 
     #
     # DRY RUN UTILITIES
@@ -3653,7 +3653,7 @@ def print_dry_run_note(loc, silent=True):
     dry_run_msg(msg, silent=silent)
 
 
-def build_and_install_one(ecdict, init_env, progressbar=None, task_id=None):
+def build_and_install_one(ecdict, init_env, progress_bar=None, task_id=None):
     """
     Build the software
     :param ecdict: dictionary contaning parsed easyconfig + metadata
@@ -3701,10 +3701,11 @@ def build_and_install_one(ecdict, init_env, progressbar=None, task_id=None):
         print_error("Failed to get application instance for %s (easyblock: %s): %s" % (name, easyblock, err.msg),
                     silent=silent)
 
-    # Setup progressbar
-    if progressbar and task_id is not None:
-        app.set_progressbar(progressbar, task_id)
-        _log.info("Updated progressbar instance for easyblock %s" % easyblock)
+    # Setup progress bar
+    if progress_bar and task_id is not None:
+        app.set_progress_bar(progress_bar, task_id)
+        _log.info("Updated progress bar instance for easyblock %s", easyblock)
+
     # application settings
     stop = build_option('stop')
     if stop is not None:
diff --git a/easybuild/main.py b/easybuild/main.py
index 987819fb0d..1a5e427917 100644
--- a/easybuild/main.py
+++ b/easybuild/main.py
@@ -73,7 +73,13 @@
 from easybuild.tools.parallelbuild import submit_jobs
 from easybuild.tools.repository.repository import init_repository
 from easybuild.tools.testing import create_test_report, overall_test_report, regtest, session_state
-from rich.progress import Progress, TextColumn, BarColumn, TimeElapsedColumn
+
+try:
+    from rich.progress import Progress, TextColumn, BarColumn, TimeElapsedColumn
+    HAVE_RICH = True
+except ImportError:
+    HAVE_RICH = False
+
 
 _log = None
 
@@ -99,13 +105,14 @@ def find_easyconfigs_by_specs(build_specs, robot_path, try_to_generate, testing=
     return [(ec_file, generated)]
 
 
-def build_and_install_software(ecs, init_session_state, exit_on_failure=True, progress=None):
+def build_and_install_software(ecs, init_session_state, exit_on_failure=True, progress_bar=None):
     """
     Build and install software for all provided parsed easyconfig files.
 
     :param ecs: easyconfig files to install software with
     :param init_session_state: initial session state, to use in test reports
     :param exit_on_failure: whether or not to exit on installation failure
+    :param progress_bar: ProgressBar instance to use to report progress
     """
     # obtain a copy of the starting environment so each build can start afresh
     # we shouldn't use the environment from init_session_state, since relevant env vars might have been set since
@@ -113,15 +120,20 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True, pr
     init_env = copy.deepcopy(os.environ)
 
     # Initialize progress bar with overall installation task
-    if progress:
-        task_id = progress.add_task("", total=len(ecs))
+    if progress_bar:
+        task_id = progress_bar.add_task("", total=len(ecs))
+    else:
+        task_id = None
+
     res = []
     for ec in ecs:
-        if progress:
-            progress.update(task_id, description=ec['short_mod_name'])
+
+        if progress_bar:
+            progress_bar.update(task_id, description=ec['short_mod_name'])
+
         ec_res = {}
         try:
-            (ec_res['success'], app_log, err) = build_and_install_one(ec, init_env, progressbar=progress,
+            (ec_res['success'], app_log, err) = build_and_install_one(ec, init_env, progress_bar=progress_bar,
                                                                       task_id=task_id)
             ec_res['log_file'] = app_log
             if not ec_res['success']:
@@ -527,18 +539,22 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
     # build software, will exit when errors occurs (except when testing)
     if not testing or (testing and do_build):
         exit_on_failure = not (options.dump_test_report or options.upload_test_report)
-        # Create progressbar around software to install
-        progress_bar = Progress(
-            TextColumn("[bold blue]Installing {task.description} ({task.completed:.0f}/{task.total})"),
-            BarColumn(),
-            "[progress.percentage]{task.percentage:>3.1f}%",
-            "•",
-            TimeElapsedColumn()
-        )
-        with progress_bar:
-            ecs_with_res = build_and_install_software(
-                ordered_ecs, init_session_state, exit_on_failure=exit_on_failure,
-                progress=progress_bar)
+
+        if HAVE_RICH:
+            # Create progressbar around software to install
+            progress_bar = Progress(
+                TextColumn("[bold blue]Installing {task.description} ({task.completed:.0f}/{task.total})"),
+                BarColumn(),
+                "[progress.percentage]{task.percentage:>3.1f}%",
+                "•",
+                TimeElapsedColumn()
+            )
+            with progress_bar:
+                ecs_with_res = build_and_install_software(ordered_ecs, init_session_state,
+                                                          exit_on_failure=exit_on_failure,
+                                                          progress_bar=progress_bar)
+        else:
+            ecs_with_res = build_and_install_software(ordered_ecs, init_session_state, exit_on_failure=exit_on_failure)
     else:
         ecs_with_res = [(ec, {}) for ec in ordered_ecs]
 

From 33f8e546ae43e933ca27434b6aa0c03aff157324 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Sat, 11 Sep 2021 14:27:12 +0200
Subject: [PATCH 020/175] add create_progress_bar function in new
 easybuild.tools.output module

---
 easybuild/main.py         | 29 ++++-----------
 easybuild/tools/output.py | 75 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 82 insertions(+), 22 deletions(-)
 create mode 100644 easybuild/tools/output.py

diff --git a/easybuild/main.py b/easybuild/main.py
index 1a5e427917..9c2243a50c 100644
--- a/easybuild/main.py
+++ b/easybuild/main.py
@@ -68,18 +68,13 @@
 from easybuild.tools.hooks import START, END, load_hooks, run_hook
 from easybuild.tools.modules import modules_tool
 from easybuild.tools.options import set_up_configuration, use_color
+from easybuild.tools.output import create_progress_bar
 from easybuild.tools.robot import check_conflicts, dry_run, missing_deps, resolve_dependencies, search_easyconfigs
 from easybuild.tools.package.utilities import check_pkg_support
 from easybuild.tools.parallelbuild import submit_jobs
 from easybuild.tools.repository.repository import init_repository
 from easybuild.tools.testing import create_test_report, overall_test_report, regtest, session_state
 
-try:
-    from rich.progress import Progress, TextColumn, BarColumn, TimeElapsedColumn
-    HAVE_RICH = True
-except ImportError:
-    HAVE_RICH = False
-
 
 _log = None
 
@@ -112,7 +107,7 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True, pr
     :param ecs: easyconfig files to install software with
     :param init_session_state: initial session state, to use in test reports
     :param exit_on_failure: whether or not to exit on installation failure
-    :param progress_bar: ProgressBar instance to use to report progress
+    :param progress_bar: progress bar to use to report progress
     """
     # obtain a copy of the starting environment so each build can start afresh
     # we shouldn't use the environment from init_session_state, since relevant env vars might have been set since
@@ -540,21 +535,11 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
     if not testing or (testing and do_build):
         exit_on_failure = not (options.dump_test_report or options.upload_test_report)
 
-        if HAVE_RICH:
-            # Create progressbar around software to install
-            progress_bar = Progress(
-                TextColumn("[bold blue]Installing {task.description} ({task.completed:.0f}/{task.total})"),
-                BarColumn(),
-                "[progress.percentage]{task.percentage:>3.1f}%",
-                "•",
-                TimeElapsedColumn()
-            )
-            with progress_bar:
-                ecs_with_res = build_and_install_software(ordered_ecs, init_session_state,
-                                                          exit_on_failure=exit_on_failure,
-                                                          progress_bar=progress_bar)
-        else:
-            ecs_with_res = build_and_install_software(ordered_ecs, init_session_state, exit_on_failure=exit_on_failure)
+        progress_bar = create_progress_bar()
+        with progress_bar:
+            ecs_with_res = build_and_install_software(ordered_ecs, init_session_state,
+                                                      exit_on_failure=exit_on_failure,
+                                                      progress_bar=progress_bar)
     else:
         ecs_with_res = [(ec, {}) for ec in ordered_ecs]
 
diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py
new file mode 100644
index 0000000000..f20f027d31
--- /dev/null
+++ b/easybuild/tools/output.py
@@ -0,0 +1,75 @@
+# -*- coding: utf-8 -*-
+# #
+# Copyright 2021-2021 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild.  If not, see <http://www.gnu.org/licenses/>.
+# #
+"""
+Tools for controlling output to terminal produced by EasyBuild.
+
+:author: Kenneth Hoste (Ghent University)
+:author: Jørgen Nordmoen (University of Oslo)
+"""
+try:
+    from rich.progress import Progress, TextColumn, BarColumn, TimeElapsedColumn
+    HAVE_RICH = True
+except ImportError:
+    HAVE_RICH = False
+
+
+class DummyProgress(object):
+    """Shim for Rich's Progress class."""
+
+    # __enter__ and __exit__ must be implemented to allow use as context manager
+    def __enter__(self, *args, **kwargs):
+        pass
+
+    def __exit__(self, *args, **kwargs):
+        pass
+
+    # dummy implementations for methods supported by rich.progress.Progress class
+    def add_task(self, *args, **kwargs):
+        pass
+
+    def update(self, *args, **kwargs):
+        pass
+
+
+def create_progress_bar():
+    """
+    Create progress bar to display overall progress.
+
+    Returns rich.progress.Progress instance if the Rich Python package is available,
+    or a shim DummyProgress instance otherwise.
+    """
+    if HAVE_RICH:
+        progress_bar = Progress(
+            TextColumn("[bold blue]Installing {task.description} ({task.completed:.0f}/{task.total})"),
+            BarColumn(),
+            "[progress.percentage]{task.percentage:>3.1f}%",
+            "•",
+            TimeElapsedColumn()
+        )
+    else:
+        progress_bar = DummyProgress()
+
+    return progress_bar

From d779a618d3fe70afc0b7f68c277bd4f55be36862 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Sat, 11 Sep 2021 14:33:20 +0200
Subject: [PATCH 021/175] fix requirements.txt: Rich only supports Python 3.6+

---
 requirements.txt | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 3f82c41b26..591fc502f3 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -63,4 +63,5 @@ archspec; python_version >= '2.7'
 cryptography==3.3.2; python_version == '2.7'
 cryptography; python_version >= '3.5'
 
-rich; python_version >= '2.7'
+# rich is only supported for Python 3.6+
+rich; python_version >= '3.6'

From 771f95d9108401582bea4a52399f8ac84b9a7515 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Sat, 11 Sep 2021 16:44:10 +0200
Subject: [PATCH 022/175] use transient progress bar, so it disappears when
 installation is complete

---
 easybuild/tools/output.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py
index f20f027d31..359e7746d2 100644
--- a/easybuild/tools/output.py
+++ b/easybuild/tools/output.py
@@ -67,7 +67,8 @@ def create_progress_bar():
             BarColumn(),
             "[progress.percentage]{task.percentage:>3.1f}%",
             "•",
-            TimeElapsedColumn()
+            TimeElapsedColumn(),
+            transient=True,
         )
     else:
         progress_bar = DummyProgress()

From 33df74b6de0e2fe400c588dfba37dfecd928e980 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Sat, 11 Sep 2021 17:05:24 +0200
Subject: [PATCH 023/175] use random spinner in progress bar

---
 easybuild/tools/output.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py
index 359e7746d2..831e0674bb 100644
--- a/easybuild/tools/output.py
+++ b/easybuild/tools/output.py
@@ -29,8 +29,10 @@
 :author: Kenneth Hoste (Ghent University)
 :author: Jørgen Nordmoen (University of Oslo)
 """
+import random
+
 try:
-    from rich.progress import Progress, TextColumn, BarColumn, TimeElapsedColumn
+    from rich.progress import BarColumn, Progress, SpinnerColumn, TextColumn, TimeElapsedColumn
     HAVE_RICH = True
 except ImportError:
     HAVE_RICH = False
@@ -62,11 +64,15 @@ def create_progress_bar():
     or a shim DummyProgress instance otherwise.
     """
     if HAVE_RICH:
+
+        # pick random spinner, from a selected subset of available spinner (see 'python3 -m rich.spinner')
+        spinner = random.choice(('aesthetic', 'arc', 'bounce', 'dots', 'line', 'monkey', 'point', 'simpleDots'))
+
         progress_bar = Progress(
             TextColumn("[bold blue]Installing {task.description} ({task.completed:.0f}/{task.total})"),
             BarColumn(),
             "[progress.percentage]{task.percentage:>3.1f}%",
-            "•",
+            SpinnerColumn(spinner),
             TimeElapsedColumn(),
             transient=True,
         )

From f714ea6132362500e3ac6f09f422e8cfbb829b10 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Sat, 11 Sep 2021 17:45:54 +0200
Subject: [PATCH 024/175] add configuration option to allow disabling progress
 bar

---
 easybuild/tools/config.py  | 1 +
 easybuild/tools/options.py | 7 ++++---
 easybuild/tools/output.py  | 4 +++-
 3 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py
index 8f660331f5..94b02d4424 100644
--- a/easybuild/tools/config.py
+++ b/easybuild/tools/config.py
@@ -292,6 +292,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
         'map_toolchains',
         'modules_tool_version_check',
         'pre_create_installdir',
+        'show_progress_bar',
     ],
     WARN: [
         'check_ebroot_env_vars',
diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py
index b1aa1dd3cb..64948af45c 100644
--- a/easybuild/tools/options.py
+++ b/easybuild/tools/options.py
@@ -406,6 +406,8 @@ def override_options(self):
             'force-download': ("Force re-downloading of sources and/or patches, "
                                "even if they are available already in source path",
                                'choice', 'store_or_None', DEFAULT_FORCE_DOWNLOAD, FORCE_DOWNLOAD_CHOICES),
+            'generate-devel-module': ("Generate a develop module file, implies --force if disabled",
+                                      None, 'store_true', True),
             'group': ("Group to be used for software installations (only verified, not set)", None, 'store', None),
             'group-writable-installdir': ("Enable group write permissions on installation directory after installation",
                                           None, 'store_true', False),
@@ -468,13 +470,12 @@ def override_options(self):
                                   None, 'store_true', False),
             'set-default-module': ("Set the generated module as default", None, 'store_true', False),
             'set-gid-bit': ("Set group ID bit on newly created directories", None, 'store_true', False),
+            'show-progress-bar': ("Show progress bar in terminal output", None, 'store_true', True),
             'silence-deprecation-warnings': ("Silence specified deprecation warnings", 'strlist', 'extend', None),
-            'sticky-bit': ("Set sticky bit on newly created directories", None, 'store_true', False),
             'skip-extensions': ("Skip installation of extensions", None, 'store_true', False),
             'skip-test-cases': ("Skip running test cases", None, 'store_true', False, 't'),
             'skip-test-step': ("Skip running the test step (e.g. unit tests)", None, 'store_true', False),
-            'generate-devel-module': ("Generate a develop module file, implies --force if disabled",
-                                      None, 'store_true', True),
+            'sticky-bit': ("Set sticky bit on newly created directories", None, 'store_true', False),
             'sysroot': ("Location root directory of system, prefix for standard paths like /usr/lib and /usr/include",
                         None, 'store', None),
             'trace': ("Provide more information in output to stdout on progress", None, 'store_true', False, 'T'),
diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py
index 831e0674bb..35c6d962a3 100644
--- a/easybuild/tools/output.py
+++ b/easybuild/tools/output.py
@@ -31,6 +31,8 @@
 """
 import random
 
+from easybuild.tools.config import build_option
+
 try:
     from rich.progress import BarColumn, Progress, SpinnerColumn, TextColumn, TimeElapsedColumn
     HAVE_RICH = True
@@ -63,7 +65,7 @@ def create_progress_bar():
     Returns rich.progress.Progress instance if the Rich Python package is available,
     or a shim DummyProgress instance otherwise.
     """
-    if HAVE_RICH:
+    if HAVE_RICH and build_option('show_progress_bar'):
 
         # pick random spinner, from a selected subset of available spinner (see 'python3 -m rich.spinner')
         spinner = random.choice(('aesthetic', 'arc', 'bounce', 'dots', 'line', 'monkey', 'point', 'simpleDots'))

From 37e6877942f46277285c51e3706b1fd7369e8e3f Mon Sep 17 00:00:00 2001
From: Simon Branford <s.j.branford@bham.ac.uk>
Date: Sun, 12 Sep 2021 08:32:27 +0100
Subject: [PATCH 025/175] expand progress bar to full screen width

---
 easybuild/tools/output.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py
index 35c6d962a3..7c936a7261 100644
--- a/easybuild/tools/output.py
+++ b/easybuild/tools/output.py
@@ -72,11 +72,12 @@ def create_progress_bar():
 
         progress_bar = Progress(
             TextColumn("[bold blue]Installing {task.description} ({task.completed:.0f}/{task.total})"),
-            BarColumn(),
+            BarColumn(bar_width=None),
             "[progress.percentage]{task.percentage:>3.1f}%",
             SpinnerColumn(spinner),
             TimeElapsedColumn(),
             transient=True,
+            expand=True,
         )
     else:
         progress_bar = DummyProgress()

From 681e53f227fe2a839c906b811bec77e83b940fb0 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Sun, 12 Sep 2021 10:07:31 +0200
Subject: [PATCH 026/175] reorder progress bar components

---
 easybuild/tools/output.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py
index 7c936a7261..52ee836b50 100644
--- a/easybuild/tools/output.py
+++ b/easybuild/tools/output.py
@@ -71,10 +71,10 @@ def create_progress_bar():
         spinner = random.choice(('aesthetic', 'arc', 'bounce', 'dots', 'line', 'monkey', 'point', 'simpleDots'))
 
         progress_bar = Progress(
-            TextColumn("[bold blue]Installing {task.description} ({task.completed:.0f}/{task.total})"),
-            BarColumn(bar_width=None),
-            "[progress.percentage]{task.percentage:>3.1f}%",
             SpinnerColumn(spinner),
+            "[progress.percentage]{task.percentage:>3.1f}%",
+            TextColumn("[bold blue]Installing {task.description} ({task.completed:.0f}/{task.total} done)"),
+            BarColumn(bar_width=None),
             TimeElapsedColumn(),
             transient=True,
             expand=True,

From 2db1befd6b45981db1fb33a9720294db7b1352d3 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Sun, 12 Sep 2021 12:20:23 +0200
Subject: [PATCH 027/175] add support for checking required/optional EasyBuild
 dependencies via 'eb --check-eb-deps'

---
 easybuild/main.py              |   5 ++
 easybuild/tools/modules.py     |  13 +++-
 easybuild/tools/options.py     |   2 +
 easybuild/tools/systemtools.py | 124 ++++++++++++++++++++++++++++++++-
 test/framework/options.py      |  22 ++++++
 5 files changed, 162 insertions(+), 4 deletions(-)

diff --git a/easybuild/main.py b/easybuild/main.py
index 9c2243a50c..84b02dc186 100644
--- a/easybuild/main.py
+++ b/easybuild/main.py
@@ -73,6 +73,7 @@
 from easybuild.tools.package.utilities import check_pkg_support
 from easybuild.tools.parallelbuild import submit_jobs
 from easybuild.tools.repository.repository import init_repository
+from easybuild.tools.systemtools import check_easybuild_deps
 from easybuild.tools.testing import create_test_report, overall_test_report, regtest, session_state
 
 
@@ -259,6 +260,9 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
         search_easyconfigs(search_query, short=options.search_short, filename_only=options.search_filename,
                            terse=options.terse)
 
+    if options.check_eb_deps:
+        print(check_easybuild_deps(modtool))
+
     # GitHub options that warrant a silent cleanup & exit
     if options.check_github:
         check_github()
@@ -297,6 +301,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
     # non-verbose cleanup after handling GitHub integration stuff or printing terse info
     early_stop_options = [
         options.add_pr_labels,
+        options.check_eb_deps,
         options.check_github,
         options.create_index,
         options.install_github_token,
diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py
index 5e48591be5..0860b810d8 100644
--- a/easybuild/tools/modules.py
+++ b/easybuild/tools/modules.py
@@ -207,6 +207,15 @@ def __init__(self, mod_paths=None, testing=False):
         self.set_and_check_version()
         self.supports_depends_on = False
 
+    def __str__(self):
+        """String representation of this ModulesTool instance."""
+        res = self.NAME
+        if self.version:
+            res += ' ' + self.version
+        else:
+            res += ' (unknown version)'
+        return res
+
     def buildstats(self):
         """Return tuple with data to be included in buildstats"""
         return (self.NAME, self.cmd, self.version)
@@ -1177,7 +1186,7 @@ def update(self):
 
 class EnvironmentModulesC(ModulesTool):
     """Interface to (C) environment modules (modulecmd)."""
-    NAME = "Environment Modules v3"
+    NAME = "Environment Modules"
     COMMAND = "modulecmd"
     REQ_VERSION = '3.2.10'
     MAX_VERSION = '3.99'
@@ -1312,7 +1321,7 @@ def remove_module_path(self, path, set_mod_paths=True):
 
 class EnvironmentModules(EnvironmentModulesTcl):
     """Interface to environment modules 4.0+"""
-    NAME = "Environment Modules v4"
+    NAME = "Environment Modules"
     COMMAND = os.path.join(os.getenv('MODULESHOME', 'MODULESHOME_NOT_DEFINED'), 'libexec', 'modulecmd.tcl')
     COMMAND_ENVIRONMENT = 'MODULES_CMD'
     REQ_VERSION = '4.0.0'
diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py
index 64948af45c..759e8d8e42 100644
--- a/easybuild/tools/options.py
+++ b/easybuild/tools/options.py
@@ -617,6 +617,8 @@ def informative_options(self):
             'avail-hooks': ("Show list of known hooks", None, 'store_true', False),
             'avail-toolchain-opts': ("Show options for toolchain", 'str', 'store', None),
             'check-conflicts': ("Check for version conflicts in dependency graphs", None, 'store_true', False),
+            'check-eb-deps': ("Check presence and version of (required and optional) EasyBuild dependencies",
+                              None, 'store_true', False),
             'dep-graph': ("Create dependency graph", None, 'store', None, {'metavar': 'depgraph.<ext>'}),
             'dump-env-script': ("Dump source script to set up build environment based on toolchain/dependencies",
                                 None, 'store_true', False),
diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py
index a43338e147..bf96fb7745 100644
--- a/easybuild/tools/systemtools.py
+++ b/easybuild/tools/systemtools.py
@@ -34,6 +34,7 @@
 import grp  # @UnresolvedImport
 import os
 import platform
+import pkg_resources
 import pwd
 import re
 import struct
@@ -152,6 +153,36 @@
 RPM = 'rpm'
 DPKG = 'dpkg'
 
+SYSTEM_TOOLS = ('7z', 'bunzip2', DPKG, 'gunzip', 'make', 'patch', RPM, 'sed', 'tar', 'unxz', 'unzip')
+
+OPT_DEPS = {
+    'archspec': "determining name of CPU microarchitecture",
+    'autopep8': "auto-formatting for dumped easyconfigs",
+    'GC3Pie': "backend for --job",
+    'GitPython': "GitHub integration + using Git repository as easyconfigs archive",
+    'graphviz-python': "rendering dependency graph with Graphviz: --dep-graph",
+    'keyring': "storing GitHub token",
+    'pep8': "fallback for code style checking: --check-style, --check-contrib",
+    'pycodestyle': "code style checking: --check-style, --check-contrib",
+    'pysvn': "using SVN repository as easyconfigs archive",
+    'python-graph-core': "creating dependency graph: --dep-graph",
+    'python-graph-dot': "saving dependency graph as dot file: --dep-graph",
+    'python-hglib': "using Mercurial repository as easyconfigs archive",
+    'requests': "fallback library for downloading files",
+    'Rich': "eb command rich terminal output",
+    'PyYAML': "easystack files and .yeb easyconfig format",
+}
+
+OPT_DEP_PKG_NAMES = {
+    'GC3Pie': 'gc3libs',
+    'GitPython': 'git',
+    'graphviz-python': 'gv',
+    'python-graph-core': 'pygraph.classes.digraph',
+    'python-graph-dot': 'pygraph.readwrite.dot',
+    'python-hglib': 'hglib',
+    'PyYAML': 'yaml',
+}
+
 
 class SystemToolsException(Exception):
     """raised when systemtools fails"""
@@ -722,14 +753,14 @@ def check_os_dependency(dep):
     return found
 
 
-def get_tool_version(tool, version_option='--version'):
+def get_tool_version(tool, version_option='--version', ignore_ec=False):
     """
     Get output of running version option for specific command line tool.
     Output is returned as a single-line string (newlines are replaced by '; ').
     """
     out, ec = run_cmd(' '.join([tool, version_option]), simple=False, log_ok=False, force_in_dry_run=True,
                       trace=False, stream_output=False)
-    if ec:
+    if not ignore_ec and ec:
         _log.warning("Failed to determine version of %s using '%s %s': %s" % (tool, tool, version_option, out))
         return UNKNOWN
     else:
@@ -1103,3 +1134,92 @@ def pick_dep_version(dep_version):
         raise EasyBuildError("Unknown value type for version: %s (%s), should be string value", typ, dep_version)
 
     return result
+
+
+def check_easybuild_deps(modtool):
+    """
+    Check presence and version of required and optional EasyBuild dependencies, and report back to terminal.
+    """
+    version_regex = re.compile(r'\s(?P<version>[0-9][0-9.]+[a-z]*)')
+
+    def extract_version(tool):
+        """Helper function to extract (only) version for specific command line tool."""
+        out = get_tool_version(tool, ignore_ec=True)
+        res = version_regex.search(out)
+        if res:
+            version = res.group('version')
+        else:
+            version = "UNKNOWN version"
+
+        return version
+
+    python_version = extract_version(sys.executable)
+
+    opt_dep_versions = {}
+    for key in OPT_DEPS:
+
+        pkg = OPT_DEP_PKG_NAMES.get(key, key.lower())
+
+        try:
+            mod = __import__(pkg)
+        except ImportError:
+            mod = None
+
+        if mod:
+            try:
+                dep_version = pkg_resources.get_distribution(pkg).version
+            except pkg_resources.DistributionNotFound:
+                try:
+                    dep_version = pkg_resources.get_distribution(key).version
+                except pkg_resources.DistributionNotFound:
+                    if hasattr(mod, '__version__'):
+                        dep_version = mod.__version__
+                    else:
+                        dep_version = '(unknown version)'
+        else:
+            dep_version = '(NOT AVAILABLE)'
+
+        opt_dep_versions[key] = dep_version
+
+    lines = [
+        '',
+        "Required dependencies:",
+        "----------------------",
+        '',
+        "* Python %s" % python_version,
+        "* %s (modules tool)" % modtool,
+        '',
+        "Optional dependencies:",
+        "----------------------",
+        '',
+    ]
+    for pkg in sorted(opt_dep_versions, key=lambda x: x.lower()):
+        line = "* %s %s" % (pkg, opt_dep_versions[pkg])
+        line = line.ljust(40) + " [%s]" % OPT_DEPS[pkg]
+        lines.append(line)
+
+    lines.extend([
+        '',
+        "System tools:",
+        "-------------",
+        '',
+    ])
+
+    tools = list(SYSTEM_TOOLS) + ['Slurm']
+    cmds = {'Slurm': 'sbatch'}
+
+    for tool in sorted(tools, key=lambda x: x.lower()):
+        line = "* %s " % tool
+        cmd = cmds.get(tool, tool)
+        if which(cmd):
+            version = extract_version(cmd)
+            if version.startswith('UNKNOWN'):
+                line += "(available, %s)" % version
+            else:
+                line += version
+        else:
+            line += "(NOT AVAILABLE)"
+
+        lines.append(line)
+
+    return '\n'.join(lines)
diff --git a/test/framework/options.py b/test/framework/options.py
index c9a64bfd0e..de338e4045 100644
--- a/test/framework/options.py
+++ b/test/framework/options.py
@@ -5845,6 +5845,28 @@ def test_show_system_info(self):
             regex = re.compile(pattern, re.M)
             self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt))
 
+    def test_check_eb_deps(self):
+        """Test for --check-eb-deps."""
+        txt, _ = self._run_mock_eb(['--check-eb-deps'], raise_error=True)
+        patterns = [
+            r"^Required dependencies:",
+            r"^\* Python [23][0-9.]+$",
+            r"^\* [A-Za-z ]+ [0-9.]+ \(modules tool\)$",
+            r"^Optional dependencies:",
+            r"^\* archspec ([0-9.]+|\(NOT AVAILABLE\))+\s+\[determining name of CPU microarchitecture\]$",
+            r"^\* GitPython ([0-9.]+|\(NOT AVAILABLE\))+\s+\[GitHub integration .*\]$",
+            r"^\* Rich ([0-9.]+|\(NOT AVAILABLE\))+\s+\[eb command rich terminal output\]$",
+            r"^System tools:",
+            r"^\* make ([0-9.]+|\(NOT AVAILABLE\)|\(available, UNKNOWN version\))$",
+            r"^\* patch ([0-9.]+|\(NOT AVAILABLE\)|\(available, UNKNOWN version\))$",
+            r"^\* sed ([0-9.]+|\(NOT AVAILABLE\)|\(available, UNKNOWN version\))$",
+            r"^\* Slurm ([0-9.]+|\(NOT AVAILABLE\)|\(available, UNKNOWN version\))$",
+        ]
+
+        for pattern in patterns:
+            regex = re.compile(pattern, re.M)
+            self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt))
+
     def test_tmp_logdir(self):
         """Test use of --tmp-logdir."""
 

From c97661a0efb2107cbd60b9c186dfd5cf35c5c8ce Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Sun, 12 Sep 2021 14:19:45 +0200
Subject: [PATCH 028/175] avoid making setuptools a required dependency by only
 using pkg_resources if it's available

---
 easybuild/tools/systemtools.py | 49 ++++++++++++++++++++++++++--------
 test/framework/options.py      |  8 +++---
 2 files changed, 43 insertions(+), 14 deletions(-)

diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py
index bf96fb7745..f8a115ba0f 100644
--- a/easybuild/tools/systemtools.py
+++ b/easybuild/tools/systemtools.py
@@ -34,7 +34,6 @@
 import grp  # @UnresolvedImport
 import os
 import platform
-import pkg_resources
 import pwd
 import re
 import struct
@@ -43,6 +42,14 @@
 from ctypes.util import find_library
 from socket import gethostname
 
+# pkg_resources is provided by the setuptools Python package,
+# which we really want to keep as an *optional* dependency
+try:
+    import pkg_resources
+    HAVE_PKG_RESOURCES = True
+except ImportError:
+    HAVE_PKG_RESOURCES = False
+
 try:
     # only needed on macOS, may not be available on Linux
     import ctypes.macholib.dyld
@@ -162,6 +169,7 @@
     'GitPython': "GitHub integration + using Git repository as easyconfigs archive",
     'graphviz-python': "rendering dependency graph with Graphviz: --dep-graph",
     'keyring': "storing GitHub token",
+    'pbs-python': "using Torque as --job backend",
     'pep8': "fallback for code style checking: --check-style, --check-contrib",
     'pycodestyle': "code style checking: --check-style, --check-contrib",
     'pysvn': "using SVN repository as easyconfigs archive",
@@ -171,12 +179,14 @@
     'requests': "fallback library for downloading files",
     'Rich': "eb command rich terminal output",
     'PyYAML': "easystack files and .yeb easyconfig format",
+    'setuptools': "obtaining information on Python packages via pkg_resources module",
 }
 
 OPT_DEP_PKG_NAMES = {
     'GC3Pie': 'gc3libs',
     'GitPython': 'git',
     'graphviz-python': 'gv',
+    'pbs-python': 'pbs',
     'python-graph-core': 'pygraph.classes.digraph',
     'python-graph-dot': 'pygraph.readwrite.dot',
     'python-hglib': 'hglib',
@@ -1136,6 +1146,30 @@ def pick_dep_version(dep_version):
     return result
 
 
+def det_pypkg_version(pkg_name, imported_pkg, import_name=None):
+    """Determine version of a Python package."""
+
+    version = None
+
+    if HAVE_PKG_RESOURCES:
+        if import_name:
+            try:
+                version = pkg_resources.get_distribution(import_name).version
+            except pkg_resources.DistributionNotFound as err:
+                _log.debug("%s Python package not found: %s", import_name, err)
+
+        if version is None:
+            try:
+                version = pkg_resources.get_distribution(pkg_name).version
+            except pkg_resources.DistributionNotFound as err:
+                _log.debug("%s Python package not found: %s", pkg_name, err)
+
+    if version is None and hasattr(imported_pkg, '__version__'):
+        version = imported_pkg.__version__
+
+    return version
+
+
 def check_easybuild_deps(modtool):
     """
     Check presence and version of required and optional EasyBuild dependencies, and report back to terminal.
@@ -1166,16 +1200,9 @@ def extract_version(tool):
             mod = None
 
         if mod:
-            try:
-                dep_version = pkg_resources.get_distribution(pkg).version
-            except pkg_resources.DistributionNotFound:
-                try:
-                    dep_version = pkg_resources.get_distribution(key).version
-                except pkg_resources.DistributionNotFound:
-                    if hasattr(mod, '__version__'):
-                        dep_version = mod.__version__
-                    else:
-                        dep_version = '(unknown version)'
+            dep_version = det_pypkg_version(key, mod, import_name=pkg)
+            if dep_version is None:
+                dep_version = '(unknown version)'
         else:
             dep_version = '(NOT AVAILABLE)'
 
diff --git a/test/framework/options.py b/test/framework/options.py
index de338e4045..b3e60904f0 100644
--- a/test/framework/options.py
+++ b/test/framework/options.py
@@ -5848,14 +5848,16 @@ def test_show_system_info(self):
     def test_check_eb_deps(self):
         """Test for --check-eb-deps."""
         txt, _ = self._run_mock_eb(['--check-eb-deps'], raise_error=True)
+        opt_dep_version_pattern = r'([0-9.]+|\(NOT AVAILABLE\)|\(unknown version\))'
         patterns = [
             r"^Required dependencies:",
             r"^\* Python [23][0-9.]+$",
             r"^\* [A-Za-z ]+ [0-9.]+ \(modules tool\)$",
             r"^Optional dependencies:",
-            r"^\* archspec ([0-9.]+|\(NOT AVAILABLE\))+\s+\[determining name of CPU microarchitecture\]$",
-            r"^\* GitPython ([0-9.]+|\(NOT AVAILABLE\))+\s+\[GitHub integration .*\]$",
-            r"^\* Rich ([0-9.]+|\(NOT AVAILABLE\))+\s+\[eb command rich terminal output\]$",
+            r"^\* archspec %s\s+\[determining name of CPU microarchitecture\]$" % opt_dep_version_pattern,
+            r"^\* GitPython %s\s+\[GitHub integration .*\]$" % opt_dep_version_pattern,
+            r"^\* Rich %s\s+\[eb command rich terminal output\]$" % opt_dep_version_pattern,
+            r"^\* setuptools %s\s+\[obtaining information on Python packages .*\]$" % opt_dep_version_pattern,
             r"^System tools:",
             r"^\* make ([0-9.]+|\(NOT AVAILABLE\)|\(available, UNKNOWN version\))$",
             r"^\* patch ([0-9.]+|\(NOT AVAILABLE\)|\(available, UNKNOWN version\))$",

From 331820fa9edbe6500b2b589f0666fca5c089b6b4 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Sun, 12 Sep 2021 15:45:03 +0200
Subject: [PATCH 029/175] add print_checks function in easybuild.tools.output
 and leverage it to produce rich output for --check-eb-deps

---
 easybuild/main.py              |  4 +-
 easybuild/tools/output.py      | 71 ++++++++++++++++++++++++++++
 easybuild/tools/systemtools.py | 86 ++++++++++++++++++----------------
 test/framework/options.py      | 31 ++++++------
 4 files changed, 136 insertions(+), 56 deletions(-)

diff --git a/easybuild/main.py b/easybuild/main.py
index 84b02dc186..1e5792fd0e 100644
--- a/easybuild/main.py
+++ b/easybuild/main.py
@@ -68,7 +68,7 @@
 from easybuild.tools.hooks import START, END, load_hooks, run_hook
 from easybuild.tools.modules import modules_tool
 from easybuild.tools.options import set_up_configuration, use_color
-from easybuild.tools.output import create_progress_bar
+from easybuild.tools.output import create_progress_bar, print_checks
 from easybuild.tools.robot import check_conflicts, dry_run, missing_deps, resolve_dependencies, search_easyconfigs
 from easybuild.tools.package.utilities import check_pkg_support
 from easybuild.tools.parallelbuild import submit_jobs
@@ -261,7 +261,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
                            terse=options.terse)
 
     if options.check_eb_deps:
-        print(check_easybuild_deps(modtool))
+        print_checks(check_easybuild_deps(modtool))
 
     # GitHub options that warrant a silent cleanup & exit
     if options.check_github:
diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py
index 35c6d962a3..e5d72119d0 100644
--- a/easybuild/tools/output.py
+++ b/easybuild/tools/output.py
@@ -32,8 +32,11 @@
 import random
 
 from easybuild.tools.config import build_option
+from easybuild.tools.py2vs3 import OrderedDict
 
 try:
+    from rich.console import Console
+    from rich.table import Table
     from rich.progress import BarColumn, Progress, SpinnerColumn, TextColumn, TimeElapsedColumn
     HAVE_RICH = True
 except ImportError:
@@ -82,3 +85,71 @@ def create_progress_bar():
         progress_bar = DummyProgress()
 
     return progress_bar
+
+
+def print_checks(checks_data):
+    """Print overview of checks that were made."""
+
+    col_titles = checks_data.pop('col_titles', ('name', 'info', 'description'))
+
+    col2_label = col_titles[1]
+
+    if HAVE_RICH:
+        console = Console()
+        # don't use console.print, which causes SyntaxError in Python 2
+        console_print = getattr(console, 'print')
+        console_print('')
+
+    for section in checks_data:
+        section_checks = checks_data[section]
+
+        if HAVE_RICH:
+            table = Table(title=section)
+            table.add_column(col_titles[0])
+            table.add_column(col_titles[1])
+            # only add 3rd column if there's any information to include in it
+            if any(x[1] for x in section_checks.values()):
+                table.add_column(col_titles[2])
+        else:
+            lines = [
+                '',
+                section + ':',
+                '-' * (len(section) + 1),
+                '',
+            ]
+
+        if isinstance(section_checks, OrderedDict):
+            check_names = section_checks.keys()
+        else:
+            check_names = sorted(section_checks, key=lambda x: x.lower())
+
+        if HAVE_RICH:
+            for check_name in check_names:
+                (info, descr) = checks_data[section][check_name]
+                if info is None:
+                    info = ':yellow_circle:  [yellow]%s?!' % col2_label
+                elif info is False:
+                    info = ':cross_mark:  [red]not found'
+                else:
+                    info = ':white_heavy_check_mark:  [green]%s' % info
+                if descr:
+                    table.add_row(check_name.rstrip(':'), info, descr)
+                else:
+                    table.add_row(check_name.rstrip(':'), info)
+        else:
+            for check_name in check_names:
+                (info, descr) = checks_data[section][check_name]
+                if info is None:
+                    info = '(found, UNKNOWN %s)' % col2_label
+                elif info is False:
+                    info = '(NOT FOUND)'
+                line = "* %s %s" % (check_name, info)
+                if descr:
+                    line = line.ljust(40) + '[%s]' % descr
+                lines.append(line)
+            lines.append('')
+
+        if HAVE_RICH:
+            console_print(table)
+        else:
+            print('\n'.join(lines))
diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py
index f8a115ba0f..2c7052f4ef 100644
--- a/easybuild/tools/systemtools.py
+++ b/easybuild/tools/systemtools.py
@@ -59,7 +59,7 @@
 from easybuild.base import fancylogger
 from easybuild.tools.build_log import EasyBuildError
 from easybuild.tools.filetools import is_readable, read_file, which
-from easybuild.tools.py2vs3 import string_type
+from easybuild.tools.py2vs3 import OrderedDict, string_type
 from easybuild.tools.run import run_cmd
 
 
@@ -160,7 +160,24 @@
 RPM = 'rpm'
 DPKG = 'dpkg'
 
-SYSTEM_TOOLS = ('7z', 'bunzip2', DPKG, 'gunzip', 'make', 'patch', RPM, 'sed', 'tar', 'unxz', 'unzip')
+SYSTEM_TOOLS = {
+    '7z': "extracting sources (.iso)",
+    'bunzip2': "decompressing sources (.bz2, .tbz, .tbz2, ...)",
+    DPKG: "checking OS dependencies (Debian, Ubuntu, ...)",
+    'gunzip': "decompressing source files (.gz, .tgz, ...)",
+    'make': "build tool",
+    'patch': "applying patch files",
+    RPM: "checking OS dependencies (CentOS, RHEL, OpenSuSE, SLES, ...)",
+    'sed': "runtime patching",
+    'Slurm': "backend for --job (sbatch command)",
+    'tar': "unpacking source files (.tar)",
+    'unxz': "decompressing source files (.xz, .txz)",
+    'unzip': "decompressing files (.zip)",
+}
+
+SYSTEM_TOOL_CMDS = {
+    'Slurm': 'sbatch',
+}
 
 OPT_DEPS = {
     'archspec': "determining name of CPU microarchitecture",
@@ -1176,6 +1193,8 @@ def check_easybuild_deps(modtool):
     """
     version_regex = re.compile(r'\s(?P<version>[0-9][0-9.]+[a-z]*)')
 
+    checks_data = OrderedDict()
+
     def extract_version(tool):
         """Helper function to extract (only) version for specific command line tool."""
         out = get_tool_version(tool, ignore_ec=True)
@@ -1201,52 +1220,39 @@ def extract_version(tool):
 
         if mod:
             dep_version = det_pypkg_version(key, mod, import_name=pkg)
-            if dep_version is None:
-                dep_version = '(unknown version)'
         else:
-            dep_version = '(NOT AVAILABLE)'
+            dep_version = False
 
         opt_dep_versions[key] = dep_version
 
-    lines = [
-        '',
-        "Required dependencies:",
-        "----------------------",
-        '',
-        "* Python %s" % python_version,
-        "* %s (modules tool)" % modtool,
-        '',
-        "Optional dependencies:",
-        "----------------------",
-        '',
-    ]
-    for pkg in sorted(opt_dep_versions, key=lambda x: x.lower()):
-        line = "* %s %s" % (pkg, opt_dep_versions[pkg])
-        line = line.ljust(40) + " [%s]" % OPT_DEPS[pkg]
-        lines.append(line)
-
-    lines.extend([
-        '',
-        "System tools:",
-        "-------------",
-        '',
-    ])
-
-    tools = list(SYSTEM_TOOLS) + ['Slurm']
-    cmds = {'Slurm': 'sbatch'}
-
-    for tool in sorted(tools, key=lambda x: x.lower()):
-        line = "* %s " % tool
-        cmd = cmds.get(tool, tool)
+    checks_data['col_titles'] = ('name', 'version', 'used for')
+
+    req_deps_key = "Required dependencies"
+    checks_data[req_deps_key] = OrderedDict()
+    checks_data[req_deps_key]['Python'] = (python_version, None)
+    checks_data[req_deps_key]['modules tool:'] = (str(modtool), None)
+
+    opt_deps_key = "Optional dependencies"
+    checks_data[opt_deps_key] = {}
+
+    for pkg in opt_dep_versions:
+        checks_data[opt_deps_key][pkg] = (opt_dep_versions[pkg], OPT_DEPS[pkg])
+
+    sys_tools_key = "System tools"
+    checks_data[sys_tools_key] = {}
+
+    for tool in SYSTEM_TOOLS:
+        tool_info = None
+        cmd = SYSTEM_TOOL_CMDS.get(tool, tool)
         if which(cmd):
             version = extract_version(cmd)
             if version.startswith('UNKNOWN'):
-                line += "(available, %s)" % version
+                tool_info = None
             else:
-                line += version
+                tool_info = version
         else:
-            line += "(NOT AVAILABLE)"
+            tool_info = False
 
-        lines.append(line)
+        checks_data[sys_tools_key][tool] = (tool_info, None)
 
-    return '\n'.join(lines)
+    return checks_data
diff --git a/test/framework/options.py b/test/framework/options.py
index b3e60904f0..2dda9e9a1c 100644
--- a/test/framework/options.py
+++ b/test/framework/options.py
@@ -5848,21 +5848,24 @@ def test_show_system_info(self):
     def test_check_eb_deps(self):
         """Test for --check-eb-deps."""
         txt, _ = self._run_mock_eb(['--check-eb-deps'], raise_error=True)
-        opt_dep_version_pattern = r'([0-9.]+|\(NOT AVAILABLE\)|\(unknown version\))'
+
+        # keep in mind that these patterns should match with both normal output and Rich output!
+        opt_dep_info_pattern = r'([0-9.]+|\(NOT FOUND\)|not found|\(unknown version\))'
+        tool_info_pattern = r'([0-9.]+|\(NOT FOUND\)|not found|\(found, UNKNOWN version\)|version\?\!)'
         patterns = [
-            r"^Required dependencies:",
-            r"^\* Python [23][0-9.]+$",
-            r"^\* [A-Za-z ]+ [0-9.]+ \(modules tool\)$",
-            r"^Optional dependencies:",
-            r"^\* archspec %s\s+\[determining name of CPU microarchitecture\]$" % opt_dep_version_pattern,
-            r"^\* GitPython %s\s+\[GitHub integration .*\]$" % opt_dep_version_pattern,
-            r"^\* Rich %s\s+\[eb command rich terminal output\]$" % opt_dep_version_pattern,
-            r"^\* setuptools %s\s+\[obtaining information on Python packages .*\]$" % opt_dep_version_pattern,
-            r"^System tools:",
-            r"^\* make ([0-9.]+|\(NOT AVAILABLE\)|\(available, UNKNOWN version\))$",
-            r"^\* patch ([0-9.]+|\(NOT AVAILABLE\)|\(available, UNKNOWN version\))$",
-            r"^\* sed ([0-9.]+|\(NOT AVAILABLE\)|\(available, UNKNOWN version\))$",
-            r"^\* Slurm ([0-9.]+|\(NOT AVAILABLE\)|\(available, UNKNOWN version\))$",
+            r"Required dependencies",
+            r"Python.* [23][0-9.]+",
+            r"modules tool.* [A-Za-z0-9.\s-]+",
+            r"Optional dependencies",
+            r"archspec.* %s.*determining name" % opt_dep_info_pattern,
+            r"GitPython.* %s.*GitHub integration" % opt_dep_info_pattern,
+            r"Rich.* %s.*eb command rich terminal output" % opt_dep_info_pattern,
+            r"setuptools.* %s.*information on Python packages" % opt_dep_info_pattern,
+            r"System tools",
+            r"make.* %s" % tool_info_pattern,
+            r"patch.* %s" % tool_info_pattern,
+            r"sed.* %s" % tool_info_pattern,
+            r"Slurm.* %s" % tool_info_pattern,
         ]
 
         for pattern in patterns:

From 93c3277f3e0d9c9b2715a9472c950aa8a8aff2ae Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Mon, 13 Sep 2021 09:04:47 +0200
Subject: [PATCH 030/175] silence the Hound on accessing console.print method
 via getattr

---
 easybuild/tools/output.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py
index e5d72119d0..bd629f3bd3 100644
--- a/easybuild/tools/output.py
+++ b/easybuild/tools/output.py
@@ -97,7 +97,7 @@ def print_checks(checks_data):
     if HAVE_RICH:
         console = Console()
         # don't use console.print, which causes SyntaxError in Python 2
-        console_print = getattr(console, 'print')
+        console_print = getattr(console, 'print')  # noqa: B009
         console_print('')
 
     for section in checks_data:

From 79cc83b96d758eb4f6ef623b492d8fcd28a5655f Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Mon, 13 Sep 2021 09:50:41 +0200
Subject: [PATCH 031/175] ensure that path configuration options have absolute
 path values

---
 easybuild/tools/options.py | 32 ++++++++++++++++++++++++++++++++
 test/framework/options.py  | 36 ++++++++++++++++++++++++++++++++++++
 2 files changed, 68 insertions(+)

diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py
index 64948af45c..47804a28ab 100644
--- a/easybuild/tools/options.py
+++ b/easybuild/tools/options.py
@@ -1047,8 +1047,40 @@ def _postprocess_checks(self):
 
         self.log.info("Checks on configuration options passed")
 
+    def _ensure_abs_path(self, opt_name):
+        """Ensure that path value for specified configuration option is an absolute path."""
+
+        def _ensure_abs_path(opt_name, path):
+            """Helper function to make path value for a configuration option an absolute path."""
+            if os.path.isabs(path):
+                abs_path = path
+            else:
+                abs_path = os.path.abspath(path)
+                self.log.info("Relative path value for '%s' configuration option resolved to absolute path: %s",
+                              path, abs_path)
+            return abs_path
+
+        opt_val = getattr(self.options, opt_name)
+        if opt_val:
+            if isinstance(opt_val, string_type):
+                setattr(self.options, opt_name, _ensure_abs_path(opt_name, opt_val))
+            elif isinstance(opt_val, list):
+                abs_paths = [_ensure_abs_path(opt_name, p) for p in opt_val]
+                setattr(self.options, opt_name, abs_paths)
+            else:
+                error_msg = "Don't know how to ensure absolute path(s) for '%s' configuration option (value type: %s)"
+                raise EasyBuildError(error_msg, opt_name, type(opt_val))
+
     def _postprocess_config(self):
         """Postprocessing of configuration options"""
+
+        # resolve relative paths for configuration options that specify a location
+        path_opt_names = ('buildpath', 'containerpath', 'git_working_dirs_path', 'installpath',
+                          'installpath_modules', 'installpath_software', 'prefix', 'packagepath',
+                          'repositorypath', 'robot_paths', 'sourcepath')
+        for opt_name in path_opt_names:
+            self._ensure_abs_path(opt_name)
+
         if self.options.prefix is not None:
             # prefix applies to all paths, and repository has to be reinitialised to take new repositorypath in account
             # in the legacy-style configuration, repository is initialised in configuration file itself
diff --git a/test/framework/options.py b/test/framework/options.py
index c9a64bfd0e..045e868790 100644
--- a/test/framework/options.py
+++ b/test/framework/options.py
@@ -6233,6 +6233,42 @@ def test_accept_eula_for(self):
         self.eb_main(args, do_build=True, raise_error=True)
         self.assertTrue(os.path.exists(toy_modfile))
 
+    def test_config_abs_path(self):
+        """Test ensuring of absolute path values for path configuration options."""
+
+        test_topdir = os.path.join(self.test_prefix, 'test_topdir')
+        test_subdir = os.path.join(test_topdir, 'test_middle_dir', 'test_subdir')
+        mkdir(test_subdir, parents=True)
+        change_dir(test_subdir)
+
+        # a relative path specified in a configuration file is positively weird, but fine :)
+        cfgfile = os.path.join(self.test_prefix, 'test.cfg')
+        cfgtxt = '\n'.join([
+            "[config]",
+            "containerpath = ..",
+        ])
+        write_file(cfgfile, cfgtxt)
+
+        os.environ['EASYBUILD_INSTALLPATH'] = '../..'
+
+        args = [
+            '--configfiles=%s' % cfgfile,
+            '--prefix=..',
+            '--sourcepath=.',
+            '--show-config',
+        ]
+        txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False, strip=True)
+
+        patterns = [
+            r"^containerpath\s+\(F\) = .*/test_topdir/test_middle_dir$",
+            r"^installpath\s+\(E\) = .*/test_topdir$",
+            r"^prefix\s+\(C\) = .*/test_topdir/test_middle_dir$",
+            r"^sourcepath\s+\(C\) = .*/test_topdir/test_middle_dir/test_subdir$",
+        ]
+        for pattern in patterns:
+            regex = re.compile(pattern, re.M)
+            self.assertTrue(regex.search(txt), "Pattern '%s' should be found in: %s" % (pattern, txt))
+
     # end-to-end testing of unknown filename
     def test_easystack_wrong_read(self):
         """Test for --easystack <easystack.yaml> when wrong name is provided"""

From 3252979f29b1b92a7223f5163cef02c7c03fa564 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Mon, 13 Sep 2021 12:16:03 +0200
Subject: [PATCH 032/175] handle repositorypath as special case when ensuring
 absolute path values

---
 easybuild/tools/options.py | 38 ++++++++++++++++++++++++--------------
 test/framework/config.py   |  2 +-
 test/framework/options.py  |  2 ++
 3 files changed, 27 insertions(+), 15 deletions(-)

diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py
index 47804a28ab..8d7b3af441 100644
--- a/easybuild/tools/options.py
+++ b/easybuild/tools/options.py
@@ -1047,25 +1047,25 @@ def _postprocess_checks(self):
 
         self.log.info("Checks on configuration options passed")
 
+    def get_cfg_opt_abs_path(self, opt_name, path):
+        """Get path value of configuration option as absolute path."""
+        if os.path.isabs(path):
+            abs_path = path
+        else:
+            abs_path = os.path.abspath(path)
+            self.log.info("Relative path value for '%s' configuration option resolved to absolute path: %s",
+                          path, abs_path)
+        return abs_path
+
     def _ensure_abs_path(self, opt_name):
         """Ensure that path value for specified configuration option is an absolute path."""
 
-        def _ensure_abs_path(opt_name, path):
-            """Helper function to make path value for a configuration option an absolute path."""
-            if os.path.isabs(path):
-                abs_path = path
-            else:
-                abs_path = os.path.abspath(path)
-                self.log.info("Relative path value for '%s' configuration option resolved to absolute path: %s",
-                              path, abs_path)
-            return abs_path
-
         opt_val = getattr(self.options, opt_name)
         if opt_val:
             if isinstance(opt_val, string_type):
-                setattr(self.options, opt_name, _ensure_abs_path(opt_name, opt_val))
+                setattr(self.options, opt_name, self.get_cfg_opt_abs_path(opt_name, opt_val))
             elif isinstance(opt_val, list):
-                abs_paths = [_ensure_abs_path(opt_name, p) for p in opt_val]
+                abs_paths = [self.get_cfg_opt_abs_path(opt_name, p) for p in opt_val]
                 setattr(self.options, opt_name, abs_paths)
             else:
                 error_msg = "Don't know how to ensure absolute path(s) for '%s' configuration option (value type: %s)"
@@ -1075,9 +1075,19 @@ def _postprocess_config(self):
         """Postprocessing of configuration options"""
 
         # resolve relative paths for configuration options that specify a location
-        path_opt_names = ('buildpath', 'containerpath', 'git_working_dirs_path', 'installpath',
+        path_opt_names = ['buildpath', 'containerpath', 'git_working_dirs_path', 'installpath',
                           'installpath_modules', 'installpath_software', 'prefix', 'packagepath',
-                          'repositorypath', 'robot_paths', 'sourcepath')
+                          'robot_paths', 'sourcepath']
+
+        # repositorypath is a special case: only first part is a path;
+        # 2nd (optional) part is a relative subdir and should not be resolved to an absolute path!
+        repositorypath = self.options.repositorypath
+        if isinstance(repositorypath, (list, tuple)) and len(repositorypath) == 2:
+            abs_path = self.get_cfg_opt_abs_path('repositorypath', repositorypath[0])
+            self.options.repositorypath = (abs_path, repositorypath[1])
+        else:
+            path_opt_names.append('repositorypath')
+
         for opt_name in path_opt_names:
             self._ensure_abs_path(opt_name)
 
diff --git a/test/framework/config.py b/test/framework/config.py
index cb13d348a5..0c4489a412 100644
--- a/test/framework/config.py
+++ b/test/framework/config.py
@@ -302,7 +302,7 @@ def test_generaloption_config_file(self):
         self.assertEqual(install_path('mod'), installpath_modules),  # via config file
         self.assertEqual(source_paths(), [testpath2])  # via command line
         self.assertEqual(build_path(), testpath1)  # via config file
-        self.assertEqual(get_repositorypath(), [os.path.join(topdir, 'ebfiles_repo'), 'somesubdir'])  # via config file
+        self.assertEqual(get_repositorypath(), (os.path.join(topdir, 'ebfiles_repo'), 'somesubdir'))  # via config file
 
         # hardcoded first entry
         self.assertEqual(options.robot_paths[0], '/tmp/foo')
diff --git a/test/framework/options.py b/test/framework/options.py
index 045e868790..a8bc6fe985 100644
--- a/test/framework/options.py
+++ b/test/framework/options.py
@@ -6246,6 +6246,7 @@ def test_config_abs_path(self):
         cfgtxt = '\n'.join([
             "[config]",
             "containerpath = ..",
+            "repositorypath = /apps/easyconfigs_archive, somesubdir",
         ])
         write_file(cfgfile, cfgtxt)
 
@@ -6263,6 +6264,7 @@ def test_config_abs_path(self):
             r"^containerpath\s+\(F\) = .*/test_topdir/test_middle_dir$",
             r"^installpath\s+\(E\) = .*/test_topdir$",
             r"^prefix\s+\(C\) = .*/test_topdir/test_middle_dir$",
+            r"^repositorypath\s+\(F\) = \('/apps/easyconfigs_archive', ' somesubdir'\)$",
             r"^sourcepath\s+\(C\) = .*/test_topdir/test_middle_dir/test_subdir$",
         ]
         for pattern in patterns:

From 25470adfb03bd407732055be8b74d2e31b899964 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Mon, 13 Sep 2021 16:40:37 +0200
Subject: [PATCH 033/175] collapse OPT_DEPS and OPT_DEP_PKG_NAMES to
 EASYBUILD_OPTIONAL_DEPENDENCIES

---
 easybuild/tools/systemtools.py | 57 ++++++++++++++--------------------
 1 file changed, 24 insertions(+), 33 deletions(-)

diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py
index 2c7052f4ef..52c6ee63c2 100644
--- a/easybuild/tools/systemtools.py
+++ b/easybuild/tools/systemtools.py
@@ -179,35 +179,24 @@
     'Slurm': 'sbatch',
 }
 
-OPT_DEPS = {
-    'archspec': "determining name of CPU microarchitecture",
-    'autopep8': "auto-formatting for dumped easyconfigs",
-    'GC3Pie': "backend for --job",
-    'GitPython': "GitHub integration + using Git repository as easyconfigs archive",
-    'graphviz-python': "rendering dependency graph with Graphviz: --dep-graph",
-    'keyring': "storing GitHub token",
-    'pbs-python': "using Torque as --job backend",
-    'pep8': "fallback for code style checking: --check-style, --check-contrib",
-    'pycodestyle': "code style checking: --check-style, --check-contrib",
-    'pysvn': "using SVN repository as easyconfigs archive",
-    'python-graph-core': "creating dependency graph: --dep-graph",
-    'python-graph-dot': "saving dependency graph as dot file: --dep-graph",
-    'python-hglib': "using Mercurial repository as easyconfigs archive",
-    'requests': "fallback library for downloading files",
-    'Rich': "eb command rich terminal output",
-    'PyYAML': "easystack files and .yeb easyconfig format",
-    'setuptools': "obtaining information on Python packages via pkg_resources module",
-}
-
-OPT_DEP_PKG_NAMES = {
-    'GC3Pie': 'gc3libs',
-    'GitPython': 'git',
-    'graphviz-python': 'gv',
-    'pbs-python': 'pbs',
-    'python-graph-core': 'pygraph.classes.digraph',
-    'python-graph-dot': 'pygraph.readwrite.dot',
-    'python-hglib': 'hglib',
-    'PyYAML': 'yaml',
+EASYBUILD_OPTIONAL_DEPENDENCIES = {
+    'archspec': (None, "determining name of CPU microarchitecture"),
+    'autopep8': (None, "auto-formatting for dumped easyconfigs"),
+    'GC3Pie': ('gc3libs', "backend for --job"),
+    'GitPython': ('git', "GitHub integration + using Git repository as easyconfigs archive"),
+    'graphviz-python': ('gv', "rendering dependency graph with Graphviz: --dep-graph"),
+    'keyring': (None, "storing GitHub token"),
+    'pbs-python': ('pbs', "using Torque as --job backend"),
+    'pep8': (None, "fallback for code style checking: --check-style, --check-contrib"),
+    'pycodestyle': (None, "code style checking: --check-style, --check-contrib"),
+    'pysvn': (None, "using SVN repository as easyconfigs archive"),
+    'python-graph-core': ('pygraph.classes.digraph', "creating dependency graph: --dep-graph"),
+    'python-graph-dot': ('pygraph.readwrite.dot', "saving dependency graph as dot file: --dep-graph"),
+    'python-hglib': ('hglib', "using Mercurial repository as easyconfigs archive"),
+    'requests': (None, "fallback library for downloading files"),
+    'Rich': (None, "eb command rich terminal output"),
+    'PyYAML': ('yaml', "easystack files and .yeb easyconfig format"),
+    'setuptools': ('pkg_resources', "obtaining information on Python packages via pkg_resources module"),
 }
 
 
@@ -1209,9 +1198,11 @@ def extract_version(tool):
     python_version = extract_version(sys.executable)
 
     opt_dep_versions = {}
-    for key in OPT_DEPS:
+    for key in EASYBUILD_OPTIONAL_DEPENDENCIES:
 
-        pkg = OPT_DEP_PKG_NAMES.get(key, key.lower())
+        pkg = EASYBUILD_OPTIONAL_DEPENDENCIES[key][0]
+        if pkg is None:
+            pkg = key.lower()
 
         try:
             mod = __import__(pkg)
@@ -1235,8 +1226,8 @@ def extract_version(tool):
     opt_deps_key = "Optional dependencies"
     checks_data[opt_deps_key] = {}
 
-    for pkg in opt_dep_versions:
-        checks_data[opt_deps_key][pkg] = (opt_dep_versions[pkg], OPT_DEPS[pkg])
+    for key in opt_dep_versions:
+        checks_data[opt_deps_key][key] = (opt_dep_versions[key], EASYBUILD_OPTIONAL_DEPENDENCIES[key][1])
 
     sys_tools_key = "System tools"
     checks_data[sys_tools_key] = {}

From dc40a2b7f671ffbcaa2824728142cfc544e59cbe Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Mon, 13 Sep 2021 17:13:18 +0200
Subject: [PATCH 034/175] fix log message in get_cfg_opt_abs_path

---
 easybuild/tools/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py
index 8d7b3af441..3dc979b149 100644
--- a/easybuild/tools/options.py
+++ b/easybuild/tools/options.py
@@ -1054,7 +1054,7 @@ def get_cfg_opt_abs_path(self, opt_name, path):
         else:
             abs_path = os.path.abspath(path)
             self.log.info("Relative path value for '%s' configuration option resolved to absolute path: %s",
-                          path, abs_path)
+                          opt_name, abs_path)
         return abs_path
 
     def _ensure_abs_path(self, opt_name):

From 5b87003ea590fdcd2d570d6e3a911345fb3048de Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Mon, 13 Sep 2021 17:40:19 +0200
Subject: [PATCH 035/175] also ensure absolute paths for 'robot' configuration
 option + enhance test_config_abs_path to also check on robot and robot-paths
 configuration options

---
 easybuild/tools/options.py |  4 ++--
 test/framework/options.py  | 32 ++++++++++++++++++++++++++++----
 2 files changed, 30 insertions(+), 6 deletions(-)

diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py
index 3dc979b149..a0592bc2fb 100644
--- a/easybuild/tools/options.py
+++ b/easybuild/tools/options.py
@@ -1077,7 +1077,7 @@ def _postprocess_config(self):
         # resolve relative paths for configuration options that specify a location
         path_opt_names = ['buildpath', 'containerpath', 'git_working_dirs_path', 'installpath',
                           'installpath_modules', 'installpath_software', 'prefix', 'packagepath',
-                          'robot_paths', 'sourcepath']
+                          'robot', 'robot_paths', 'sourcepath']
 
         # repositorypath is a special case: only first part is a path;
         # 2nd (optional) part is a relative subdir and should not be resolved to an absolute path!
@@ -1133,7 +1133,7 @@ def _postprocess_config(self):
 
             # paths specified to --robot have preference over --robot-paths
             # keep both values in sync if robot is enabled, which implies enabling dependency resolver
-            self.options.robot_paths = [os.path.abspath(path) for path in self.options.robot + self.options.robot_paths]
+            self.options.robot_paths = self.options.robot + self.options.robot_paths
             self.options.robot = self.options.robot_paths
 
         # Update the search_paths (if any) to absolute paths
diff --git a/test/framework/options.py b/test/framework/options.py
index a8bc6fe985..0425402049 100644
--- a/test/framework/options.py
+++ b/test/framework/options.py
@@ -6250,7 +6250,10 @@ def test_config_abs_path(self):
         ])
         write_file(cfgfile, cfgtxt)
 
+        # relative paths in environment variables is also weird,
+        # but OK for the sake of testing...
         os.environ['EASYBUILD_INSTALLPATH'] = '../..'
+        os.environ['EASYBUILD_ROBOT_PATHS'] = '../..'
 
         args = [
             '--configfiles=%s' % cfgfile,
@@ -6258,19 +6261,40 @@ def test_config_abs_path(self):
             '--sourcepath=.',
             '--show-config',
         ]
+
         txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False, strip=True)
 
         patterns = [
-            r"^containerpath\s+\(F\) = .*/test_topdir/test_middle_dir$",
-            r"^installpath\s+\(E\) = .*/test_topdir$",
-            r"^prefix\s+\(C\) = .*/test_topdir/test_middle_dir$",
+            r"^containerpath\s+\(F\) = /.*/test_topdir/test_middle_dir$",
+            r"^installpath\s+\(E\) = /.*/test_topdir$",
+            r"^prefix\s+\(C\) = /.*/test_topdir/test_middle_dir$",
             r"^repositorypath\s+\(F\) = \('/apps/easyconfigs_archive', ' somesubdir'\)$",
-            r"^sourcepath\s+\(C\) = .*/test_topdir/test_middle_dir/test_subdir$",
+            r"^sourcepath\s+\(C\) = /.*/test_topdir/test_middle_dir/test_subdir$",
+            r"^robot-paths\s+\(E\) = /.*/test_topdir$",
         ]
         for pattern in patterns:
             regex = re.compile(pattern, re.M)
             self.assertTrue(regex.search(txt), "Pattern '%s' should be found in: %s" % (pattern, txt))
 
+        # if --robot is also used, that wins and $EASYBUILD_ROBOT_PATHS doesn't matter anymore
+        change_dir(test_subdir)
+        args.append('--robot=..:.')
+        txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False, strip=True)
+
+        patterns.pop(-1)
+        robot_value_pattern = ', '.join([
+            r'/.*/test_topdir/test_middle_dir',  # via --robot (first path)
+            r'/.*/test_topdir/test_middle_dir/test_subdir',  # via --robot (second path)
+            r'/.*/test_topdir',  # via $EASYBUILD_ROBOT_PATHS
+        ])
+        patterns.extend([
+            r"^robot-paths\s+\(C\) = %s$" % robot_value_pattern,
+            r"^robot\s+\(C\) = %s$" % robot_value_pattern,
+        ])
+        for pattern in patterns:
+            regex = re.compile(pattern, re.M)
+            self.assertTrue(regex.search(txt), "Pattern '%s' should be found in: %s" % (pattern, txt))
+
     # end-to-end testing of unknown filename
     def test_easystack_wrong_read(self):
         """Test for --easystack <easystack.yaml> when wrong name is provided"""

From 74175a78cd6754c9420c8d18329c119bed2c772b Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Mon, 13 Sep 2021 18:37:08 +0200
Subject: [PATCH 036/175] fix comment in test_config_abs_path

---
 test/framework/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/framework/options.py b/test/framework/options.py
index 0425402049..c547f2328d 100644
--- a/test/framework/options.py
+++ b/test/framework/options.py
@@ -6276,7 +6276,7 @@ def test_config_abs_path(self):
             regex = re.compile(pattern, re.M)
             self.assertTrue(regex.search(txt), "Pattern '%s' should be found in: %s" % (pattern, txt))
 
-        # if --robot is also used, that wins and $EASYBUILD_ROBOT_PATHS doesn't matter anymore
+        # paths specified via --robot have precedence over those specified via $EASYBUILD_ROBOT_PATHS
         change_dir(test_subdir)
         args.append('--robot=..:.')
         txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False, strip=True)

From 593105e058c29d68719ad7f65ace4428beddf2eb Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Mon, 13 Sep 2021 19:37:39 +0200
Subject: [PATCH 037/175] add --output-style configuration option, which can be
 used to disable use of Rich or type of any colored output

---
 easybuild/tools/config.py  | 32 ++++++++++++++++++++++++++++++++
 easybuild/tools/options.py |  6 +++++-
 easybuild/tools/output.py  | 20 ++++++++++++--------
 3 files changed, 49 insertions(+), 9 deletions(-)

diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py
index 94b02d4424..fc583e7c4f 100644
--- a/easybuild/tools/config.py
+++ b/easybuild/tools/config.py
@@ -48,6 +48,12 @@
 from easybuild.tools.build_log import EasyBuildError
 from easybuild.tools.py2vs3 import ascii_letters, create_base_metaclass, string_type
 
+try:
+    import rich
+    HAVE_RICH = True
+except ImportError:
+    HAVE_RICH = False
+
 
 _log = fancylogger.getLogger('config', fname=False)
 
@@ -137,6 +143,13 @@
 LOCAL_VAR_NAMING_CHECKS = [LOCAL_VAR_NAMING_CHECK_ERROR, LOCAL_VAR_NAMING_CHECK_LOG, LOCAL_VAR_NAMING_CHECK_WARN]
 
 
+OUTPUT_STYLE_AUTO = 'auto'
+OUTPUT_STYLE_BASIC = 'basic'
+OUTPUT_STYLE_NO_COLOR = 'no_color'
+OUTPUT_STYLE_RICH = 'rich'
+OUTPUT_STYLES = (OUTPUT_STYLE_AUTO, OUTPUT_STYLE_BASIC, OUTPUT_STYLE_NO_COLOR, OUTPUT_STYLE_RICH)
+
+
 class Singleton(ABCMeta):
     """Serves as metaclass for classes that should implement the Singleton pattern.
 
@@ -342,6 +355,9 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
     DEFAULT_WAIT_ON_LOCK_INTERVAL: [
         'wait_on_lock_interval',
     ],
+    OUTPUT_STYLE_AUTO: [
+        'output_style',
+    ],
 }
 # build option that do not have a perfectly matching command line option
 BUILD_OPTIONS_OTHER = {
@@ -688,6 +704,22 @@ def get_module_syntax():
     return ConfigurationVariables()['module_syntax']
 
 
+def get_output_style():
+    """Return output style to use."""
+    output_style = build_option('output_style')
+
+    if output_style == OUTPUT_STYLE_AUTO:
+        if HAVE_RICH:
+            output_style = OUTPUT_STYLE_RICH
+        else:
+            output_style = OUTPUT_STYLE_BASIC
+
+    if output_style == OUTPUT_STYLE_RICH and not HAVE_RICH:
+        raise EasyBuildError("Can't use '%s' output style, Rich Python package is not available!", OUTPUT_STYLE_RICH)
+
+    return output_style
+
+
 def log_file_format(return_directory=False, ec=None, date=None, timestamp=None):
     """
     Return the format for the logfile or the directory
diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py
index 759e8d8e42..0d4cec6787 100644
--- a/easybuild/tools/options.py
+++ b/easybuild/tools/options.py
@@ -69,7 +69,8 @@
 from easybuild.tools.config import DEFAULT_REPOSITORY, DEFAULT_WAIT_ON_LOCK_INTERVAL, DEFAULT_WAIT_ON_LOCK_LIMIT
 from easybuild.tools.config import EBROOT_ENV_VAR_ACTIONS, ERROR, FORCE_DOWNLOAD_CHOICES, GENERAL_CLASS, IGNORE
 from easybuild.tools.config import JOB_DEPS_TYPE_ABORT_ON_ERROR, JOB_DEPS_TYPE_ALWAYS_RUN, LOADED_MODULES_ACTIONS
-from easybuild.tools.config import LOCAL_VAR_NAMING_CHECK_WARN, LOCAL_VAR_NAMING_CHECKS, WARN
+from easybuild.tools.config import LOCAL_VAR_NAMING_CHECK_WARN, LOCAL_VAR_NAMING_CHECKS
+from easybuild.tools.config import OUTPUT_STYLE_AUTO, OUTPUT_STYLES, WARN
 from easybuild.tools.config import get_pretend_installpath, init, init_build_options, mk_full_default_path
 from easybuild.tools.configobj import ConfigObj, ConfigObjError
 from easybuild.tools.docs import FORMAT_TXT, FORMAT_RST
@@ -448,6 +449,9 @@ def override_options(self):
             'optarch': ("Set architecture optimization, overriding native architecture optimizations",
                         None, 'store', None),
             'output-format': ("Set output format", 'choice', 'store', FORMAT_TXT, [FORMAT_TXT, FORMAT_RST]),
+            'output-style': ("Control output style; auto implies using Rich if available to produce rich output, "
+                             "with fallback to basic colored output",
+                             'choice', 'store', OUTPUT_STYLE_AUTO, OUTPUT_STYLES),
             'parallel': ("Specify (maximum) level of parallellism used during build procedure",
                          'int', 'store', None),
             'pre-create-installdir': ("Create installation directory before submitting build jobs",
diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py
index 5b9c9ba3dd..fb9ad176c4 100644
--- a/easybuild/tools/output.py
+++ b/easybuild/tools/output.py
@@ -31,16 +31,15 @@
 """
 import random
 
-from easybuild.tools.config import build_option
+from easybuild.tools.config import OUTPUT_STYLE_RICH, build_option, get_output_style
 from easybuild.tools.py2vs3 import OrderedDict
 
 try:
     from rich.console import Console
     from rich.table import Table
     from rich.progress import BarColumn, Progress, SpinnerColumn, TextColumn, TimeElapsedColumn
-    HAVE_RICH = True
 except ImportError:
-    HAVE_RICH = False
+    pass
 
 
 class DummyProgress(object):
@@ -61,6 +60,11 @@ def update(self, *args, **kwargs):
         pass
 
 
+def use_rich():
+    """Return whether or not to use Rich to produce rich output."""
+    return get_output_style() == OUTPUT_STYLE_RICH
+
+
 def create_progress_bar():
     """
     Create progress bar to display overall progress.
@@ -68,7 +72,7 @@ def create_progress_bar():
     Returns rich.progress.Progress instance if the Rich Python package is available,
     or a shim DummyProgress instance otherwise.
     """
-    if HAVE_RICH and build_option('show_progress_bar'):
+    if use_rich() and build_option('show_progress_bar'):
 
         # pick random spinner, from a selected subset of available spinner (see 'python3 -m rich.spinner')
         spinner = random.choice(('aesthetic', 'arc', 'bounce', 'dots', 'line', 'monkey', 'point', 'simpleDots'))
@@ -95,7 +99,7 @@ def print_checks(checks_data):
 
     col2_label = col_titles[1]
 
-    if HAVE_RICH:
+    if use_rich():
         console = Console()
         # don't use console.print, which causes SyntaxError in Python 2
         console_print = getattr(console, 'print')  # noqa: B009
@@ -104,7 +108,7 @@ def print_checks(checks_data):
     for section in checks_data:
         section_checks = checks_data[section]
 
-        if HAVE_RICH:
+        if use_rich():
             table = Table(title=section)
             table.add_column(col_titles[0])
             table.add_column(col_titles[1])
@@ -124,7 +128,7 @@ def print_checks(checks_data):
         else:
             check_names = sorted(section_checks, key=lambda x: x.lower())
 
-        if HAVE_RICH:
+        if use_rich():
             for check_name in check_names:
                 (info, descr) = checks_data[section][check_name]
                 if info is None:
@@ -150,7 +154,7 @@ def print_checks(checks_data):
                 lines.append(line)
             lines.append('')
 
-        if HAVE_RICH:
+        if use_rich():
             console_print(table)
         else:
             print('\n'.join(lines))

From f538359a8ce653f14e534562639de1c6ebfacf2d Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Mon, 13 Sep 2021 21:25:09 +0200
Subject: [PATCH 038/175] silence the Hound on unused import of rich in
 tools.config

---
 easybuild/tools/config.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py
index fc583e7c4f..18902ae799 100644
--- a/easybuild/tools/config.py
+++ b/easybuild/tools/config.py
@@ -49,7 +49,7 @@
 from easybuild.tools.py2vs3 import ascii_letters, create_base_metaclass, string_type
 
 try:
-    import rich
+    import rich  # noqa
     HAVE_RICH = True
 except ImportError:
     HAVE_RICH = False

From 476a826c6f18ffbe610fd4c45f7fb782580a450e Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Tue, 14 Sep 2021 09:07:05 +0200
Subject: [PATCH 039/175] add tests for function provided by
 easybuild.tools.output

---
 test/framework/output.py | 110 +++++++++++++++++++++++++++++++++++++++
 test/framework/suite.py  |   3 +-
 2 files changed, 112 insertions(+), 1 deletion(-)
 create mode 100644 test/framework/output.py

diff --git a/test/framework/output.py b/test/framework/output.py
new file mode 100644
index 0000000000..b69a90cd35
--- /dev/null
+++ b/test/framework/output.py
@@ -0,0 +1,110 @@
+# #
+# Copyright 2021-2021 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild.  If not, see <http://www.gnu.org/licenses/>.
+# #
+"""
+Unit tests for functionality in easybuild.tools.output
+
+@author: Kenneth Hoste (Ghent University)
+"""
+import sys
+from unittest import TextTestRunner
+from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered
+
+from easybuild.tools.build_log import EasyBuildError
+from easybuild.tools.config import build_option, get_output_style, update_build_option
+from easybuild.tools.output import DummyProgress, create_progress_bar, use_rich
+
+try:
+    import rich.progress
+    HAVE_RICH = True
+except ImportError:
+    HAVE_RICH = False
+
+
+class OutputTest(EnhancedTestCase):
+    """Tests for functions controlling terminal output."""
+
+    def test_create_progress_bar(self):
+        """Test create_progress_bar function."""
+
+        progress_bar = create_progress_bar()
+        if HAVE_RICH:
+            progress_bar_class = rich.progress.ProgressBar
+        else:
+            progress_bar_class = DummyProgress
+        self.assertTrue(isinstance(progress_bar, progress_bar_class))
+
+        update_build_option('output_style', 'basic')
+        self.assertTrue(isinstance(progress_bar, DummyProgress))
+
+        update_build_option('output_style', 'rich')
+        self.assertTrue(isinstance(progress_bar, progress_bar_class))
+
+        update_build_option('show_progress_bar', False)
+        self.assertTrue(isinstance(progress_bar, DummyProgress))
+
+    def test_get_output_style(self):
+        """Test get_output_style function."""
+
+        self.assertEqual(build_option('output_style'), 'auto')
+
+        for style in (None, 'auto'):
+            if style:
+                update_build_option('output_style', style)
+
+            if HAVE_RICH:
+                self.assertEqual(get_output_style(), 'rich')
+            else:
+                self.assertEqual(get_output_style(), 'basic')
+
+        test_styles = ['basic', 'no_color']
+        if HAVE_RICH:
+            test_styles.append('rich')
+
+        for style in test_styles:
+            update_build_option('output_style', style)
+            self.assertEqual(get_output_style(), style)
+
+        if not HAVE_RICH:
+            update_build_option('output_style', 'rich')
+            error_pattern = "Can't use 'rich' output style, Rich Python package is not available!"
+            self.assertErrorRegex(EasyBuildError, error_pattern, get_output_style)
+
+    def test_use_rich(self):
+        """Test use_rich function."""
+        try:
+            import rich  # noqa
+            self.assertTrue(use_rich())
+        except ImportError:
+            self.assertFalse(use_rich())
+
+
+def suite():
+    """ returns all the testcases in this module """
+    return TestLoaderFiltered().loadTestsFromTestCase(OutputTest, sys.argv[1:])
+
+
+if __name__ == '__main__':
+    res = TextTestRunner(verbosity=1).run(suite())
+    sys.exit(len(res.failures))
diff --git a/test/framework/suite.py b/test/framework/suite.py
index 1633bba103..80bce4983f 100755
--- a/test/framework/suite.py
+++ b/test/framework/suite.py
@@ -66,6 +66,7 @@
 import test.framework.modules as m
 import test.framework.modulestool as mt
 import test.framework.options as o
+import test.framework.output as ou
 import test.framework.parallelbuild as p
 import test.framework.package as pkg
 import test.framework.repository as r
@@ -120,7 +121,7 @@
 # call suite() for each module and then run them all
 # note: make sure the options unit tests run first, to avoid running some of them with a readily initialized config
 tests = [gen, bl, o, r, ef, ev, ebco, ep, e, mg, m, mt, f, run, a, robot, b, v, g, tcv, tc, t, c, s, lic, f_c,
-         tw, p, i, pkg, d, env, et, y, st, h, ct, lib, u, es]
+         tw, p, i, pkg, d, env, et, y, st, h, ct, lib, u, es, ou]
 
 SUITE = unittest.TestSuite([x.suite() for x in tests])
 res = unittest.TextTestRunner().run(SUITE)

From 4c9f72dc0eea2e31cc9ae62384292bc9d6a2e822 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Tue, 14 Sep 2021 09:49:23 +0200
Subject: [PATCH 040/175] fix test_create_progress_bar

---
 test/framework/output.py | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/test/framework/output.py b/test/framework/output.py
index b69a90cd35..174f8164d2 100644
--- a/test/framework/output.py
+++ b/test/framework/output.py
@@ -48,20 +48,27 @@ class OutputTest(EnhancedTestCase):
     def test_create_progress_bar(self):
         """Test create_progress_bar function."""
 
-        progress_bar = create_progress_bar()
         if HAVE_RICH:
-            progress_bar_class = rich.progress.ProgressBar
+            expected_progress_bar_class = rich.progress.Progress
         else:
-            progress_bar_class = DummyProgress
-        self.assertTrue(isinstance(progress_bar, progress_bar_class))
+            expected_progress_bar_class = DummyProgress
+
+        progress_bar = create_progress_bar()
+        error_msg = "%s should be instance of class %s" % (progress_bar, expected_progress_bar_class)
+        self.assertTrue(isinstance(progress_bar, expected_progress_bar_class), error_msg)
 
         update_build_option('output_style', 'basic')
+        progress_bar = create_progress_bar()
         self.assertTrue(isinstance(progress_bar, DummyProgress))
 
-        update_build_option('output_style', 'rich')
-        self.assertTrue(isinstance(progress_bar, progress_bar_class))
+        if HAVE_RICH:
+            update_build_option('output_style', 'rich')
+            progress_bar = create_progress_bar()
+            error_msg = "%s should be instance of class %s" % (progress_bar, expected_progress_bar_class)
+            self.assertTrue(isinstance(progress_bar, expected_progress_bar_class), error_msg)
 
         update_build_option('show_progress_bar', False)
+        progress_bar = create_progress_bar()
         self.assertTrue(isinstance(progress_bar, DummyProgress))
 
     def test_get_output_style(self):

From 6934dd84c5d1de61851bf9cd73d2791a206ea90a Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Tue, 14 Sep 2021 10:37:23 +0200
Subject: [PATCH 041/175] handle ensuring of absolute paths in 'robot'
 configuration value separately, due to special treatment needed for --robot
 argument

---
 easybuild/tools/options.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py
index a0592bc2fb..06ce7f6013 100644
--- a/easybuild/tools/options.py
+++ b/easybuild/tools/options.py
@@ -1074,10 +1074,12 @@ def _ensure_abs_path(self, opt_name):
     def _postprocess_config(self):
         """Postprocessing of configuration options"""
 
-        # resolve relative paths for configuration options that specify a location
+        # resolve relative paths for configuration options that specify a location;
+        # ensuring absolute paths for 'robot' is handled separately below,
+        # because we need to be careful with the argument pass to --robot
         path_opt_names = ['buildpath', 'containerpath', 'git_working_dirs_path', 'installpath',
                           'installpath_modules', 'installpath_software', 'prefix', 'packagepath',
-                          'robot', 'robot_paths', 'sourcepath']
+                          'robot_paths', 'sourcepath']
 
         # repositorypath is a special case: only first part is a path;
         # 2nd (optional) part is a relative subdir and should not be resolved to an absolute path!
@@ -1133,7 +1135,7 @@ def _postprocess_config(self):
 
             # paths specified to --robot have preference over --robot-paths
             # keep both values in sync if robot is enabled, which implies enabling dependency resolver
-            self.options.robot_paths = self.options.robot + self.options.robot_paths
+            self.options.robot_paths = [os.path.abspath(p) for p in self.options.robot + self.options.robot_paths]
             self.options.robot = self.options.robot_paths
 
         # Update the search_paths (if any) to absolute paths

From 88034abbd152d283edd51edf8af0b28152242a94 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B8rgen=20Nordmoen?= <j.h.nordmoen@usit.uio.no>
Date: Wed, 15 Sep 2021 08:57:05 +0200
Subject: [PATCH 042/175] Explicitly check if a file exist when copying

Previous behavior would simply skip copying a file if it did not exist
and report copy success. This commit changes that behavior to error out
early if the file does not exist and explicitly warn if trying to copy
something that is neither a file nor a symbolic link.
---
 easybuild/tools/filetools.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py
index 0e4afabe1a..99096c34fc 100644
--- a/easybuild/tools/filetools.py
+++ b/easybuild/tools/filetools.py
@@ -2270,6 +2270,8 @@ def copy_file(path, target_path, force_in_dry_run=False):
     :param target_path: path to copy the file to
     :param force_in_dry_run: force copying of file during dry run
     """
+    if not os.path.exists(path):
+        raise EasyBuildError("could not copy '%s' it does not exist!" % path)
     if not force_in_dry_run and build_option('extended_dry_run'):
         dry_run_msg("copied file %s to %s" % (path, target_path))
     else:
@@ -2285,13 +2287,16 @@ def copy_file(path, target_path, force_in_dry_run=False):
                 _log.info("Copied contents of file %s to %s", path, target_path)
             else:
                 mkdir(os.path.dirname(target_path), parents=True)
-                if os.path.exists(path):
+                if os.path.isfile(path) and not os.path.islink(path):
                     shutil.copy2(path, target_path)
+                    _log.info("%s copied to %s", path, target_path)
                 elif os.path.islink(path):
                     # special care for copying broken symlinks
                     link_target = os.readlink(path)
                     symlink(link_target, target_path)
-                _log.info("%s copied to %s", path, target_path)
+                    _log.info("created symlink from %s to %s", path, target_path)
+                else:
+                    _log.warn("ignoring '%s' since it is neither a file nor a symlink" % path)
         except (IOError, OSError, shutil.Error) as err:
             raise EasyBuildError("Failed to copy file %s to %s: %s", path, target_path, err)
 

From a7dde5fcfd98defd59325a53bb295a3a527ec79e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B8rgen=20Nordmoen?= <j.h.nordmoen@usit.uio.no>
Date: Wed, 15 Sep 2021 09:20:21 +0200
Subject: [PATCH 043/175] Do not check if path is a file

Previous intention is simply to let the error bubble up if the copy
tries to affect a directory.
---
 easybuild/tools/filetools.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py
index 99096c34fc..89579b188a 100644
--- a/easybuild/tools/filetools.py
+++ b/easybuild/tools/filetools.py
@@ -2287,16 +2287,14 @@ def copy_file(path, target_path, force_in_dry_run=False):
                 _log.info("Copied contents of file %s to %s", path, target_path)
             else:
                 mkdir(os.path.dirname(target_path), parents=True)
-                if os.path.isfile(path) and not os.path.islink(path):
-                    shutil.copy2(path, target_path)
-                    _log.info("%s copied to %s", path, target_path)
-                elif os.path.islink(path):
+                if os.path.islink(path):
                     # special care for copying broken symlinks
                     link_target = os.readlink(path)
                     symlink(link_target, target_path)
                     _log.info("created symlink from %s to %s", path, target_path)
                 else:
-                    _log.warn("ignoring '%s' since it is neither a file nor a symlink" % path)
+                    shutil.copy2(path, target_path)
+                    _log.info("%s copied to %s", path, target_path)
         except (IOError, OSError, shutil.Error) as err:
             raise EasyBuildError("Failed to copy file %s to %s: %s", path, target_path, err)
 

From 94e9876038530c89dcd3ad6b81287ee15cb0264d Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 15 Sep 2021 09:59:41 +0200
Subject: [PATCH 044/175] only call os.path.abspath on paths passed to robot
 paths, robot_paths is already handled

---
 easybuild/tools/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py
index 06ce7f6013..614ba78f97 100644
--- a/easybuild/tools/options.py
+++ b/easybuild/tools/options.py
@@ -1135,7 +1135,7 @@ def _postprocess_config(self):
 
             # paths specified to --robot have preference over --robot-paths
             # keep both values in sync if robot is enabled, which implies enabling dependency resolver
-            self.options.robot_paths = [os.path.abspath(p) for p in self.options.robot + self.options.robot_paths]
+            self.options.robot_paths = [os.path.abspath(p) for p in self.options.robot] + self.options.robot_paths
             self.options.robot = self.options.robot_paths
 
         # Update the search_paths (if any) to absolute paths

From a9745d99c91da5898666038872229076257ca7e9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B8rgen=20Nordmoen?= <j.h.nordmoen@usit.uio.no>
Date: Wed, 15 Sep 2021 10:37:42 +0200
Subject: [PATCH 045/175] Added check if path is a broken symlink

---
 easybuild/tools/filetools.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py
index 89579b188a..66b2249b3a 100644
--- a/easybuild/tools/filetools.py
+++ b/easybuild/tools/filetools.py
@@ -2270,7 +2270,8 @@ def copy_file(path, target_path, force_in_dry_run=False):
     :param target_path: path to copy the file to
     :param force_in_dry_run: force copying of file during dry run
     """
-    if not os.path.exists(path):
+    # NOTE: 'exists' will return False if 'path' is a broken symlink
+    if not os.path.exists(path) and not os.path.islink(path):
         raise EasyBuildError("could not copy '%s' it does not exist!" % path)
     if not force_in_dry_run and build_option('extended_dry_run'):
         dry_run_msg("copied file %s to %s" % (path, target_path))

From 467c29c6a1d2ff1c6162c43176d6c8b1bc83fc76 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Tue, 14 Sep 2021 21:06:11 +0200
Subject: [PATCH 046/175] disable progress bars when running the tests to avoid
 messing up test suite output

---
 test/framework/output.py    | 3 +++
 test/framework/utilities.py | 7 ++++++-
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/test/framework/output.py b/test/framework/output.py
index 174f8164d2..be5d4d5046 100644
--- a/test/framework/output.py
+++ b/test/framework/output.py
@@ -48,6 +48,9 @@ class OutputTest(EnhancedTestCase):
     def test_create_progress_bar(self):
         """Test create_progress_bar function."""
 
+        # restore default (was disabled in EnhancedTestCase.setUp to avoid messing up test output)
+        update_build_option('show_progress_bar', True)
+
         if HAVE_RICH:
             expected_progress_bar_class = rich.progress.Progress
         else:
diff --git a/test/framework/utilities.py b/test/framework/utilities.py
index 4a82aaf6a8..e6188991b4 100644
--- a/test/framework/utilities.py
+++ b/test/framework/utilities.py
@@ -48,7 +48,7 @@
 from easybuild.framework.easyblock import EasyBlock
 from easybuild.main import main
 from easybuild.tools import config
-from easybuild.tools.config import GENERAL_CLASS, Singleton, module_classes
+from easybuild.tools.config import GENERAL_CLASS, Singleton, module_classes, update_build_option
 from easybuild.tools.configobj import ConfigObj
 from easybuild.tools.environment import modify_env
 from easybuild.tools.filetools import copy_dir, mkdir, read_file, which
@@ -137,6 +137,11 @@ def setUp(self):
 
         init_config()
 
+        # disable progress bars when running the tests,
+        # since it messes up with test suite progress output when test installations are performed
+        os.environ['EASYBUILD_DISABLE_SHOW_PROGRESS_BAR'] = '1'
+        update_build_option('show_progress_bar', False)
+
         import easybuild
         # try to import easybuild.easyblocks(.generic) packages
         # it's OK if it fails here, but important to import first before fiddling with sys.path

From 90c24677edde832e82e8e06c9d0a88299e14540c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B8rgen=20Nordmoen?= <j.h.nordmoen@usit.uio.no>
Date: Thu, 16 Sep 2021 08:01:39 +0200
Subject: [PATCH 047/175] Added test for copy of non-existing file

---
 easybuild/tools/filetools.py | 2 +-
 test/framework/filetools.py  | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py
index 66b2249b3a..2cd50d2791 100644
--- a/easybuild/tools/filetools.py
+++ b/easybuild/tools/filetools.py
@@ -2272,7 +2272,7 @@ def copy_file(path, target_path, force_in_dry_run=False):
     """
     # NOTE: 'exists' will return False if 'path' is a broken symlink
     if not os.path.exists(path) and not os.path.islink(path):
-        raise EasyBuildError("could not copy '%s' it does not exist!" % path)
+        raise EasyBuildError("Could not copy '%s' it does not exist!" % path)
     if not force_in_dry_run and build_option('extended_dry_run'):
         dry_run_msg("copied file %s to %s" % (path, target_path))
     else:
diff --git a/test/framework/filetools.py b/test/framework/filetools.py
index f3eed568f8..55684fa56e 100644
--- a/test/framework/filetools.py
+++ b/test/framework/filetools.py
@@ -1661,6 +1661,10 @@ def test_copy_file(self):
         self.assertTrue(ft.read_file(to_copy) == ft.read_file(target_path))
         self.assertEqual(txt, '')
 
+        # Test that a non-existing file raises an exception
+        src, target = os.path.join(self.test_prefix, 'this_file_does_not_exist'), os.path.join(self.test_prefix, 'toy')
+        self.assertErrorRegex(EasyBuildError, "Could not copy *", ft.copy_file, src, target)
+
     def test_copy_files(self):
         """Test copy_files function."""
         test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')

From d5ccae9252369d6d5b3bb8e2d66b33512e3f3199 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Fri, 17 Sep 2021 08:39:37 +0200
Subject: [PATCH 048/175] fix comment

---
 test/framework/utilities.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/framework/utilities.py b/test/framework/utilities.py
index e6188991b4..14248243d3 100644
--- a/test/framework/utilities.py
+++ b/test/framework/utilities.py
@@ -138,7 +138,7 @@ def setUp(self):
         init_config()
 
         # disable progress bars when running the tests,
-        # since it messes up with test suite progress output when test installations are performed
+        # since it messes with test suite progress output when test installations are performed
         os.environ['EASYBUILD_DISABLE_SHOW_PROGRESS_BAR'] = '1'
         update_build_option('show_progress_bar', False)
 

From cf12d627a4fb6d6a883636a4baa94b192e3e2e9d Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Fri, 17 Sep 2021 10:47:13 +0200
Subject: [PATCH 049/175] take into account changed error raised by Python
 3.9.7 when copying directory via shutil.copyfile

---
 test/framework/filetools.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/test/framework/filetools.py b/test/framework/filetools.py
index f3eed568f8..722710430f 100644
--- a/test/framework/filetools.py
+++ b/test/framework/filetools.py
@@ -1610,7 +1610,9 @@ def test_copy_file(self):
 
         # clean error when trying to copy a directory with copy_file
         src, target = os.path.dirname(to_copy), os.path.join(self.test_prefix, 'toy')
-        self.assertErrorRegex(EasyBuildError, "Failed to copy file.*Is a directory", ft.copy_file, src, target)
+        # error message was changed in Python 3.9.7 to "FileNotFoundError: Directory does not exist"
+        error_pattern = "Failed to copy file.*(Is a directory|Directory does not exist)"
+        self.assertErrorRegex(EasyBuildError, error_pattern, ft.copy_file, src, target)
 
         # test overwriting of existing file owned by someone else,
         # which should make copy_file use shutil.copyfile rather than shutil.copy2

From c951010574a82030eaa9d129739f5b937697945d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B8rgen=20Nordmoen?= <j.h.nordmoen@usit.uio.no>
Date: Fri, 17 Sep 2021 12:03:32 +0200
Subject: [PATCH 050/175] Do not raise exception in dry run mode

---
 easybuild/tools/filetools.py |  8 +++++---
 test/framework/filetools.py  | 10 ++++++++++
 2 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py
index 2cd50d2791..9d589bd90a 100644
--- a/easybuild/tools/filetools.py
+++ b/easybuild/tools/filetools.py
@@ -2272,9 +2272,11 @@ def copy_file(path, target_path, force_in_dry_run=False):
     """
     # NOTE: 'exists' will return False if 'path' is a broken symlink
     if not os.path.exists(path) and not os.path.islink(path):
-        raise EasyBuildError("Could not copy '%s' it does not exist!" % path)
-    if not force_in_dry_run and build_option('extended_dry_run'):
-        dry_run_msg("copied file %s to %s" % (path, target_path))
+        if force_in_dry_run:
+            raise EasyBuildError("Could not copy '%s' it does not exist!", path)
+        else:
+            _log.debug("Ignoring non-existing file in 'copy_file' because of dry run mode: %s", path)
+            dry_run_msg("copied file %s to %s" % (path, target_path))
     else:
         try:
             target_exists = os.path.exists(target_path)
diff --git a/test/framework/filetools.py b/test/framework/filetools.py
index 55684fa56e..9af1f8a1a7 100644
--- a/test/framework/filetools.py
+++ b/test/framework/filetools.py
@@ -1662,8 +1662,18 @@ def test_copy_file(self):
         self.assertEqual(txt, '')
 
         # Test that a non-existing file raises an exception
+        update_build_option('extended_dry_run', False)
         src, target = os.path.join(self.test_prefix, 'this_file_does_not_exist'), os.path.join(self.test_prefix, 'toy')
         self.assertErrorRegex(EasyBuildError, "Could not copy *", ft.copy_file, src, target)
+        # Test that copying a non-existing file in 'dry_run' mode does noting
+        update_build_option('extended_dry_run', True)
+        self.mock_stdout(True)
+        ft.copy_file(src, target, force_in_dry_run=False)
+        txt = self.get_stdout()
+        self.mock_stdout(False)
+        self.assertTrue(re.search("^copied file %s to %s" % (src, target), txt))
+        # However, if we add 'force_in_dry_run=True' it should throw an exception
+        self.assertErrorRegex(EasyBuildError, "Could not copy *", ft.copy_file, src, target, force_in_dry_run=True)
 
     def test_copy_files(self):
         """Test copy_files function."""

From f0ab45899917aecb1749269008d09cf2ab0024d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B8rgen=20Nordmoen?= <j.h.nordmoen@usit.uio.no>
Date: Fri, 17 Sep 2021 12:08:50 +0200
Subject: [PATCH 051/175] Added import of 'update_build_options'

---
 test/framework/filetools.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/framework/filetools.py b/test/framework/filetools.py
index 9af1f8a1a7..9bea250b75 100644
--- a/test/framework/filetools.py
+++ b/test/framework/filetools.py
@@ -45,7 +45,7 @@
 from easybuild.tools import run
 import easybuild.tools.filetools as ft
 from easybuild.tools.build_log import EasyBuildError
-from easybuild.tools.config import IGNORE, ERROR
+from easybuild.tools.config import IGNORE, ERROR, update_build_option
 from easybuild.tools.multidiff import multidiff
 from easybuild.tools.py2vs3 import std_urllib
 

From 9313e40eb97faccd17c3360731bc1e5dc092c9c8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B8rgen=20Nordmoen?= <j.h.nordmoen@usit.uio.no>
Date: Fri, 17 Sep 2021 13:54:26 +0200
Subject: [PATCH 052/175] Fixed so that copy_file adheres to dry run

---
 easybuild/tools/filetools.py | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py
index 9d589bd90a..723ac5d6e1 100644
--- a/easybuild/tools/filetools.py
+++ b/easybuild/tools/filetools.py
@@ -2270,13 +2270,12 @@ def copy_file(path, target_path, force_in_dry_run=False):
     :param target_path: path to copy the file to
     :param force_in_dry_run: force copying of file during dry run
     """
-    # NOTE: 'exists' will return False if 'path' is a broken symlink
-    if not os.path.exists(path) and not os.path.islink(path):
-        if force_in_dry_run:
-            raise EasyBuildError("Could not copy '%s' it does not exist!", path)
-        else:
-            _log.debug("Ignoring non-existing file in 'copy_file' because of dry run mode: %s", path)
-            dry_run_msg("copied file %s to %s" % (path, target_path))
+    if not force_in_dry_run and build_option('extended_dry_run'):
+        # If in dry run mode, do not copy any files, just lie about it
+        dry_run_msg("copied file %s to %s" % (path, target_path))
+    elif not os.path.exists(path) and not os.path.islink(path):
+        # NOTE: 'exists' will return False if 'path' is a broken symlink
+        raise EasyBuildError("Could not copy '%s' it does not exist!", path)
     else:
         try:
             target_exists = os.path.exists(target_path)

From 76471d9d34fa0bb0b0e85b9a9663cf12cf3e0c6c Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Tue, 21 Sep 2021 10:44:13 +0200
Subject: [PATCH 053/175] use separate progress bars for overall progress,
 installation steps, downloading sources

---
 easybuild/framework/easyblock.py |  29 +----
 easybuild/main.py                |  28 ++---
 easybuild/tools/filetools.py     |  40 ++++++-
 easybuild/tools/output.py        | 194 +++++++++++++++++++++++++++----
 4 files changed, 223 insertions(+), 68 deletions(-)

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index 58bed6afde..a719073333 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -89,6 +89,7 @@
 from easybuild.tools.modules import ROOT_ENV_VAR_NAME_PREFIX, VERSION_ENV_VAR_NAME_PREFIX, DEVEL_ENV_VAR_NAME_PREFIX
 from easybuild.tools.modules import Lmod, curr_module_paths, invalidate_module_caches_for, get_software_root
 from easybuild.tools.modules import get_software_root_env_var_name, get_software_version_env_var_name
+from easybuild.tools.output import PROGRESS_BAR_EASYCONFIG, start_progress_bar, stop_progress_bar, update_progress_bar
 from easybuild.tools.package.utilities import package
 from easybuild.tools.py2vs3 import extract_method_name, string_type
 from easybuild.tools.repository.repository import init_repository
@@ -304,21 +305,6 @@ def close_log(self):
         self.log.info("Closing log for application name %s version %s" % (self.name, self.version))
         fancylogger.logToFile(self.logfile, enable=False)
 
-    def set_progress_bar(self, progress_bar, task_id):
-        """
-        Set progress bar, the progress bar is needed when writing messages so
-        that the progress counter is always at the bottom
-        """
-        self.progress_bar = progress_bar
-        self.pbar_task = task_id
-
-    def advance_progress(self, tick=1.0):
-        """
-        Advance the progress bar forward with `tick`
-        """
-        if self.progress_bar and self.pbar_task is not None:
-            self.progress_bar.advance(self.pbar_task, tick)
-
     #
     # DRY RUN UTILITIES
     #
@@ -3587,8 +3573,8 @@ def run_all_steps(self, run_test_cases):
             return True
 
         steps = self.get_steps(run_test_cases=run_test_cases, iteration_count=self.det_iter_cnt())
-        # Calculate progress bar tick
-        tick = 1.0 / float(len(steps))
+
+        start_progress_bar(PROGRESS_BAR_EASYCONFIG, len(steps), label=self.full_mod_name)
 
         print_msg("building and installing %s..." % self.full_mod_name, log=self.log, silent=self.silent)
         trace_msg("installation prefix: %s" % self.installdir)
@@ -3627,7 +3613,7 @@ def run_all_steps(self, run_test_cases):
                                 print_msg("... (took %s)", time2str(step_duration), log=self.log, silent=self.silent)
                             elif self.logdebug or build_option('trace'):
                                 print_msg("... (took < 1 sec)", log=self.log, silent=self.silent)
-                self.advance_progress(tick)
+                update_progress_bar(PROGRESS_BAR_EASYCONFIG, progress_size=1)
 
         except StopException:
             pass
@@ -3653,7 +3639,7 @@ def print_dry_run_note(loc, silent=True):
     dry_run_msg(msg, silent=silent)
 
 
-def build_and_install_one(ecdict, init_env, progress_bar=None, task_id=None):
+def build_and_install_one(ecdict, init_env):
     """
     Build the software
     :param ecdict: dictionary contaning parsed easyconfig + metadata
@@ -3701,11 +3687,6 @@ def build_and_install_one(ecdict, init_env, progress_bar=None, task_id=None):
         print_error("Failed to get application instance for %s (easyblock: %s): %s" % (name, easyblock, err.msg),
                     silent=silent)
 
-    # Setup progress bar
-    if progress_bar and task_id is not None:
-        app.set_progress_bar(progress_bar, task_id)
-        _log.info("Updated progress bar instance for easyblock %s", easyblock)
-
     # application settings
     stop = build_option('stop')
     if stop is not None:
diff --git a/easybuild/main.py b/easybuild/main.py
index 1e5792fd0e..59d7c60c20 100644
--- a/easybuild/main.py
+++ b/easybuild/main.py
@@ -68,7 +68,8 @@
 from easybuild.tools.hooks import START, END, load_hooks, run_hook
 from easybuild.tools.modules import modules_tool
 from easybuild.tools.options import set_up_configuration, use_color
-from easybuild.tools.output import create_progress_bar, print_checks
+from easybuild.tools.output import PROGRESS_BAR_OVERALL, print_checks, rich_live_cm
+from easybuild.tools.output import start_progress_bar, stop_progress_bar, update_progress_bar
 from easybuild.tools.robot import check_conflicts, dry_run, missing_deps, resolve_dependencies, search_easyconfigs
 from easybuild.tools.package.utilities import check_pkg_support
 from easybuild.tools.parallelbuild import submit_jobs
@@ -101,36 +102,27 @@ def find_easyconfigs_by_specs(build_specs, robot_path, try_to_generate, testing=
     return [(ec_file, generated)]
 
 
-def build_and_install_software(ecs, init_session_state, exit_on_failure=True, progress_bar=None):
+def build_and_install_software(ecs, init_session_state, exit_on_failure=True):
     """
     Build and install software for all provided parsed easyconfig files.
 
     :param ecs: easyconfig files to install software with
     :param init_session_state: initial session state, to use in test reports
     :param exit_on_failure: whether or not to exit on installation failure
-    :param progress_bar: progress bar to use to report progress
     """
     # obtain a copy of the starting environment so each build can start afresh
     # we shouldn't use the environment from init_session_state, since relevant env vars might have been set since
     # e.g. via easyconfig.handle_allowed_system_deps
     init_env = copy.deepcopy(os.environ)
 
-    # Initialize progress bar with overall installation task
-    if progress_bar:
-        task_id = progress_bar.add_task("", total=len(ecs))
-    else:
-        task_id = None
+    start_progress_bar(PROGRESS_BAR_OVERALL, size=len(ecs))
 
     res = []
     for ec in ecs:
 
-        if progress_bar:
-            progress_bar.update(task_id, description=ec['short_mod_name'])
-
         ec_res = {}
         try:
-            (ec_res['success'], app_log, err) = build_and_install_one(ec, init_env, progress_bar=progress_bar,
-                                                                      task_id=task_id)
+            (ec_res['success'], app_log, err) = build_and_install_one(ec, init_env)
             ec_res['log_file'] = app_log
             if not ec_res['success']:
                 ec_res['err'] = EasyBuildError(err)
@@ -169,6 +161,10 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True, pr
 
         res.append((ec, ec_res))
 
+        update_progress_bar(PROGRESS_BAR_OVERALL, progress_size=1)
+
+    stop_progress_bar(PROGRESS_BAR_OVERALL, visible=True)
+
     return res
 
 
@@ -540,11 +536,9 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
     if not testing or (testing and do_build):
         exit_on_failure = not (options.dump_test_report or options.upload_test_report)
 
-        progress_bar = create_progress_bar()
-        with progress_bar:
+        with rich_live_cm():
             ecs_with_res = build_and_install_software(ordered_ecs, init_session_state,
-                                                      exit_on_failure=exit_on_failure,
-                                                      progress_bar=progress_bar)
+                                                      exit_on_failure=exit_on_failure)
     else:
         ecs_with_res = [(ec, {}) for ec in ordered_ecs]
 
diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py
index 0e4afabe1a..125975bdfd 100644
--- a/easybuild/tools/filetools.py
+++ b/easybuild/tools/filetools.py
@@ -54,6 +54,7 @@
 import tempfile
 import time
 import zlib
+from functools import partial
 
 from easybuild.base import fancylogger
 from easybuild.tools import run
@@ -61,6 +62,7 @@
 from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_msg, print_warning
 from easybuild.tools.config import DEFAULT_WAIT_ON_LOCK_INTERVAL, ERROR, GENERIC_EASYBLOCK_PKG, IGNORE, WARN
 from easybuild.tools.config import build_option, install_path
+from easybuild.tools.output import PROGRESS_BAR_DOWNLOAD, start_progress_bar, stop_progress_bar, update_progress_bar
 from easybuild.tools.py2vs3 import HTMLParser, std_urllib, string_type
 from easybuild.tools.utilities import natural_keys, nub, remove_unwanted_chars
 
@@ -215,7 +217,8 @@ def read_file(path, log_error=True, mode='r'):
     return txt
 
 
-def write_file(path, data, append=False, forced=False, backup=False, always_overwrite=True, verbose=False):
+def write_file(path, data, append=False, forced=False, backup=False, always_overwrite=True, verbose=False,
+               show_progress=False, size=None):
     """
     Write given contents to file at given path;
     overwrites current file contents without backup by default!
@@ -227,6 +230,8 @@ def write_file(path, data, append=False, forced=False, backup=False, always_over
     :param backup: back up existing file before overwriting or modifying it
     :param always_overwrite: don't require --force to overwrite an existing file
     :param verbose: be verbose, i.e. inform where backup file was created
+    :param show_progress: show progress bar while writing file
+    :param size: size (in bytes) of data to write (used for progress bar)
     """
     # early exit in 'dry run' mode
     if not forced and build_option('extended_dry_run'):
@@ -256,15 +261,30 @@ def write_file(path, data, append=False, forced=False, backup=False, always_over
     if sys.version_info[0] >= 3 and (isinstance(data, bytes) or data_is_file_obj):
         mode += 'b'
 
+    # don't bother showing a progress bar for small files
+    if size and size < 1024:
+        _log.info("Not showing progress bar for downloading small file (size %s)", size)
+        show_progress = False
+
+    if show_progress:
+        start_progress_bar(PROGRESS_BAR_DOWNLOAD, size, label=os.path.basename(path))
+
     # note: we can't use try-except-finally, because Python 2.4 doesn't support it as a single block
     try:
         mkdir(os.path.dirname(path), parents=True)
         with open_file(path, mode) as fh:
             if data_is_file_obj:
-                # if a file-like object was provided, use copyfileobj (which reads the file in chunks)
-                shutil.copyfileobj(data, fh)
+                # if a file-like object was provided, read file in 1MB chunks
+                for chunk in iter(partial(data.read, 1024 ** 2), b''):
+                    fh.write(chunk)
+                    if show_progress:
+                        update_progress_bar(PROGRESS_BAR_DOWNLOAD, progress_size=len(chunk))
             else:
                 fh.write(data)
+
+        if show_progress:
+            stop_progress_bar(PROGRESS_BAR_DOWNLOAD)
+
     except IOError as err:
         raise EasyBuildError("Failed to write to %s: %s", path, err)
 
@@ -752,11 +772,21 @@ def download_file(filename, url, path, forced=False):
     while not downloaded and attempt_cnt < max_attempts:
         attempt_cnt += 1
         try:
+            size = None
             if used_urllib is std_urllib:
                 # urllib2 (Python 2) / urllib.request (Python 3) does the right thing for http proxy setups,
                 # urllib does not!
                 url_fd = std_urllib.urlopen(url_req, timeout=timeout)
                 status_code = url_fd.getcode()
+                http_header = url_fd.info()
+                len_key = 'Content-Length'
+                if len_key in http_header:
+                    size = http_header[len_key]
+                    try:
+                        size = int(size)
+                    except (ValueError, TypeError) as err:
+                        _log.warning("Failed to interpret size '%s' as integer value: %s", size, err)
+                        size = None
             else:
                 response = requests.get(url, headers=headers, stream=True, timeout=timeout)
                 status_code = response.status_code
@@ -768,8 +798,8 @@ def download_file(filename, url, path, forced=False):
             # to ensure the data is read in chunks (which prevents problems in Python 3.9+);
             # cfr. https://github.com/easybuilders/easybuild-framework/issues/3455
             # and https://bugs.python.org/issue42853
-            write_file(path, url_fd, forced=forced, backup=True)
-            _log.info("Downloaded file %s from url %s to %s" % (filename, url, path))
+            write_file(path, url_fd, forced=forced, backup=True, show_progress=True, size=size)
+            _log.info("Downloaded file %s from url %s to %s", filename, url, path)
             downloaded = True
             url_fd.close()
         except used_urllib.HTTPError as err:
diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py
index fb9ad176c4..81cd31db05 100644
--- a/easybuild/tools/output.py
+++ b/easybuild/tools/output.py
@@ -29,21 +29,35 @@
 :author: Kenneth Hoste (Ghent University)
 :author: Jørgen Nordmoen (University of Oslo)
 """
+import functools
 import random
 
+from easybuild.tools.build_log import EasyBuildError
 from easybuild.tools.config import OUTPUT_STYLE_RICH, build_option, get_output_style
 from easybuild.tools.py2vs3 import OrderedDict
 
 try:
-    from rich.console import Console
+    from rich.console import Console, RenderGroup
+    from rich.live import Live
     from rich.table import Table
     from rich.progress import BarColumn, Progress, SpinnerColumn, TextColumn, TimeElapsedColumn
+    from rich.progress import DownloadColumn, FileSizeColumn, TransferSpeedColumn, TimeRemainingColumn
 except ImportError:
     pass
 
 
-class DummyProgress(object):
-    """Shim for Rich's Progress class."""
+PROGRESS_BAR_DOWNLOAD = 'download'
+PROGRESS_BAR_EASYCONFIG = 'easyconfig'
+PROGRESS_BAR_OVERALL = 'overall'
+
+_progress_bar_cache = {}
+
+
+class DummyRich(object):
+    """
+    Dummy shim for Rich classes.
+    Used in case Rich is not available, or when EasyBuild is not configured to use rich output style.
+    """
 
     # __enter__ and __exit__ must be implemented to allow use as context manager
     def __enter__(self, *args, **kwargs):
@@ -61,37 +75,173 @@ def update(self, *args, **kwargs):
 
 
 def use_rich():
-    """Return whether or not to use Rich to produce rich output."""
+    """
+    Return whether or not to use Rich to produce rich output.
+    """
     return get_output_style() == OUTPUT_STYLE_RICH
 
 
-def create_progress_bar():
+def rich_live_cm():
     """
-    Create progress bar to display overall progress.
-
-    Returns rich.progress.Progress instance if the Rich Python package is available,
-    or a shim DummyProgress instance otherwise.
+    Return Live instance to use as context manager.
     """
     if use_rich() and build_option('show_progress_bar'):
-
-        # pick random spinner, from a selected subset of available spinner (see 'python3 -m rich.spinner')
-        spinner = random.choice(('aesthetic', 'arc', 'bounce', 'dots', 'line', 'monkey', 'point', 'simpleDots'))
-
-        progress_bar = Progress(
-            SpinnerColumn(spinner),
-            "[progress.percentage]{task.percentage:>3.1f}%",
-            TextColumn("[bold blue]Installing {task.description} ({task.completed:.0f}/{task.total} done)"),
-            BarColumn(bar_width=None),
-            TimeElapsedColumn(),
-            transient=True,
-            expand=True,
+        overall_pbar = overall_progress_bar()
+        easyconfig_pbar = easyconfig_progress_bar()
+        download_pbar = download_progress_bar()
+        download_pbar_bis = download_progress_bar_unknown_size()
+        pbar_group = RenderGroup(
+                download_pbar,
+                download_pbar_bis,
+                easyconfig_pbar,
+                overall_pbar
         )
+        live = Live(pbar_group)
     else:
-        progress_bar = DummyProgress()
+        live = DummyRich()
+
+    return live
+
+
+def progress_bar_cache(func):
+    """
+    Function decorator to cache created progress bars for easy retrieval.
+    """
+    @functools.wraps(func)
+    def new_func():
+        if hasattr(func, 'cached'):
+            progress_bar = func.cached
+        elif use_rich() and build_option('show_progress_bar'):
+            progress_bar = func()
+        else:
+            progress_bar = DummyRich()
+
+        func.cached = progress_bar
+        return func.cached
+
+    return new_func
+
+
+@progress_bar_cache
+def overall_progress_bar():
+    """
+    Get progress bar to display overall progress.
+    """
+    progress_bar = Progress(
+        TimeElapsedColumn(),
+        TextColumn("{task.description}({task.completed} out of {task.total} easyconfigs done)"),
+        BarColumn(bar_width=None),
+        expand=True,
+    )
+
+    return progress_bar
+
+
+@progress_bar_cache
+def easyconfig_progress_bar():
+    """
+    Get progress bar to display progress for installing a single easyconfig file.
+    """
+    progress_bar = Progress(
+        TextColumn("[bold blue]{task.description} ({task.completed} out of {task.total} steps done)"),
+        BarColumn(),
+        TimeElapsedColumn(),
+    )
+
+    return progress_bar
+
+
+@progress_bar_cache
+def download_progress_bar():
+    """
+    Get progress bar to show progress for downloading a file of known size.
+    """
+    progress_bar = Progress(
+        TextColumn('[bold yellow]Downloading {task.description}'),
+        BarColumn(),
+        DownloadColumn(),
+        TransferSpeedColumn(),
+        TimeRemainingColumn(),
+    )
 
     return progress_bar
 
 
+@progress_bar_cache
+def download_progress_bar_unknown_size():
+    """
+    Get progress bar to show progress for downloading a file of unknown size.
+    """
+    progress_bar = Progress(
+        TextColumn('[bold yellow]Downloading {task.description}'),
+        FileSizeColumn(),
+        TransferSpeedColumn(),
+    )
+
+    return progress_bar
+
+
+def get_progress_bar(bar_type, size=None):
+    """
+    Get progress bar of given type.
+    """
+    progress_bar_types = {
+        PROGRESS_BAR_DOWNLOAD: download_progress_bar,
+        PROGRESS_BAR_EASYCONFIG: easyconfig_progress_bar,
+        PROGRESS_BAR_OVERALL: overall_progress_bar,
+    }
+
+    if bar_type == PROGRESS_BAR_DOWNLOAD and not size:
+        pbar = download_progress_bar_unknown_size()
+    elif bar_type in progress_bar_types:
+        pbar = progress_bar_types[bar_type]()
+    else:
+        raise EasyBuildError("Unknown progress bar type: %s", bar_type)
+
+    return pbar
+
+
+def start_progress_bar(bar_type, size, label=None):
+    """
+    Start progress bar of given type.
+
+    :param label: label for progress bar
+    :param size: total target size of progress bar
+    """
+    pbar = get_progress_bar(bar_type, size=size)
+    task_id = pbar.add_task('')
+    _progress_bar_cache[bar_type] = (pbar, task_id)
+    if size:
+        pbar.update(task_id, total=size)
+    if label:
+        pbar.update(task_id, description=label)
+
+
+def update_progress_bar(bar_type, label=None, progress_size=None):
+    """
+    Update progress bar of given type, add progress of given size.
+
+    :param bar_type: type of progress bar
+    :param label: label for progress bar
+    :param progress_size: size of progress made
+    """
+    (pbar, task_id) = _progress_bar_cache[bar_type]
+    if label:
+        pbar.update(task_id, description=label)
+    if progress_size:
+        pbar.update(task_id, advance=progress_size)
+
+
+def stop_progress_bar(bar_type, visible=False):
+    """
+    Stop progress bar of given type.
+    """
+    (pbar, task_id) = _progress_bar_cache[bar_type]
+    pbar.stop_task(task_id)
+    if not visible:
+        pbar.update(task_id, visible=False)
+
+
 def print_checks(checks_data):
     """Print overview of checks that were made."""
 

From b0b64f7f03e9e918b073f991914c9c306e4254c7 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Tue, 21 Sep 2021 11:19:30 +0200
Subject: [PATCH 054/175] set progress bar label to 'done' for finished
 installations

---
 easybuild/framework/easyblock.py | 14 +++++++++++---
 easybuild/tools/output.py        |  5 ++---
 2 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index a719073333..766085ece0 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -3574,7 +3574,11 @@ def run_all_steps(self, run_test_cases):
 
         steps = self.get_steps(run_test_cases=run_test_cases, iteration_count=self.det_iter_cnt())
 
-        start_progress_bar(PROGRESS_BAR_EASYCONFIG, len(steps), label=self.full_mod_name)
+        progress_label_tmpl = "%s (%d out of %d steps done)"
+
+        n_steps = len(steps)
+        progress_label = progress_label_tmpl % (self.full_mod_name, 0, n_steps)
+        start_progress_bar(PROGRESS_BAR_EASYCONFIG, n_steps, label=progress_label)
 
         print_msg("building and installing %s..." % self.full_mod_name, log=self.log, silent=self.silent)
         trace_msg("installation prefix: %s" % self.installdir)
@@ -3594,7 +3598,7 @@ def run_all_steps(self, run_test_cases):
             create_lock(lock_name)
 
         try:
-            for (step_name, descr, step_methods, skippable) in steps:
+            for step_id, (step_name, descr, step_methods, skippable) in enumerate(steps):
                 if self.skip_step(step_name, skippable):
                     print_msg("%s [skipped]" % descr, log=self.log, silent=self.silent)
                 else:
@@ -3613,7 +3617,9 @@ def run_all_steps(self, run_test_cases):
                                 print_msg("... (took %s)", time2str(step_duration), log=self.log, silent=self.silent)
                             elif self.logdebug or build_option('trace'):
                                 print_msg("... (took < 1 sec)", log=self.log, silent=self.silent)
-                update_progress_bar(PROGRESS_BAR_EASYCONFIG, progress_size=1)
+
+                progress_label = progress_label_tmpl % (self.full_mod_name, step_id, n_steps)
+                update_progress_bar(PROGRESS_BAR_EASYCONFIG, progress_size=1, label=progress_label)
 
         except StopException:
             pass
@@ -3621,6 +3627,8 @@ def run_all_steps(self, run_test_cases):
             if not ignore_locks:
                 remove_lock(lock_name)
 
+        update_progress_bar(PROGRESS_BAR_EASYCONFIG, label="%s done!" % self.full_mod_name)
+
         # return True for successfull build (or stopped build)
         return True
 
diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py
index 81cd31db05..8743441dd4 100644
--- a/easybuild/tools/output.py
+++ b/easybuild/tools/output.py
@@ -30,7 +30,6 @@
 :author: Jørgen Nordmoen (University of Oslo)
 """
 import functools
-import random
 
 from easybuild.tools.build_log import EasyBuildError
 from easybuild.tools.config import OUTPUT_STYLE_RICH, build_option, get_output_style
@@ -40,7 +39,7 @@
     from rich.console import Console, RenderGroup
     from rich.live import Live
     from rich.table import Table
-    from rich.progress import BarColumn, Progress, SpinnerColumn, TextColumn, TimeElapsedColumn
+    from rich.progress import BarColumn, Progress, TextColumn, TimeElapsedColumn
     from rich.progress import DownloadColumn, FileSizeColumn, TransferSpeedColumn, TimeRemainingColumn
 except ImportError:
     pass
@@ -143,7 +142,7 @@ def easyconfig_progress_bar():
     Get progress bar to display progress for installing a single easyconfig file.
     """
     progress_bar = Progress(
-        TextColumn("[bold blue]{task.description} ({task.completed} out of {task.total} steps done)"),
+        TextColumn("[bold blue]{task.description}"),
         BarColumn(),
         TimeElapsedColumn(),
     )

From 42398de3ac10284683f0626e67b09fe1dc38c0a8 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Tue, 21 Sep 2021 11:20:21 +0200
Subject: [PATCH 055/175] change default progress size for update_progress_bar
 function

---
 easybuild/framework/easyblock.py | 2 +-
 easybuild/tools/output.py        | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index 766085ece0..7e886e748d 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -3619,7 +3619,7 @@ def run_all_steps(self, run_test_cases):
                                 print_msg("... (took < 1 sec)", log=self.log, silent=self.silent)
 
                 progress_label = progress_label_tmpl % (self.full_mod_name, step_id, n_steps)
-                update_progress_bar(PROGRESS_BAR_EASYCONFIG, progress_size=1, label=progress_label)
+                update_progress_bar(PROGRESS_BAR_EASYCONFIG, label=progress_label)
 
         except StopException:
             pass
diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py
index 8743441dd4..10457b1b93 100644
--- a/easybuild/tools/output.py
+++ b/easybuild/tools/output.py
@@ -216,13 +216,13 @@ def start_progress_bar(bar_type, size, label=None):
         pbar.update(task_id, description=label)
 
 
-def update_progress_bar(bar_type, label=None, progress_size=None):
+def update_progress_bar(bar_type, label=None, progress_size=1):
     """
     Update progress bar of given type, add progress of given size.
 
     :param bar_type: type of progress bar
     :param label: label for progress bar
-    :param progress_size: size of progress made
+    :param progress_size: amount of progress made
     """
     (pbar, task_id) = _progress_bar_cache[bar_type]
     if label:

From 3808f298c6a5a7c7d8b1ffa09874ab4e42ba9217 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Tue, 21 Sep 2021 11:21:37 +0200
Subject: [PATCH 056/175] fix unused import

---
 easybuild/framework/easyblock.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index 7e886e748d..e33c9e796c 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -89,7 +89,7 @@
 from easybuild.tools.modules import ROOT_ENV_VAR_NAME_PREFIX, VERSION_ENV_VAR_NAME_PREFIX, DEVEL_ENV_VAR_NAME_PREFIX
 from easybuild.tools.modules import Lmod, curr_module_paths, invalidate_module_caches_for, get_software_root
 from easybuild.tools.modules import get_software_root_env_var_name, get_software_version_env_var_name
-from easybuild.tools.output import PROGRESS_BAR_EASYCONFIG, start_progress_bar, stop_progress_bar, update_progress_bar
+from easybuild.tools.output import PROGRESS_BAR_EASYCONFIG, start_progress_bar, update_progress_bar
 from easybuild.tools.package.utilities import package
 from easybuild.tools.py2vs3 import extract_method_name, string_type
 from easybuild.tools.repository.repository import init_repository

From f8be1c5a99c0381f0437d5b7aab00f1df635f69e Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Sat, 25 Sep 2021 16:11:22 +0200
Subject: [PATCH 057/175] fix tests for overall_progress_bar

---
 easybuild/tools/output.py |  4 ++--
 test/framework/output.py  | 20 ++++++++++----------
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py
index 10457b1b93..e99301c8bb 100644
--- a/easybuild/tools/output.py
+++ b/easybuild/tools/output.py
@@ -107,8 +107,8 @@ def progress_bar_cache(func):
     Function decorator to cache created progress bars for easy retrieval.
     """
     @functools.wraps(func)
-    def new_func():
-        if hasattr(func, 'cached'):
+    def new_func(ignore_cache=False):
+        if hasattr(func, 'cached') and not ignore_cache:
             progress_bar = func.cached
         elif use_rich() and build_option('show_progress_bar'):
             progress_bar = func()
diff --git a/test/framework/output.py b/test/framework/output.py
index 174f8164d2..28ec84ab3e 100644
--- a/test/framework/output.py
+++ b/test/framework/output.py
@@ -33,7 +33,7 @@
 
 from easybuild.tools.build_log import EasyBuildError
 from easybuild.tools.config import build_option, get_output_style, update_build_option
-from easybuild.tools.output import DummyProgress, create_progress_bar, use_rich
+from easybuild.tools.output import DummyRich, overall_progress_bar, use_rich
 
 try:
     import rich.progress
@@ -45,31 +45,31 @@
 class OutputTest(EnhancedTestCase):
     """Tests for functions controlling terminal output."""
 
-    def test_create_progress_bar(self):
-        """Test create_progress_bar function."""
+    def test_overall_progress_bar(self):
+        """Test overall_progress_bar function."""
 
         if HAVE_RICH:
             expected_progress_bar_class = rich.progress.Progress
         else:
-            expected_progress_bar_class = DummyProgress
+            expected_progress_bar_class = DummyRich
 
-        progress_bar = create_progress_bar()
+        progress_bar = overall_progress_bar()
         error_msg = "%s should be instance of class %s" % (progress_bar, expected_progress_bar_class)
         self.assertTrue(isinstance(progress_bar, expected_progress_bar_class), error_msg)
 
         update_build_option('output_style', 'basic')
-        progress_bar = create_progress_bar()
-        self.assertTrue(isinstance(progress_bar, DummyProgress))
+        progress_bar = overall_progress_bar(ignore_cache=True)
+        self.assertTrue(isinstance(progress_bar, DummyRich))
 
         if HAVE_RICH:
             update_build_option('output_style', 'rich')
-            progress_bar = create_progress_bar()
+            progress_bar = overall_progress_bar(ignore_cache=True)
             error_msg = "%s should be instance of class %s" % (progress_bar, expected_progress_bar_class)
             self.assertTrue(isinstance(progress_bar, expected_progress_bar_class), error_msg)
 
         update_build_option('show_progress_bar', False)
-        progress_bar = create_progress_bar()
-        self.assertTrue(isinstance(progress_bar, DummyProgress))
+        progress_bar = overall_progress_bar(ignore_cache=True)
+        self.assertTrue(isinstance(progress_bar, DummyRich))
 
     def test_get_output_style(self):
         """Test get_output_style function."""

From d5950ecbc60bd116b6ba6618925318d6f242b4d1 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Sat, 25 Sep 2021 17:25:52 +0200
Subject: [PATCH 058/175] add dummy implementation of stop_task to DummyRich

---
 easybuild/tools/output.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py
index e99301c8bb..dca4e8fe0c 100644
--- a/easybuild/tools/output.py
+++ b/easybuild/tools/output.py
@@ -69,6 +69,9 @@ def __exit__(self, *args, **kwargs):
     def add_task(self, *args, **kwargs):
         pass
 
+    def stop_task(self, *args, **kwargs):
+        pass
+
     def update(self, *args, **kwargs):
         pass
 

From 4080cbba9f77cec2ae0019fabede8eefe06dc589 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Sun, 26 Sep 2021 10:20:08 +0200
Subject: [PATCH 059/175] add dummy implementation to __rich_console__ method
 to DummyRich

---
 easybuild/tools/output.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py
index dca4e8fe0c..1e31613183 100644
--- a/easybuild/tools/output.py
+++ b/easybuild/tools/output.py
@@ -75,6 +75,10 @@ def stop_task(self, *args, **kwargs):
     def update(self, *args, **kwargs):
         pass
 
+    # internal Rich methods
+    def __rich_console__(self, *args, **kwargs):
+        pass
+
 
 def use_rich():
     """

From ff9e9723d5e89e44654656bf762ceaf30d83a161 Mon Sep 17 00:00:00 2001
From: Alan O'Cais <a.ocais@fz-juelich.de>
Date: Mon, 27 Sep 2021 15:35:53 +0200
Subject: [PATCH 060/175] Ensure newer location of CUDA stubs is taken into
 account by rpath  filter

---
 easybuild/tools/toolchain/toolchain.py | 8 +++++---
 test/framework/toolchain.py            | 8 ++++++++
 2 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py
index 782d6634b3..45229ba722 100644
--- a/easybuild/tools/toolchain/toolchain.py
+++ b/easybuild/tools/toolchain/toolchain.py
@@ -960,9 +960,11 @@ def prepare_rpath_wrappers(self, rpath_filter_dirs=None, rpath_include_dirs=None
 
         # always include filter for 'stubs' library directory,
         # cfr. https://github.com/easybuilders/easybuild-framework/issues/2683
-        lib_stubs_pattern = '.*/lib(64)?/stubs/?'
-        if lib_stubs_pattern not in rpath_filter_dirs:
-            rpath_filter_dirs.append(lib_stubs_pattern)
+        # (since CUDA 11.something the stubs are in $EBROOTCUDA/stubs/lib64)
+        lib_stubs_patterns = ['.*/lib(64)?/stubs/?', '.*/stubs/lib(64)?/?']
+        for lib_stubs_pattern in lib_stubs_patterns:
+            if lib_stubs_pattern not in rpath_filter_dirs:
+                rpath_filter_dirs.append(lib_stubs_pattern)
 
         # directory where all wrappers will be placed
         wrappers_dir = os.path.join(tempfile.mkdtemp(), RPATH_WRAPPERS_SUBDIR)
diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py
index 11a6c083d0..403f6774ca 100644
--- a/test/framework/toolchain.py
+++ b/test/framework/toolchain.py
@@ -2437,12 +2437,16 @@ def test_toolchain_prepare_rpath(self):
         # check whether 'stubs' library directory are correctly filtered out
         paths = [
             'prefix/software/CUDA/1.2.3/lib/stubs/',  # should be filtered (no -rpath)
+            'prefix/software/CUDA/1.2.3/stubs/lib/',  # should be filtered (no -rpath)
             'tmp/foo/',
             'prefix/software/stubs/1.2.3/lib',  # should NOT be filtered
             'prefix/software/CUDA/1.2.3/lib/stubs',  # should be filtered (no -rpath)
+            'prefix/software/CUDA/1.2.3/stubs/lib',  # should be filtered (no -rpath)
             'prefix/software/CUDA/1.2.3/lib64/stubs/',  # should be filtered (no -rpath)
+            'prefix/software/CUDA/1.2.3/stubs/lib64/',  # should be filtered (no -rpath)
             'prefix/software/foobar/4.5/notreallystubs',  # should NOT be filtered
             'prefix/software/CUDA/1.2.3/lib64/stubs',  # should be filtered (no -rpath)
+            'prefix/software/CUDA/1.2.3/stubs/lib64',  # should be filtered (no -rpath)
             'prefix/software/zlib/1.2.11/lib',
             'prefix/software/bleh/0/lib/stubs',  # should be filtered (no -rpath)
             'prefix/software/foobar/4.5/stubsbutnotreally',  # should NOT be filtered
@@ -2465,12 +2469,16 @@ def test_toolchain_prepare_rpath(self):
             '-Wl,-rpath=%s/prefix/software/foobar/4.5/stubsbutnotreally' % self.test_prefix,
             '%(user)s.c',
             '-L%s/prefix/software/CUDA/1.2.3/lib/stubs/' % self.test_prefix,
+            '-L%s/prefix/software/CUDA/1.2.3/stubs/lib/' % self.test_prefix,
             '-L%s/tmp/foo/' % self.test_prefix,
             '-L%s/prefix/software/stubs/1.2.3/lib' % self.test_prefix,
             '-L%s/prefix/software/CUDA/1.2.3/lib/stubs' % self.test_prefix,
+            '-L%s/prefix/software/CUDA/1.2.3/stubs/lib' % self.test_prefix,
             '-L%s/prefix/software/CUDA/1.2.3/lib64/stubs/' % self.test_prefix,
+            '-L%s/prefix/software/CUDA/1.2.3/stubs/lib64/' % self.test_prefix,
             '-L%s/prefix/software/foobar/4.5/notreallystubs' % self.test_prefix,
             '-L%s/prefix/software/CUDA/1.2.3/lib64/stubs' % self.test_prefix,
+            '-L%s/prefix/software/CUDA/1.2.3/stubs/lib64' % self.test_prefix,
             '-L%s/prefix/software/zlib/1.2.11/lib' % self.test_prefix,
             '-L%s/prefix/software/bleh/0/lib/stubs' % self.test_prefix,
             '-L%s/prefix/software/foobar/4.5/stubsbutnotreally' % self.test_prefix,

From b01f1414625b7d07c93c47aa5294ef75a3035685 Mon Sep 17 00:00:00 2001
From: Simon Branford <s.j.branford@bham.ac.uk>
Date: Tue, 28 Sep 2021 11:38:25 +0100
Subject: [PATCH 061/175] GPU Info

---
 easybuild/tools/options.py     | 13 ++++++++++++-
 easybuild/tools/systemtools.py | 27 +++++++++++++++++++++++++++
 easybuild/tools/testing.py     | 15 +++++++++++++--
 3 files changed, 52 insertions(+), 3 deletions(-)

diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py
index 8ed73b66d5..5d313b8a13 100644
--- a/easybuild/tools/options.py
+++ b/easybuild/tools/options.py
@@ -99,7 +99,7 @@
 from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME
 from easybuild.tools.repository.repository import avail_repositories
 from easybuild.tools.systemtools import UNKNOWN, check_python_version, get_cpu_architecture, get_cpu_family
-from easybuild.tools.systemtools import get_cpu_features, get_system_info
+from easybuild.tools.systemtools import get_cpu_features, get_gpu_info, get_system_info
 from easybuild.tools.version import this_is_easybuild
 
 
@@ -1292,6 +1292,7 @@ def show_system_info(self):
         """Show system information."""
         system_info = get_system_info()
         cpu_features = get_cpu_features()
+        gpu_info = get_gpu_info()
         cpu_arch_name = system_info['cpu_arch_name']
         lines = [
             "System information (%s):" % system_info['hostname'],
@@ -1324,6 +1325,16 @@ def show_system_info(self):
             "  -> Python version: %s" % sys.version.split(' ')[0],
         ])
 
+        if gpu_info:
+            lines.extend([
+                '',
+                "* GPU:",
+            ])
+            for vendor in gpu_info:
+                lines.append("  -> %s" % vendor)
+                for gpu, num in gpu_info[vendor].items():
+                    lines.append("    -> %s: %s" % (gpu, num))
+
         return '\n'.join(lines)
 
     def show_config(self):
diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py
index 52c6ee63c2..e177d62583 100644
--- a/easybuild/tools/systemtools.py
+++ b/easybuild/tools/systemtools.py
@@ -574,6 +574,33 @@ def get_cpu_features():
     return cpu_feat
 
 
+def get_gpu_info():
+    """
+    Get the GPU info
+    """
+    gpu_info = {}
+    os_type = get_os_type()
+
+    if os_type == LINUX:
+        try:
+            cmd = "nvidia-smi --query-gpu=gpu_name,driver_version --format=csv,noheader"
+            _log.debug("Trying to determine NVIDIA GPU info on Linux via cmd '%s'", cmd)
+            out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False)
+            if ec == 0:
+                for line in out.strip().split('\n'):
+                    if 'NVIDIA' in gpu_info and line in gpu_info['NVIDIA']:
+                        gpu_info['NVIDIA'][line] += 1
+                    else:
+                        gpu_info['NVIDIA'] = {}
+                        gpu_info['NVIDIA'][line] = 1
+        except Exception:
+            _log.debug("No NVIDIA GPUs detected")
+    else:
+        _log.debug("Only know how to get GPU info on Linux")
+
+    return gpu_info
+
+
 def get_kernel_name():
     """NO LONGER SUPPORTED: use get_os_type() instead"""
     _log.nosupport("get_kernel_name() is replaced by get_os_type()", '2.0')
diff --git a/easybuild/tools/testing.py b/easybuild/tools/testing.py
index 5d765b6b82..f9e4d0886c 100644
--- a/easybuild/tools/testing.py
+++ b/easybuild/tools/testing.py
@@ -50,7 +50,7 @@
 from easybuild.tools.jenkins import aggregate_xml_in_dirs
 from easybuild.tools.parallelbuild import build_easyconfigs_in_parallel
 from easybuild.tools.robot import resolve_dependencies
-from easybuild.tools.systemtools import UNKNOWN, get_system_info
+from easybuild.tools.systemtools import UNKNOWN, get_gpu_info, get_system_info
 from easybuild.tools.version import FRAMEWORK_VERSION, EASYBLOCKS_VERSION
 
 
@@ -295,11 +295,22 @@ def post_pr_test_report(pr_nrs, repo_type, test_report, msg, init_session_state,
     if system_info['cpu_arch_name'] != UNKNOWN:
         system_info['cpu_model'] += " (%s)" % system_info['cpu_arch_name']
 
+    # add GPU info, if known
+    gpu_info = get_gpu_info()
+    if gpu_info:
+        gpu_str = " gpu: "
+        for vendor in gpu_info:
+            for gpu, num in gpu_info[vendor].items():
+                gpu_str += "%s %s x %s; " % (vendor, num, gpu)
+    else:
+        gpu_str = ""
+
     os_info = '%(hostname)s - %(os_type)s %(os_name)s %(os_version)s' % system_info
-    short_system_info = "%(os_info)s, %(cpu_arch)s, %(cpu_model)s, Python %(pyver)s" % {
+    short_system_info = "%(os_info)s, %(cpu_arch)s, %(cpu_model)s%(gpu)s, Python %(pyver)s" % {
         'os_info': os_info,
         'cpu_arch': system_info['cpu_arch'],
         'cpu_model': system_info['cpu_model'],
+        'gpu': gpu_str,
         'pyver': system_info['python_version'].split(' ')[0],
     }
 

From 2e344200c1671bdbdf2c9e79815416186462f683 Mon Sep 17 00:00:00 2001
From: Simon Branford <s.j.branford@bham.ac.uk>
Date: Tue, 28 Sep 2021 17:12:42 +0100
Subject: [PATCH 062/175] improve code, from review

---
 easybuild/tools/systemtools.py | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py
index e177d62583..284aa5c5ee 100644
--- a/easybuild/tools/systemtools.py
+++ b/easybuild/tools/systemtools.py
@@ -588,15 +588,16 @@ def get_gpu_info():
             out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False)
             if ec == 0:
                 for line in out.strip().split('\n'):
-                    if 'NVIDIA' in gpu_info and line in gpu_info['NVIDIA']:
-                        gpu_info['NVIDIA'][line] += 1
-                    else:
-                        gpu_info['NVIDIA'] = {}
-                        gpu_info['NVIDIA'][line] = 1
-        except Exception:
-            _log.debug("No NVIDIA GPUs detected")
+                    nvidia_gpu_info = gpu_info.setdefault('NVIDIA', {})
+                    nvidia_gpu_info.setdefault(line, 0)
+                    nvidia_gpu_info[line] += 1
+            else:
+                _log.debug("None zero exit (%s) from nvidia-smi: %s" % (ec, out))
+        except Exception as err:
+            _log.debug("Exception was raised when running nvidia-smi: %s", err)
+            _log.info("No NVIDIA GPUs detected")
     else:
-        _log.debug("Only know how to get GPU info on Linux")
+        _log.info("Only know how to get GPU info on Linux, assuming no GPUs are present")
 
     return gpu_info
 

From dc0ae01804d20cf76b123341ef36c3db57c2918c Mon Sep 17 00:00:00 2001
From: Simon Branford <s.j.branford@bham.ac.uk>
Date: Tue, 28 Sep 2021 17:46:00 +0100
Subject: [PATCH 063/175] improve PR test report message formatting

---
 easybuild/tools/testing.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/easybuild/tools/testing.py b/easybuild/tools/testing.py
index f9e4d0886c..f9edc8d865 100644
--- a/easybuild/tools/testing.py
+++ b/easybuild/tools/testing.py
@@ -297,13 +297,11 @@ def post_pr_test_report(pr_nrs, repo_type, test_report, msg, init_session_state,
 
     # add GPU info, if known
     gpu_info = get_gpu_info()
+    gpu_str = ""
     if gpu_info:
-        gpu_str = " gpu: "
         for vendor in gpu_info:
             for gpu, num in gpu_info[vendor].items():
-                gpu_str += "%s %s x %s; " % (vendor, num, gpu)
-    else:
-        gpu_str = ""
+                gpu_str += ", %s %s x %s" % (vendor, num, gpu)
 
     os_info = '%(hostname)s - %(os_type)s %(os_name)s %(os_version)s' % system_info
     short_system_info = "%(os_info)s, %(cpu_arch)s, %(cpu_model)s%(gpu)s, Python %(pyver)s" % {

From db8a157b893b551dfa4275274e0cf9a42e505d24 Mon Sep 17 00:00:00 2001
From: Simon Branford <4967+branfosj@users.noreply.github.com>
Date: Wed, 29 Sep 2021 06:09:48 +0100
Subject: [PATCH 064/175] Improve info

Co-authored-by: SebastianAchilles <achilles@aices.rwth-aachen.de>
---
 easybuild/tools/testing.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/easybuild/tools/testing.py b/easybuild/tools/testing.py
index f9edc8d865..8f2e4d3529 100644
--- a/easybuild/tools/testing.py
+++ b/easybuild/tools/testing.py
@@ -301,7 +301,7 @@ def post_pr_test_report(pr_nrs, repo_type, test_report, msg, init_session_state,
     if gpu_info:
         for vendor in gpu_info:
             for gpu, num in gpu_info[vendor].items():
-                gpu_str += ", %s %s x %s" % (vendor, num, gpu)
+                gpu_str += ", %s x %s %s" % (num, vendor, gpu)
 
     os_info = '%(hostname)s - %(os_type)s %(os_name)s %(os_version)s' % system_info
     short_system_info = "%(os_info)s, %(cpu_arch)s, %(cpu_model)s%(gpu)s, Python %(pyver)s" % {

From e92a2acf0ea8edb1b39dd208495b542adaa834b6 Mon Sep 17 00:00:00 2001
From: Christoph Siegert <christoph.siegert@uni-leipzig.de>
Date: Wed, 29 Sep 2021 14:29:34 +0200
Subject: [PATCH 065/175] replace which by command -v to avoid dependency

Same rational as on line 65, see  #2976
---
 eb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/eb b/eb
index 53a345f08f..880759f62c 100755
--- a/eb
+++ b/eb
@@ -104,7 +104,7 @@ if [ -z $PYTHON ]; then
     echo "(EasyBuild requires Python 2.${REQ_MIN_PY2VER}+ or 3.${REQ_MIN_PY3VER}+)" >&2
     exit 1
 else
-    verbose "Selected Python command: $python_cmd (`which $python_cmd`)"
+    verbose "Selected Python command: $python_cmd (`command -v $python_cmd`)"
 fi
 
 # enable optimization, unless $PYTHONOPTIMIZE is defined (use "export PYTHONOPTIMIZE=0" to disable optimization)

From 9f3a957507e6aae68fbfa7164535e000fcf8f5d3 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 29 Sep 2021 21:05:05 +0200
Subject: [PATCH 066/175] tweak GPU part of output produced by
 --show-system-info

---
 easybuild/tools/options.py | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py
index 5d313b8a13..e4fc7661a4 100644
--- a/easybuild/tools/options.py
+++ b/easybuild/tools/options.py
@@ -1318,11 +1318,6 @@ def show_system_info(self):
             "  -> speed: %s" % system_info['cpu_speed'],
             "  -> cores: %s" % system_info['core_count'],
             "  -> features: %s" % ','.join(cpu_features),
-            '',
-            "* software:",
-            "  -> glibc version: %s" % system_info['glibc_version'],
-            "  -> Python binary: %s" % sys.executable,
-            "  -> Python version: %s" % sys.version.split(' ')[0],
         ])
 
         if gpu_info:
@@ -1333,7 +1328,15 @@ def show_system_info(self):
             for vendor in gpu_info:
                 lines.append("  -> %s" % vendor)
                 for gpu, num in gpu_info[vendor].items():
-                    lines.append("    -> %s: %s" % (gpu, num))
+                    lines.append("    -> %sx %s" % (num, gpu))
+
+        lines.extend([
+            '',
+            "* software:",
+            "  -> glibc version: %s" % system_info['glibc_version'],
+            "  -> Python binary: %s" % sys.executable,
+            "  -> Python version: %s" % sys.version.split(' ')[0],
+        ])
 
         return '\n'.join(lines)
 

From ab90fa194f07276709201d46b547db5c879cb419 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 29 Sep 2021 21:09:49 +0200
Subject: [PATCH 067/175] use lazy logging in get_gpu_info

---
 easybuild/tools/systemtools.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py
index 284aa5c5ee..6783b49eaa 100644
--- a/easybuild/tools/systemtools.py
+++ b/easybuild/tools/systemtools.py
@@ -592,7 +592,7 @@ def get_gpu_info():
                     nvidia_gpu_info.setdefault(line, 0)
                     nvidia_gpu_info[line] += 1
             else:
-                _log.debug("None zero exit (%s) from nvidia-smi: %s" % (ec, out))
+                _log.debug("None zero exit (%s) from nvidia-smi: %s", ec, out)
         except Exception as err:
             _log.debug("Exception was raised when running nvidia-smi: %s", err)
             _log.info("No NVIDIA GPUs detected")

From 80381ad82b7747d9ae5a0d8be59593784c926da5 Mon Sep 17 00:00:00 2001
From: Ake Sandgren <ake.sandgren@hpc2n.umu.se>
Date: Thu, 30 Sep 2021 08:52:14 +0200
Subject: [PATCH 068/175] Fix copy_file so it doesn't fail if the target_path
 is an existing dir.

---
 easybuild/tools/filetools.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py
index 723ac5d6e1..788e554dc9 100644
--- a/easybuild/tools/filetools.py
+++ b/easybuild/tools/filetools.py
@@ -2277,6 +2277,8 @@ def copy_file(path, target_path, force_in_dry_run=False):
         # NOTE: 'exists' will return False if 'path' is a broken symlink
         raise EasyBuildError("Could not copy '%s' it does not exist!", path)
     else:
+        if os.path.isdir(target_path):
+            target_path = os.path.join(target_path, os.path.basename(path))
         try:
             target_exists = os.path.exists(target_path)
             if target_exists and os.path.samefile(path, target_path):

From e04a838e806eaa02e1819f5361534b2138eeaf74 Mon Sep 17 00:00:00 2001
From: Ake Sandgren <ake.sandgren@hpc2n.umu.se>
Date: Thu, 30 Sep 2021 09:03:39 +0200
Subject: [PATCH 069/175] Add test for copy_file that it shouldn't fail if it
 copies a new file into a partially occupied target_dir

---
 test/framework/filetools.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/test/framework/filetools.py b/test/framework/filetools.py
index ee9eb2ab48..10328f006e 100644
--- a/test/framework/filetools.py
+++ b/test/framework/filetools.py
@@ -1608,6 +1608,13 @@ def test_copy_file(self):
         self.assertTrue(os.path.exists(target_path))
         self.assertTrue(ft.read_file(to_copy) == ft.read_file(target_path))
 
+        # Make sure it doesn't fail if target_path is a dir with some other file(s) in it
+        to_copy = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-deps.eb')
+        target_path = self.test_prefix
+        ft.copy_file(to_copy, target_path)
+        self.assertTrue(os.path.exists(os.path.join(target_path, os.path.basename(to_copy))))
+        self.assertTrue(ft.read_file(to_copy) == ft.read_file(os.path.join(target_path, os.path.basename(to_copy))))
+
         # clean error when trying to copy a directory with copy_file
         src, target = os.path.dirname(to_copy), os.path.join(self.test_prefix, 'toy')
         # error message was changed in Python 3.9.7 to "FileNotFoundError: Directory does not exist"

From 44035e78cb32a13c88c2d45a9fc51c70af8a8c4c Mon Sep 17 00:00:00 2001
From: Ake Sandgren <ake.sandgren@hpc2n.umu.se>
Date: Thu, 30 Sep 2021 09:23:26 +0200
Subject: [PATCH 070/175] Adjust copy_file test to test the actual problem from
 issue https://github.com/easybuilders/easybuild-framework/issues/3854 I.e.,
 copying a symlink into a dir

---
 test/framework/filetools.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/test/framework/filetools.py b/test/framework/filetools.py
index 10328f006e..89631d07c0 100644
--- a/test/framework/filetools.py
+++ b/test/framework/filetools.py
@@ -1608,11 +1608,12 @@ def test_copy_file(self):
         self.assertTrue(os.path.exists(target_path))
         self.assertTrue(ft.read_file(to_copy) == ft.read_file(target_path))
 
-        # Make sure it doesn't fail if target_path is a dir with some other file(s) in it
-        to_copy = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-deps.eb')
+        # Make sure it doesn't fail if path is a symlink and target_path is a dir
+        to_copy = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-link-0.0.eb')
         target_path = self.test_prefix
         ft.copy_file(to_copy, target_path)
         self.assertTrue(os.path.exists(os.path.join(target_path, os.path.basename(to_copy))))
+        self.assertTrue(os.path.islink(os.path.join(target_path, os.path.basename(to_copy))))
         self.assertTrue(ft.read_file(to_copy) == ft.read_file(os.path.join(target_path, os.path.basename(to_copy))))
 
         # clean error when trying to copy a directory with copy_file

From 3b341973926117f6f5de631c338a1ae98d500b00 Mon Sep 17 00:00:00 2001
From: Ake Sandgren <ake.sandgren@hpc2n.umu.se>
Date: Thu, 30 Sep 2021 09:24:41 +0200
Subject: [PATCH 071/175] Add the symlink file for the new copy_file test

---
 test/framework/easyconfigs/test_ecs/t/toy/toy-link-0.0.eb | 1 +
 1 file changed, 1 insertion(+)
 create mode 120000 test/framework/easyconfigs/test_ecs/t/toy/toy-link-0.0.eb

diff --git a/test/framework/easyconfigs/test_ecs/t/toy/toy-link-0.0.eb b/test/framework/easyconfigs/test_ecs/t/toy/toy-link-0.0.eb
new file mode 120000
index 0000000000..72dfcf609f
--- /dev/null
+++ b/test/framework/easyconfigs/test_ecs/t/toy/toy-link-0.0.eb
@@ -0,0 +1 @@
+toy-0.0.eb
\ No newline at end of file

From 79013ed08e02f53ce79c177a4994272c3978fd79 Mon Sep 17 00:00:00 2001
From: Ake Sandgren <ake.sandgren@hpc2n.umu.se>
Date: Thu, 30 Sep 2021 10:34:37 +0200
Subject: [PATCH 072/175] Only change target_path in copy_file if the source
 path is a symlink Change test to correctly test for this situation.

---
 easybuild/tools/filetools.py |  9 +++++----
 test/framework/filetools.py  | 14 ++++++++------
 2 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py
index 788e554dc9..4dd40bd4f4 100644
--- a/easybuild/tools/filetools.py
+++ b/easybuild/tools/filetools.py
@@ -2277,8 +2277,6 @@ def copy_file(path, target_path, force_in_dry_run=False):
         # NOTE: 'exists' will return False if 'path' is a broken symlink
         raise EasyBuildError("Could not copy '%s' it does not exist!", path)
     else:
-        if os.path.isdir(target_path):
-            target_path = os.path.join(target_path, os.path.basename(path))
         try:
             target_exists = os.path.exists(target_path)
             if target_exists and os.path.samefile(path, target_path):
@@ -2292,10 +2290,13 @@ def copy_file(path, target_path, force_in_dry_run=False):
             else:
                 mkdir(os.path.dirname(target_path), parents=True)
                 if os.path.islink(path):
+                    if os.path.isdir(target_path):
+                        target_path = os.path.join(target_path, os.path.basename(path))
+                        _log.info("target_path changed to %s", target_path)
                     # special care for copying broken symlinks
                     link_target = os.readlink(path)
-                    symlink(link_target, target_path)
-                    _log.info("created symlink from %s to %s", path, target_path)
+                    symlink(link_target, target_path, use_abspath_source=False)
+                    _log.info("created symlink %s to %s", link_target, target_path)
                 else:
                     shutil.copy2(path, target_path)
                     _log.info("%s copied to %s", path, target_path)
diff --git a/test/framework/filetools.py b/test/framework/filetools.py
index 89631d07c0..4b25de75cf 100644
--- a/test/framework/filetools.py
+++ b/test/framework/filetools.py
@@ -1609,12 +1609,14 @@ def test_copy_file(self):
         self.assertTrue(ft.read_file(to_copy) == ft.read_file(target_path))
 
         # Make sure it doesn't fail if path is a symlink and target_path is a dir
-        to_copy = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-link-0.0.eb')
-        target_path = self.test_prefix
-        ft.copy_file(to_copy, target_path)
-        self.assertTrue(os.path.exists(os.path.join(target_path, os.path.basename(to_copy))))
-        self.assertTrue(os.path.islink(os.path.join(target_path, os.path.basename(to_copy))))
-        self.assertTrue(ft.read_file(to_copy) == ft.read_file(os.path.join(target_path, os.path.basename(to_copy))))
+        link_to_copy = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-link-0.0.eb')
+        base_dir = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy')
+        link_to_copy = 'toy-link-0.0.eb'
+        dir_target_path = self.test_prefix
+        ft.copy_file(os.path.join(base_dir, link_to_copy), dir_target_path)
+        self.assertTrue(os.path.islink(os.path.join(dir_target_path, link_to_copy)))
+        self.assertTrue(os.readlink(os.path.join(dir_target_path, link_to_copy)) == os.readlink(os.path.join(base_dir, link_to_copy)))
+        os.remove(os.path.join(dir_target_path, link_to_copy))
 
         # clean error when trying to copy a directory with copy_file
         src, target = os.path.dirname(to_copy), os.path.join(self.test_prefix, 'toy')

From c273b105df0c28be0845daed390cc14a7e7d709f Mon Sep 17 00:00:00 2001
From: Ake Sandgren <ake.sandgren@hpc2n.umu.se>
Date: Thu, 30 Sep 2021 10:44:30 +0200
Subject: [PATCH 073/175] Forgot to update the check of the number of .eb files
 in the test_ecs tree after adding the test-link-0.0.eb

---
 test/framework/filetools.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/framework/filetools.py b/test/framework/filetools.py
index 4b25de75cf..c9fee0b707 100644
--- a/test/framework/filetools.py
+++ b/test/framework/filetools.py
@@ -2203,7 +2203,7 @@ def test_index_functions(self):
         # test with specified path with and without trailing '/'s
         for path in [test_ecs, test_ecs + '/', test_ecs + '//']:
             index = ft.create_index(path)
-            self.assertEqual(len(index), 89)
+            self.assertEqual(len(index), 90)
 
             expected = [
                 os.path.join('b', 'bzip2', 'bzip2-1.0.6-GCC-4.9.2.eb'),

From d5ee6f40ac7f643919918fae24263f119bdf5a65 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Thu, 30 Sep 2021 10:47:35 +0200
Subject: [PATCH 074/175] only create symlink to test copy_file with locally in
 test_copy_file

---
 .../test_ecs/t/toy/toy-link-0.0.eb            |  1 -
 test/framework/filetools.py                   | 33 ++++++++++---------
 2 files changed, 17 insertions(+), 17 deletions(-)
 delete mode 120000 test/framework/easyconfigs/test_ecs/t/toy/toy-link-0.0.eb

diff --git a/test/framework/easyconfigs/test_ecs/t/toy/toy-link-0.0.eb b/test/framework/easyconfigs/test_ecs/t/toy/toy-link-0.0.eb
deleted file mode 120000
index 72dfcf609f..0000000000
--- a/test/framework/easyconfigs/test_ecs/t/toy/toy-link-0.0.eb
+++ /dev/null
@@ -1 +0,0 @@
-toy-0.0.eb
\ No newline at end of file
diff --git a/test/framework/filetools.py b/test/framework/filetools.py
index c9fee0b707..2f8052bc03 100644
--- a/test/framework/filetools.py
+++ b/test/framework/filetools.py
@@ -1602,24 +1602,25 @@ def test_apply_patch(self):
     def test_copy_file(self):
         """Test copy_file function."""
         testdir = os.path.dirname(os.path.abspath(__file__))
-        to_copy = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')
+        toy_ec = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')
         target_path = os.path.join(self.test_prefix, 'toy.eb')
-        ft.copy_file(to_copy, target_path)
+        ft.copy_file(toy_ec, target_path)
         self.assertTrue(os.path.exists(target_path))
-        self.assertTrue(ft.read_file(to_copy) == ft.read_file(target_path))
+        self.assertTrue(ft.read_file(toy_ec) == ft.read_file(target_path))
 
         # Make sure it doesn't fail if path is a symlink and target_path is a dir
-        link_to_copy = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-link-0.0.eb')
-        base_dir = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy')
-        link_to_copy = 'toy-link-0.0.eb'
-        dir_target_path = self.test_prefix
-        ft.copy_file(os.path.join(base_dir, link_to_copy), dir_target_path)
-        self.assertTrue(os.path.islink(os.path.join(dir_target_path, link_to_copy)))
-        self.assertTrue(os.readlink(os.path.join(dir_target_path, link_to_copy)) == os.readlink(os.path.join(base_dir, link_to_copy)))
-        os.remove(os.path.join(dir_target_path, link_to_copy))
+        toy_link_fn = 'toy-link-0.0.eb'
+        toy_link = os.path.join(self.test_prefix, toy_link_fn)
+        ft.symlink(toy_ec, toy_link)
+        dir_target_path = os.path.join(self.test_prefix, 'subdir')
+        ft.mkdir(dir_target_path)
+        ft.copy_file(toy_link, dir_target_path)
+        self.assertTrue(os.path.islink(os.path.join(dir_target_path, toy_link_fn)))
+        self.assertEqual(os.readlink(os.path.join(dir_target_path, toy_link_fn)), os.readlink(toy_link))
+        os.remove(os.path.join(dir_target_path, toy_link))
 
         # clean error when trying to copy a directory with copy_file
-        src, target = os.path.dirname(to_copy), os.path.join(self.test_prefix, 'toy')
+        src, target = os.path.dirname(toy_ec), os.path.join(self.test_prefix, 'toy')
         # error message was changed in Python 3.9.7 to "FileNotFoundError: Directory does not exist"
         error_pattern = "Failed to copy file.*(Is a directory|Directory does not exist)"
         self.assertErrorRegex(EasyBuildError, error_pattern, ft.copy_file, src, target)
@@ -1656,7 +1657,7 @@ def test_copy_file(self):
         os.remove(target_path)
 
         self.mock_stdout(True)
-        ft.copy_file(to_copy, target_path)
+        ft.copy_file(toy_ec, target_path)
         txt = self.get_stdout()
         self.mock_stdout(False)
 
@@ -1665,12 +1666,12 @@ def test_copy_file(self):
 
         # forced copy, even in dry run mode
         self.mock_stdout(True)
-        ft.copy_file(to_copy, target_path, force_in_dry_run=True)
+        ft.copy_file(toy_ec, target_path, force_in_dry_run=True)
         txt = self.get_stdout()
         self.mock_stdout(False)
 
         self.assertTrue(os.path.exists(target_path))
-        self.assertTrue(ft.read_file(to_copy) == ft.read_file(target_path))
+        self.assertTrue(ft.read_file(toy_ec) == ft.read_file(target_path))
         self.assertEqual(txt, '')
 
         # Test that a non-existing file raises an exception
@@ -2203,7 +2204,7 @@ def test_index_functions(self):
         # test with specified path with and without trailing '/'s
         for path in [test_ecs, test_ecs + '/', test_ecs + '//']:
             index = ft.create_index(path)
-            self.assertEqual(len(index), 90)
+            self.assertEqual(len(index), 89)
 
             expected = [
                 os.path.join('b', 'bzip2', 'bzip2-1.0.6-GCC-4.9.2.eb'),

From ea161c15f4e24ef3401ffb1bffceb639491b73dd Mon Sep 17 00:00:00 2001
From: Ake Sandgren <ake.sandgren@hpc2n.umu.se>
Date: Thu, 30 Sep 2021 10:48:43 +0200
Subject: [PATCH 075/175] Fix long line

---
 test/framework/filetools.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/test/framework/filetools.py b/test/framework/filetools.py
index c9fee0b707..91f037fd75 100644
--- a/test/framework/filetools.py
+++ b/test/framework/filetools.py
@@ -1615,7 +1615,9 @@ def test_copy_file(self):
         dir_target_path = self.test_prefix
         ft.copy_file(os.path.join(base_dir, link_to_copy), dir_target_path)
         self.assertTrue(os.path.islink(os.path.join(dir_target_path, link_to_copy)))
-        self.assertTrue(os.readlink(os.path.join(dir_target_path, link_to_copy)) == os.readlink(os.path.join(base_dir, link_to_copy)))
+        link_source = os.readlink(os.path.join(base_dir, link_to_copy))
+        link_target = os.readlink(os.path.join(dir_target_path, link_to_copy))
+        self.assertTrue(link_target == link_source)
         os.remove(os.path.join(dir_target_path, link_to_copy))
 
         # clean error when trying to copy a directory with copy_file

From 0cdf0c5fd86bad44aea7184fbfb6543d6d379b06 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 13 Oct 2021 08:40:56 +0200
Subject: [PATCH 076/175] extend test for use_rich

---
 test/framework/output.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/test/framework/output.py b/test/framework/output.py
index 9fbe3ec5d9..32da43ebcc 100644
--- a/test/framework/output.py
+++ b/test/framework/output.py
@@ -103,12 +103,20 @@ def test_get_output_style(self):
 
     def test_use_rich(self):
         """Test use_rich function."""
-        try:
-            import rich  # noqa
+
+        self.assertEqual(build_option('output_style'), 'auto')
+
+        if HAVE_RICH:
+            self.assertTrue(use_rich())
+
+            update_build_option('output_style', 'rich')
             self.assertTrue(use_rich())
-        except ImportError:
+        else:
             self.assertFalse(use_rich())
 
+        update_build_option('output_style', 'basic')
+        self.assertFalse(use_rich())
+
 
 def suite():
     """ returns all the testcases in this module """

From a7a5910651032e064e2cc54bb499234591a578c5 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 13 Oct 2021 08:46:56 +0200
Subject: [PATCH 077/175] always ignore cache when testing overall_progress_bar
 function

---
 test/framework/output.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/framework/output.py b/test/framework/output.py
index 32da43ebcc..ad0280179d 100644
--- a/test/framework/output.py
+++ b/test/framework/output.py
@@ -56,7 +56,7 @@ def test_overall_progress_bar(self):
         else:
             expected_progress_bar_class = DummyRich
 
-        progress_bar = overall_progress_bar()
+        progress_bar = overall_progress_bar(ignore_cache=True)
         error_msg = "%s should be instance of class %s" % (progress_bar, expected_progress_bar_class)
         self.assertTrue(isinstance(progress_bar, expected_progress_bar_class), error_msg)
 

From 0dead5fe36e043699eaf8ffc90b5f093e0adb836 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 13 Oct 2021 10:15:35 +0200
Subject: [PATCH 078/175] show progress bar for extensions

---
 easybuild/framework/easyblock.py | 12 +++++++++-
 easybuild/tools/output.py        | 40 ++++++++++++++++++++++++--------
 2 files changed, 41 insertions(+), 11 deletions(-)

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index e33c9e796c..57551c04c6 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -89,7 +89,8 @@
 from easybuild.tools.modules import ROOT_ENV_VAR_NAME_PREFIX, VERSION_ENV_VAR_NAME_PREFIX, DEVEL_ENV_VAR_NAME_PREFIX
 from easybuild.tools.modules import Lmod, curr_module_paths, invalidate_module_caches_for, get_software_root
 from easybuild.tools.modules import get_software_root_env_var_name, get_software_version_env_var_name
-from easybuild.tools.output import PROGRESS_BAR_EASYCONFIG, start_progress_bar, update_progress_bar
+from easybuild.tools.output import PROGRESS_BAR_EASYCONFIG, PROGRESS_BAR_EXTENSIONS
+from easybuild.tools.output import start_progress_bar, stop_progress_bar, update_progress_bar
 from easybuild.tools.package.utilities import package
 from easybuild.tools.py2vs3 import extract_method_name, string_type
 from easybuild.tools.repository.repository import init_repository
@@ -2415,6 +2416,9 @@ def extensions_step(self, fetch=False, install=True):
             self.skip_extensions()
 
         exts_cnt = len(self.ext_instances)
+
+        start_progress_bar(PROGRESS_BAR_EXTENSIONS, exts_cnt)
+
         for idx, ext in enumerate(self.ext_instances):
 
             self.log.debug("Starting extension %s" % ext.name)
@@ -2422,8 +2426,12 @@ def extensions_step(self, fetch=False, install=True):
             # always go back to original work dir to avoid running stuff from a dir that no longer exists
             change_dir(self.orig_workdir)
 
+            progress_label = "Installing '%s' extension" % ext.name
+            update_progress_bar(PROGRESS_BAR_EXTENSIONS, label=progress_label)
+
             tup = (ext.name, ext.version or '', idx + 1, exts_cnt)
             print_msg("installing extension %s %s (%d/%d)..." % tup, silent=self.silent)
+
             start_time = datetime.now()
 
             if self.dry_run:
@@ -2459,6 +2467,8 @@ def extensions_step(self, fetch=False, install=True):
                         elif self.logdebug or build_option('trace'):
                             print_msg("\t... (took < 1 sec)", log=self.log, silent=self.silent)
 
+        stop_progress_bar(PROGRESS_BAR_EXTENSIONS, visible=False)
+
         # cleanup (unload fake module, remove fake module dir)
         if fake_mod_data:
             self.clean_up_fake_module(fake_mod_data)
diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py
index 1e31613183..dd895af1c0 100644
--- a/easybuild/tools/output.py
+++ b/easybuild/tools/output.py
@@ -46,6 +46,7 @@
 
 
 PROGRESS_BAR_DOWNLOAD = 'download'
+PROGRESS_BAR_EXTENSIONS = 'extensions'
 PROGRESS_BAR_EASYCONFIG = 'easyconfig'
 PROGRESS_BAR_OVERALL = 'overall'
 
@@ -87,20 +88,24 @@ def use_rich():
     return get_output_style() == OUTPUT_STYLE_RICH
 
 
+def show_progress_bars():
+    """
+    Return whether or not to show progress bars.
+    """
+    return use_rich() and build_option('show_progress_bar')
+
+
 def rich_live_cm():
     """
     Return Live instance to use as context manager.
     """
-    if use_rich() and build_option('show_progress_bar'):
-        overall_pbar = overall_progress_bar()
-        easyconfig_pbar = easyconfig_progress_bar()
-        download_pbar = download_progress_bar()
-        download_pbar_bis = download_progress_bar_unknown_size()
+    if show_progress_bars():
         pbar_group = RenderGroup(
-                download_pbar,
-                download_pbar_bis,
-                easyconfig_pbar,
-                overall_pbar
+                download_progress_bar(),
+                download_progress_bar_unknown_size(),
+                extensions_progress_bar(),
+                easyconfig_progress_bar(),
+                overall_progress_bar(),
         )
         live = Live(pbar_group)
     else:
@@ -149,7 +154,7 @@ def easyconfig_progress_bar():
     Get progress bar to display progress for installing a single easyconfig file.
     """
     progress_bar = Progress(
-        TextColumn("[bold blue]{task.description}"),
+        TextColumn("[bold green]{task.description}"),
         BarColumn(),
         TimeElapsedColumn(),
     )
@@ -187,12 +192,27 @@ def download_progress_bar_unknown_size():
     return progress_bar
 
 
+@progress_bar_cache
+def extensions_progress_bar():
+    """
+    Get progress bar to show progress for installing extensions.
+    """
+    progress_bar = Progress(
+        TextColumn("[bold blue]{task.description} ({task.completed}/{task.total})"),
+        BarColumn(),
+        TimeElapsedColumn(),
+    )
+
+    return progress_bar
+
+
 def get_progress_bar(bar_type, size=None):
     """
     Get progress bar of given type.
     """
     progress_bar_types = {
         PROGRESS_BAR_DOWNLOAD: download_progress_bar,
+        PROGRESS_BAR_EXTENSIONS: extensions_progress_bar,
         PROGRESS_BAR_EASYCONFIG: easyconfig_progress_bar,
         PROGRESS_BAR_OVERALL: overall_progress_bar,
     }

From b956e8b040ea33e0791054a22668173ecd9d0890 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 13 Oct 2021 11:15:52 +0200
Subject: [PATCH 079/175] show separate progress bar to report progress on
 fetching of sources/patches + only show download progress for files >= 10MB

---
 easybuild/framework/easyblock.py | 27 ++++++++++++++++++++++++-
 easybuild/tools/filetools.py     | 12 +++++------
 easybuild/tools/output.py        | 34 ++++++++++++++++++++++++--------
 3 files changed, 58 insertions(+), 15 deletions(-)

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index 57551c04c6..3b2994f2c2 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -89,7 +89,7 @@
 from easybuild.tools.modules import ROOT_ENV_VAR_NAME_PREFIX, VERSION_ENV_VAR_NAME_PREFIX, DEVEL_ENV_VAR_NAME_PREFIX
 from easybuild.tools.modules import Lmod, curr_module_paths, invalidate_module_caches_for, get_software_root
 from easybuild.tools.modules import get_software_root_env_var_name, get_software_version_env_var_name
-from easybuild.tools.output import PROGRESS_BAR_EASYCONFIG, PROGRESS_BAR_EXTENSIONS
+from easybuild.tools.output import PROGRESS_BAR_DOWNLOAD_ALL, PROGRESS_BAR_EASYCONFIG, PROGRESS_BAR_EXTENSIONS
 from easybuild.tools.output import start_progress_bar, stop_progress_bar, update_progress_bar
 from easybuild.tools.package.utilities import package
 from easybuild.tools.py2vs3 import extract_method_name, string_type
@@ -690,6 +690,8 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
         """
         srcpaths = source_paths()
 
+        update_progress_bar(PROGRESS_BAR_DOWNLOAD_ALL, label=filename)
+
         # should we download or just try and find it?
         if re.match(r"^(https?|ftp)://", filename):
             # URL detected, so let's try and download it
@@ -1921,6 +1923,27 @@ def fetch_step(self, skip_checksums=False):
                 raise EasyBuildError("EasyBuild-version %s is newer than the currently running one. Aborting!",
                                      easybuild_version)
 
+        # count number of files to download: sources + patches (incl. extensions)
+        # FIXME: to make this count fully correct, we need the Extension instances first,
+        # which requires significant refactoring in fetch_extension_sources
+        # (also needed to fix https://github.com/easybuilders/easybuild-framework/issues/3849)
+        cnt = len(self.cfg['sources']) + len(self.cfg['patches'])
+        if self.cfg['exts_list']:
+            for ext in self.cfg['exts_list']:
+                if isinstance(ext, tuple) and len(ext) >= 3:
+                    ext_opts = ext[2]
+                    if 'source' in ext_opts:
+                        cnt += 1
+                    elif 'sources' in ext_opts:
+                        cnt += len(ext_opts['sources'])
+                    else:
+                        # assume there's always one source file;
+                        # for extensions using PythonPackage, no 'source' or 'sources' may be specified
+                        cnt += 1
+                    cnt += len(ext_opts.get('patches', []))
+
+        start_progress_bar(PROGRESS_BAR_DOWNLOAD_ALL, cnt)
+
         if self.dry_run:
 
             self.dry_run_msg("Available download URLs for sources/patches:")
@@ -2003,6 +2026,8 @@ def fetch_step(self, skip_checksums=False):
         else:
             self.log.info("Skipped installation dirs check per user request")
 
+        stop_progress_bar(PROGRESS_BAR_DOWNLOAD_ALL)
+
     def checksum_step(self):
         """Verify checksum of sources and patches, if a checksum is available."""
         for fil in self.src + self.patches:
diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py
index f26cc73385..ebeb9a9618 100644
--- a/easybuild/tools/filetools.py
+++ b/easybuild/tools/filetools.py
@@ -62,7 +62,7 @@
 from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_msg, print_warning
 from easybuild.tools.config import DEFAULT_WAIT_ON_LOCK_INTERVAL, ERROR, GENERIC_EASYBLOCK_PKG, IGNORE, WARN
 from easybuild.tools.config import build_option, install_path
-from easybuild.tools.output import PROGRESS_BAR_DOWNLOAD, start_progress_bar, stop_progress_bar, update_progress_bar
+from easybuild.tools.output import PROGRESS_BAR_DOWNLOAD_ONE, start_progress_bar, stop_progress_bar, update_progress_bar
 from easybuild.tools.py2vs3 import HTMLParser, std_urllib, string_type
 from easybuild.tools.utilities import natural_keys, nub, remove_unwanted_chars
 
@@ -261,13 +261,13 @@ def write_file(path, data, append=False, forced=False, backup=False, always_over
     if sys.version_info[0] >= 3 and (isinstance(data, bytes) or data_is_file_obj):
         mode += 'b'
 
-    # don't bother showing a progress bar for small files
-    if size and size < 1024:
+    # don't bother showing a progress bar for small files (< 10MB)
+    if size and size < 10 * (1024 ** 2):
         _log.info("Not showing progress bar for downloading small file (size %s)", size)
         show_progress = False
 
     if show_progress:
-        start_progress_bar(PROGRESS_BAR_DOWNLOAD, size, label=os.path.basename(path))
+        start_progress_bar(PROGRESS_BAR_DOWNLOAD_ONE, size, label=os.path.basename(path))
 
     # note: we can't use try-except-finally, because Python 2.4 doesn't support it as a single block
     try:
@@ -278,12 +278,12 @@ def write_file(path, data, append=False, forced=False, backup=False, always_over
                 for chunk in iter(partial(data.read, 1024 ** 2), b''):
                     fh.write(chunk)
                     if show_progress:
-                        update_progress_bar(PROGRESS_BAR_DOWNLOAD, progress_size=len(chunk))
+                        update_progress_bar(PROGRESS_BAR_DOWNLOAD_ONE, progress_size=len(chunk))
             else:
                 fh.write(data)
 
         if show_progress:
-            stop_progress_bar(PROGRESS_BAR_DOWNLOAD)
+            stop_progress_bar(PROGRESS_BAR_DOWNLOAD_ONE)
 
     except IOError as err:
         raise EasyBuildError("Failed to write to %s: %s", path, err)
diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py
index dd895af1c0..225385fc4b 100644
--- a/easybuild/tools/output.py
+++ b/easybuild/tools/output.py
@@ -45,7 +45,8 @@
     pass
 
 
-PROGRESS_BAR_DOWNLOAD = 'download'
+PROGRESS_BAR_DOWNLOAD_ALL = 'download_all'
+PROGRESS_BAR_DOWNLOAD_ONE = 'download_one'
 PROGRESS_BAR_EXTENSIONS = 'extensions'
 PROGRESS_BAR_EASYCONFIG = 'easyconfig'
 PROGRESS_BAR_OVERALL = 'overall'
@@ -101,8 +102,9 @@ def rich_live_cm():
     """
     if show_progress_bars():
         pbar_group = RenderGroup(
-                download_progress_bar(),
-                download_progress_bar_unknown_size(),
+                download_one_progress_bar(),
+                download_one_progress_bar_unknown_size(),
+                download_all_progress_bar(),
                 extensions_progress_bar(),
                 easyconfig_progress_bar(),
                 overall_progress_bar(),
@@ -163,7 +165,22 @@ def easyconfig_progress_bar():
 
 
 @progress_bar_cache
-def download_progress_bar():
+def download_all_progress_bar():
+    """
+    Get progress bar to show progress on downloading of all source files.
+    """
+    progress_bar = Progress(
+        TextColumn("[bold blue]Fetching files: {task.percentage:>3.0f}% ({task.completed}/{task.total})"),
+        BarColumn(),
+        TimeElapsedColumn(),
+        TextColumn("({task.description})"),
+    )
+
+    return progress_bar
+
+
+@progress_bar_cache
+def download_one_progress_bar():
     """
     Get progress bar to show progress for downloading a file of known size.
     """
@@ -179,7 +196,7 @@ def download_progress_bar():
 
 
 @progress_bar_cache
-def download_progress_bar_unknown_size():
+def download_one_progress_bar_unknown_size():
     """
     Get progress bar to show progress for downloading a file of unknown size.
     """
@@ -211,14 +228,15 @@ def get_progress_bar(bar_type, size=None):
     Get progress bar of given type.
     """
     progress_bar_types = {
-        PROGRESS_BAR_DOWNLOAD: download_progress_bar,
+        PROGRESS_BAR_DOWNLOAD_ALL: download_all_progress_bar,
+        PROGRESS_BAR_DOWNLOAD_ONE: download_one_progress_bar,
         PROGRESS_BAR_EXTENSIONS: extensions_progress_bar,
         PROGRESS_BAR_EASYCONFIG: easyconfig_progress_bar,
         PROGRESS_BAR_OVERALL: overall_progress_bar,
     }
 
-    if bar_type == PROGRESS_BAR_DOWNLOAD and not size:
-        pbar = download_progress_bar_unknown_size()
+    if bar_type == PROGRESS_BAR_DOWNLOAD_ONE and not size:
+        pbar = download_one_progress_bar_unknown_size()
     elif bar_type in progress_bar_types:
         pbar = progress_bar_types[bar_type]()
     else:

From 8d5bf09abf61c8542a0ed4d70843dfc09e720a0b Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 13 Oct 2021 11:24:13 +0200
Subject: [PATCH 080/175] don't show progress bar if there's only a single task

---
 easybuild/tools/output.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py
index 225385fc4b..7d235a41f9 100644
--- a/easybuild/tools/output.py
+++ b/easybuild/tools/output.py
@@ -255,8 +255,13 @@ def start_progress_bar(bar_type, size, label=None):
     pbar = get_progress_bar(bar_type, size=size)
     task_id = pbar.add_task('')
     _progress_bar_cache[bar_type] = (pbar, task_id)
-    if size:
+
+    # don't bother showing progress bar if there's only 1 item to make progress on
+    if size == 1:
+        pbar.update(task_id, visible=False)
+    elif size:
         pbar.update(task_id, total=size)
+
     if label:
         pbar.update(task_id, description=label)
 

From a9a9d23a78d31ab20392ce9720102a3c75dfb95d Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 13 Oct 2021 11:40:51 +0200
Subject: [PATCH 081/175] take into account --stop, --fetch, --module-only when
 determining number of steps that will be run

---
 easybuild/framework/easyblock.py | 18 +++++++++++-------
 easybuild/tools/output.py        |  2 +-
 2 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index 3b2994f2c2..b8d22fb106 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -3496,6 +3496,7 @@ def run_step(self, step, step_methods):
         run_hook(step, self.hooks, post_step_hook=True, args=[self])
 
         if self.cfg['stop'] == step:
+            update_progress_bar(PROGRESS_BAR_EASYCONFIG)
             self.log.info("Stopping after %s step.", step)
             raise StopException(step)
 
@@ -3609,11 +3610,15 @@ def run_all_steps(self, run_test_cases):
 
         steps = self.get_steps(run_test_cases=run_test_cases, iteration_count=self.det_iter_cnt())
 
-        progress_label_tmpl = "%s (%d out of %d steps done)"
+        # figure out how many steps will actually be run (not be skipped)
+        step_cnt = 0
+        for (step_name, _, _, skippable) in steps:
+            if not self.skip_step(step_name, skippable):
+                step_cnt += 1
+            if self.cfg['stop'] == step_name:
+                break
 
-        n_steps = len(steps)
-        progress_label = progress_label_tmpl % (self.full_mod_name, 0, n_steps)
-        start_progress_bar(PROGRESS_BAR_EASYCONFIG, n_steps, label=progress_label)
+        start_progress_bar(PROGRESS_BAR_EASYCONFIG, step_cnt, label=self.full_mod_name)
 
         print_msg("building and installing %s..." % self.full_mod_name, log=self.log, silent=self.silent)
         trace_msg("installation prefix: %s" % self.installdir)
@@ -3653,8 +3658,7 @@ def run_all_steps(self, run_test_cases):
                             elif self.logdebug or build_option('trace'):
                                 print_msg("... (took < 1 sec)", log=self.log, silent=self.silent)
 
-                progress_label = progress_label_tmpl % (self.full_mod_name, step_id, n_steps)
-                update_progress_bar(PROGRESS_BAR_EASYCONFIG, label=progress_label)
+                    update_progress_bar(PROGRESS_BAR_EASYCONFIG)
 
         except StopException:
             pass
@@ -3662,7 +3666,7 @@ def run_all_steps(self, run_test_cases):
             if not ignore_locks:
                 remove_lock(lock_name)
 
-        update_progress_bar(PROGRESS_BAR_EASYCONFIG, label="%s done!" % self.full_mod_name)
+        update_progress_bar(PROGRESS_BAR_EASYCONFIG, label="%s done!" % self.full_mod_name, progress_size=0)
 
         # return True for successfull build (or stopped build)
         return True
diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py
index 7d235a41f9..bb87b63bff 100644
--- a/easybuild/tools/output.py
+++ b/easybuild/tools/output.py
@@ -156,7 +156,7 @@ def easyconfig_progress_bar():
     Get progress bar to display progress for installing a single easyconfig file.
     """
     progress_bar = Progress(
-        TextColumn("[bold green]{task.description}"),
+        TextColumn("[bold green]{task.description} ({task.completed} out of {task.total} steps done)"),
         BarColumn(),
         TimeElapsedColumn(),
     )

From 9e2e0b0daaf6b311a823427b3b19132562643718 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 13 Oct 2021 12:06:28 +0200
Subject: [PATCH 082/175] also determine file size when downloading file via
 requests module

---
 easybuild/tools/filetools.py | 33 ++++++++++++++++++++++-----------
 test/framework/filetools.py  | 28 ++++++++++++++++++++++++++++
 2 files changed, 50 insertions(+), 11 deletions(-)

diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py
index ebeb9a9618..52ac6ac748 100644
--- a/easybuild/tools/filetools.py
+++ b/easybuild/tools/filetools.py
@@ -721,6 +721,22 @@ def parse_http_header_fields_urlpat(arg, urlpat=None, header=None, urlpat_header
     return urlpat_headers
 
 
+def det_file_size(http_header):
+    """
+    Determine size of file from provided HTTP header info (without downloading it).
+    """
+    res = None
+    len_key = 'Content-Length'
+    if len_key in http_header:
+        size = http_header[len_key]
+        try:
+            res = int(size)
+        except (ValueError, TypeError) as err:
+            _log.warning("Failed to interpret size '%s' as integer value: %s", size, err)
+
+    return res
+
+
 def download_file(filename, url, path, forced=False):
     """Download a file from the given URL, to the specified path."""
 
@@ -772,28 +788,23 @@ def download_file(filename, url, path, forced=False):
     while not downloaded and attempt_cnt < max_attempts:
         attempt_cnt += 1
         try:
-            size = None
             if used_urllib is std_urllib:
                 # urllib2 (Python 2) / urllib.request (Python 3) does the right thing for http proxy setups,
                 # urllib does not!
                 url_fd = std_urllib.urlopen(url_req, timeout=timeout)
                 status_code = url_fd.getcode()
-                http_header = url_fd.info()
-                len_key = 'Content-Length'
-                if len_key in http_header:
-                    size = http_header[len_key]
-                    try:
-                        size = int(size)
-                    except (ValueError, TypeError) as err:
-                        _log.warning("Failed to interpret size '%s' as integer value: %s", size, err)
-                        size = None
+                size = det_file_size(url_fd.info())
             else:
                 response = requests.get(url, headers=headers, stream=True, timeout=timeout)
                 status_code = response.status_code
                 response.raise_for_status()
+                size = det_file_size(response.headers)
                 url_fd = response.raw
                 url_fd.decode_content = True
-            _log.debug('response code for given url %s: %s' % (url, status_code))
+
+            _log.debug("HTTP response code for given url %s: %s", url, status_code)
+            _log.info("File size for %s: %s", url, size)
+
             # note: we pass the file object to write_file rather than reading the file first,
             # to ensure the data is read in chunks (which prevents problems in Python 3.9+);
             # cfr. https://github.com/easybuilders/easybuild-framework/issues/3455
diff --git a/test/framework/filetools.py b/test/framework/filetools.py
index ee9eb2ab48..fdaa934165 100644
--- a/test/framework/filetools.py
+++ b/test/framework/filetools.py
@@ -376,6 +376,34 @@ def test_normalize_path(self):
         self.assertEqual(ft.normalize_path('/././foo//bar/././baz/'), '/foo/bar/baz')
         self.assertEqual(ft.normalize_path('//././foo//bar/././baz/'), '//foo/bar/baz')
 
+    def test_det_file_size(self):
+        """Test det_file_size function."""
+
+        self.assertEqual(ft.det_file_size({'Content-Length': '12345'}), 12345)
+
+        # missing content length, or invalid value
+        self.assertEqual(ft.det_file_size({}), None)
+        self.assertEqual(ft.det_file_size({'Content-Length': 'foo'}), None)
+
+        test_url = 'https://github.com/easybuilders/easybuild-framework/raw/develop/'
+        test_url += 'test/framework/sandbox/sources/toy/toy-0.0.tar.gz'
+        expected_size = 273
+
+        # also try with actual HTTP header
+        try:
+            with std_urllib.urlopen(test_url) as fh:
+                self.assertEqual(ft.det_file_size(fh.info()), expected_size)
+
+            # also try using requests, which is used as a fallback in download_file
+            try:
+                import requests
+                with requests.get(test_url) as res:
+                    self.assertEqual(ft.det_file_size(res.headers), expected_size)
+            except ImportError:
+                pass
+        except std_urllib.URLError:
+            print("Skipping online test for det_file_size (working offline)")
+
     def test_download_file(self):
         """Test download_file function."""
         fn = 'toy-0.0.tar.gz'

From 5350e5967c82ad17344f5b0ffb1e705eb68f5a2e Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 13 Oct 2021 13:52:00 +0200
Subject: [PATCH 083/175] fix broken test_toy_multi_deps by re-disabling
 showing of progress bars

---
 test/framework/toy_build.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py
index 265c83f22a..6c047c9fe6 100644
--- a/test/framework/toy_build.py
+++ b/test/framework/toy_build.py
@@ -2902,13 +2902,16 @@ def check_toy_load(depends_on=False):
         modify_env(os.environ, self.orig_environ, verbose=False)
         self.modtool.use(test_mod_path)
 
+        # disable showing of progress bars (again), doesn't make sense when running tests
+        os.environ['EASYBUILD_DISABLE_SHOW_PROGRESS_BAR'] = '1'
+
         write_file(test_ec, test_ec_txt)
 
         # also check behaviour when using 'depends_on' rather than 'load' statements (requires Lmod 7.6.1 or newer)
         if self.modtool.supports_depends_on:
 
             remove_file(toy_mod_file)
-            self.test_toy_build(ec_file=test_ec, extra_args=['--module-depends-on'])
+            self.test_toy_build(ec_file=test_ec, extra_args=['--module-depends-on'], raise_error=True)
 
             toy_mod_txt = read_file(toy_mod_file)
 

From a4215be02acf4be66d925078e6005f80cc3772cc Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 13 Oct 2021 14:38:50 +0200
Subject: [PATCH 084/175] add test for toy build with showing of progress bars
 enabled

---
 test/framework/toy_build.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py
index 6c047c9fe6..2cbba1cebf 100644
--- a/test/framework/toy_build.py
+++ b/test/framework/toy_build.py
@@ -57,6 +57,7 @@
 from easybuild.tools.module_generator import ModuleGeneratorTcl
 from easybuild.tools.modules import Lmod
 from easybuild.tools.py2vs3 import reload, string_type
+from easybuild.tools.output import show_progress_bars
 from easybuild.tools.run import run_cmd
 from easybuild.tools.systemtools import get_shared_lib_ext
 from easybuild.tools.version import VERSION as EASYBUILD_VERSION
@@ -3523,6 +3524,19 @@ def test_toy_ignore_test_failure(self):
         self.assertTrue("Build succeeded (with --ignore-test-failure) for 1 out of 1" in stdout)
         self.assertFalse(stderr)
 
+    def test_toy_build_with_progress_bars(self):
+        """Test installation with showing of progress bars enabled."""
+
+        # don't disable showing of progress bars, to ensure we catch any problems
+        # that are only triggered when progress bars are being shown...
+        del os.environ['EASYBUILD_DISABLE_SHOW_PROGRESS_BAR']
+
+        stdout, _ = self.run_test_toy_build_with_output()
+
+        if show_progress_bars():
+            regex = re.compile(r"^toy/0.0 done! \(17 out of 17 steps done\) ━+ [0-9:]+", re.M)
+            self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout))
+
 
 def suite():
     """ return all the tests in this file """

From c61e8487e66103d9e0e90891b11306f4230067bc Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 13 Oct 2021 14:39:02 +0200
Subject: [PATCH 085/175] also test show_progress_bars() function in output
 tests

---
 test/framework/output.py | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/test/framework/output.py b/test/framework/output.py
index ad0280179d..d27be8aa0f 100644
--- a/test/framework/output.py
+++ b/test/framework/output.py
@@ -33,7 +33,7 @@
 
 from easybuild.tools.build_log import EasyBuildError
 from easybuild.tools.config import build_option, get_output_style, update_build_option
-from easybuild.tools.output import DummyRich, overall_progress_bar, use_rich
+from easybuild.tools.output import DummyRich, overall_progress_bar, show_progress_bars, use_rich
 
 try:
     import rich.progress
@@ -101,21 +101,28 @@ def test_get_output_style(self):
             error_pattern = "Can't use 'rich' output style, Rich Python package is not available!"
             self.assertErrorRegex(EasyBuildError, error_pattern, get_output_style)
 
-    def test_use_rich(self):
-        """Test use_rich function."""
+    def test_use_rich_show_progress_bars(self):
+        """Test use_rich and show_progress_bar functions."""
+
+        # restore default configuration to show progress bars (disabled to avoid mangled test output)
+        update_build_option('show_progress_bar', True)
 
         self.assertEqual(build_option('output_style'), 'auto')
 
         if HAVE_RICH:
             self.assertTrue(use_rich())
+            self.assertTrue(show_progress_bars())
 
             update_build_option('output_style', 'rich')
             self.assertTrue(use_rich())
+            self.assertTrue(show_progress_bars())
         else:
             self.assertFalse(use_rich())
+            self.assertFalse(show_progress_bars())
 
         update_build_option('output_style', 'basic')
         self.assertFalse(use_rich())
+        self.assertFalse(show_progress_bars())
 
 
 def suite():

From 4e09da4f6e41c8a7a31a6fb3dc907dabeecf92ab Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 13 Oct 2021 14:41:44 +0200
Subject: [PATCH 086/175] remove unused step_id variable in
 EasyBlock.run_all_steps

---
 easybuild/framework/easyblock.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index b8d22fb106..5d47884789 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -3638,7 +3638,7 @@ def run_all_steps(self, run_test_cases):
             create_lock(lock_name)
 
         try:
-            for step_id, (step_name, descr, step_methods, skippable) in enumerate(steps):
+            for step_name, descr, step_methods, skippable in steps:
                 if self.skip_step(step_name, skippable):
                     print_msg("%s [skipped]" % descr, log=self.log, silent=self.silent)
                 else:

From 56660a229a091d96f4c3b57db2af0ebe4e9c0a3a Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 13 Oct 2021 19:53:45 +0200
Subject: [PATCH 087/175] fix test_det_file_size for Python 2.7 (result of
 urllib2.urlopen can't be used as context manager)

---
 test/framework/filetools.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/test/framework/filetools.py b/test/framework/filetools.py
index fdaa934165..0b868d9278 100644
--- a/test/framework/filetools.py
+++ b/test/framework/filetools.py
@@ -391,14 +391,16 @@ def test_det_file_size(self):
 
         # also try with actual HTTP header
         try:
-            with std_urllib.urlopen(test_url) as fh:
-                self.assertEqual(ft.det_file_size(fh.info()), expected_size)
+            fh = std_urllib.urlopen(test_url)
+            self.assertEqual(ft.det_file_size(fh.info()), expected_size)
+            fh.close()
 
             # also try using requests, which is used as a fallback in download_file
             try:
                 import requests
-                with requests.get(test_url) as res:
-                    self.assertEqual(ft.det_file_size(res.headers), expected_size)
+                res = requests.get(test_url)
+                self.assertEqual(ft.det_file_size(res.headers), expected_size)
+                res.close()
             except ImportError:
                 pass
         except std_urllib.URLError:

From ab103630c29e1b046a54ef00f9d71eec3c9b9439 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 13 Oct 2021 22:51:50 +0200
Subject: [PATCH 088/175] remove test_toy_build_with_progress_bars, fails in CI
 due to 'err: object None is not renderable'

---
 test/framework/toy_build.py | 14 --------------
 1 file changed, 14 deletions(-)

diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py
index 2cbba1cebf..6c047c9fe6 100644
--- a/test/framework/toy_build.py
+++ b/test/framework/toy_build.py
@@ -57,7 +57,6 @@
 from easybuild.tools.module_generator import ModuleGeneratorTcl
 from easybuild.tools.modules import Lmod
 from easybuild.tools.py2vs3 import reload, string_type
-from easybuild.tools.output import show_progress_bars
 from easybuild.tools.run import run_cmd
 from easybuild.tools.systemtools import get_shared_lib_ext
 from easybuild.tools.version import VERSION as EASYBUILD_VERSION
@@ -3524,19 +3523,6 @@ def test_toy_ignore_test_failure(self):
         self.assertTrue("Build succeeded (with --ignore-test-failure) for 1 out of 1" in stdout)
         self.assertFalse(stderr)
 
-    def test_toy_build_with_progress_bars(self):
-        """Test installation with showing of progress bars enabled."""
-
-        # don't disable showing of progress bars, to ensure we catch any problems
-        # that are only triggered when progress bars are being shown...
-        del os.environ['EASYBUILD_DISABLE_SHOW_PROGRESS_BAR']
-
-        stdout, _ = self.run_test_toy_build_with_output()
-
-        if show_progress_bars():
-            regex = re.compile(r"^toy/0.0 done! \(17 out of 17 steps done\) ━+ [0-9:]+", re.M)
-            self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout))
-
 
 def suite():
     """ return all the tests in this file """

From 4337df1dad790dbb7a84e353d29e8dc7b687a971 Mon Sep 17 00:00:00 2001
From: Joris Rommelse <Joris.Rommelse@shell.com>
Date: Thu, 14 Oct 2021 18:18:27 +0200
Subject: [PATCH 089/175] Added support for --insecure-download option. Similar
 to --no-check-certificate for wget or --insecure for curl.

---
 easybuild/framework/easyblock.py | 17 +++++++++++------
 easybuild/tools/config.py        |  1 +
 easybuild/tools/filetools.py     | 12 +++++++++---
 easybuild/tools/options.py       |  2 ++
 4 files changed, 23 insertions(+), 9 deletions(-)

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index 58bed6afde..e37d96a63d 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -400,8 +400,10 @@ def fetch_source(self, source, checksum=None, extension=False):
 
         # check if the sources can be located
         force_download = build_option('force_download') in [FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_SOURCES]
+        insecure_download = build_option('insecure_download')
         path = self.obtain_file(filename, extension=extension, download_filename=download_filename,
-                                force_download=force_download, urls=source_urls, git_config=git_config)
+                                force_download=force_download, insecure_download=insecure_download,
+                                urls=source_urls, git_config=git_config)
         if path is None:
             raise EasyBuildError('No file found for source %s', filename)
 
@@ -487,7 +489,8 @@ def fetch_patches(self, patch_specs=None, extension=False, checksums=None):
                 patch_file = patch_spec
 
             force_download = build_option('force_download') in [FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_PATCHES]
-            path = self.obtain_file(patch_file, extension=extension, force_download=force_download)
+            insecure_download = build_option('insecure_download')
+            path = self.obtain_file(patch_file, extension=extension, force_download=force_download, insecure_download=insecure_download)
             if path:
                 self.log.debug('File %s found for patch %s' % (path, patch_spec))
                 patchspec = {
@@ -527,6 +530,7 @@ def fetch_extension_sources(self, skip_checksums=False):
             self.dry_run_msg("\nList of sources/patches for extensions:")
 
         force_download = build_option('force_download') in [FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_SOURCES]
+        insecure_download = build_option('insecure_download')
 
         for ext in exts_list:
             if (isinstance(ext, list) or isinstance(ext, tuple)) and ext:
@@ -620,7 +624,7 @@ def fetch_extension_sources(self, skip_checksums=False):
                             raise EasyBuildError(error_msg, type(src_fn).__name__, src_fn)
 
                         src_path = self.obtain_file(src_fn, extension=True, urls=source_urls,
-                                                    force_download=force_download)
+                                                    force_download=force_download, insecure_download=insecure_download)
                         if src_path:
                             ext_src.update({'src': src_path})
                         else:
@@ -689,7 +693,7 @@ def fetch_extension_sources(self, skip_checksums=False):
         return exts_sources
 
     def obtain_file(self, filename, extension=False, urls=None, download_filename=None, force_download=False,
-                    git_config=None):
+                    insecure_download=False, git_config=None):
         """
         Locate the file with the given name
         - searches in different subdirectories of source path
@@ -699,6 +703,7 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
         :param urls: list of source URLs where this file may be available
         :param download_filename: filename with which the file should be downloaded, and then renamed to <filename>
         :param force_download: always try to download file, even if it's already available in source path
+        :param insecure_download: don't check the server certificate against the available certificate authorities
         :param git_config: dictionary to define how to download a git repository
         """
         srcpaths = source_paths()
@@ -728,7 +733,7 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
                         self.log.info("Found file %s at %s, no need to download it", filename, filepath)
                         return fullpath
 
-                if download_file(filename, url, fullpath):
+                if download_file(filename, url, fullpath, insecure=insecure_download):
                     return fullpath
 
             except IOError as err:
@@ -855,7 +860,7 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
                         self.log.debug("Trying to download file %s from %s to %s ..." % (filename, fullurl, targetpath))
                         downloaded = False
                         try:
-                            if download_file(filename, fullurl, targetpath):
+                            if download_file(filename, fullurl, targetpath, insecure=insecure_download):
                                 downloaded = True
 
                         except IOError as err:
diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py
index 18902ae799..7b669764f4 100644
--- a/easybuild/tools/config.py
+++ b/easybuild/tools/config.py
@@ -199,6 +199,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
         'hide_toolchains',
         'http_header_fields_urlpat',
         'force_download',
+        'insecure_download',
         'from_pr',
         'git_working_dirs_path',
         'github_user',
diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py
index 4dd40bd4f4..a84696321a 100644
--- a/easybuild/tools/filetools.py
+++ b/easybuild/tools/filetools.py
@@ -50,6 +50,7 @@
 import shutil
 import signal
 import stat
+import ssl
 import sys
 import tempfile
 import time
@@ -701,7 +702,7 @@ def parse_http_header_fields_urlpat(arg, urlpat=None, header=None, urlpat_header
     return urlpat_headers
 
 
-def download_file(filename, url, path, forced=False):
+def download_file(filename, url, path, forced=False, insecure=False):
     """Download a file from the given URL, to the specified path."""
 
     _log.debug("Trying to download %s from %s to %s", filename, url, path)
@@ -752,13 +753,18 @@ def download_file(filename, url, path, forced=False):
     while not downloaded and attempt_cnt < max_attempts:
         attempt_cnt += 1
         try:
+            if insecure:
+                print_warning("Not checking server certificates while downloading %s from %s." % (filename, url))
             if used_urllib is std_urllib:
                 # urllib2 (Python 2) / urllib.request (Python 3) does the right thing for http proxy setups,
                 # urllib does not!
-                url_fd = std_urllib.urlopen(url_req, timeout=timeout)
+                if insecure:
+                    url_fd = std_urllib.urlopen(url_req, timeout=timeout, context=ssl._create_unverified_context())
+                else:
+                    url_fd = std_urllib.urlopen(url_req, timeout=timeout)
                 status_code = url_fd.getcode()
             else:
-                response = requests.get(url, headers=headers, stream=True, timeout=timeout)
+                response = requests.get(url, headers=headers, stream=True, timeout=timeout, verify=( not insecure ))
                 status_code = response.status_code
                 response.raise_for_status()
                 url_fd = response.raw
diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py
index e4fc7661a4..8055221d87 100644
--- a/easybuild/tools/options.py
+++ b/easybuild/tools/options.py
@@ -407,6 +407,8 @@ def override_options(self):
             'force-download': ("Force re-downloading of sources and/or patches, "
                                "even if they are available already in source path",
                                'choice', 'store_or_None', DEFAULT_FORCE_DOWNLOAD, FORCE_DOWNLOAD_CHOICES),
+            'insecure-download': ("Don't check the server certificate against the available certificate authorities.",
+                                   None, 'store_true', False),
             'generate-devel-module': ("Generate a develop module file, implies --force if disabled",
                                       None, 'store_true', True),
             'group': ("Group to be used for software installations (only verified, not set)", None, 'store', None),

From f3020f75a6e1d0510618d7a7fc3b87911562137a Mon Sep 17 00:00:00 2001
From: Joris Rommelse <Joris.Rommelse@shell.com>
Date: Thu, 14 Oct 2021 21:03:09 +0200
Subject: [PATCH 090/175] Resolved 3 minor issues that were flagged by the
 hound bot in pull request 3859.

---
 easybuild/framework/easyblock.py | 3 ++-
 easybuild/tools/filetools.py     | 2 +-
 easybuild/tools/options.py       | 2 +-
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index e37d96a63d..672b928ee3 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -490,7 +490,8 @@ def fetch_patches(self, patch_specs=None, extension=False, checksums=None):
 
             force_download = build_option('force_download') in [FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_PATCHES]
             insecure_download = build_option('insecure_download')
-            path = self.obtain_file(patch_file, extension=extension, force_download=force_download, insecure_download=insecure_download)
+            path = self.obtain_file(patch_file, extension=extension, force_download=force_download,
+                                    insecure_download=insecure_download)
             if path:
                 self.log.debug('File %s found for patch %s' % (path, patch_spec))
                 patchspec = {
diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py
index a84696321a..9ba9675f70 100644
--- a/easybuild/tools/filetools.py
+++ b/easybuild/tools/filetools.py
@@ -764,7 +764,7 @@ def download_file(filename, url, path, forced=False, insecure=False):
                     url_fd = std_urllib.urlopen(url_req, timeout=timeout)
                 status_code = url_fd.getcode()
             else:
-                response = requests.get(url, headers=headers, stream=True, timeout=timeout, verify=( not insecure ))
+                response = requests.get(url, headers=headers, stream=True, timeout=timeout, verify=(not insecure))
                 status_code = response.status_code
                 response.raise_for_status()
                 url_fd = response.raw
diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py
index 8055221d87..4c2f7091d9 100644
--- a/easybuild/tools/options.py
+++ b/easybuild/tools/options.py
@@ -408,7 +408,7 @@ def override_options(self):
                                "even if they are available already in source path",
                                'choice', 'store_or_None', DEFAULT_FORCE_DOWNLOAD, FORCE_DOWNLOAD_CHOICES),
             'insecure-download': ("Don't check the server certificate against the available certificate authorities.",
-                                   None, 'store_true', False),
+                                  None, 'store_true', False),
             'generate-devel-module': ("Generate a develop module file, implies --force if disabled",
                                       None, 'store_true', True),
             'group': ("Group to be used for software installations (only verified, not set)", None, 'store', None),

From eb2e71db8fbdb52a5b1d73c536fe403ec0991df3 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Thu, 14 Oct 2021 21:08:25 +0200
Subject: [PATCH 091/175] add back spinner, mention step name in easyconfig
 progress bar, stop/hide easyconfig progress bar when done

---
 easybuild/framework/easyblock.py | 7 ++++---
 easybuild/tools/output.py        | 3 ++-
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index 5d47884789..775d43b5d2 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -3618,7 +3618,7 @@ def run_all_steps(self, run_test_cases):
             if self.cfg['stop'] == step_name:
                 break
 
-        start_progress_bar(PROGRESS_BAR_EASYCONFIG, step_cnt, label=self.full_mod_name)
+        start_progress_bar(PROGRESS_BAR_EASYCONFIG, step_cnt, label="Installing %s" % self.full_mod_name)
 
         print_msg("building and installing %s..." % self.full_mod_name, log=self.log, silent=self.silent)
         trace_msg("installation prefix: %s" % self.installdir)
@@ -3658,7 +3658,8 @@ def run_all_steps(self, run_test_cases):
                             elif self.logdebug or build_option('trace'):
                                 print_msg("... (took < 1 sec)", log=self.log, silent=self.silent)
 
-                    update_progress_bar(PROGRESS_BAR_EASYCONFIG)
+                    progress_label = "Installing %s: %s" % (self.full_mod_name, descr)
+                    update_progress_bar(PROGRESS_BAR_EASYCONFIG, label=progress_label)
 
         except StopException:
             pass
@@ -3666,7 +3667,7 @@ def run_all_steps(self, run_test_cases):
             if not ignore_locks:
                 remove_lock(lock_name)
 
-        update_progress_bar(PROGRESS_BAR_EASYCONFIG, label="%s done!" % self.full_mod_name, progress_size=0)
+        stop_progress_bar(PROGRESS_BAR_EASYCONFIG)
 
         # return True for successfull build (or stopped build)
         return True
diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py
index bb87b63bff..e838f1fd52 100644
--- a/easybuild/tools/output.py
+++ b/easybuild/tools/output.py
@@ -39,7 +39,7 @@
     from rich.console import Console, RenderGroup
     from rich.live import Live
     from rich.table import Table
-    from rich.progress import BarColumn, Progress, TextColumn, TimeElapsedColumn
+    from rich.progress import BarColumn, Progress, SpinnerColumn, TextColumn, TimeElapsedColumn
     from rich.progress import DownloadColumn, FileSizeColumn, TransferSpeedColumn, TimeRemainingColumn
 except ImportError:
     pass
@@ -156,6 +156,7 @@ def easyconfig_progress_bar():
     Get progress bar to display progress for installing a single easyconfig file.
     """
     progress_bar = Progress(
+        SpinnerColumn('line'),
         TextColumn("[bold green]{task.description} ({task.completed} out of {task.total} steps done)"),
         BarColumn(),
         TimeElapsedColumn(),

From 230aaeab0afda61c62a1f79abc21ea162286d679 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Thu, 14 Oct 2021 21:10:08 +0200
Subject: [PATCH 092/175] stop showing overall progress bar when done

---
 easybuild/main.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/easybuild/main.py b/easybuild/main.py
index 59d7c60c20..cf514d619d 100644
--- a/easybuild/main.py
+++ b/easybuild/main.py
@@ -161,9 +161,9 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True):
 
         res.append((ec, ec_res))
 
-        update_progress_bar(PROGRESS_BAR_OVERALL, progress_size=1)
+        update_progress_bar(PROGRESS_BAR_OVERALL)
 
-    stop_progress_bar(PROGRESS_BAR_OVERALL, visible=True)
+    stop_progress_bar(PROGRESS_BAR_OVERALL)
 
     return res
 

From 13fe3467367913bd386e9807c40c132427b69bd6 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Thu, 14 Oct 2021 21:13:53 +0200
Subject: [PATCH 093/175] don't show progress bars in dry run mode

---
 easybuild/tools/output.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py
index e838f1fd52..f9f0cef9b6 100644
--- a/easybuild/tools/output.py
+++ b/easybuild/tools/output.py
@@ -93,7 +93,7 @@ def show_progress_bars():
     """
     Return whether or not to show progress bars.
     """
-    return use_rich() and build_option('show_progress_bar')
+    return use_rich() and build_option('show_progress_bar') and not build_option('extended_dry_run')
 
 
 def rich_live_cm():

From c791b0b0884d58070d44aa8705ebc17220653642 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Thu, 14 Oct 2021 22:18:19 +0200
Subject: [PATCH 094/175] use nicer spinner in easyconfig progress bar

---
 easybuild/tools/output.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py
index f9f0cef9b6..3652ea9682 100644
--- a/easybuild/tools/output.py
+++ b/easybuild/tools/output.py
@@ -156,7 +156,7 @@ def easyconfig_progress_bar():
     Get progress bar to display progress for installing a single easyconfig file.
     """
     progress_bar = Progress(
-        SpinnerColumn('line'),
+        SpinnerColumn('point'),
         TextColumn("[bold green]{task.description} ({task.completed} out of {task.total} steps done)"),
         BarColumn(),
         TimeElapsedColumn(),

From c84b82dcfbeb23ca7f0762577b99b7b01a7dacfa Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Sat, 16 Oct 2021 14:11:46 +0200
Subject: [PATCH 095/175] don't expand overall progress bar, to be consistent
 with other progress bars which aren't extended either

---
 easybuild/tools/output.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py
index 3652ea9682..ddac5bc718 100644
--- a/easybuild/tools/output.py
+++ b/easybuild/tools/output.py
@@ -144,7 +144,6 @@ def overall_progress_bar():
         TimeElapsedColumn(),
         TextColumn("{task.description}({task.completed} out of {task.total} easyconfigs done)"),
         BarColumn(bar_width=None),
-        expand=True,
     )
 
     return progress_bar

From 5065be65c39e9b2e139bac672cde25f25156d547 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Sat, 16 Oct 2021 14:45:21 +0200
Subject: [PATCH 096/175] add EasyConfig.count_files method, and leverage it in
 EasyBlock.fetch_step for download progrss counter

---
 easybuild/framework/easyblock.py             | 21 +-------
 easybuild/framework/easyconfig/easyconfig.py | 22 ++++++++
 test/framework/easyconfig.py                 | 53 ++++++++++++++++++++
 3 files changed, 76 insertions(+), 20 deletions(-)

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index 775d43b5d2..dc5eeb41a0 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -1923,26 +1923,7 @@ def fetch_step(self, skip_checksums=False):
                 raise EasyBuildError("EasyBuild-version %s is newer than the currently running one. Aborting!",
                                      easybuild_version)
 
-        # count number of files to download: sources + patches (incl. extensions)
-        # FIXME: to make this count fully correct, we need the Extension instances first,
-        # which requires significant refactoring in fetch_extension_sources
-        # (also needed to fix https://github.com/easybuilders/easybuild-framework/issues/3849)
-        cnt = len(self.cfg['sources']) + len(self.cfg['patches'])
-        if self.cfg['exts_list']:
-            for ext in self.cfg['exts_list']:
-                if isinstance(ext, tuple) and len(ext) >= 3:
-                    ext_opts = ext[2]
-                    if 'source' in ext_opts:
-                        cnt += 1
-                    elif 'sources' in ext_opts:
-                        cnt += len(ext_opts['sources'])
-                    else:
-                        # assume there's always one source file;
-                        # for extensions using PythonPackage, no 'source' or 'sources' may be specified
-                        cnt += 1
-                    cnt += len(ext_opts.get('patches', []))
-
-        start_progress_bar(PROGRESS_BAR_DOWNLOAD_ALL, cnt)
+        start_progress_bar(PROGRESS_BAR_DOWNLOAD_ALL, self.cfg.count_files())
 
         if self.dry_run:
 
diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py
index 8bc89606ce..0cbcb11ef3 100644
--- a/easybuild/framework/easyconfig/easyconfig.py
+++ b/easybuild/framework/easyconfig/easyconfig.py
@@ -773,6 +773,28 @@ def remove_false_versions(deps):
         # indicate that this is a parsed easyconfig
         self._config['parsed'] = [True, "This is a parsed easyconfig", "HIDDEN"]
 
+    def count_files(self):
+        """
+        Determine number of files (sources + patches) required for this easyconfig.
+        """
+        cnt = len(self['sources']) + len(self['patches'])
+
+        for ext in self['exts_list']:
+            if isinstance(ext, tuple) and len(ext) >= 3:
+                ext_opts = ext[2]
+                # check for 'sources' first, since that's also considered first by EasyBlock.fetch_extension_sources
+                if 'sources' in ext_opts:
+                    cnt += len(ext_opts['sources'])
+                elif 'source_tmpl' in ext_opts:
+                    cnt += 1
+                else:
+                    # assume there's always one source file;
+                    # for extensions using PythonPackage, no 'source' or 'sources' may be specified
+                    cnt += 1
+                cnt += len(ext_opts.get('patches', []))
+
+        return cnt
+
     def local_var_naming(self, local_var_naming_check):
         """Deal with local variables that do not follow the recommended naming scheme (if any)."""
 
diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py
index 48aaeb86a4..7491c109e0 100644
--- a/test/framework/easyconfig.py
+++ b/test/framework/easyconfig.py
@@ -4537,6 +4537,59 @@ def test_get_cuda_cc_template_value(self):
         for key in cuda_template_values:
             self.assertEqual(ec.get_cuda_cc_template_value(key), cuda_template_values[key])
 
+    def test_count_files(self):
+        """Tests for EasyConfig.count_files method."""
+        test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
+
+        foss = os.path.join(test_ecs_dir, 'f', 'foss', 'foss-2018a.eb')
+        toy = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb')
+        toy_exts = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb')
+
+        # no sources or patches for toolchain => 0
+        foss_ec = EasyConfig(foss)
+        self.assertEqual(foss_ec['sources'], [])
+        self.assertEqual(foss_ec['patches'], [])
+        self.assertEqual(foss_ec.count_files(), 0)
+        # 1 source + 2 patches => 3
+        toy_ec = EasyConfig(toy)
+        self.assertEqual(len(toy_ec['sources']), 1)
+        self.assertEqual(len(toy_ec['patches']), 2)
+        self.assertEqual(toy_ec['exts_list'], [])
+        self.assertEqual(toy_ec.count_files(), 3)
+        # 1 source + 1 patch
+        # 4 extensions
+        # * ls: no sources/patches (only name is specified)
+        # * bar: 1 source (implied, using default source_tmpl) + 2 patches
+        # * barbar: 1 source (implied, using default source_tmpl)
+        # * toy: 1 source (implied, using default source_tmpl)
+        # => 7 files in total
+        toy_exts_ec = EasyConfig(toy_exts)
+        self.assertEqual(len(toy_exts_ec['sources']), 1)
+        self.assertEqual(len(toy_exts_ec['patches']), 1)
+        self.assertEqual(len(toy_exts_ec['exts_list']), 4)
+        self.assertEqual(toy_exts_ec.count_files(), 7)
+
+        test_ec = os.path.join(self.test_prefix, 'test.eb')
+        copy_file(toy_exts, test_ec)
+        # add a couple of additional extensions to verify correct file count
+        test_ec_extra = '\n'.join([
+            'exts_list += [',
+            '    ("test-ext-one", "0.0", {',
+            '        "sources": ["test-ext-one-0.0-part1.tgz", "test-ext-one-0.0-part2.zip"],',
+            # if both 'sources' and 'source_tmpl' are specified, 'source_tmpl' is ignored,
+            # see EasyBlock.fetch_extension_sources, so it should be too when counting files
+            '        "source_tmpl": "test-ext-one-%(version)s.tar.gz",',
+            '    }),',
+            '    ("test-ext-two", "0.0", {',
+            '        "source_tmpl": "test-ext-two-0.0-part1.tgz",',
+            '        "patches": ["test-ext-two.patch"],',
+            '    }),',
+            ']',
+        ])
+        write_file(test_ec, test_ec_extra, append=True)
+        test_ec = EasyConfig(test_ec)
+        self.assertEqual(test_ec.count_files(), 11)
+
 
 def suite():
     """ returns all the testcases in this module """

From 988e813dc7cd774e9b5ba6b738e7d4cb8e2c6830 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Sat, 16 Oct 2021 15:29:37 +0200
Subject: [PATCH 097/175] tweak test_module_only_extensions to catch bug
 reported in https://github.com/easybuilders/easybuild-framework/issues/3849

---
 test/framework/toy_build.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py
index 265c83f22a..8f8af2b139 100644
--- a/test/framework/toy_build.py
+++ b/test/framework/toy_build.py
@@ -1746,6 +1746,16 @@ def test_module_only_extensions(self):
         # remove module file so we can try --module-only
         remove_file(toy_mod)
 
+        # make sure that sources for extensions can't be found,
+        # they should not be needed when using --module-only
+        # (cfr. https://github.com/easybuilders/easybuild-framework/issues/3849)
+        del os.environ['EASYBUILD_SOURCEPATH']
+
+        # first try normal --module-only, should work fine
+        self.eb_main([test_ec, '--module-only'], do_build=True, raise_error=True)
+        self.assertTrue(os.path.exists(toy_mod))
+        remove_file(toy_mod)
+
         # rename file required for barbar extension, so we can check whether sanity check catches it
         libbarbar = os.path.join(self.test_installpath, 'software', 'toy', '0.0', 'lib', 'libbarbar.a')
         move_file(libbarbar, libbarbar + '.foobar')

From b7edf1d2f804aa82a5e4dd587dd4fa4e60295e95 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Sat, 16 Oct 2021 16:10:51 +0200
Subject: [PATCH 098/175] refactor EasyBlock to decouple collecting of
 information on extension source/patch files from downloading them (fixes
 #3849) includes deprecating EasyBlock.fetch_extension_sources and replacing
 it with EasyBlock.collect_exts_file_info

---
 easybuild/framework/easyblock.py | 120 ++++++++++++++-----------------
 easybuild/tools/filetools.py     |  38 ++++++++++
 test/framework/easyblock.py      |  83 ++++++++++++++++++++-
 test/framework/easyconfig.py     |   2 +-
 test/framework/filetools.py      |  17 +++++
 5 files changed, 192 insertions(+), 68 deletions(-)

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index 58bed6afde..5a3a57409a 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -72,9 +72,10 @@
 from easybuild.tools.config import install_path, log_path, package_path, source_paths
 from easybuild.tools.environment import restore_env, sanitize_env
 from easybuild.tools.filetools import CHECKSUM_TYPE_MD5, CHECKSUM_TYPE_SHA256
-from easybuild.tools.filetools import adjust_permissions, apply_patch, back_up_file, change_dir, convert_name
-from easybuild.tools.filetools import compute_checksum, copy_file, check_lock, create_lock, derive_alt_pypi_url
-from easybuild.tools.filetools import diff_files, dir_contains_files, download_file, encode_class_name, extract_file
+from easybuild.tools.filetools import adjust_permissions, apply_patch, back_up_file, change_dir, create_patch_info
+from easybuild.tools.filetools import convert_name, compute_checksum, copy_file, check_lock, create_lock
+from easybuild.tools.filetools import derive_alt_pypi_url, diff_files, dir_contains_files, download_file
+from easybuild.tools.filetools import encode_class_name, extract_file
 from easybuild.tools.filetools import find_backup_name_candidate, get_source_tarball_from_git, is_alt_pypi_url
 from easybuild.tools.filetools import is_binary, is_sha256_checksum, mkdir, move_file, move_logs, read_file, remove_dir
 from easybuild.tools.filetools import remove_file, remove_lock, verify_checksum, weld_paths, write_file, symlink
@@ -372,7 +373,7 @@ def fetch_source(self, source, checksum=None, extension=False):
 
         :param source: source to be found (single dictionary in 'sources' list, or filename)
         :param checksum: checksum corresponding to source
-        :param extension: flag if being called from fetch_extension_sources()
+        :param extension: flag if being called from collect_exts_file_info()
         """
         filename, download_filename, extract_cmd, source_urls, git_config = None, None, None, None, None
 
@@ -461,52 +462,19 @@ def fetch_patches(self, patch_specs=None, extension=False, checksums=None):
         patches = []
         for index, patch_spec in enumerate(patch_specs):
 
-            # check if the patches can be located
-            copy_file = False
-            suff = None
-            level = None
-            if isinstance(patch_spec, (list, tuple)):
-                if not len(patch_spec) == 2:
-                    raise EasyBuildError("Unknown patch specification '%s', only 2-element lists/tuples are supported!",
-                                         str(patch_spec))
-                patch_file = patch_spec[0]
-
-                # this *must* be of typ int, nothing else
-                # no 'isinstance(..., int)', since that would make True/False also acceptable
-                if isinstance(patch_spec[1], int):
-                    level = patch_spec[1]
-                elif isinstance(patch_spec[1], string_type):
-                    # non-patch files are assumed to be files to copy
-                    if not patch_spec[0].endswith('.patch'):
-                        copy_file = True
-                    suff = patch_spec[1]
-                else:
-                    raise EasyBuildError("Wrong patch spec '%s', only int/string are supported as 2nd element",
-                                         str(patch_spec))
-            else:
-                patch_file = patch_spec
+            patch_info = create_patch_info(patch_spec)
 
             force_download = build_option('force_download') in [FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_PATCHES]
-            path = self.obtain_file(patch_file, extension=extension, force_download=force_download)
+            path = self.obtain_file(patch_info['name'], extension=extension, force_download=force_download)
             if path:
-                self.log.debug('File %s found for patch %s' % (path, patch_spec))
-                patchspec = {
-                    'name': patch_file,
-                    'path': path,
-                    'checksum': self.get_checksum_for(checksums, index=index),
-                }
-                if suff:
-                    if copy_file:
-                        patchspec['copy'] = suff
-                    else:
-                        patchspec['sourcepath'] = suff
-                if level is not None:
-                    patchspec['level'] = level
+                self.log.debug('File %s found for patch %s', path, patch_spec)
+                patch_info['path'] = path
+                patch_info['checksum'] = self.get_checksum_for(checksums, index=index)
 
                 if extension:
-                    patches.append(patchspec)
+                    patches.append(patch_info)
                 else:
-                    self.patches.append(patchspec)
+                    self.patches.append(patch_info)
             else:
                 raise EasyBuildError('No file found for patch %s', patch_spec)
 
@@ -514,25 +482,38 @@ def fetch_patches(self, patch_specs=None, extension=False, checksums=None):
             self.log.info("Fetched extension patches: %s", patches)
             return patches
         else:
-            self.log.info("Added patches: %s" % self.patches)
+            self.log.info("Added patches: %s", self.patches)
 
     def fetch_extension_sources(self, skip_checksums=False):
         """
-        Find source file for extensions.
+        Fetch source and patch files for extensions (DEPRECATED, use collect_exts_file_info instead).
+        """
+        depr_msg = "EasyBlock.fetch_extension_sources is deprecated, use EasyBlock.collect_exts_file_info instead"
+        self.log.deprecated(depr_msg, '5.0')
+        return self.collect_exts_file_info(fetch_files=True, verify_checksums=not skip_checksums)
+
+    def collect_exts_file_info(self, fetch_files=True, verify_checksums=True):
+        """
+        Collect information on source and patch files for extensions.
+
+        :param fetch_files: whether or not to fetch files (if False, path to files will be missing from info)
+        :param verify_checksums: whether or not to verify checksums
+        :return: list of dict values, one per extension, with information on source/patch files.
         """
         exts_sources = []
         exts_list = self.cfg.get_ref('exts_list')
 
+        if verify_checksums and not fetch_files:
+            raise EasyBuildError("Can't verify checksums for extension files if they are not being fetched")
+
         if self.dry_run:
             self.dry_run_msg("\nList of sources/patches for extensions:")
 
         force_download = build_option('force_download') in [FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_SOURCES]
 
         for ext in exts_list:
-            if (isinstance(ext, list) or isinstance(ext, tuple)) and ext:
-
+            if isinstance(ext, (list, tuple)) and ext:
                 # expected format: (name, version, options (dict))
-
                 ext_name = ext[0]
                 if len(ext) == 1:
                     exts_sources.append({'name': ext_name})
@@ -602,10 +583,10 @@ def fetch_extension_sources(self, skip_checksums=False):
                         if 'source_urls' not in source:
                             source['source_urls'] = source_urls
 
-                        src = self.fetch_source(source, checksums, extension=True)
-
-                        # copy 'path' entry to 'src' for use with extensions
-                        ext_src.update({'src': src['path']})
+                        if fetch_files:
+                            src = self.fetch_source(source, checksums, extension=True)
+                            # copy 'path' entry to 'src' for use with extensions
+                            ext_src.update({'src': src['path']})
 
                     else:
                         # use default template for name of source file if none is specified
@@ -619,15 +600,16 @@ def fetch_extension_sources(self, skip_checksums=False):
                             error_msg = "source_tmpl value must be a string! (found value of type '%s'): %s"
                             raise EasyBuildError(error_msg, type(src_fn).__name__, src_fn)
 
-                        src_path = self.obtain_file(src_fn, extension=True, urls=source_urls,
-                                                    force_download=force_download)
-                        if src_path:
-                            ext_src.update({'src': src_path})
-                        else:
-                            raise EasyBuildError("Source for extension %s not found.", ext)
+                        if fetch_files:
+                            src_path = self.obtain_file(src_fn, extension=True, urls=source_urls,
+                                                        force_download=force_download)
+                            if src_path:
+                                ext_src.update({'src': src_path})
+                            else:
+                                raise EasyBuildError("Source for extension %s not found.", ext)
 
                     # verify checksum for extension sources
-                    if 'src' in ext_src and not skip_checksums:
+                    if verify_checksums and 'src' in ext_src:
                         src_path = ext_src['src']
                         src_fn = os.path.basename(src_path)
 
@@ -647,12 +629,17 @@ def fetch_extension_sources(self, skip_checksums=False):
                             raise EasyBuildError('Checksum verification for extension source %s failed', src_fn)
 
                     # locate extension patches (if any), and verify checksums
-                    ext_patches = self.fetch_patches(patch_specs=ext_options.get('patches', []), extension=True)
+                    ext_patches = ext_options.get('patches', [])
+                    if fetch_files:
+                        ext_patches = self.fetch_patches(patch_specs=ext_patches, extension=True)
+                    else:
+                        ext_patches = [create_patch_info(p) for p in ext_patches]
+
                     if ext_patches:
                         self.log.debug('Found patches for extension %s: %s', ext_name, ext_patches)
                         ext_src.update({'patches': ext_patches})
 
-                        if not skip_checksums:
+                        if verify_checksums:
                             for patch in ext_patches:
                                 patch = patch['path']
                                 # report both MD5 and SHA256 checksums,
@@ -1995,7 +1982,7 @@ def fetch_step(self, skip_checksums=False):
 
         # fetch extensions
         if self.cfg.get_ref('exts_list'):
-            self.exts = self.fetch_extension_sources(skip_checksums=skip_checksums)
+            self.exts = self.collect_exts_file_info(fetch_files=True, verify_checksums=not skip_checksums)
 
         # create parent dirs in install and modules path already
         # this is required when building in parallel
@@ -2311,8 +2298,11 @@ def init_ext_instances(self):
         self.ext_instances = []
         exts_classmap = self.cfg['exts_classmap']
 
+        # self.exts may already be populated at this point through collect_exts_file_info;
+        # if it's not, we do it lightweight here, by skipping fetching of the files;
+        # information on location of source/patch files will be lacking in that case (but that should be fine)
         if exts_list and not self.exts:
-            self.exts = self.fetch_extension_sources()
+            self.exts = self.collect_exts_file_info(fetch_files=False, verify_checksums=False)
 
         # obtain name and module path for default extention class
         exts_defaultclass = self.cfg['exts_defaultclass']
@@ -2410,7 +2400,7 @@ def extensions_step(self, fetch=False, install=True):
         self.prepare_for_extensions()
 
         if fetch:
-            self.exts = self.fetch_extension_sources()
+            self.exts = self.collect_exts_file_info(fetch_files=True)
 
         self.exts_all = self.exts[:]  # retain a copy of all extensions, regardless of filtering/skipping
 
diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py
index 723ac5d6e1..b6a4bc623a 100644
--- a/easybuild/tools/filetools.py
+++ b/easybuild/tools/filetools.py
@@ -1409,6 +1409,44 @@ def guess_patch_level(patched_files, parent_dir):
     return patch_level
 
 
+def create_patch_info(patch_spec):
+    """
+    Create info dictionary from specified patch spec.
+    """
+    if isinstance(patch_spec, (list, tuple)):
+        if not len(patch_spec) == 2:
+            error_msg = "Unknown patch specification '%s', only 2-element lists/tuples are supported!"
+            raise EasyBuildError(error_msg, str(patch_spec))
+
+        patch_info = {'name': patch_spec[0]}
+
+        patch_arg = patch_spec[1]
+        # patch level *must* be of type int, nothing else (not True/False!)
+        # note that 'isinstance(..., int)' returns True for True/False values...
+        if isinstance(patch_arg, int) and not isinstance(patch_arg, bool):
+            patch_info['level'] = patch_arg
+
+        # string value as patch argument can be either path where patch should be applied,
+        # or path to where a non-patch file should be copied
+        elif isinstance(patch_arg, string_type):
+            if patch_spec[0].endswith('.patch'):
+                patch_info['sourcepath'] = patch_arg
+            # non-patch files are assumed to be files to copy
+            else:
+                patch_info['copy'] = patch_arg
+        else:
+            raise EasyBuildError("Wrong patch spec '%s', only int/string are supported as 2nd element",
+                                 str(patch_spec))
+
+    elif isinstance(patch_spec, string_type):
+        patch_info = {'name': patch_spec}
+    else:
+        error_msg = "Wrong patch spec, should be string of 2-tuple with patch name + argument: %s"
+        raise EasyBuildError(error_msg, patch_spec)
+
+    return patch_info
+
+
 def apply_patch(patch_file, dest, fn=None, copy=False, level=None, use_git_am=False, use_git=False):
     """
     Apply a patch to source code in directory dest
diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py
index 18bfcf07b0..241e533690 100644
--- a/test/framework/easyblock.py
+++ b/test/framework/easyblock.py
@@ -1617,6 +1617,65 @@ def test_fallback_source_url(self):
 
         self.assertTrue(verify_checksum(expected_path, eb.cfg['checksums'][0]))
 
+    def test_collect_exts_file_info(self):
+        """Test collect_exts_file_info method."""
+        testdir = os.path.abspath(os.path.dirname(__file__))
+        toy_sources = os.path.join(testdir, 'sandbox', 'sources', 'toy')
+        toy_ext_sources = os.path.join(toy_sources, 'extensions')
+        toy_ec_file = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-gompi-2018a-test.eb')
+        toy_ec = process_easyconfig(toy_ec_file)[0]
+        toy_eb = EasyBlock(toy_ec['ec'])
+
+        exts_file_info = toy_eb.collect_exts_file_info()
+
+        self.assertTrue(isinstance(exts_file_info, list))
+        self.assertEqual(len(exts_file_info), 4)
+
+        self.assertEqual(exts_file_info[0], {'name': 'ls'})
+
+        self.assertEqual(exts_file_info[1]['name'], 'bar')
+        self.assertEqual(exts_file_info[1]['src'], os.path.join(toy_ext_sources, 'bar-0.0.tar.gz'))
+        bar_patch1 = 'bar-0.0_fix-silly-typo-in-printf-statement.patch'
+        self.assertEqual(exts_file_info[1]['patches'][0]['name'], bar_patch1)
+        self.assertEqual(exts_file_info[1]['patches'][0]['path'], os.path.join(toy_ext_sources, bar_patch1))
+        bar_patch2 = 'bar-0.0_fix-very-silly-typo-in-printf-statement.patch'
+        self.assertEqual(exts_file_info[1]['patches'][1]['name'], bar_patch2)
+        self.assertEqual(exts_file_info[1]['patches'][1]['path'], os.path.join(toy_ext_sources, bar_patch2))
+
+        self.assertEqual(exts_file_info[2]['name'], 'barbar')
+        self.assertEqual(exts_file_info[2]['src'], os.path.join(toy_ext_sources, 'barbar-0.0.tar.gz'))
+        self.assertFalse('patches' in exts_file_info[2])
+
+        self.assertEqual(exts_file_info[3]['name'], 'toy')
+        self.assertEqual(exts_file_info[3]['src'], os.path.join(toy_sources, 'toy-0.0.tar.gz'))
+        self.assertFalse('patches' in exts_file_info[3])
+
+        # location of files is missing when fetch_files is set to False
+        exts_file_info = toy_eb.collect_exts_file_info(fetch_files=False, verify_checksums=False)
+
+        self.assertTrue(isinstance(exts_file_info, list))
+        self.assertEqual(len(exts_file_info), 4)
+
+        self.assertEqual(exts_file_info[0], {'name': 'ls'})
+
+        self.assertEqual(exts_file_info[1]['name'], 'bar')
+        self.assertFalse('src' in exts_file_info[1])
+        self.assertEqual(exts_file_info[1]['patches'][0]['name'], bar_patch1)
+        self.assertFalse('path' in exts_file_info[1]['patches'][0])
+        self.assertEqual(exts_file_info[1]['patches'][1]['name'], bar_patch2)
+        self.assertFalse('path' in exts_file_info[1]['patches'][1])
+
+        self.assertEqual(exts_file_info[2]['name'], 'barbar')
+        self.assertFalse('src' in exts_file_info[2])
+        self.assertFalse('patches' in exts_file_info[2])
+
+        self.assertEqual(exts_file_info[3]['name'], 'toy')
+        self.assertFalse('src' in exts_file_info[3])
+        self.assertFalse('patches' in exts_file_info[3])
+
+        error_msg = "Can't verify checksums for extension files if they are not being fetched"
+        self.assertErrorRegex(EasyBuildError, error_msg, toy_eb.collect_exts_file_info, fetch_files=False)
+
     def test_obtain_file_extension(self):
         """Test use of obtain_file method on an extension."""
 
@@ -2062,9 +2121,16 @@ def test_checksum_step(self):
         error_msg = "Checksum verification for .*/toy-0.0.tar.gz using .* failed"
         self.assertErrorRegex(EasyBuildError, error_msg, eb.checksum_step)
 
-        # also check verification of checksums for extensions, which is part of fetch_extension_sources
+        # also check verification of checksums for extensions, which is part of collect_exts_file_info
         error_msg = "Checksum verification for extension source bar-0.0.tar.gz failed"
+        self.assertErrorRegex(EasyBuildError, error_msg, eb.collect_exts_file_info)
+
+        # also check with deprecated fetch_extension_sources method
+        self.allow_deprecated_behaviour()
+        self.mock_stderr(True)
         self.assertErrorRegex(EasyBuildError, error_msg, eb.fetch_extension_sources)
+        self.mock_stderr(False)
+        self.disallow_deprecated_behaviour()
 
         # if --ignore-checksums is enabled, faulty checksums are reported but otherwise ignored (no error)
         build_options = {
@@ -2084,7 +2150,7 @@ def test_checksum_step(self):
 
         self.mock_stderr(True)
         self.mock_stdout(True)
-        eb.fetch_extension_sources()
+        eb.collect_exts_file_info()
         stderr = self.get_stderr()
         stdout = self.get_stdout()
         self.mock_stderr(False)
@@ -2092,6 +2158,19 @@ def test_checksum_step(self):
         self.assertEqual(stdout, '')
         self.assertEqual(stderr.strip(), "WARNING: Ignoring failing checksum verification for bar-0.0.tar.gz")
 
+        # also check with deprecated fetch_extension_sources method
+        self.allow_deprecated_behaviour()
+        self.mock_stderr(True)
+        self.mock_stdout(True)
+        eb.fetch_extension_sources()
+        stderr = self.get_stderr()
+        stdout = self.get_stdout()
+        self.mock_stderr(False)
+        self.mock_stdout(False)
+        self.assertEqual(stdout, '')
+        self.assertTrue(stderr.strip().endswith("WARNING: Ignoring failing checksum verification for bar-0.0.tar.gz"))
+        self.disallow_deprecated_behaviour()
+
     def test_check_checksums(self):
         """Test for check_checksums_for and check_checksums methods."""
         testdir = os.path.abspath(os.path.dirname(__file__))
diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py
index 48aaeb86a4..ab11c94c50 100644
--- a/test/framework/easyconfig.py
+++ b/test/framework/easyconfig.py
@@ -474,7 +474,7 @@ def test_exts_list(self):
         self.prep()
         ec = EasyConfig(self.eb_file)
         eb = EasyBlock(ec)
-        exts_sources = eb.fetch_extension_sources()
+        exts_sources = eb.collect_exts_file_info()
 
         self.assertEqual(len(exts_sources), 2)
         self.assertEqual(exts_sources[0]['name'], 'ext1')
diff --git a/test/framework/filetools.py b/test/framework/filetools.py
index ee9eb2ab48..fafd1e54d3 100644
--- a/test/framework/filetools.py
+++ b/test/framework/filetools.py
@@ -1510,6 +1510,23 @@ def test_derive_alt_pypi_url(self):
         url = 'https://pypi.python.org/packages/source/n/nosuchpackageonpypiever/nosuchpackageonpypiever-0.0.0.tar.gz'
         self.assertEqual(ft.derive_alt_pypi_url(url), None)
 
+    def test_create_patch_info(self):
+        """Test create_patch_info function."""
+
+        self.assertEqual(ft.create_patch_info('foo.patch'), {'name': 'foo.patch'})
+        self.assertEqual(ft.create_patch_info('foo.txt'), {'name': 'foo.txt'})
+        self.assertEqual(ft.create_patch_info(('foo.patch', 1)), {'name': 'foo.patch', 'level': 1})
+        self.assertEqual(ft.create_patch_info(('foo.patch', 'subdir')), {'name': 'foo.patch', 'sourcepath': 'subdir'})
+        self.assertEqual(ft.create_patch_info(('foo.txt', 'subdir')), {'name': 'foo.txt', 'copy': 'subdir'})
+
+        # faulty input
+        error_msg = "Wrong patch spec"
+        self.assertErrorRegex(EasyBuildError, error_msg, ft.create_patch_info, None)
+        self.assertErrorRegex(EasyBuildError, error_msg, ft.create_patch_info, {'name': 'foo.patch'})
+        self.assertErrorRegex(EasyBuildError, error_msg, ft.create_patch_info, ('foo.patch', [1, 2]))
+        error_msg = "Unknown patch specification"
+        self.assertErrorRegex(EasyBuildError, error_msg, ft.create_patch_info, ('foo.patch', 1, 'subdir'))
+
     def test_apply_patch(self):
         """ Test apply_patch """
         testdir = os.path.dirname(os.path.abspath(__file__))

From 1fa4c08202a2d565f8462ddad2c381b3c969de8a Mon Sep 17 00:00:00 2001
From: Bart Oldeman <bart.oldeman@calculquebec.ca>
Date: Sun, 17 Oct 2021 01:30:29 +0000
Subject: [PATCH 099/175] Make toolchain logic aware of imkl-FFTW module

When this module is detected use it to obtain the cluster (cdft)
MPI FFTW3 interfaces, but obtain the non-MPI FFTW3 interfaces from
the main MKL libraries from the imkl binary distribution, which
have provided them since MKL 10.2
---
 easybuild/toolchains/fft/intelfftw.py         |  6 ++
 test/framework/modules/impi/2021.4.0          | 42 +++++++++
 .../modules/intel-compilers/2021.4.0          | 41 +++++++++
 test/framework/modules/intel/2021b            | 36 ++++++++
 test/framework/toolchain.py                   | 85 ++++++++++++++++++-
 5 files changed, 206 insertions(+), 4 deletions(-)
 create mode 100644 test/framework/modules/impi/2021.4.0
 create mode 100644 test/framework/modules/intel-compilers/2021.4.0
 create mode 100644 test/framework/modules/intel/2021b

diff --git a/easybuild/toolchains/fft/intelfftw.py b/easybuild/toolchains/fft/intelfftw.py
index b748fd5e7e..6181903c4d 100644
--- a/easybuild/toolchains/fft/intelfftw.py
+++ b/easybuild/toolchains/fft/intelfftw.py
@@ -95,6 +95,12 @@ def _set_fftw_variables(self):
         # so make sure libraries are there before FFT_LIB is set
         imklroot = get_software_root(self.FFT_MODULE_NAME[0])
         fft_lib_dirs = [os.path.join(imklroot, d) for d in self.FFT_LIB_DIR]
+        imklfftwroot = get_software_root('imkl-FFTW')
+        if imklfftwroot:
+            # only get cluster_interface_lib from seperate module imkl-FFTW, rest via libmkl_gf/libmkl_intel
+            fft_lib_dirs += [os.path.join(imklfftwroot, 'lib')]
+            fftw_libs.remove(interface_lib)
+            fftw_mt_libs.remove(interface_lib)
 
         def fftw_lib_exists(libname):
             """Helper function to check whether FFTW library with specified name exists."""
diff --git a/test/framework/modules/impi/2021.4.0 b/test/framework/modules/impi/2021.4.0
new file mode 100644
index 0000000000..a8003ced0e
--- /dev/null
+++ b/test/framework/modules/impi/2021.4.0
@@ -0,0 +1,42 @@
+#%Module
+proc ModulesHelp { } {
+    puts stderr {
+
+Description
+===========
+Intel MPI Library, compatible with MPICH ABI
+
+
+More information
+================
+ - Homepage: https://software.intel.com/content/www/us/en/develop/tools/mpi-library.html
+    }
+}
+
+module-whatis {Description: Intel MPI Library, compatible with MPICH ABI}
+module-whatis {Homepage: https://software.intel.com/content/www/us/en/develop/tools/mpi-library.html}
+module-whatis {URL: https://software.intel.com/content/www/us/en/develop/tools/mpi-library.html}
+
+set root /tmp/impi/2021.4.0
+
+conflict impi
+
+prepend-path	CMAKE_PREFIX_PATH		$root
+prepend-path	CPATH		$root/mpi/2021.4.0/include
+prepend-path	FI_PROVIDER_PATH		$root/mpi/2021.4.0/libfabric/lib/prov
+prepend-path	LD_LIBRARY_PATH		$root/mpi/2021.4.0/lib
+prepend-path	LD_LIBRARY_PATH		$root/mpi/2021.4.0/lib/release
+prepend-path	LD_LIBRARY_PATH		$root/mpi/2021.4.0/libfabric/lib
+prepend-path	LIBRARY_PATH		$root/mpi/2021.4.0/lib
+prepend-path	LIBRARY_PATH		$root/mpi/2021.4.0/lib/release
+prepend-path	LIBRARY_PATH		$root/mpi/2021.4.0/libfabric/lib
+prepend-path	MANPATH		$root/mpi/2021.4.0/man
+prepend-path	PATH		$root/mpi/2021.4.0/bin
+prepend-path	PATH		$root/mpi/2021.4.0/libfabric/bin
+setenv	EBROOTIMPI		"$root"
+setenv	EBVERSIONIMPI		"2021.4.0"
+setenv	EBDEVELIMPI		"$root/easybuild/impi-2021.4.0-easybuild-devel"
+
+setenv	I_MPI_ROOT		"$root/mpi/2021.4.0"
+setenv	UCX_TLS		"all"
+# Built with EasyBuild version 4.5.0dev
diff --git a/test/framework/modules/intel-compilers/2021.4.0 b/test/framework/modules/intel-compilers/2021.4.0
new file mode 100644
index 0000000000..b9e93096d1
--- /dev/null
+++ b/test/framework/modules/intel-compilers/2021.4.0
@@ -0,0 +1,41 @@
+#%Module
+proc ModulesHelp { } {
+    puts stderr {
+
+Description
+===========
+Intel C, C++ & Fortran compilers (classic and oneAPI)
+
+
+More information
+================
+ - Homepage: https://software.intel.com/content/www/us/en/develop/tools/oneapi/hpc-toolkit.html
+    }
+}
+
+module-whatis {Description: Intel C, C++ & Fortran compilers (classic and oneAPI)}
+module-whatis {Homepage: https://software.intel.com/content/www/us/en/develop/tools/oneapi/hpc-toolkit.html}
+module-whatis {URL: https://software.intel.com/content/www/us/en/develop/tools/oneapi/hpc-toolkit.html}
+
+set root /tmp/intel-compilers/2021.4.0
+
+conflict intel-compilers
+
+prepend-path	CPATH		$root/tbb/2021.4.0/include
+prepend-path	LD_LIBRARY_PATH		$root/compiler/2021.4.0/linux/lib
+prepend-path	LD_LIBRARY_PATH		$root/compiler/2021.4.0/linux/lib/x64
+prepend-path	LD_LIBRARY_PATH		$root/compiler/2021.4.0/linux/compiler/lib/intel64_lin
+prepend-path	LD_LIBRARY_PATH		$root/tbb/2021.4.0/lib/intel64/gcc4.8
+prepend-path	LIBRARY_PATH		$root/compiler/2021.4.0/linux/lib
+prepend-path	LIBRARY_PATH		$root/compiler/2021.4.0/linux/lib/x64
+prepend-path	LIBRARY_PATH		$root/compiler/2021.4.0/linux/compiler/lib/intel64_lin
+prepend-path	LIBRARY_PATH		$root/tbb/2021.4.0/lib/intel64/gcc4.8
+prepend-path	OCL_ICD_FILENAMES		$root/compiler/2021.4.0/linux/lib/x64/libintelocl.so
+prepend-path	PATH		$root/compiler/2021.4.0/linux/bin
+prepend-path	PATH		$root/compiler/2021.4.0/linux/bin/intel64
+prepend-path	TBBROOT		$root/tbb/2021.4.0
+setenv	EBROOTINTELMINCOMPILERS		"$root"
+setenv	EBVERSIONINTELMINCOMPILERS		"2021.4.0"
+setenv	EBDEVELINTELMINCOMPILERS		"$root/easybuild/Core-intel-compilers-2021.4.0-easybuild-devel"
+
+# Built with EasyBuild version 4.5.0dev
diff --git a/test/framework/modules/intel/2021b b/test/framework/modules/intel/2021b
new file mode 100644
index 0000000000..5695e2e0b3
--- /dev/null
+++ b/test/framework/modules/intel/2021b
@@ -0,0 +1,36 @@
+#%Module
+
+proc ModulesHelp { } {
+    puts stderr {   Intel Cluster Toolkit Compiler Edition provides Intel C/C++ and Fortran compilers, Intel MPI & Intel MKL. - Homepage: http://software.intel.com/en-us/intel-cluster-toolkit-compiler/
+    }
+}
+
+module-whatis {Intel Cluster Toolkit Compiler Edition provides Intel C/C++ and Fortran compilers, Intel MPI & Intel MKL. - Homepage: http://software.intel.com/en-us/intel-cluster-toolkit-compiler/}
+
+set root    /tmp/intel/2021b
+
+conflict   intel
+
+if { ![is-loaded intel-compilers/2021.4.0] } {
+    module load intel-compilers/2021.4.0
+}
+
+if { ![is-loaded impi/2021.4.0] } {
+    module load impi/2021.4.0
+}
+
+if { ![is-loaded imkl/2021.4.0] } {
+    module load imkl/2021.4.0
+}
+
+if { ![is-loaded imkl-FFTW/2021.4.0] } {
+    module load imkl-FFTW/2021.4.0
+}
+
+
+setenv	EBROOTINTEL		"$root"
+setenv	EBVERSIONINTEL		"2021b"
+setenv	EBDEVELINTEL		"$root/easybuild/intel-2021b-easybuild-devel"
+
+
+# built with EasyBuild version 4.5.0dev
diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py
index 403f6774ca..8bad3a7a8d 100644
--- a/test/framework/toolchain.py
+++ b/test/framework/toolchain.py
@@ -1079,6 +1079,63 @@ def test_fft_env_vars_intel(self):
         libfft_mt += '-Wl,-Bdynamic -liomp5 -lpthread'
         self.assertEqual(tc.get_variable('LIBFFT_MT'), libfft_mt)
 
+        self.setup_sandbox_for_intel_fftw(self.test_prefix, imklver='2021.4.0')
+        tc = self.get_toolchain('intel', version='2021b')
+        tc.prepare()
+
+        fft_static_libs = 'libmkl_intel_lp64.a,libmkl_sequential.a,libmkl_core.a'
+        self.assertEqual(tc.get_variable('FFT_STATIC_LIBS'), fft_static_libs)
+        self.assertEqual(tc.get_variable('FFTW_STATIC_LIBS'), fft_static_libs)
+
+        fft_static_libs_mt = 'libmkl_intel_lp64.a,libmkl_intel_thread.a,libmkl_core.a,'
+        fft_static_libs_mt += 'libiomp5.a,libpthread.a'
+        self.assertEqual(tc.get_variable('FFT_STATIC_LIBS_MT'), fft_static_libs_mt)
+        self.assertEqual(tc.get_variable('FFTW_STATIC_LIBS_MT'), fft_static_libs_mt)
+
+        libfft = "-Wl,-Bstatic -Wl,--start-group -lmkl_intel_lp64 -lmkl_sequential -lmkl_core "
+        libfft += "-Wl,--end-group -Wl,-Bdynamic"
+        self.assertEqual(tc.get_variable('LIBFFT'), libfft)
+
+        libfft_mt = "-Wl,-Bstatic -Wl,--start-group -lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core "
+        libfft_mt += "-Wl,--end-group -Wl,-Bdynamic -liomp5 -lpthread"
+        self.assertEqual(tc.get_variable('LIBFFT_MT'), libfft_mt)
+
+        tc = self.get_toolchain('intel', version='2021b')
+        tc.set_options({'openmp': True})
+        tc.prepare()
+
+        self.assertEqual(tc.get_variable('FFT_STATIC_LIBS'), fft_static_libs)
+        self.assertEqual(tc.get_variable('FFTW_STATIC_LIBS'), fft_static_libs)
+
+        self.assertEqual(tc.get_variable('FFT_STATIC_LIBS_MT'), fft_static_libs_mt)
+        self.assertEqual(tc.get_variable('FFTW_STATIC_LIBS_MT'), fft_static_libs_mt)
+
+        self.assertEqual(tc.get_variable('LIBFFT'), libfft)
+        self.assertEqual(tc.get_variable('LIBFFT_MT'), libfft_mt)
+
+        tc = self.get_toolchain('intel', version='2021b')
+        tc.set_options({'usempi': True})
+        tc.prepare()
+
+        fft_static_libs = 'libfftw3x_cdft_lp64.a,libmkl_cdft_core.a,libmkl_blacs_intelmpi_lp64.a,'
+        fft_static_libs += 'libmkl_intel_lp64.a,libmkl_sequential.a,libmkl_core.a'
+        self.assertEqual(tc.get_variable('FFT_STATIC_LIBS'), fft_static_libs)
+        self.assertEqual(tc.get_variable('FFTW_STATIC_LIBS'), fft_static_libs)
+
+        fft_static_libs_mt = 'libfftw3x_cdft_lp64.a,libmkl_cdft_core.a,libmkl_blacs_intelmpi_lp64.a,'
+        fft_static_libs_mt += 'libmkl_intel_lp64.a,libmkl_intel_thread.a,libmkl_core.a,libiomp5.a,libpthread.a'
+        self.assertEqual(tc.get_variable('FFT_STATIC_LIBS_MT'), fft_static_libs_mt)
+        self.assertEqual(tc.get_variable('FFTW_STATIC_LIBS_MT'), fft_static_libs_mt)
+
+        libfft = '-Wl,-Bstatic -Wl,--start-group -lfftw3x_cdft_lp64 -lmkl_cdft_core '
+        libfft += '-lmkl_blacs_intelmpi_lp64 -lmkl_intel_lp64 -lmkl_sequential -lmkl_core -Wl,--end-group -Wl,-Bdynamic'
+        self.assertEqual(tc.get_variable('LIBFFT'), libfft)
+
+        libfft_mt = '-Wl,-Bstatic -Wl,--start-group -lfftw3x_cdft_lp64 -lmkl_cdft_core '
+        libfft_mt += '-lmkl_blacs_intelmpi_lp64 -lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core -Wl,--end-group '
+        libfft_mt += '-Wl,-Bdynamic -liomp5 -lpthread'
+        self.assertEqual(tc.get_variable('LIBFFT_MT'), libfft_mt)
+
     def test_fosscuda(self):
         """Test whether fosscuda is handled properly."""
         tc = self.get_toolchain("fosscuda", version="2018a")
@@ -1115,7 +1172,9 @@ def setup_sandbox_for_intel_fftw(self, moddir, imklver='2018.1.163'):
         # create dummy imkl module and put required lib*.a files in place
 
         imkl_module_path = os.path.join(moddir, 'imkl', imklver)
+        imkl_fftw_module_path = os.path.join(moddir, 'imkl-FFTW', imklver)
         imkl_dir = os.path.join(self.test_prefix, 'software', 'imkl', imklver)
+        imkl_fftw_dir = os.path.join(self.test_prefix, 'software', 'imkl-FFTW', imklver)
 
         imkl_mod_txt = '\n'.join([
             "#%Module",
@@ -1124,17 +1183,35 @@ def setup_sandbox_for_intel_fftw(self, moddir, imklver='2018.1.163'):
         ])
         write_file(imkl_module_path, imkl_mod_txt)
 
-        fftw_libs = ['fftw3xc_intel', 'fftw3xc_pgi', 'mkl_cdft_core', 'mkl_blacs_intelmpi_lp64']
-        fftw_libs += ['mkl_intel_lp64', 'mkl_sequential', 'mkl_core', 'mkl_intel_ilp64']
+        imkl_fftw_mod_txt = '\n'.join([
+            "#%Module",
+            "setenv EBROOTIMKLMINFFTW %s" % imkl_fftw_dir,
+            "setenv EBVERSIONIMKLMINFFTW %s" % imklver,
+        ])
+        write_file(imkl_fftw_module_path, imkl_fftw_mod_txt)
+
+        mkl_libs = ['mkl_cdft_core', 'mkl_blacs_intelmpi_lp64']
+        mkl_libs += ['mkl_intel_lp64', 'mkl_sequential', 'mkl_core', 'mkl_intel_ilp64']
+        fftw_libs = ['fftw3xc_intel', 'fftw3xc_pgi']
         if LooseVersion(imklver) >= LooseVersion('11'):
             fftw_libs.extend(['fftw3x_cdft_ilp64', 'fftw3x_cdft_lp64'])
         else:
             fftw_libs.append('fftw3x_cdft')
 
-        for subdir in ['mkl/lib/intel64', 'compiler/lib/intel64', 'lib/em64t']:
+        if LooseVersion(imklver) >= LooseVersion('2021.4.0'):
+            subdir = 'mkl/%s/lib/intel64' % imklver
             os.makedirs(os.path.join(imkl_dir, subdir))
-            for fftlib in fftw_libs:
+            for fftlib in mkl_libs:
                 write_file(os.path.join(imkl_dir, subdir, 'lib%s.a' % fftlib), 'foo')
+            subdir = 'lib'
+            os.makedirs(os.path.join(imkl_fftw_dir, subdir))
+            for fftlib in fftw_libs:
+                write_file(os.path.join(imkl_fftw_dir, subdir, 'lib%s.a' % fftlib), 'foo')
+        else:
+            for subdir in ['mkl/lib/intel64', 'compiler/lib/intel64', 'lib/em64t']:
+                os.makedirs(os.path.join(imkl_dir, subdir))
+                for fftlib in mkl_libs + fftw_libs:
+                    write_file(os.path.join(imkl_dir, subdir, 'lib%s.a' % fftlib), 'foo')
 
     def test_intel_toolchain(self):
         """Test for intel toolchain."""

From e15e748bdeb520b4ed5a7fa84826e5679b97f566 Mon Sep 17 00:00:00 2001
From: Bart Oldeman <bart.oldeman@calculquebec.ca>
Date: Sun, 17 Oct 2021 02:10:26 +0000
Subject: [PATCH 100/175] Add explicit test modules for imkl,
 imkl-FFTW/2021.4.0

Without these the test_avail check fails when checking dependencies
for intel/2021b.
---
 test/framework/modules/imkl-FFTW/2021.4.0 | 31 +++++++++++++++++++
 test/framework/modules/imkl/2021.4.0      | 37 +++++++++++++++++++++++
 2 files changed, 68 insertions(+)
 create mode 100644 test/framework/modules/imkl-FFTW/2021.4.0
 create mode 100644 test/framework/modules/imkl/2021.4.0

diff --git a/test/framework/modules/imkl-FFTW/2021.4.0 b/test/framework/modules/imkl-FFTW/2021.4.0
new file mode 100644
index 0000000000..955bf68727
--- /dev/null
+++ b/test/framework/modules/imkl-FFTW/2021.4.0
@@ -0,0 +1,31 @@
+#%Module
+proc ModulesHelp { } {
+    puts stderr {
+
+Description
+===========
+FFTW interfaces using Intel oneAPI Math Kernel Library
+
+
+More information
+================
+ - Homepage: https://software.intel.com/content/www/us/en/develop/tools/oneapi/components/onemkl.html
+    }
+}
+
+module-whatis {Description: FFTW interfaces using Intel oneAPI Math Kernel Library}
+module-whatis {Homepage: https://software.intel.com/content/www/us/en/develop/tools/oneapi/components/onemkl.html}
+module-whatis {URL: https://software.intel.com/content/www/us/en/develop/tools/oneapi/components/onemkl.html}
+
+set root /tmp/imkl-FFTW/2021.4.0
+
+conflict imkl-FFTW
+
+prepend-path	CMAKE_PREFIX_PATH		$root
+prepend-path	LD_LIBRARY_PATH		$root/lib
+prepend-path	LIBRARY_PATH		$root/lib
+setenv	EBROOTIMKLMINFFTW		"$root"
+setenv	EBVERSIONIMKLMINFFTW		"2021.4.0"
+setenv	EBDEVELIMKLMINFFTW		"$root/easybuild/imkl-FFTW-2021.4.0-easybuild-devel"
+
+# Built with EasyBuild version 4.5.0dev
diff --git a/test/framework/modules/imkl/2021.4.0 b/test/framework/modules/imkl/2021.4.0
new file mode 100644
index 0000000000..f188251b48
--- /dev/null
+++ b/test/framework/modules/imkl/2021.4.0
@@ -0,0 +1,37 @@
+#%Module
+proc ModulesHelp { } {
+    puts stderr {
+
+Description
+===========
+Intel oneAPI Math Kernel Library
+
+
+More information
+================
+ - Homepage: https://software.intel.com/content/www/us/en/develop/tools/oneapi/components/onemkl.html
+    }
+}
+
+module-whatis {Description: Intel oneAPI Math Kernel Library}
+module-whatis {Homepage: https://software.intel.com/content/www/us/en/develop/tools/oneapi/components/onemkl.html}
+module-whatis {URL: https://software.intel.com/content/www/us/en/develop/tools/oneapi/components/onemkl.html}
+
+set root	/tmp/eb-bI0pBy/eb-DmuEpJ/eb-leoYDw/eb-UtJJqp/tmp8P3FOY
+
+conflict imkl
+
+prepend-path	CMAKE_PREFIX_PATH		$root
+prepend-path	CPATH		$root/mkl/2021.4.0/include
+prepend-path	CPATH		$root/mkl/2021.4.0/include/fftw
+prepend-path	LD_LIBRARY_PATH		$root/compiler/2021.4.0/linux/compiler/lib/intel64_lin
+prepend-path	LD_LIBRARY_PATH		$root/mkl/2021.4.0/lib/intel64
+prepend-path	LIBRARY_PATH		$root/compiler/2021.4.0/linux/compiler/lib/intel64_lin
+prepend-path	LIBRARY_PATH		$root/mkl/2021.4.0/lib/intel64
+setenv	EBROOTIMKL		"$root"
+setenv	EBVERSIONIMKL		"2021.4.0"
+setenv	EBDEVELIMKL		"$root/easybuild/Core-imkl-2021.4.0-easybuild-devel"
+
+setenv	MKL_EXAMPLES		"$root/mkl/2021.4.0/examples"
+setenv	MKLROOT		"$root/mkl/2021.4.0"
+# Built with EasyBuild version 4.5.0dev

From fcca35487b42914f6ef0a697bee4966b9fcafc5c Mon Sep 17 00:00:00 2001
From: Bart Oldeman <bart.oldeman@calculquebec.ca>
Date: Sun, 17 Oct 2021 02:27:36 +0000
Subject: [PATCH 101/175] TEST_MODULES_COUNT now at 87 with 5 new ones

---
 test/framework/modules.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/framework/modules.py b/test/framework/modules.py
index b56c08bb2f..9c834477ee 100644
--- a/test/framework/modules.py
+++ b/test/framework/modules.py
@@ -54,7 +54,7 @@
 
 
 # number of modules included for testing purposes
-TEST_MODULES_COUNT = 82
+TEST_MODULES_COUNT = 87
 
 
 class ModulesTest(EnhancedTestCase):

From 7a635124d20f68c7d1eabe346108b2ad6e86e8f2 Mon Sep 17 00:00:00 2001
From: Ake Sandgren <ake.sandgren@hpc2n.umu.se>
Date: Sun, 17 Oct 2021 11:46:56 +0200
Subject: [PATCH 102/175] Make sure the contrib/hooks tree is included in the
 distribution.

---
 setup.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/setup.py b/setup.py
index e95b620a65..4b2d936ec6 100644
--- a/setup.py
+++ b/setup.py
@@ -101,6 +101,7 @@ def find_rel_test():
     data_files=[
         ('easybuild/scripts', glob.glob('easybuild/scripts/*')),
         ('etc', glob.glob('etc/*')),
+        ('contrib/hooks', glob.glob('contrib/hooks/*')),
     ],
     long_description=read('README.rst'),
     classifiers=[

From af5a22245e3a0f48dcadf106c0e1391c9ec69557 Mon Sep 17 00:00:00 2001
From: Bart Oldeman <bart.oldeman@calculquebec.ca>
Date: Sun, 17 Oct 2021 17:25:04 +0000
Subject: [PATCH 103/175] Move imkl_fftw logic under if statement and else
 remove the module

---
 test/framework/toolchain.py | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py
index 8bad3a7a8d..14b26cfbf3 100644
--- a/test/framework/toolchain.py
+++ b/test/framework/toolchain.py
@@ -1172,9 +1172,7 @@ def setup_sandbox_for_intel_fftw(self, moddir, imklver='2018.1.163'):
         # create dummy imkl module and put required lib*.a files in place
 
         imkl_module_path = os.path.join(moddir, 'imkl', imklver)
-        imkl_fftw_module_path = os.path.join(moddir, 'imkl-FFTW', imklver)
         imkl_dir = os.path.join(self.test_prefix, 'software', 'imkl', imklver)
-        imkl_fftw_dir = os.path.join(self.test_prefix, 'software', 'imkl-FFTW', imklver)
 
         imkl_mod_txt = '\n'.join([
             "#%Module",
@@ -1183,13 +1181,6 @@ def setup_sandbox_for_intel_fftw(self, moddir, imklver='2018.1.163'):
         ])
         write_file(imkl_module_path, imkl_mod_txt)
 
-        imkl_fftw_mod_txt = '\n'.join([
-            "#%Module",
-            "setenv EBROOTIMKLMINFFTW %s" % imkl_fftw_dir,
-            "setenv EBVERSIONIMKLMINFFTW %s" % imklver,
-        ])
-        write_file(imkl_fftw_module_path, imkl_fftw_mod_txt)
-
         mkl_libs = ['mkl_cdft_core', 'mkl_blacs_intelmpi_lp64']
         mkl_libs += ['mkl_intel_lp64', 'mkl_sequential', 'mkl_core', 'mkl_intel_ilp64']
         fftw_libs = ['fftw3xc_intel', 'fftw3xc_pgi']
@@ -1199,6 +1190,15 @@ def setup_sandbox_for_intel_fftw(self, moddir, imklver='2018.1.163'):
             fftw_libs.append('fftw3x_cdft')
 
         if LooseVersion(imklver) >= LooseVersion('2021.4.0'):
+            imkl_fftw_module_path = os.path.join(moddir, 'imkl-FFTW', imklver)
+            imkl_fftw_dir = os.path.join(self.test_prefix, 'software', 'imkl-FFTW', imklver)
+            imkl_fftw_mod_txt = '\n'.join([
+                "#%Module",
+                "setenv EBROOTIMKLMINFFTW %s" % imkl_fftw_dir,
+                "setenv EBVERSIONIMKLMINFFTW %s" % imklver,
+            ])
+            write_file(imkl_fftw_module_path, imkl_fftw_mod_txt)
+
             subdir = 'mkl/%s/lib/intel64' % imklver
             os.makedirs(os.path.join(imkl_dir, subdir))
             for fftlib in mkl_libs:
@@ -1208,6 +1208,7 @@ def setup_sandbox_for_intel_fftw(self, moddir, imklver='2018.1.163'):
             for fftlib in fftw_libs:
                 write_file(os.path.join(imkl_fftw_dir, subdir, 'lib%s.a' % fftlib), 'foo')
         else:
+            self.modtool.unload(['imkl-FFTW'])
             for subdir in ['mkl/lib/intel64', 'compiler/lib/intel64', 'lib/em64t']:
                 os.makedirs(os.path.join(imkl_dir, subdir))
                 for fftlib in mkl_libs + fftw_libs:

From 5d840ce339a1f664057fd2d396bac08c7723be62 Mon Sep 17 00:00:00 2001
From: Bart Oldeman <bart.oldeman@calculquebec.ca>
Date: Sun, 17 Oct 2021 18:38:17 +0000
Subject: [PATCH 104/175] Purge modules before testing with different intel
 module.

Otherwise auto-swap functionality doesn't work so well with
environment modules (it does work with Lmod)
---
 test/framework/toolchain.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py
index 14b26cfbf3..8139b274d5 100644
--- a/test/framework/toolchain.py
+++ b/test/framework/toolchain.py
@@ -1020,6 +1020,7 @@ def test_fft_env_vars_foss(self):
     def test_fft_env_vars_intel(self):
         """Test setting of $FFT* environment variables using intel toolchain."""
 
+        self.modtool.purge()
         self.setup_sandbox_for_intel_fftw(self.test_prefix)
         self.modtool.prepend_module_path(self.test_prefix)
 
@@ -1079,6 +1080,7 @@ def test_fft_env_vars_intel(self):
         libfft_mt += '-Wl,-Bdynamic -liomp5 -lpthread'
         self.assertEqual(tc.get_variable('LIBFFT_MT'), libfft_mt)
 
+        self.modtool.purge()
         self.setup_sandbox_for_intel_fftw(self.test_prefix, imklver='2021.4.0')
         tc = self.get_toolchain('intel', version='2021b')
         tc.prepare()
@@ -1208,7 +1210,6 @@ def setup_sandbox_for_intel_fftw(self, moddir, imklver='2018.1.163'):
             for fftlib in fftw_libs:
                 write_file(os.path.join(imkl_fftw_dir, subdir, 'lib%s.a' % fftlib), 'foo')
         else:
-            self.modtool.unload(['imkl-FFTW'])
             for subdir in ['mkl/lib/intel64', 'compiler/lib/intel64', 'lib/em64t']:
                 os.makedirs(os.path.join(imkl_dir, subdir))
                 for fftlib in mkl_libs + fftw_libs:

From 7a4d8ce161808869d9935b59c9dc42ab19477ba4 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 20 Oct 2021 11:26:54 +0200
Subject: [PATCH 105/175] fix alphabetical ordering of filetools import
 statements in easyblock.py

---
 easybuild/framework/easyblock.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index 5a3a57409a..a4d83d3f9d 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -72,8 +72,8 @@
 from easybuild.tools.config import install_path, log_path, package_path, source_paths
 from easybuild.tools.environment import restore_env, sanitize_env
 from easybuild.tools.filetools import CHECKSUM_TYPE_MD5, CHECKSUM_TYPE_SHA256
-from easybuild.tools.filetools import adjust_permissions, apply_patch, back_up_file, change_dir, create_patch_info
-from easybuild.tools.filetools import convert_name, compute_checksum, copy_file, check_lock, create_lock
+from easybuild.tools.filetools import adjust_permissions, apply_patch, back_up_file, change_dir, check_lock
+from easybuild.tools.filetools import compute_checksum, convert_name, copy_file, create_lock, create_patch_info
 from easybuild.tools.filetools import derive_alt_pypi_url, diff_files, dir_contains_files, download_file
 from easybuild.tools.filetools import encode_class_name, extract_file
 from easybuild.tools.filetools import find_backup_name_candidate, get_source_tarball_from_git, is_alt_pypi_url

From 28d107476bb702692d95fa1c3de264559cc0fbfa Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 20 Oct 2021 14:14:42 +0200
Subject: [PATCH 106/175] fix step description in easyconfig progress bar

---
 easybuild/framework/easyblock.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index 989fb34ec2..e29f9cda3b 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -3613,6 +3613,9 @@ def run_all_steps(self, run_test_cases):
                 if self.skip_step(step_name, skippable):
                     print_msg("%s [skipped]" % descr, log=self.log, silent=self.silent)
                 else:
+                    progress_label = "Installing %s: %s" % (self.full_mod_name, descr)
+                    update_progress_bar(PROGRESS_BAR_EASYCONFIG, label=progress_label, progress_size=0)
+
                     if self.dry_run:
                         self.dry_run_msg("%s... [DRY RUN]\n", descr)
                     else:
@@ -3629,8 +3632,7 @@ def run_all_steps(self, run_test_cases):
                             elif self.logdebug or build_option('trace'):
                                 print_msg("... (took < 1 sec)", log=self.log, silent=self.silent)
 
-                    progress_label = "Installing %s: %s" % (self.full_mod_name, descr)
-                    update_progress_bar(PROGRESS_BAR_EASYCONFIG, label=progress_label)
+                    update_progress_bar(PROGRESS_BAR_EASYCONFIG)
 
         except StopException:
             pass

From 106d9a9ebba0b844239a1b57dc49162c45bb16f2 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 20 Oct 2021 14:21:34 +0200
Subject: [PATCH 107/175] always stop easyconfig progress bar, also when
 installation failed (or when lock could not be created)

---
 easybuild/framework/easyblock.py | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index e29f9cda3b..426306a3fa 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -3596,19 +3596,19 @@ def run_all_steps(self, run_test_cases):
 
         ignore_locks = build_option('ignore_locks')
 
-        if ignore_locks:
-            self.log.info("Ignoring locks...")
-        else:
-            lock_name = self.installdir.replace('/', '_')
+        try:
+            if ignore_locks:
+                self.log.info("Ignoring locks...")
+            else:
+                lock_name = self.installdir.replace('/', '_')
 
-            # check if lock already exists;
-            # either aborts with an error or waits until it disappears (depends on --wait-on-lock)
-            check_lock(lock_name)
+                # check if lock already exists;
+                # either aborts with an error or waits until it disappears (depends on --wait-on-lock)
+                check_lock(lock_name)
 
-            # create lock to avoid that another installation running in parallel messes things up
-            create_lock(lock_name)
+                # create lock to avoid that another installation running in parallel messes things up
+                create_lock(lock_name)
 
-        try:
             for step_name, descr, step_methods, skippable in steps:
                 if self.skip_step(step_name, skippable):
                     print_msg("%s [skipped]" % descr, log=self.log, silent=self.silent)
@@ -3640,7 +3640,7 @@ def run_all_steps(self, run_test_cases):
             if not ignore_locks:
                 remove_lock(lock_name)
 
-        stop_progress_bar(PROGRESS_BAR_EASYCONFIG)
+            stop_progress_bar(PROGRESS_BAR_EASYCONFIG)
 
         # return True for successfull build (or stopped build)
         return True

From 04f16288666d97b0bd5a0b679b866e0e91b0fc4c Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 20 Oct 2021 14:42:56 +0200
Subject: [PATCH 108/175] use Group rather than RenderGroup, since latter is
 deprecated in Rich

---
 easybuild/tools/output.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py
index ddac5bc718..9b66060f76 100644
--- a/easybuild/tools/output.py
+++ b/easybuild/tools/output.py
@@ -36,7 +36,7 @@
 from easybuild.tools.py2vs3 import OrderedDict
 
 try:
-    from rich.console import Console, RenderGroup
+    from rich.console import Console, Group
     from rich.live import Live
     from rich.table import Table
     from rich.progress import BarColumn, Progress, SpinnerColumn, TextColumn, TimeElapsedColumn
@@ -101,7 +101,7 @@ def rich_live_cm():
     Return Live instance to use as context manager.
     """
     if show_progress_bars():
-        pbar_group = RenderGroup(
+        pbar_group = Group(
                 download_one_progress_bar(),
                 download_one_progress_bar_unknown_size(),
                 download_all_progress_bar(),

From 447c96f3c54e521d19bde301b88d56317ef838ca Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 20 Oct 2021 15:02:26 +0200
Subject: [PATCH 109/175] tweak overall progress bar: show result for
 easyconfigs that are already handled (ok vs failed)

---
 easybuild/main.py         | 11 ++++++++---
 easybuild/tools/output.py | 33 ++++++++++++++++++++++++++++++---
 2 files changed, 38 insertions(+), 6 deletions(-)

diff --git a/easybuild/main.py b/easybuild/main.py
index cf514d619d..05b4f2a1ad 100644
--- a/easybuild/main.py
+++ b/easybuild/main.py
@@ -68,7 +68,7 @@
 from easybuild.tools.hooks import START, END, load_hooks, run_hook
 from easybuild.tools.modules import modules_tool
 from easybuild.tools.options import set_up_configuration, use_color
-from easybuild.tools.output import PROGRESS_BAR_OVERALL, print_checks, rich_live_cm
+from easybuild.tools.output import COLOR_GREEN, COLOR_RED, PROGRESS_BAR_OVERALL, colorize, print_checks, rich_live_cm
 from easybuild.tools.output import start_progress_bar, stop_progress_bar, update_progress_bar
 from easybuild.tools.robot import check_conflicts, dry_run, missing_deps, resolve_dependencies, search_easyconfigs
 from easybuild.tools.package.utilities import check_pkg_support
@@ -118,19 +118,24 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True):
     start_progress_bar(PROGRESS_BAR_OVERALL, size=len(ecs))
 
     res = []
+    ec_results = []
     for ec in ecs:
 
         ec_res = {}
         try:
             (ec_res['success'], app_log, err) = build_and_install_one(ec, init_env)
             ec_res['log_file'] = app_log
-            if not ec_res['success']:
+            if ec_res['success']:
+                ec_results.append(ec['full_mod_name'] + ' (' + colorize('OK', COLOR_GREEN) + ')')
+            else:
                 ec_res['err'] = EasyBuildError(err)
+                ec_results.append(ec['full_mod_name'] + ' (' + colorize('FAILED', COLOR_RED) + ')')
         except Exception as err:
             # purposely catch all exceptions
             ec_res['success'] = False
             ec_res['err'] = err
             ec_res['traceback'] = traceback.format_exc()
+            ec_results.append(ec['full_mod_name'] + ' (' + colorize('FAILED', COLOR_RED) + ')')
 
         # keep track of success/total count
         if ec_res['success']:
@@ -161,7 +166,7 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True):
 
         res.append((ec, ec_res))
 
-        update_progress_bar(PROGRESS_BAR_OVERALL)
+        update_progress_bar(PROGRESS_BAR_OVERALL, label=': ' + ', '.join(ec_results))
 
     stop_progress_bar(PROGRESS_BAR_OVERALL)
 
diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py
index 9b66060f76..ef24dc4ea6 100644
--- a/easybuild/tools/output.py
+++ b/easybuild/tools/output.py
@@ -45,6 +45,18 @@
     pass
 
 
+COLOR_GREEN = 'green'
+COLOR_RED = 'red'
+COLOR_YELLOW = 'yellow'
+
+# map known colors to ANSII color codes
+KNOWN_COLORS = {
+    COLOR_GREEN: '\033[0;32m',
+    COLOR_RED: '\033[0;31m',
+    COLOR_YELLOW: '\033[1;33m',
+}
+COLOR_END = '\033[0m'
+
 PROGRESS_BAR_DOWNLOAD_ALL = 'download_all'
 PROGRESS_BAR_DOWNLOAD_ONE = 'download_one'
 PROGRESS_BAR_EXTENSIONS = 'extensions'
@@ -54,6 +66,21 @@
 _progress_bar_cache = {}
 
 
+def colorize(txt, color):
+    """
+    Colorize given text, with specified color.
+    """
+    if color in KNOWN_COLORS:
+        if use_rich():
+            coltxt = '[bold %s]%s[/bold %s]' % (color, txt, color)
+        else:
+            coltxt = KNOWN_COLORS[color] + txt + COLOR_END
+    else:
+        raise EasyBuildError("Unknown color: %s", color)
+
+    return coltxt
+
+
 class DummyRich(object):
     """
     Dummy shim for Rich classes.
@@ -142,8 +169,7 @@ def overall_progress_bar():
     """
     progress_bar = Progress(
         TimeElapsedColumn(),
-        TextColumn("{task.description}({task.completed} out of {task.total} easyconfigs done)"),
-        BarColumn(bar_width=None),
+        TextColumn("{task.completed} out of {task.total} easyconfigs done{task.description}"),
     )
 
     return progress_bar
@@ -155,10 +181,11 @@ def easyconfig_progress_bar():
     Get progress bar to display progress for installing a single easyconfig file.
     """
     progress_bar = Progress(
-        SpinnerColumn('point'),
+        SpinnerColumn('point', speed=0.2),
         TextColumn("[bold green]{task.description} ({task.completed} out of {task.total} steps done)"),
         BarColumn(),
         TimeElapsedColumn(),
+        refresh_per_second=1,
     )
 
     return progress_bar

From 5d1c0177cd124743a1798487571271d31e2388d9 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 20 Oct 2021 15:18:12 +0200
Subject: [PATCH 110/175] add tests for colorize function

---
 test/framework/output.py | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/test/framework/output.py b/test/framework/output.py
index d27be8aa0f..f35b281a66 100644
--- a/test/framework/output.py
+++ b/test/framework/output.py
@@ -33,7 +33,7 @@
 
 from easybuild.tools.build_log import EasyBuildError
 from easybuild.tools.config import build_option, get_output_style, update_build_option
-from easybuild.tools.output import DummyRich, overall_progress_bar, show_progress_bars, use_rich
+from easybuild.tools.output import DummyRich, colorize, overall_progress_bar, show_progress_bars, use_rich
 
 try:
     import rich.progress
@@ -124,6 +124,20 @@ def test_use_rich_show_progress_bars(self):
         self.assertFalse(use_rich())
         self.assertFalse(show_progress_bars())
 
+    def test_colorize(self):
+        """
+        Test colorize function
+        """
+        if HAVE_RICH:
+            for color in ('green', 'red', 'yellow'):
+                self.assertEqual(colorize('test', color), '[bold %s]test[/bold %s]' % (color, color))
+        else:
+            self.assertEqual(colorize('test', 'green'), '\x1b[0;32mtest\x1b[0m')
+            self.assertEqual(colorize('test', 'red'), '\x1b[0;31mtest\x1b[0m')
+            self.assertEqual(colorize('test', 'yellow'), '\x1b[1;33mtest\x1b[0m')
+
+        self.assertErrorRegex(EasyBuildError, "Unknown color: nosuchcolor", colorize, 'test', 'nosuchcolor')
+
 
 def suite():
     """ returns all the testcases in this module """

From 99afb2ad658b31ffa95e5ea5179ee5bda02722dc Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 20 Oct 2021 16:34:33 +0200
Subject: [PATCH 111/175] show processed easyconfigs in reverse order (since
 status bar doesn't line wrap if it becomes too long), show failed easyconfigs
 first, mention number of failed easyconfigs so far

---
 easybuild/main.py | 28 +++++++++++++++++++++++-----
 1 file changed, 23 insertions(+), 5 deletions(-)

diff --git a/easybuild/main.py b/easybuild/main.py
index 05b4f2a1ad..baab1f1db3 100644
--- a/easybuild/main.py
+++ b/easybuild/main.py
@@ -119,23 +119,32 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True):
 
     res = []
     ec_results = []
+    failed_cnt = 0
+
+    def collect_result(mod_name, success):
+        """
+        Keep track of failed easyconfig
+        """
+
     for ec in ecs:
 
         ec_res = {}
         try:
             (ec_res['success'], app_log, err) = build_and_install_one(ec, init_env)
             ec_res['log_file'] = app_log
-            if ec_res['success']:
-                ec_results.append(ec['full_mod_name'] + ' (' + colorize('OK', COLOR_GREEN) + ')')
-            else:
+            if not ec_res['success']:
                 ec_res['err'] = EasyBuildError(err)
-                ec_results.append(ec['full_mod_name'] + ' (' + colorize('FAILED', COLOR_RED) + ')')
         except Exception as err:
             # purposely catch all exceptions
             ec_res['success'] = False
             ec_res['err'] = err
             ec_res['traceback'] = traceback.format_exc()
+
+        if ec_res['success']:
+            ec_results.append(ec['full_mod_name'] + ' (' + colorize('OK', COLOR_GREEN) + ')')
+        else:
             ec_results.append(ec['full_mod_name'] + ' (' + colorize('FAILED', COLOR_RED) + ')')
+            failed_cnt += 1
 
         # keep track of success/total count
         if ec_res['success']:
@@ -166,7 +175,16 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True):
 
         res.append((ec, ec_res))
 
-        update_progress_bar(PROGRESS_BAR_OVERALL, label=': ' + ', '.join(ec_results))
+        if failed_cnt:
+            # if installations failed: indicate th
+            status_label = '(%s): ' % colorize('%s failed!' % failed_cnt, COLOR_RED)
+            failed_ecs = [x for x in ec_results[::-1] if 'FAILED' in x]
+            ok_ecs = [x for x in ec_results[::-1] if x not in failed_ecs]
+            status_label += ', '.join(failed_ecs + ok_ecs)
+        else:
+            status_label = ': ' + ', '.join(ec_results[::-1])
+
+        update_progress_bar(PROGRESS_BAR_OVERALL, label=status_label)
 
     stop_progress_bar(PROGRESS_BAR_OVERALL)
 

From d440c118e021c577821e49f2e0f3db11e8827e4e Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 20 Oct 2021 16:43:00 +0200
Subject: [PATCH 112/175] add extra space to status bar in case of failed
 builds

---
 easybuild/main.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/easybuild/main.py b/easybuild/main.py
index baab1f1db3..13ed75e941 100644
--- a/easybuild/main.py
+++ b/easybuild/main.py
@@ -177,7 +177,7 @@ def collect_result(mod_name, success):
 
         if failed_cnt:
             # if installations failed: indicate th
-            status_label = '(%s): ' % colorize('%s failed!' % failed_cnt, COLOR_RED)
+            status_label = ' (%s): ' % colorize('%s failed!' % failed_cnt, COLOR_RED)
             failed_ecs = [x for x in ec_results[::-1] if 'FAILED' in x]
             ok_ecs = [x for x in ec_results[::-1] if x not in failed_ecs]
             status_label += ', '.join(failed_ecs + ok_ecs)

From 0f6fb795b345f4a46e56d370ace9e0cc2c93ffaf Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 20 Oct 2021 16:46:16 +0200
Subject: [PATCH 113/175] rename overall_progress_bar to status_bar

---
 easybuild/main.py         |  8 ++++----
 easybuild/tools/output.py |  8 ++++----
 test/framework/output.py  | 14 +++++++-------
 3 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/easybuild/main.py b/easybuild/main.py
index 13ed75e941..ddc3cce156 100644
--- a/easybuild/main.py
+++ b/easybuild/main.py
@@ -68,7 +68,7 @@
 from easybuild.tools.hooks import START, END, load_hooks, run_hook
 from easybuild.tools.modules import modules_tool
 from easybuild.tools.options import set_up_configuration, use_color
-from easybuild.tools.output import COLOR_GREEN, COLOR_RED, PROGRESS_BAR_OVERALL, colorize, print_checks, rich_live_cm
+from easybuild.tools.output import COLOR_GREEN, COLOR_RED, STATUS_BAR, colorize, print_checks, rich_live_cm
 from easybuild.tools.output import start_progress_bar, stop_progress_bar, update_progress_bar
 from easybuild.tools.robot import check_conflicts, dry_run, missing_deps, resolve_dependencies, search_easyconfigs
 from easybuild.tools.package.utilities import check_pkg_support
@@ -115,7 +115,7 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True):
     # e.g. via easyconfig.handle_allowed_system_deps
     init_env = copy.deepcopy(os.environ)
 
-    start_progress_bar(PROGRESS_BAR_OVERALL, size=len(ecs))
+    start_progress_bar(STATUS_BAR, size=len(ecs))
 
     res = []
     ec_results = []
@@ -184,9 +184,9 @@ def collect_result(mod_name, success):
         else:
             status_label = ': ' + ', '.join(ec_results[::-1])
 
-        update_progress_bar(PROGRESS_BAR_OVERALL, label=status_label)
+        update_progress_bar(STATUS_BAR, label=status_label)
 
-    stop_progress_bar(PROGRESS_BAR_OVERALL)
+    stop_progress_bar(STATUS_BAR)
 
     return res
 
diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py
index ef24dc4ea6..542f7e3fd0 100644
--- a/easybuild/tools/output.py
+++ b/easybuild/tools/output.py
@@ -61,7 +61,7 @@
 PROGRESS_BAR_DOWNLOAD_ONE = 'download_one'
 PROGRESS_BAR_EXTENSIONS = 'extensions'
 PROGRESS_BAR_EASYCONFIG = 'easyconfig'
-PROGRESS_BAR_OVERALL = 'overall'
+STATUS_BAR = 'status'
 
 _progress_bar_cache = {}
 
@@ -134,7 +134,7 @@ def rich_live_cm():
                 download_all_progress_bar(),
                 extensions_progress_bar(),
                 easyconfig_progress_bar(),
-                overall_progress_bar(),
+                status_bar(),
         )
         live = Live(pbar_group)
     else:
@@ -163,7 +163,7 @@ def new_func(ignore_cache=False):
 
 
 @progress_bar_cache
-def overall_progress_bar():
+def status_bar():
     """
     Get progress bar to display overall progress.
     """
@@ -259,7 +259,7 @@ def get_progress_bar(bar_type, size=None):
         PROGRESS_BAR_DOWNLOAD_ONE: download_one_progress_bar,
         PROGRESS_BAR_EXTENSIONS: extensions_progress_bar,
         PROGRESS_BAR_EASYCONFIG: easyconfig_progress_bar,
-        PROGRESS_BAR_OVERALL: overall_progress_bar,
+        STATUS_BAR: status_bar,
     }
 
     if bar_type == PROGRESS_BAR_DOWNLOAD_ONE and not size:
diff --git a/test/framework/output.py b/test/framework/output.py
index f35b281a66..00c46185cf 100644
--- a/test/framework/output.py
+++ b/test/framework/output.py
@@ -33,7 +33,7 @@
 
 from easybuild.tools.build_log import EasyBuildError
 from easybuild.tools.config import build_option, get_output_style, update_build_option
-from easybuild.tools.output import DummyRich, colorize, overall_progress_bar, show_progress_bars, use_rich
+from easybuild.tools.output import DummyRich, colorize, status_bar, show_progress_bars, use_rich
 
 try:
     import rich.progress
@@ -45,8 +45,8 @@
 class OutputTest(EnhancedTestCase):
     """Tests for functions controlling terminal output."""
 
-    def test_overall_progress_bar(self):
-        """Test overall_progress_bar function."""
+    def test_status_bar(self):
+        """Test status_bar function."""
 
         # restore default (was disabled in EnhancedTestCase.setUp to avoid messing up test output)
         update_build_option('show_progress_bar', True)
@@ -56,22 +56,22 @@ def test_overall_progress_bar(self):
         else:
             expected_progress_bar_class = DummyRich
 
-        progress_bar = overall_progress_bar(ignore_cache=True)
+        progress_bar = status_bar(ignore_cache=True)
         error_msg = "%s should be instance of class %s" % (progress_bar, expected_progress_bar_class)
         self.assertTrue(isinstance(progress_bar, expected_progress_bar_class), error_msg)
 
         update_build_option('output_style', 'basic')
-        progress_bar = overall_progress_bar(ignore_cache=True)
+        progress_bar = status_bar(ignore_cache=True)
         self.assertTrue(isinstance(progress_bar, DummyRich))
 
         if HAVE_RICH:
             update_build_option('output_style', 'rich')
-            progress_bar = overall_progress_bar(ignore_cache=True)
+            progress_bar = status_bar(ignore_cache=True)
             error_msg = "%s should be instance of class %s" % (progress_bar, expected_progress_bar_class)
             self.assertTrue(isinstance(progress_bar, expected_progress_bar_class), error_msg)
 
         update_build_option('show_progress_bar', False)
-        progress_bar = overall_progress_bar(ignore_cache=True)
+        progress_bar = status_bar(ignore_cache=True)
         self.assertTrue(isinstance(progress_bar, DummyRich))
 
     def test_get_output_style(self):

From 8984aa55ae8d99eab6c4306bbe01d43593bf0565 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 20 Oct 2021 17:30:31 +0200
Subject: [PATCH 114/175] avoid disappearing of time elapsed in status bar by
 specifying minimal width

---
 easybuild/tools/output.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py
index 542f7e3fd0..23270fecc4 100644
--- a/easybuild/tools/output.py
+++ b/easybuild/tools/output.py
@@ -38,9 +38,9 @@
 try:
     from rich.console import Console, Group
     from rich.live import Live
-    from rich.table import Table
     from rich.progress import BarColumn, Progress, SpinnerColumn, TextColumn, TimeElapsedColumn
     from rich.progress import DownloadColumn, FileSizeColumn, TransferSpeedColumn, TimeRemainingColumn
+    from rich.table import Column, Table
 except ImportError:
     pass
 
@@ -168,7 +168,7 @@ def status_bar():
     Get progress bar to display overall progress.
     """
     progress_bar = Progress(
-        TimeElapsedColumn(),
+        TimeElapsedColumn(Column(min_width=7, no_wrap=True)),
         TextColumn("{task.completed} out of {task.total} easyconfigs done{task.description}"),
     )
 

From 619f70ee63a0c5cfffaa1460ca945b1e4b0c0c2b Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 20 Oct 2021 19:55:11 +0200
Subject: [PATCH 115/175] remove unused helper function in
 build_and_install_software

---
 easybuild/main.py | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/easybuild/main.py b/easybuild/main.py
index ddc3cce156..b41df476c7 100644
--- a/easybuild/main.py
+++ b/easybuild/main.py
@@ -121,11 +121,6 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True):
     ec_results = []
     failed_cnt = 0
 
-    def collect_result(mod_name, success):
-        """
-        Keep track of failed easyconfig
-        """
-
     for ec in ecs:
 
         ec_res = {}

From 563cbf4b0dad5387ed70f22aeb9557a064d779bd Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 20 Oct 2021 19:56:17 +0200
Subject: [PATCH 116/175] fix import order in tests for tools/output.py

---
 test/framework/output.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/framework/output.py b/test/framework/output.py
index 00c46185cf..6cacc80fd6 100644
--- a/test/framework/output.py
+++ b/test/framework/output.py
@@ -33,7 +33,7 @@
 
 from easybuild.tools.build_log import EasyBuildError
 from easybuild.tools.config import build_option, get_output_style, update_build_option
-from easybuild.tools.output import DummyRich, colorize, status_bar, show_progress_bars, use_rich
+from easybuild.tools.output import DummyRich, colorize, show_progress_bars, status_bar, use_rich
 
 try:
     import rich.progress

From 55481e1a94a7b47363a519e4804e6348b7cc4aa3 Mon Sep 17 00:00:00 2001
From: Miguel Dias Costa <migueldiascosta@gmail.com>
Date: Thu, 21 Oct 2021 16:56:34 +0800
Subject: [PATCH 117/175] don't sort alphabetically the result of
 find_related_easyconfigs

---
 easybuild/framework/easyconfig/tools.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py
index ff6882f33a..b8cd49f404 100644
--- a/easybuild/framework/easyconfig/tools.py
+++ b/easybuild/framework/easyconfig/tools.py
@@ -483,7 +483,7 @@ def find_related_easyconfigs(path, ec):
         else:
             _log.debug("No related easyconfigs in potential paths using '%s'" % regex)
 
-    return sorted(res)
+    return res
 
 
 def review_pr(paths=None, pr=None, colored=True, branch='develop', testing=False, max_ecs=None, filter_ecs=None):

From 950e24f30e397cba0a363fd3153b9a338d8ea0e6 Mon Sep 17 00:00:00 2001
From: Miguel Dias Costa <migueldiascosta@gmail.com>
Date: Thu, 21 Oct 2021 17:58:06 +0800
Subject: [PATCH 118/175] fix order of easyconfigs in test of
 find_related_easyconfigs after removing sort

---
 test/framework/easyconfig.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py
index bf207bf8f4..5a8ec46868 100644
--- a/test/framework/easyconfig.py
+++ b/test/framework/easyconfig.py
@@ -2826,7 +2826,7 @@ def test_find_related_easyconfigs(self):
         ec['toolchain'] = {'name': 'gompi', 'version': '1.5.16'}
         ec['versionsuffix'] = '-foobar'
         res = [os.path.basename(x) for x in find_related_easyconfigs(test_easyconfigs, ec)]
-        self.assertEqual(res, ['toy-0.0-gompi-2018a-test.eb', 'toy-0.0-gompi-2018a.eb'])
+        self.assertEqual(res, ['toy-0.0-gompi-2018a.eb', 'toy-0.0-gompi-2018a-test.eb'])
 
         # restore original versionsuffix => matching versionsuffix wins over matching toolchain (name)
         ec['versionsuffix'] = '-deps'

From 6053c0cb1d768d0e5970e1cf1ebf21dafa2d9e13 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Thu, 21 Oct 2021 14:36:44 +0200
Subject: [PATCH 119/175] also update extensions progress bar when installing
 extensions in parallel

---
 easybuild/framework/easyblock.py | 21 ++++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index e3b34f5c2b..a2383b6a0d 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -1733,6 +1733,21 @@ def install_extensions_parallel(self, install=True):
         exts_cnt = len(all_ext_names)
         exts_queue = self.ext_instances[:]
 
+        start_progress_bar(PROGRESS_BAR_EXTENSIONS, exts_cnt)
+
+        def update_exts_progress_bar(running_exts, progress_size):
+            """Helper function to update extensions progress bar."""
+            running_exts_cnt = len(running_exts)
+            if running_exts_cnt > 1:
+                progress_label = "Installing %d extensions: " % running_exts_cnt
+            elif running_exts_cnt == 1:
+                progress_label = "Installing extension "
+            else:
+                progress_label = "Not installing extensions (yet)"
+
+            progress_label += ' '.join(e.name for e in running_exts)
+            update_progress_bar(PROGRESS_BAR_EXTENSIONS, label=progress_label, progress_size=progress_size)
+
         iter_id = 0
         while exts_queue or running_exts:
 
@@ -1750,6 +1765,7 @@ def install_extensions_parallel(self, install=True):
                         ext.postrun()
                         running_exts.remove(ext)
                         installed_ext_names.append(ext.name)
+                        update_exts_progress_bar(running_exts, 1)
                     else:
                         self.log.debug("Installation of %s is still running...", ext.name)
 
@@ -1811,7 +1827,10 @@ def install_extensions_parallel(self, install=True):
                         ext.prerun()
                         ext.run(asynchronous=True)
                         running_exts.append(ext)
-                        self.log.debug("Started installation of extension %s in the background...", ext.name)
+                        self.log.info("Started installation of extension %s in the background...", ext.name)
+                        update_exts_progress_bar(running_exts, 0)
+
+        stop_progress_bar(PROGRESS_BAR_EXTENSIONS, visible=False)
 
     #
     # MISCELLANEOUS UTILITY FUNCTIONS

From 3fe1f688b3903433c36ed538a18fdf7a693dc2e9 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Thu, 21 Oct 2021 14:59:01 +0200
Subject: [PATCH 120/175] mark support for installing extensions in parallel as
 experimental + add --parallel-extensions-install configuration option to
 opt-in to it

---
 easybuild/framework/easyblock.py | 3 ++-
 easybuild/tools/config.py        | 1 +
 easybuild/tools/options.py       | 2 ++
 3 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index a2383b6a0d..dbc31f1ca6 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -1646,7 +1646,8 @@ def install_extensions(self, install=True, parallel=False):
         """
         self.log.debug("List of loaded modules: %s", self.modules_tool.list())
 
-        if parallel:
+        if build_option('parallel_extensions_install') and parallel:
+            self.log.experimental("installing extensions in parallel")
             self.install_extensions_parallel(install=install)
         else:
             self.install_extensions_sequential(install=install)
diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py
index 18902ae799..32642ac224 100644
--- a/easybuild/tools/config.py
+++ b/easybuild/tools/config.py
@@ -269,6 +269,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
         'module_extensions',
         'module_only',
         'package',
+        'parallel_extensions_install',
         'read_only_installdir',
         'remove_ghost_install_dirs',
         'rebuild',
diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py
index e4fc7661a4..7ef8e7232e 100644
--- a/easybuild/tools/options.py
+++ b/easybuild/tools/options.py
@@ -454,6 +454,8 @@ def override_options(self):
                              'choice', 'store', OUTPUT_STYLE_AUTO, OUTPUT_STYLES),
             'parallel': ("Specify (maximum) level of parallellism used during build procedure",
                          'int', 'store', None),
+            'parallel-extensions-install': ("Install list of extensions in parallel (if supported)",
+                                            None, 'store_true', False),
             'pre-create-installdir': ("Create installation directory before submitting build jobs",
                                       None, 'store_true', True),
             'pretend': (("Does the build/installation in a test directory located in $HOME/easybuildinstall"),

From 2e5c6ab8db7440b56a063cf3672280205c96a691 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Thu, 21 Oct 2021 15:14:24 +0200
Subject: [PATCH 121/175] start extensions progress bar a bit earlier, also
 mention preparatory steps (like creating of Extension instances)

---
 easybuild/framework/easyblock.py | 17 +++++++++--------
 easybuild/tools/output.py        |  2 +-
 2 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index dbc31f1ca6..636c0d30a6 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -1661,7 +1661,6 @@ def install_extensions_sequential(self, install=True):
         self.log.info("Installing extensions sequentially...")
 
         exts_cnt = len(self.ext_instances)
-        start_progress_bar(PROGRESS_BAR_EXTENSIONS, exts_cnt)
 
         for idx, ext in enumerate(self.ext_instances):
 
@@ -1670,7 +1669,7 @@ def install_extensions_sequential(self, install=True):
             # always go back to original work dir to avoid running stuff from a dir that no longer exists
             change_dir(self.orig_workdir)
 
-            progress_label = "Installing '%s' extension" % ext.name
+            progress_label = "Installing '%s' extension (%s/%s)" % (ext.name, idx + 1, exts_cnt)
             update_progress_bar(PROGRESS_BAR_EXTENSIONS, label=progress_label)
 
             tup = (ext.name, ext.version or '', idx + 1, exts_cnt)
@@ -1711,8 +1710,6 @@ def install_extensions_sequential(self, install=True):
                         elif self.logdebug or build_option('trace'):
                             print_msg("\t... (took < 1 sec)", log=self.log, silent=self.silent)
 
-        stop_progress_bar(PROGRESS_BAR_EXTENSIONS, visible=False)
-
     def install_extensions_parallel(self, install=True):
         """
         Install extensions in parallel.
@@ -1734,8 +1731,6 @@ def install_extensions_parallel(self, install=True):
         exts_cnt = len(all_ext_names)
         exts_queue = self.ext_instances[:]
 
-        start_progress_bar(PROGRESS_BAR_EXTENSIONS, exts_cnt)
-
         def update_exts_progress_bar(running_exts, progress_size):
             """Helper function to update extensions progress bar."""
             running_exts_cnt = len(running_exts)
@@ -1747,6 +1742,7 @@ def update_exts_progress_bar(running_exts, progress_size):
                 progress_label = "Not installing extensions (yet)"
 
             progress_label += ' '.join(e.name for e in running_exts)
+            progress_label += "(%d/%d done)" % (len(installed_ext_names), exts_cnt)
             update_progress_bar(PROGRESS_BAR_EXTENSIONS, label=progress_label, progress_size=progress_size)
 
         iter_id = 0
@@ -1831,8 +1827,6 @@ def update_exts_progress_bar(running_exts, progress_size):
                         self.log.info("Started installation of extension %s in the background...", ext.name)
                         update_exts_progress_bar(running_exts, 0)
 
-        stop_progress_bar(PROGRESS_BAR_EXTENSIONS, visible=False)
-
     #
     # MISCELLANEOUS UTILITY FUNCTIONS
     #
@@ -2587,9 +2581,12 @@ def extensions_step(self, fetch=False, install=True):
 
             fake_mod_data = self.load_fake_module(purge=True, extra_modules=build_dep_mods)
 
+        start_progress_bar(PROGRESS_BAR_EXTENSIONS, len(self.cfg['exts_list']))
+
         self.prepare_for_extensions()
 
         if fetch:
+            update_progress_bar(PROGRESS_BAR_EXTENSIONS, label="fetching extension sources/patches", progress_size=0)
             self.exts = self.collect_exts_file_info(fetch_files=True)
 
         self.exts_all = self.exts[:]  # retain a copy of all extensions, regardless of filtering/skipping
@@ -2603,9 +2600,11 @@ def extensions_step(self, fetch=False, install=True):
             self.clean_up_fake_module(fake_mod_data)
             raise EasyBuildError("ERROR: No default extension class set for %s", self.name)
 
+        update_progress_bar(PROGRESS_BAR_EXTENSIONS, label="creating Extension instances", progress_size=0)
         self.init_ext_instances()
 
         if self.skip:
+            update_progress_bar(PROGRESS_BAR_EXTENSIONS, label="skipping installed extensions", progress_size=0)
             self.skip_extensions()
 
         self.install_extensions(install=install)
@@ -2614,6 +2613,8 @@ def extensions_step(self, fetch=False, install=True):
         if fake_mod_data:
             self.clean_up_fake_module(fake_mod_data)
 
+        stop_progress_bar(PROGRESS_BAR_EXTENSIONS, visible=False)
+
     def package_step(self):
         """Package installed software (e.g., into an RPM), if requested, using selected package tool."""
 
diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py
index 23270fecc4..ff8250b7b2 100644
--- a/easybuild/tools/output.py
+++ b/easybuild/tools/output.py
@@ -242,7 +242,7 @@ def extensions_progress_bar():
     Get progress bar to show progress for installing extensions.
     """
     progress_bar = Progress(
-        TextColumn("[bold blue]{task.description} ({task.completed}/{task.total})"),
+        TextColumn("[bold blue]{task.description}"),
         BarColumn(),
         TimeElapsedColumn(),
     )

From 3748d9c444a290c3b7649821376a34c504b10854 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Thu, 21 Oct 2021 15:26:00 +0200
Subject: [PATCH 122/175] add and use update_exts_progress_bar method to
 EasyBlock

---
 easybuild/framework/easyblock.py | 36 +++++++++++++++++++-------------
 1 file changed, 22 insertions(+), 14 deletions(-)

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index 636c0d30a6..39b06987dc 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -1669,8 +1669,8 @@ def install_extensions_sequential(self, install=True):
             # always go back to original work dir to avoid running stuff from a dir that no longer exists
             change_dir(self.orig_workdir)
 
-            progress_label = "Installing '%s' extension (%s/%s)" % (ext.name, idx + 1, exts_cnt)
-            update_progress_bar(PROGRESS_BAR_EXTENSIONS, label=progress_label)
+            progress_info = "Installing '%s' extension (%s/%s)" % (ext.name, idx + 1, exts_cnt)
+            self.update_exts_progress_bar(progress_info)
 
             tup = (ext.name, ext.version or '', idx + 1, exts_cnt)
             print_msg("installing extension %s %s (%d/%d)..." % tup, silent=self.silent, log=self.log)
@@ -1710,6 +1710,8 @@ def install_extensions_sequential(self, install=True):
                         elif self.logdebug or build_option('trace'):
                             print_msg("\t... (took < 1 sec)", log=self.log, silent=self.silent)
 
+            self.update_exts_progress_bar(progress_info, progress_size=1)
+
     def install_extensions_parallel(self, install=True):
         """
         Install extensions in parallel.
@@ -1731,19 +1733,19 @@ def install_extensions_parallel(self, install=True):
         exts_cnt = len(all_ext_names)
         exts_queue = self.ext_instances[:]
 
-        def update_exts_progress_bar(running_exts, progress_size):
+        def update_exts_progress_bar_helper(running_exts, progress_size):
             """Helper function to update extensions progress bar."""
             running_exts_cnt = len(running_exts)
             if running_exts_cnt > 1:
-                progress_label = "Installing %d extensions: " % running_exts_cnt
+                progress_info = "Installing %d extensions: " % running_exts_cnt
             elif running_exts_cnt == 1:
-                progress_label = "Installing extension "
+                progress_info = "Installing extension "
             else:
-                progress_label = "Not installing extensions (yet)"
+                progress_info = "Not installing extensions (yet)"
 
-            progress_label += ' '.join(e.name for e in running_exts)
-            progress_label += "(%d/%d done)" % (len(installed_ext_names), exts_cnt)
-            update_progress_bar(PROGRESS_BAR_EXTENSIONS, label=progress_label, progress_size=progress_size)
+            progress_info += ' '.join(e.name for e in running_exts)
+            progress_info += "(%d/%d done)" % (len(installed_ext_names), exts_cnt)
+            self.update_exts_progress_bar(progress_info, progress_size=progress_size)
 
         iter_id = 0
         while exts_queue or running_exts:
@@ -1762,7 +1764,7 @@ def update_exts_progress_bar(running_exts, progress_size):
                         ext.postrun()
                         running_exts.remove(ext)
                         installed_ext_names.append(ext.name)
-                        update_exts_progress_bar(running_exts, 1)
+                        update_exts_progress_bar_helper(running_exts, 1)
                     else:
                         self.log.debug("Installation of %s is still running...", ext.name)
 
@@ -1825,7 +1827,7 @@ def update_exts_progress_bar(running_exts, progress_size):
                         ext.run(asynchronous=True)
                         running_exts.append(ext)
                         self.log.info("Started installation of extension %s in the background...", ext.name)
-                        update_exts_progress_bar(running_exts, 0)
+                        update_exts_progress_bar_helper(running_exts, 0)
 
     #
     # MISCELLANEOUS UTILITY FUNCTIONS
@@ -2560,6 +2562,12 @@ def init_ext_instances(self):
 
             self.ext_instances.append(inst)
 
+    def update_exts_progress_bar(self, info, progress_size=0):
+        """
+        Update extensions progress bar with specified info and amount of progress made
+        """
+        update_progress_bar(PROGRESS_BAR_EXTENSIONS, label=info, progress_size=progress_size)
+
     def extensions_step(self, fetch=False, install=True):
         """
         After make install, run this.
@@ -2586,7 +2594,7 @@ def extensions_step(self, fetch=False, install=True):
         self.prepare_for_extensions()
 
         if fetch:
-            update_progress_bar(PROGRESS_BAR_EXTENSIONS, label="fetching extension sources/patches", progress_size=0)
+            self.update_exts_progress_bar("fetching extension sources/patches")
             self.exts = self.collect_exts_file_info(fetch_files=True)
 
         self.exts_all = self.exts[:]  # retain a copy of all extensions, regardless of filtering/skipping
@@ -2600,11 +2608,11 @@ def extensions_step(self, fetch=False, install=True):
             self.clean_up_fake_module(fake_mod_data)
             raise EasyBuildError("ERROR: No default extension class set for %s", self.name)
 
-        update_progress_bar(PROGRESS_BAR_EXTENSIONS, label="creating Extension instances", progress_size=0)
+        self.update_exts_progress_bar("creating internal datastructures")
         self.init_ext_instances()
 
         if self.skip:
-            update_progress_bar(PROGRESS_BAR_EXTENSIONS, label="skipping installed extensions", progress_size=0)
+            self.update_exts_progress_bar("skipping install extensions")
             self.skip_extensions()
 
         self.install_extensions(install=install)

From 45a2627a382df986bc28d2330fb6a8734e04f0ac Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Thu, 21 Oct 2021 16:57:38 +0200
Subject: [PATCH 123/175] fix formatting for extension progress bar when
 installing extensions in parallel

---
 easybuild/framework/easyblock.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index 39b06987dc..e0fca04973 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -1743,8 +1743,8 @@ def update_exts_progress_bar_helper(running_exts, progress_size):
             else:
                 progress_info = "Not installing extensions (yet)"
 
-            progress_info += ' '.join(e.name for e in running_exts)
-            progress_info += "(%d/%d done)" % (len(installed_ext_names), exts_cnt)
+            progress_info += ', '.join(e.name for e in running_exts)
+            progress_info += " (%d/%d done)" % (len(installed_ext_names), exts_cnt)
             self.update_exts_progress_bar(progress_info, progress_size=progress_size)
 
         iter_id = 0

From e47fead521fec0f87a07d8229c0a7c19e0f05e4d Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Thu, 21 Oct 2021 17:54:46 +0200
Subject: [PATCH 124/175] add check_async_cmd function to facilitate checking
 on asynchronously running commands

---
 easybuild/tools/run.py | 36 +++++++++++++++++++++++++++++++
 test/framework/run.py  | 49 +++++++++++++++++++++++++++++++++++++-----
 2 files changed, 80 insertions(+), 5 deletions(-)

diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py
index ce04900204..aa13a14f77 100644
--- a/easybuild/tools/run.py
+++ b/easybuild/tools/run.py
@@ -248,6 +248,42 @@ def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True
                             regexp=regexp, stream_output=stream_output, trace=trace)
 
 
+def check_async_cmd(proc, cmd, owd, start_time, cmd_log, output_read_size=1024, output=''):
+    """
+    Check status of command that was started asynchronously.
+
+    :param proc: subprocess.Popen instance representing asynchronous command
+    :param cmd: command being run
+    :param owd: original working directory
+    :param start_time: start time of command (datetime instance)
+    :param cmd_log: log file to print command output to
+    :param output_read_size: number of bytes to read from output
+    :param output: already collected output for this command
+
+    :result: dict value with result of the check (boolean 'done', 'exit_code', 'output')
+    """
+    # use small read size, to avoid waiting for a long time until sufficient output is produced
+    add_out = get_output_from_process(proc, read_size=output_read_size)
+    _log.debug("Additional output from asynchronous command '%s': %s" % (cmd, add_out))
+    output += add_out
+
+    exit_code = proc.poll()
+    if exit_code is None:
+        _log.debug("Asynchronous command '%s' still running..." % cmd)
+        done = False
+    else:
+        _log.debug("Asynchronous command '%s' completed!", cmd)
+        output, _ = complete_cmd(proc, cmd, owd, start_time, cmd_log, output=output, simple=False)
+        done = True
+
+    res = {
+        'done': done,
+        'exit_code': exit_code,
+        'output': output,
+    }
+    return res
+
+
 def complete_cmd(proc, cmd, owd, start_time, cmd_log, log_ok=True, log_all=False, simple=False,
                  regexp=True, stream_output=None, trace=True, output=''):
     """
diff --git a/test/framework/run.py b/test/framework/run.py
index 368a4b14ed..1f48129f2f 100644
--- a/test/framework/run.py
+++ b/test/framework/run.py
@@ -47,7 +47,7 @@
 import easybuild.tools.utilities
 from easybuild.tools.build_log import EasyBuildError, init_logging, stop_logging
 from easybuild.tools.filetools import adjust_permissions, read_file, write_file
-from easybuild.tools.run import check_log_for_errors, complete_cmd, get_output_from_process
+from easybuild.tools.run import check_async_cmd, check_log_for_errors, complete_cmd, get_output_from_process
 from easybuild.tools.run import parse_log_for_error, run_cmd, run_cmd_qa
 from easybuild.tools.config import ERROR, IGNORE, WARN
 
@@ -575,7 +575,8 @@ def test_run_cmd_async(self):
 
         os.environ['TEST'] = 'test123'
 
-        cmd_info = run_cmd("sleep 2; echo $TEST", asynchronous=True)
+        test_cmd = "echo 'sleeping...'; sleep 2; echo $TEST"
+        cmd_info = run_cmd(test_cmd, asynchronous=True)
         proc = cmd_info[0]
 
         # change value of $TEST to check that command is completed with correct environment
@@ -585,18 +586,41 @@ def test_run_cmd_async(self):
         ec = proc.poll()
         self.assertEqual(ec, None)
 
+        # wait until command is done
         while ec is None:
             time.sleep(1)
             ec = proc.poll()
 
         out, ec = complete_cmd(*cmd_info, simple=False)
         self.assertEqual(ec, 0)
-        self.assertEqual(out, 'test123\n')
+        self.assertEqual(out, 'sleeping...\ntest123\n')
+
+        # also test use of check_async_cmd function
+        os.environ['TEST'] = 'test123'
+        cmd_info = run_cmd(test_cmd, asynchronous=True)
+
+        # first check, only read first 12 output characters
+        # (otherwise we'll be waiting until command is completed)
+        res = check_async_cmd(*cmd_info, output_read_size=12)
+        self.assertEqual(res, {'done': False, 'exit_code': None, 'output': 'sleeping...\n'})
+
+        # 2nd check with default output size (1024) gets full output
+        res = check_async_cmd(*cmd_info, output=res['output'])
+        self.assertEqual(res, {'done': True, 'exit_code': 0, 'output': 'sleeping...\ntest123\n'})
 
         # also test with a command that produces a lot of output,
         # since that tends to lock up things unless we frequently grab some output...
-        cmd = "echo start; for i in $(seq 1 50); do sleep 0.1; for j in $(seq 1000); do echo foo; done; done; echo done"
-        cmd_info = run_cmd(cmd, asynchronous=True)
+        verbose_test_cmd = ';'.join([
+            "echo start",
+            "for i in $(seq 1 50)",
+            "do sleep 0.1",
+            "for j in $(seq 1000)",
+            "do echo foo",
+            "done",
+            "done",
+            "echo done",
+        ])
+        cmd_info = run_cmd(verbose_test_cmd, asynchronous=True)
         proc = cmd_info[0]
 
         output = ''
@@ -613,6 +637,21 @@ def test_run_cmd_async(self):
         self.assertTrue(out.startswith('start\n'))
         self.assertTrue(out.endswith('\ndone\n'))
 
+        # also test use of check_async_cmd on verbose test command
+        cmd_info = run_cmd(verbose_test_cmd, asynchronous=True)
+        res = check_async_cmd(*cmd_info)
+        self.assertEqual(res['done'], False)
+        self.assertEqual(res['exit_code'], None)
+        self.assertTrue(res['output'].startswith('start\n'))
+        self.assertFalse(res['output'].endswith('\ndone\n'))
+        # keep checking until command is complete
+        while not res['done']:
+            res = check_async_cmd(*cmd_info, output=res['output'])
+        self.assertEqual(res['done'], True)
+        self.assertEqual(res['exit_code'], 0)
+        self.assertTrue(res['output'].startswith('start\n'))
+        self.assertTrue(res['output'].endswith('\ndone\n'))
+
     def test_check_log_for_errors(self):
         fd, logfile = tempfile.mkstemp(suffix='.log', prefix='eb-test-')
         os.close(fd)

From 471ff3ec649a832feec9acf5f4251bc41f83ff74 Mon Sep 17 00:00:00 2001
From: Ake Sandgren <ake.sandgren@hpc2n.umu.se>
Date: Thu, 21 Oct 2021 18:24:56 +0200
Subject: [PATCH 125/175] Fix IntelCompilers libpaths to actually point to the
 compiler libs

---
 easybuild/toolchains/compiler/intel_compilers.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/easybuild/toolchains/compiler/intel_compilers.py b/easybuild/toolchains/compiler/intel_compilers.py
index 98b4ea5b7f..8bc4bdc763 100644
--- a/easybuild/toolchains/compiler/intel_compilers.py
+++ b/easybuild/toolchains/compiler/intel_compilers.py
@@ -47,11 +47,11 @@ def _set_compiler_vars(self):
         Compiler._set_compiler_vars(self)
 
         root = self.get_software_root(self.COMPILER_MODULE_NAME)[0]
+        version = self.get_software_version(self.COMPILER_MODULE_NAME)[0]
 
+        libbase = os.path.join('compiler', version, 'linux')
         libpaths = [
-            'lib',
-            os.path.join('lib', 'x64'),
-            os.path.join('compiler', 'lib', 'intel64_lin'),
+            os.path.join(libbase, 'compiler', 'lib', 'intel64'),
         ]
 
         self.variables.append_subdirs("LDFLAGS", root, subdirs=libpaths)

From 0825d425d028bfaa230534b9aadbf1a8ef22ca62 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Thu, 21 Oct 2021 18:27:58 +0200
Subject: [PATCH 126/175] don't read any output in check_async_cmd if
 output_read_size is set to 0

---
 easybuild/tools/run.py | 7 ++++---
 test/framework/run.py  | 7 +++++++
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py
index aa13a14f77..5340c76b76 100644
--- a/easybuild/tools/run.py
+++ b/easybuild/tools/run.py
@@ -263,9 +263,10 @@ def check_async_cmd(proc, cmd, owd, start_time, cmd_log, output_read_size=1024,
     :result: dict value with result of the check (boolean 'done', 'exit_code', 'output')
     """
     # use small read size, to avoid waiting for a long time until sufficient output is produced
-    add_out = get_output_from_process(proc, read_size=output_read_size)
-    _log.debug("Additional output from asynchronous command '%s': %s" % (cmd, add_out))
-    output += add_out
+    if output_read_size:
+        add_out = get_output_from_process(proc, read_size=output_read_size)
+        _log.debug("Additional output from asynchronous command '%s': %s" % (cmd, add_out))
+        output += add_out
 
     exit_code = proc.poll()
     if exit_code is None:
diff --git a/test/framework/run.py b/test/framework/run.py
index 1f48129f2f..6a348e3eb2 100644
--- a/test/framework/run.py
+++ b/test/framework/run.py
@@ -639,6 +639,13 @@ def test_run_cmd_async(self):
 
         # also test use of check_async_cmd on verbose test command
         cmd_info = run_cmd(verbose_test_cmd, asynchronous=True)
+
+        # with output_read_size set to 0, no output is read yet, only status of command is checked
+        res = check_async_cmd(*cmd_info, output_read_size=0)
+        self.assertEqual(res['done'], False)
+        self.assertEqual(res['exit_code'], None)
+        self.assertEqual(res['output'], '')
+
         res = check_async_cmd(*cmd_info)
         self.assertEqual(res['done'], False)
         self.assertEqual(res['exit_code'], None)

From 3ea4d147795f2cb5b5c5b668444bfbd019cf5523 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Thu, 21 Oct 2021 19:27:55 +0200
Subject: [PATCH 127/175] verify that output_read_size passed to
 check_async_cmd is a positive integer value

---
 easybuild/tools/run.py | 2 ++
 test/framework/run.py  | 4 ++++
 2 files changed, 6 insertions(+)

diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py
index 5340c76b76..e70e6702ce 100644
--- a/easybuild/tools/run.py
+++ b/easybuild/tools/run.py
@@ -264,6 +264,8 @@ def check_async_cmd(proc, cmd, owd, start_time, cmd_log, output_read_size=1024,
     """
     # use small read size, to avoid waiting for a long time until sufficient output is produced
     if output_read_size:
+        if not isinstance(output_read_size, int) or output_read_size < 0:
+            raise EasyBuildError("Number of output bytes to read should be a positive integer value")
         add_out = get_output_from_process(proc, read_size=output_read_size)
         _log.debug("Additional output from asynchronous command '%s': %s" % (cmd, add_out))
         output += add_out
diff --git a/test/framework/run.py b/test/framework/run.py
index 6a348e3eb2..d84fbf2711 100644
--- a/test/framework/run.py
+++ b/test/framework/run.py
@@ -640,6 +640,10 @@ def test_run_cmd_async(self):
         # also test use of check_async_cmd on verbose test command
         cmd_info = run_cmd(verbose_test_cmd, asynchronous=True)
 
+        error_pattern = "Number of output bytes to read should be a positive integer value"
+        self.assertErrorRegex(EasyBuildError, error_pattern, check_async_cmd, *cmd_info, output_read_size=-1)
+        self.assertErrorRegex(EasyBuildError, error_pattern, check_async_cmd, *cmd_info, output_read_size='foo')
+
         # with output_read_size set to 0, no output is read yet, only status of command is checked
         res = check_async_cmd(*cmd_info, output_read_size=0)
         self.assertEqual(res['done'], False)

From 78faeabafa37aef7fc9b8a3c5a66ad1a388e6ae9 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Thu, 21 Oct 2021 19:55:25 +0200
Subject: [PATCH 128/175] update extensions progress bar with more detailed
 info when creating Extension instances + checking for extensions to skip

---
 easybuild/framework/easyblock.py | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index e0fca04973..b22322544c 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -1616,14 +1616,18 @@ def skip_extensions(self):
         - use this to detect existing extensions and to remove them from self.ext_instances
         - based on initial R version
         """
+        self.update_exts_progress_bar("skipping installed extensions")
+
         # obtaining untemplated reference value is required here to support legacy string templates like name/version
         exts_filter = self.cfg.get_ref('exts_filter')
 
         if not exts_filter or len(exts_filter) == 0:
             raise EasyBuildError("Skipping of extensions, but no exts_filter set in easyconfig")
 
+        exts_cnt = len(self.ext_instances)
+
         res = []
-        for ext_inst in self.ext_instances:
+        for idx, ext_inst in enumerate(self.ext_instances):
             cmd, stdin = resolve_exts_filter_template(exts_filter, ext_inst)
             (cmdstdouterr, ec) = run_cmd(cmd, log_all=False, log_ok=False, simple=False, inp=stdin, regexp=False)
             self.log.info("exts_filter result %s %s", cmdstdouterr, ec)
@@ -1634,6 +1638,8 @@ def skip_extensions(self):
             else:
                 print_msg("skipping extension %s" % ext_inst.name, silent=self.silent, log=self.log)
 
+            self.update_exts_progress_bar("skipping installed extensions (%d/%d checked)" % (idx + 1, exts_cnt))
+
         self.ext_instances = res
 
     def install_extensions(self, install=True, parallel=False):
@@ -2477,6 +2483,8 @@ def init_ext_instances(self):
         """
         exts_list = self.cfg.get_ref('exts_list')
 
+        self.update_exts_progress_bar("creating internal datastructures for extensions")
+
         # early exit if there are no extensions
         if not exts_list:
             return
@@ -2500,7 +2508,9 @@ def init_ext_instances(self):
             error_msg = "Improper default extension class specification, should be string: %s (%s)"
             raise EasyBuildError(error_msg, exts_defaultclass, type(exts_defaultclass))
 
-        for ext in self.exts:
+        exts_cnt = len(self.exts)
+
+        for idx, ext in enumerate(self.exts):
             ext_name = ext['name']
             self.log.debug("Creating class instance for extension %s...", ext_name)
 
@@ -2561,6 +2571,9 @@ def init_ext_instances(self):
                 self.log.debug("Installing extension %s with class %s (from %s)", ext_name, class_name, mod_path)
 
             self.ext_instances.append(inst)
+            pbar_label = "creating internal datastructures for extensions "
+            pbar_label += "(%d/%d done)" % (idx + 1, exts_cnt)
+            self.update_exts_progress_bar(pbar_label)
 
     def update_exts_progress_bar(self, info, progress_size=0):
         """
@@ -2608,11 +2621,9 @@ def extensions_step(self, fetch=False, install=True):
             self.clean_up_fake_module(fake_mod_data)
             raise EasyBuildError("ERROR: No default extension class set for %s", self.name)
 
-        self.update_exts_progress_bar("creating internal datastructures")
         self.init_ext_instances()
 
         if self.skip:
-            self.update_exts_progress_bar("skipping install extensions")
             self.skip_extensions()
 
         self.install_extensions(install=install)

From 6b431687fb75101b444ddb30c96f9354aa844810 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Thu, 21 Oct 2021 20:26:41 +0200
Subject: [PATCH 129/175] also mention zero as valid value for output_read_size
 in check_async_cmd

---
 easybuild/tools/run.py | 2 +-
 test/framework/run.py  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py
index e70e6702ce..dbfcf4d2d4 100644
--- a/easybuild/tools/run.py
+++ b/easybuild/tools/run.py
@@ -265,7 +265,7 @@ def check_async_cmd(proc, cmd, owd, start_time, cmd_log, output_read_size=1024,
     # use small read size, to avoid waiting for a long time until sufficient output is produced
     if output_read_size:
         if not isinstance(output_read_size, int) or output_read_size < 0:
-            raise EasyBuildError("Number of output bytes to read should be a positive integer value")
+            raise EasyBuildError("Number of output bytes to read should be a positive integer value (or zero)")
         add_out = get_output_from_process(proc, read_size=output_read_size)
         _log.debug("Additional output from asynchronous command '%s': %s" % (cmd, add_out))
         output += add_out
diff --git a/test/framework/run.py b/test/framework/run.py
index d84fbf2711..d421d72cc1 100644
--- a/test/framework/run.py
+++ b/test/framework/run.py
@@ -640,7 +640,7 @@ def test_run_cmd_async(self):
         # also test use of check_async_cmd on verbose test command
         cmd_info = run_cmd(verbose_test_cmd, asynchronous=True)
 
-        error_pattern = "Number of output bytes to read should be a positive integer value"
+        error_pattern = r"Number of output bytes to read should be a positive integer value \(or zero\)"
         self.assertErrorRegex(EasyBuildError, error_pattern, check_async_cmd, *cmd_info, output_read_size=-1)
         self.assertErrorRegex(EasyBuildError, error_pattern, check_async_cmd, *cmd_info, output_read_size='foo')
 

From e9ca5f7e35d3fde6b02c948ab0f233e6d38e43fd Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Thu, 21 Oct 2021 20:28:35 +0200
Subject: [PATCH 130/175] disable trace output when calling complete_cmd in
 check_async_cmd

---
 easybuild/tools/run.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py
index dbfcf4d2d4..2713b4b007 100644
--- a/easybuild/tools/run.py
+++ b/easybuild/tools/run.py
@@ -276,7 +276,7 @@ def check_async_cmd(proc, cmd, owd, start_time, cmd_log, output_read_size=1024,
         done = False
     else:
         _log.debug("Asynchronous command '%s' completed!", cmd)
-        output, _ = complete_cmd(proc, cmd, owd, start_time, cmd_log, output=output, simple=False)
+        output, _ = complete_cmd(proc, cmd, owd, start_time, cmd_log, output=output, simple=False, trace=False)
         done = True
 
     res = {

From 6c19701ac2777bc94bda33a6134d8291ae5b2b67 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Thu, 21 Oct 2021 20:39:48 +0200
Subject: [PATCH 131/175] add support to check_async_cmd for not failing when
 command exited with an error

---
 easybuild/tools/run.py |  5 +++--
 test/framework/run.py  | 10 ++++++++++
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py
index 2713b4b007..69cebc57a9 100644
--- a/easybuild/tools/run.py
+++ b/easybuild/tools/run.py
@@ -248,7 +248,7 @@ def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True
                             regexp=regexp, stream_output=stream_output, trace=trace)
 
 
-def check_async_cmd(proc, cmd, owd, start_time, cmd_log, output_read_size=1024, output=''):
+def check_async_cmd(proc, cmd, owd, start_time, cmd_log, fail_on_error=True, output_read_size=1024, output=''):
     """
     Check status of command that was started asynchronously.
 
@@ -276,7 +276,8 @@ def check_async_cmd(proc, cmd, owd, start_time, cmd_log, output_read_size=1024,
         done = False
     else:
         _log.debug("Asynchronous command '%s' completed!", cmd)
-        output, _ = complete_cmd(proc, cmd, owd, start_time, cmd_log, output=output, simple=False, trace=False)
+        output, _ = complete_cmd(proc, cmd, owd, start_time, cmd_log, output=output,
+                                 simple=False, trace=False, log_ok=fail_on_error, log_all=fail_on_error)
         done = True
 
     res = {
diff --git a/test/framework/run.py b/test/framework/run.py
index d421d72cc1..cfcaac2805 100644
--- a/test/framework/run.py
+++ b/test/framework/run.py
@@ -608,6 +608,16 @@ def test_run_cmd_async(self):
         res = check_async_cmd(*cmd_info, output=res['output'])
         self.assertEqual(res, {'done': True, 'exit_code': 0, 'output': 'sleeping...\ntest123\n'})
 
+        # check asynchronous running of failing command
+        error_test_cmd = "sleep 2; echo 'FAIL!' >&2; exit 123"
+        cmd_info = run_cmd(error_test_cmd, asynchronous=True)
+        error_pattern = 'cmd ".*" exited with exit code 123'
+        self.assertErrorRegex(EasyBuildError, error_pattern, check_async_cmd, *cmd_info)
+
+        cmd_info = run_cmd(error_test_cmd, asynchronous=True)
+        res = check_async_cmd(*cmd_info, fail_on_error=False)
+        self.assertEqual(res, {'done': True, 'exit_code': 123, 'output': "FAIL!\n"})
+
         # also test with a command that produces a lot of output,
         # since that tends to lock up things unless we frequently grab some output...
         verbose_test_cmd = ';'.join([

From d64ecae91335be8a7e0556755067da6addf6f69a Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Thu, 21 Oct 2021 22:19:30 +0200
Subject: [PATCH 132/175] make update_progress_bar a bit more robust by just
 doing nothing if the corresponding progress bar was not started (and making
 stopping of a non-started progress bar fatal)

---
 easybuild/tools/output.py | 24 ++++++++++++++----------
 test/framework/output.py  | 31 ++++++++++++++++++++++++++++++-
 2 files changed, 44 insertions(+), 11 deletions(-)

diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py
index 23270fecc4..193a6c05ef 100644
--- a/easybuild/tools/output.py
+++ b/easybuild/tools/output.py
@@ -295,27 +295,31 @@ def start_progress_bar(bar_type, size, label=None):
 
 def update_progress_bar(bar_type, label=None, progress_size=1):
     """
-    Update progress bar of given type, add progress of given size.
+    Update progress bar of given type (if it was started), add progress of given size.
 
     :param bar_type: type of progress bar
     :param label: label for progress bar
     :param progress_size: amount of progress made
     """
-    (pbar, task_id) = _progress_bar_cache[bar_type]
-    if label:
-        pbar.update(task_id, description=label)
-    if progress_size:
-        pbar.update(task_id, advance=progress_size)
+    if bar_type in _progress_bar_cache:
+        (pbar, task_id) = _progress_bar_cache[bar_type]
+        if label:
+            pbar.update(task_id, description=label)
+        if progress_size:
+            pbar.update(task_id, advance=progress_size)
 
 
 def stop_progress_bar(bar_type, visible=False):
     """
     Stop progress bar of given type.
     """
-    (pbar, task_id) = _progress_bar_cache[bar_type]
-    pbar.stop_task(task_id)
-    if not visible:
-        pbar.update(task_id, visible=False)
+    if bar_type in _progress_bar_cache:
+        (pbar, task_id) = _progress_bar_cache[bar_type]
+        pbar.stop_task(task_id)
+        if not visible:
+            pbar.update(task_id, visible=False)
+    else:
+        raise EasyBuildError("Failed to stop %s progress bar, since it was never started?!", bar_type)
 
 
 def print_checks(checks_data):
diff --git a/test/framework/output.py b/test/framework/output.py
index 6cacc80fd6..b3ff7e558a 100644
--- a/test/framework/output.py
+++ b/test/framework/output.py
@@ -33,7 +33,8 @@
 
 from easybuild.tools.build_log import EasyBuildError
 from easybuild.tools.config import build_option, get_output_style, update_build_option
-from easybuild.tools.output import DummyRich, colorize, show_progress_bars, status_bar, use_rich
+from easybuild.tools.output import PROGRESS_BAR_EXTENSIONS, DummyRich, colorize, get_progress_bar, show_progress_bars
+from easybuild.tools.output import start_progress_bar, status_bar, stop_progress_bar, update_progress_bar, use_rich
 
 try:
     import rich.progress
@@ -138,6 +139,34 @@ def test_colorize(self):
 
         self.assertErrorRegex(EasyBuildError, "Unknown color: nosuchcolor", colorize, 'test', 'nosuchcolor')
 
+    def test_get_start_update_stop_progress_bar(self):
+        """
+        Test starting/updating/stopping of progress bars.
+        """
+        # restore default configuration to show progress bars (disabled to avoid mangled test output)
+        update_build_option('show_progress_bar', True)
+
+        # stopping a progress bar that never was started results in an error
+        error_pattern = "Failed to stop extensions progress bar, since it was never started"
+        self.assertErrorRegex(EasyBuildError, error_pattern, stop_progress_bar, PROGRESS_BAR_EXTENSIONS)
+
+        # updating a progress bar that never was started is silently ignored on purpose
+        update_progress_bar(PROGRESS_BAR_EXTENSIONS)
+        update_progress_bar(PROGRESS_BAR_EXTENSIONS, label="foo")
+        update_progress_bar(PROGRESS_BAR_EXTENSIONS, progress_size=100)
+
+        # also test normal cycle: start, update, stop
+        start_progress_bar(PROGRESS_BAR_EXTENSIONS, 100)
+        update_progress_bar(PROGRESS_BAR_EXTENSIONS)  # single step progress
+        update_progress_bar(PROGRESS_BAR_EXTENSIONS, label="test123", progress_size=5)
+        stop_progress_bar(PROGRESS_BAR_EXTENSIONS)
+
+        pbar = get_progress_bar(PROGRESS_BAR_EXTENSIONS)
+        if HAVE_RICH:
+            self.assertTrue(isinstance(pbar, rich.progress.Progress))
+        else:
+            self.assertTrue(isinstance(pbar, DummyRich))
+
 
 def suite():
     """ returns all the testcases in this module """

From 6e0097ffb7f5e1ae1f0b2060e5857e7a3936d778 Mon Sep 17 00:00:00 2001
From: Miguel Dias Costa <migueldiascosta@gmail.com>
Date: Fri, 22 Oct 2021 09:48:26 +0800
Subject: [PATCH 133/175] reverse sort potential_paths in
 find_related_easyconfigs and fix order of easyconfigs in respective tests

---
 easybuild/framework/easyconfig/tools.py | 2 +-
 test/framework/easyconfig.py            | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py
index a08b016bd4..595481d74a 100644
--- a/easybuild/framework/easyconfig/tools.py
+++ b/easybuild/framework/easyconfig/tools.py
@@ -457,7 +457,7 @@ def find_related_easyconfigs(path, ec):
         toolchain_pattern = ''
 
     potential_paths = [glob.glob(ec_path) for ec_path in create_paths(path, name, '*')]
-    potential_paths = sum(potential_paths, [])  # flatten
+    potential_paths = sorted(sum(potential_paths, []), reverse=True)  # flatten
     _log.debug("found these potential paths: %s" % potential_paths)
 
     parsed_version = LooseVersion(version).version
diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py
index 5a8ec46868..c62b767069 100644
--- a/test/framework/easyconfig.py
+++ b/test/framework/easyconfig.py
@@ -2807,12 +2807,12 @@ def test_find_related_easyconfigs(self):
         # tweak version to 4.6.1, GCC/4.6.x easyconfigs are found as closest match
         ec['version'] = '4.6.1'
         res = [os.path.basename(x) for x in find_related_easyconfigs(test_easyconfigs, ec)]
-        self.assertEqual(res, ['GCC-4.6.3.eb', 'GCC-4.6.4.eb'])
+        self.assertEqual(res, ['GCC-4.6.4.eb', 'GCC-4.6.3.eb'])
 
         # tweak version to 4.5.0, GCC/4.x easyconfigs are found as closest match
         ec['version'] = '4.5.0'
         res = [os.path.basename(x) for x in find_related_easyconfigs(test_easyconfigs, ec)]
-        expected = ['GCC-4.6.3.eb', 'GCC-4.6.4.eb', 'GCC-4.8.2.eb', 'GCC-4.8.3.eb', 'GCC-4.9.2.eb']
+        expected = ['GCC-4.9.2.eb', 'GCC-4.8.3.eb', 'GCC-4.8.2.eb', 'GCC-4.6.4.eb', 'GCC-4.6.3.eb']
         self.assertEqual(res, expected)
 
         ec_file = os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0-deps.eb')

From 8946355c4019f23f1a1f456132a79ee495458312 Mon Sep 17 00:00:00 2001
From: Miguel Dias Costa <migueldiascosta@gmail.com>
Date: Fri, 22 Oct 2021 12:23:54 +0800
Subject: [PATCH 134/175] also test --review-pr-max and --review-pr-filter in
 test_github_review_pr

---
 test/framework/options.py | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/test/framework/options.py b/test/framework/options.py
index 7f0eb39d40..4e78999742 100644
--- a/test/framework/options.py
+++ b/test/framework/options.py
@@ -3786,6 +3786,36 @@ def test_github_review_pr(self):
         self.mock_stderr(False)
         self.assertTrue("This PR should be labelled with 'update'" in txt)
 
+        # test --review-pr-max
+        self.mock_stdout(True)
+        self.mock_stderr(True)
+        args = [
+            '--color=never',
+            '--github-user=%s' % GITHUB_TEST_ACCOUNT,
+            '--review-pr=5365',
+            '--review-pr-max=1',
+        ]
+        self.eb_main(args, raise_error=True, testing=True)
+        txt = self.get_stdout()
+        self.mock_stdout(False)
+        self.mock_stderr(False)
+        self.assertTrue("2016.04" not in txt)
+
+        # test --review-pr-filter
+        self.mock_stdout(True)
+        self.mock_stderr(True)
+        args = [
+            '--color=never',
+            '--github-user=%s' % GITHUB_TEST_ACCOUNT,
+            '--review-pr=5365',
+            '--review-pr-filter=2016a',
+        ]
+        self.eb_main(args, raise_error=True, testing=True)
+        txt = self.get_stdout()
+        self.mock_stdout(False)
+        self.mock_stderr(False)
+        self.assertTrue("2016.04" not in txt)
+
     def test_set_tmpdir(self):
         """Test set_tmpdir config function."""
         self.purge_environment()

From c2c7557e6d86810e0734ac3ae2ef936bdb698177 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Fri, 22 Oct 2021 08:25:18 +0200
Subject: [PATCH 135/175] fix remarks for check_async_cmd

---
 easybuild/tools/run.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py
index 69cebc57a9..322a9fddf5 100644
--- a/easybuild/tools/run.py
+++ b/easybuild/tools/run.py
@@ -257,6 +257,7 @@ def check_async_cmd(proc, cmd, owd, start_time, cmd_log, fail_on_error=True, out
     :param owd: original working directory
     :param start_time: start time of command (datetime instance)
     :param cmd_log: log file to print command output to
+    :param fail_on_error: raise EasyBuildError when command exited with an error
     :param output_read_size: number of bytes to read from output
     :param output: already collected output for this command
 
@@ -277,7 +278,7 @@ def check_async_cmd(proc, cmd, owd, start_time, cmd_log, fail_on_error=True, out
     else:
         _log.debug("Asynchronous command '%s' completed!", cmd)
         output, _ = complete_cmd(proc, cmd, owd, start_time, cmd_log, output=output,
-                                 simple=False, trace=False, log_ok=fail_on_error, log_all=fail_on_error)
+                                 simple=False, trace=False, log_ok=fail_on_error)
         done = True
 
     res = {

From e6a82ca571856c9a2e9f5cbf1dfbf6fc153bdf4a Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Fri, 22 Oct 2021 09:14:56 +0200
Subject: [PATCH 136/175] clear progress bar cache first in
 test_get_start_update_stop_progress_bar

---
 test/framework/output.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/test/framework/output.py b/test/framework/output.py
index b3ff7e558a..a34c3d2de1 100644
--- a/test/framework/output.py
+++ b/test/framework/output.py
@@ -31,6 +31,7 @@
 from unittest import TextTestRunner
 from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered
 
+import easybuild.tools.output
 from easybuild.tools.build_log import EasyBuildError
 from easybuild.tools.config import build_option, get_output_style, update_build_option
 from easybuild.tools.output import PROGRESS_BAR_EXTENSIONS, DummyRich, colorize, get_progress_bar, show_progress_bars
@@ -143,6 +144,9 @@ def test_get_start_update_stop_progress_bar(self):
         """
         Test starting/updating/stopping of progress bars.
         """
+        # clear progress bar cache first, this test assumes we start with a clean slate
+        easybuild.tools.output._progress_bar_cache.clear()
+
         # restore default configuration to show progress bars (disabled to avoid mangled test output)
         update_build_option('show_progress_bar', True)
 

From a7a970aed65caee67195494d385316dce43f6505 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Fri, 22 Oct 2021 09:37:04 +0200
Subject: [PATCH 137/175] add constant for progress bar types + support use of
 ignore_cache in get_progress_bar

---
 easybuild/tools/output.py | 25 ++++++++++++++-----------
 test/framework/output.py  | 24 +++++++++++++++++-------
 2 files changed, 31 insertions(+), 18 deletions(-)

diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py
index 193a6c05ef..e408962338 100644
--- a/easybuild/tools/output.py
+++ b/easybuild/tools/output.py
@@ -250,22 +250,15 @@ def extensions_progress_bar():
     return progress_bar
 
 
-def get_progress_bar(bar_type, size=None):
+def get_progress_bar(bar_type, ignore_cache=False, size=None):
     """
     Get progress bar of given type.
     """
-    progress_bar_types = {
-        PROGRESS_BAR_DOWNLOAD_ALL: download_all_progress_bar,
-        PROGRESS_BAR_DOWNLOAD_ONE: download_one_progress_bar,
-        PROGRESS_BAR_EXTENSIONS: extensions_progress_bar,
-        PROGRESS_BAR_EASYCONFIG: easyconfig_progress_bar,
-        STATUS_BAR: status_bar,
-    }
 
     if bar_type == PROGRESS_BAR_DOWNLOAD_ONE and not size:
-        pbar = download_one_progress_bar_unknown_size()
-    elif bar_type in progress_bar_types:
-        pbar = progress_bar_types[bar_type]()
+        pbar = download_one_progress_bar_unknown_size(ignore_cache=ignore_cache)
+    elif bar_type in PROGRESS_BAR_TYPES:
+        pbar = PROGRESS_BAR_TYPES[bar_type](ignore_cache=ignore_cache)
     else:
         raise EasyBuildError("Unknown progress bar type: %s", bar_type)
 
@@ -388,3 +381,13 @@ def print_checks(checks_data):
             console_print(table)
         else:
             print('\n'.join(lines))
+
+
+# this constant must be defined at the end, since functions used as values need to be defined
+PROGRESS_BAR_TYPES = {
+    PROGRESS_BAR_DOWNLOAD_ALL: download_all_progress_bar,
+    PROGRESS_BAR_DOWNLOAD_ONE: download_one_progress_bar,
+    PROGRESS_BAR_EXTENSIONS: extensions_progress_bar,
+    PROGRESS_BAR_EASYCONFIG: easyconfig_progress_bar,
+    STATUS_BAR: status_bar,
+}
diff --git a/test/framework/output.py b/test/framework/output.py
index a34c3d2de1..ea574d043a 100644
--- a/test/framework/output.py
+++ b/test/framework/output.py
@@ -34,7 +34,8 @@
 import easybuild.tools.output
 from easybuild.tools.build_log import EasyBuildError
 from easybuild.tools.config import build_option, get_output_style, update_build_option
-from easybuild.tools.output import PROGRESS_BAR_EXTENSIONS, DummyRich, colorize, get_progress_bar, show_progress_bars
+from easybuild.tools.output import PROGRESS_BAR_EXTENSIONS, PROGRESS_BAR_TYPES
+from easybuild.tools.output import DummyRich, colorize, get_progress_bar, show_progress_bars
 from easybuild.tools.output import start_progress_bar, status_bar, stop_progress_bar, update_progress_bar, use_rich
 
 try:
@@ -140,6 +141,21 @@ def test_colorize(self):
 
         self.assertErrorRegex(EasyBuildError, "Unknown color: nosuchcolor", colorize, 'test', 'nosuchcolor')
 
+    def test_get_progress_bar(self):
+        """
+        Test get_progress_bar.
+        """
+        # restore default configuration to show progress bars (disabled to avoid mangled test output),
+        # to ensure we'll get actual Progress instances when Rich is available
+        update_build_option('show_progress_bar', True)
+
+        for pbar_type in PROGRESS_BAR_TYPES:
+            pbar = get_progress_bar(pbar_type, ignore_cache=True)
+            if HAVE_RICH:
+                self.assertTrue(isinstance(pbar, rich.progress.Progress))
+            else:
+                self.assertTrue(isinstance(pbar, DummyRich))
+
     def test_get_start_update_stop_progress_bar(self):
         """
         Test starting/updating/stopping of progress bars.
@@ -165,12 +181,6 @@ def test_get_start_update_stop_progress_bar(self):
         update_progress_bar(PROGRESS_BAR_EXTENSIONS, label="test123", progress_size=5)
         stop_progress_bar(PROGRESS_BAR_EXTENSIONS)
 
-        pbar = get_progress_bar(PROGRESS_BAR_EXTENSIONS)
-        if HAVE_RICH:
-            self.assertTrue(isinstance(pbar, rich.progress.Progress))
-        else:
-            self.assertTrue(isinstance(pbar, DummyRich))
-
 
 def suite():
     """ returns all the testcases in this module """

From fdc8a1aa15dc92dcf6e9623eac7912515d0c653a Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Fri, 22 Oct 2021 10:04:26 +0200
Subject: [PATCH 138/175] only update extensions progress bar in
 init_ext_instances if there actually are extensions

---
 easybuild/framework/easyblock.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index b22322544c..e4b1254d0e 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -2483,8 +2483,6 @@ def init_ext_instances(self):
         """
         exts_list = self.cfg.get_ref('exts_list')
 
-        self.update_exts_progress_bar("creating internal datastructures for extensions")
-
         # early exit if there are no extensions
         if not exts_list:
             return
@@ -2510,6 +2508,8 @@ def init_ext_instances(self):
 
         exts_cnt = len(self.exts)
 
+        self.update_exts_progress_bar("creating internal datastructures for extensions")
+
         for idx, ext in enumerate(self.exts):
             ext_name = ext['name']
             self.log.debug("Creating class instance for extension %s...", ext_name)

From bdf1e208b1e0d00068ce9d8f16a7236f05e74f77 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Fri, 22 Oct 2021 11:00:35 +0200
Subject: [PATCH 139/175] drop sleep from failing test command in
 test_run_cmd_async

---
 test/framework/run.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/framework/run.py b/test/framework/run.py
index cfcaac2805..a6a88b638c 100644
--- a/test/framework/run.py
+++ b/test/framework/run.py
@@ -609,7 +609,7 @@ def test_run_cmd_async(self):
         self.assertEqual(res, {'done': True, 'exit_code': 0, 'output': 'sleeping...\ntest123\n'})
 
         # check asynchronous running of failing command
-        error_test_cmd = "sleep 2; echo 'FAIL!' >&2; exit 123"
+        error_test_cmd = "echo 'FAIL!' >&2; exit 123"
         cmd_info = run_cmd(error_test_cmd, asynchronous=True)
         error_pattern = 'cmd ".*" exited with exit code 123'
         self.assertErrorRegex(EasyBuildError, error_pattern, check_async_cmd, *cmd_info)

From 78552da47d0a3044de08c361b8360f96335b0c85 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Fri, 22 Oct 2021 11:57:49 +0200
Subject: [PATCH 140/175] add test for insecure downloading with
 filetools.download_file

---
 test/framework/filetools.py | 79 ++++++++++++++++++++++++++++++++++++-
 1 file changed, 77 insertions(+), 2 deletions(-)

diff --git a/test/framework/filetools.py b/test/framework/filetools.py
index dd46966071..0425f10ac1 100644
--- a/test/framework/filetools.py
+++ b/test/framework/filetools.py
@@ -47,7 +47,7 @@
 from easybuild.tools.build_log import EasyBuildError
 from easybuild.tools.config import IGNORE, ERROR, update_build_option
 from easybuild.tools.multidiff import multidiff
-from easybuild.tools.py2vs3 import std_urllib
+from easybuild.tools.py2vs3 import StringIO, std_urllib
 
 
 class FileToolsTest(EnhancedTestCase):
@@ -66,12 +66,18 @@ def setUp(self):
         super(FileToolsTest, self).setUp()
 
         self.orig_filetools_std_urllib_urlopen = ft.std_urllib.urlopen
+        if ft.HAVE_REQUESTS:
+            self.orig_filetools_requests_get = ft.requests.get
+        self.orig_filetools_HAVE_REQUESTS = ft.HAVE_REQUESTS
 
     def tearDown(self):
         """Cleanup."""
         super(FileToolsTest, self).tearDown()
 
         ft.std_urllib.urlopen = self.orig_filetools_std_urllib_urlopen
+        ft.HAVE_REQUESTS = self.orig_filetools_HAVE_REQUESTS
+        if ft.HAVE_REQUESTS:
+            ft.requests.get = self.orig_filetools_requests_get
 
     def test_extract_cmd(self):
         """Test various extract commands."""
@@ -508,7 +514,6 @@ def fake_urllib_open(*args, **kwargs):
 
         # replaceurlopen with function that raises HTTP error 403
         def fake_urllib_open(*args, **kwargs):
-            from easybuild.tools.py2vs3 import StringIO
             raise ft.std_urllib.HTTPError(url, 403, "Forbidden", "", StringIO())
 
         ft.std_urllib.urlopen = fake_urllib_open
@@ -523,6 +528,76 @@ def fake_urllib_open(*args, **kwargs):
         ft.HAVE_REQUESTS = False
         self.assertErrorRegex(EasyBuildError, "SSL issues with urllib2", ft.download_file, fn, url, target)
 
+    def test_download_file_insecure(self):
+        """
+        Test downloading of file via insecure URL
+        """
+        # replace urlopen with function that raises IOError
+        def fake_urllib_open(url, *args, **kwargs):
+            if kwargs.get('context') is None:
+                error_msg = " <urlopen error [SSL: CERTIFICATE_VERIFY_FAILED] "
+                error_msg += "certificate verify failed (_ssl.c:618)>"
+                raise IOError(error_msg)
+
+            return self.orig_filetools_std_urllib_urlopen(url, *args, **kwargs)
+
+        fn = 'toy-0.0.eb'
+        test_dir = os.path.abspath(os.path.dirname(__file__))
+        toy_dir = os.path.join(test_dir, 'easyconfigs', 'test_ecs', 't', 'toy')
+        url = 'file://%s/%s' % (toy_dir, fn)
+
+        ft.std_urllib.urlopen = fake_urllib_open
+
+        target_path = os.path.join(self.test_prefix, fn)
+
+        # first try without allowing insecure downloads (default)
+        res = ft.download_file(fn, url, target_path)
+        self.assertEqual(res, None)
+
+        self.mock_stderr(True)
+        res = ft.download_file(fn, url, target_path, insecure=True)
+        stderr = self.get_stderr()
+        self.mock_stderr(False)
+
+        self.assertTrue("WARNING: Not checking server certificates while downloading toy-0.0.eb" in stderr)
+        self.assertTrue(os.path.exists(res))
+        self.assertTrue(ft.read_file(res).startswith("name = 'toy'"))
+
+        # also test insecure download via requests fallback
+        if ft.HAVE_REQUESTS:
+
+            # need to use actual URL here, requests doesn't like file:// URLs
+            url = 'https://raw.githubusercontent.com/easybuilders/easybuild-framework/master/README.rst'
+            fn = os.path.basename(url)
+            target_path = os.path.join(self.test_prefix, fn)
+
+            # replaceurlopen with function that raises HTTP error 403
+            def fake_urllib_open(url, *args, **kwargs):
+                raise ft.std_urllib.HTTPError(url, 403, "Forbidden", "", StringIO())
+
+            ft.std_urllib.urlopen = fake_urllib_open
+
+            def fake_requests_get(url, *args, **kwargs):
+                verify = kwargs.get('verify')
+                if verify:
+                    raise IOError("failing SSL certificate!")
+
+                return self.orig_filetools_requests_get(url, *args, **kwargs)
+
+            ft.requests.get = fake_requests_get
+
+            res = ft.download_file(fn, url, target_path)
+            self.assertEqual(res, None)
+
+            self.mock_stderr(True)
+            res = ft.download_file(fn, url, target_path, insecure=True)
+            stderr = self.get_stderr()
+            self.mock_stderr(False)
+
+            self.assertTrue("WARNING: Not checking server certificates while downloading README.rst" in stderr)
+            self.assertTrue(os.path.exists(res))
+            self.assertTrue("https://easybuilders.github.io/easybuild" in ft.read_file(res))
+
     def test_mkdir(self):
         """Test mkdir function."""
 

From d2f607f8f39a8f147ae68aff25faf7101498663e Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Fri, 22 Oct 2021 12:05:16 +0200
Subject: [PATCH 141/175] pick up insecure_download build option directly in
 download_file

---
 easybuild/framework/easyblock.py | 19 ++++++-------------
 easybuild/tools/filetools.py     |  4 +++-
 test/framework/filetools.py      | 12 +++++++++---
 3 files changed, 18 insertions(+), 17 deletions(-)

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index dd24b1e431..2ec0ec7a7a 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -388,10 +388,8 @@ def fetch_source(self, source, checksum=None, extension=False):
 
         # check if the sources can be located
         force_download = build_option('force_download') in [FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_SOURCES]
-        insecure_download = build_option('insecure_download')
         path = self.obtain_file(filename, extension=extension, download_filename=download_filename,
-                                force_download=force_download, insecure_download=insecure_download,
-                                urls=source_urls, git_config=git_config)
+                                force_download=force_download, urls=source_urls, git_config=git_config)
         if path is None:
             raise EasyBuildError('No file found for source %s', filename)
 
@@ -454,9 +452,7 @@ def fetch_patches(self, patch_specs=None, extension=False, checksums=None):
             patch_info = create_patch_info(patch_spec)
 
             force_download = build_option('force_download') in [FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_PATCHES]
-            insecure_download = build_option('insecure_download')
-            path = self.obtain_file(patch_info['name'], extension=extension, force_download=force_download,
-                                    insecure_download=insecure_download)
+            path = self.obtain_file(patch_info['name'], extension=extension, force_download=force_download)
             if path:
                 self.log.debug('File %s found for patch %s', path, patch_spec)
                 patch_info['path'] = path
@@ -501,7 +497,6 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True):
             self.dry_run_msg("\nList of sources/patches for extensions:")
 
         force_download = build_option('force_download') in [FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_SOURCES]
-        insecure_download = build_option('insecure_download')
 
         for ext in exts_list:
             if isinstance(ext, (list, tuple)) and ext:
@@ -594,8 +589,7 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True):
 
                         if fetch_files:
                             src_path = self.obtain_file(src_fn, extension=True, urls=source_urls,
-                                                        force_download=force_download,
-                                                        insecure_download=insecure_download)
+                                                        force_download=force_download)
                             if src_path:
                                 ext_src.update({'src': src_path})
                             else:
@@ -669,7 +663,7 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True):
         return exts_sources
 
     def obtain_file(self, filename, extension=False, urls=None, download_filename=None, force_download=False,
-                    insecure_download=False, git_config=None):
+                    git_config=None):
         """
         Locate the file with the given name
         - searches in different subdirectories of source path
@@ -679,7 +673,6 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
         :param urls: list of source URLs where this file may be available
         :param download_filename: filename with which the file should be downloaded, and then renamed to <filename>
         :param force_download: always try to download file, even if it's already available in source path
-        :param insecure_download: don't check the server certificate against the available certificate authorities
         :param git_config: dictionary to define how to download a git repository
         """
         srcpaths = source_paths()
@@ -711,7 +704,7 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
                         self.log.info("Found file %s at %s, no need to download it", filename, filepath)
                         return fullpath
 
-                if download_file(filename, url, fullpath, insecure=insecure_download):
+                if download_file(filename, url, fullpath):
                     return fullpath
 
             except IOError as err:
@@ -838,7 +831,7 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
                         self.log.debug("Trying to download file %s from %s to %s ..." % (filename, fullurl, targetpath))
                         downloaded = False
                         try:
-                            if download_file(filename, fullurl, targetpath, insecure=insecure_download):
+                            if download_file(filename, fullurl, targetpath):
                                 downloaded = True
 
                         except IOError as err:
diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py
index 7c5439e6c1..b313928df1 100644
--- a/easybuild/tools/filetools.py
+++ b/easybuild/tools/filetools.py
@@ -738,9 +738,11 @@ def det_file_size(http_header):
     return res
 
 
-def download_file(filename, url, path, forced=False, insecure=False):
+def download_file(filename, url, path, forced=False):
     """Download a file from the given URL, to the specified path."""
 
+    insecure = build_option('insecure_download')
+
     _log.debug("Trying to download %s from %s to %s", filename, url, path)
 
     timeout = build_option('download_timeout')
diff --git a/test/framework/filetools.py b/test/framework/filetools.py
index 0425f10ac1..1438a5482a 100644
--- a/test/framework/filetools.py
+++ b/test/framework/filetools.py
@@ -45,7 +45,7 @@
 from easybuild.tools import run
 import easybuild.tools.filetools as ft
 from easybuild.tools.build_log import EasyBuildError
-from easybuild.tools.config import IGNORE, ERROR, update_build_option
+from easybuild.tools.config import IGNORE, ERROR, build_option, update_build_option
 from easybuild.tools.multidiff import multidiff
 from easybuild.tools.py2vs3 import StringIO, std_urllib
 
@@ -532,6 +532,9 @@ def test_download_file_insecure(self):
         """
         Test downloading of file via insecure URL
         """
+
+        self.assertFalse(build_option('insecure_download'))
+
         # replace urlopen with function that raises IOError
         def fake_urllib_open(url, *args, **kwargs):
             if kwargs.get('context') is None:
@@ -554,8 +557,9 @@ def fake_urllib_open(url, *args, **kwargs):
         res = ft.download_file(fn, url, target_path)
         self.assertEqual(res, None)
 
+        update_build_option('insecure_download', True)
         self.mock_stderr(True)
-        res = ft.download_file(fn, url, target_path, insecure=True)
+        res = ft.download_file(fn, url, target_path)
         stderr = self.get_stderr()
         self.mock_stderr(False)
 
@@ -586,11 +590,13 @@ def fake_requests_get(url, *args, **kwargs):
 
             ft.requests.get = fake_requests_get
 
+            update_build_option('insecure_download', False)
             res = ft.download_file(fn, url, target_path)
             self.assertEqual(res, None)
 
+            update_build_option('insecure_download', True)
             self.mock_stderr(True)
-            res = ft.download_file(fn, url, target_path, insecure=True)
+            res = ft.download_file(fn, url, target_path)
             stderr = self.get_stderr()
             self.mock_stderr(False)
 

From 85273d039ccb8926d95102045321efd7005e111e Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Fri, 22 Oct 2021 14:10:18 +0200
Subject: [PATCH 142/175] use check_async_cmd in Extension.async_cmd_check

---
 easybuild/framework/extension.py | 20 +++++++++-----------
 1 file changed, 9 insertions(+), 11 deletions(-)

diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py
index eda2393fa0..024261c332 100644
--- a/easybuild/framework/extension.py
+++ b/easybuild/framework/extension.py
@@ -40,7 +40,7 @@
 from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP, template_constant_dict
 from easybuild.tools.build_log import EasyBuildError, raise_nosupport
 from easybuild.tools.filetools import change_dir
-from easybuild.tools.run import complete_cmd, get_output_from_process, run_cmd
+from easybuild.tools.run import check_async_cmd, complete_cmd, run_cmd
 from easybuild.tools.py2vs3 import string_type
 
 
@@ -196,23 +196,21 @@ def async_cmd_check(self):
             raise EasyBuildError("No installation command running asynchronously for %s", self.name)
         else:
             self.log.debug("Checking on installation of extension %s...", self.name)
-            proc = self.async_cmd_info[0]
             # use small read size, to avoid waiting for a long time until sufficient output is produced
-            self.async_cmd_output += get_output_from_process(proc, read_size=self.async_cmd_read_size)
-            ec = proc.poll()
-            if ec is None:
-                res = False
+            res = check_async_cmd(*self.async_cmd_info, output_read_size=self.async_cmd_read_size)
+            self.async_cmd_output += res['output']
+            if res['done']:
+                self.log.info("Installation of extension %s completed!", self.name)
+            else:
                 self.async_cmd_check_cnt += 1
+                self.log.debug("Installation of extension %s still running (checked %d times)",
+                               self.name, self.async_cmd_check_cnt)
                 # increase read size after sufficient checks,
                 # to avoid that installation hangs due to output buffer filling up...
                 if self.async_cmd_check_cnt % 10 == 0 and self.async_cmd_read_size < (1024 ** 2):
                     self.async_cmd_read_size *= 2
-            else:
-                self.log.debug("Completing installation of extension %s...", self.name)
-                self.async_cmd_output, _ = complete_cmd(*self.async_cmd_info, output=self.async_cmd_output)
-                res = True
 
-        return res
+            return res['done']
 
     @property
     def required_deps(self):

From b2938dde42c2848df277fc8b2db401ff80f68dfe Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Fri, 22 Oct 2021 14:27:29 +0200
Subject: [PATCH 143/175] remove import for unused complete_cmd from
 framework/extension.py

---
 easybuild/framework/extension.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py
index 024261c332..f78d1c63e6 100644
--- a/easybuild/framework/extension.py
+++ b/easybuild/framework/extension.py
@@ -40,7 +40,7 @@
 from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP, template_constant_dict
 from easybuild.tools.build_log import EasyBuildError, raise_nosupport
 from easybuild.tools.filetools import change_dir
-from easybuild.tools.run import check_async_cmd, complete_cmd, run_cmd
+from easybuild.tools.run import check_async_cmd, run_cmd
 from easybuild.tools.py2vs3 import string_type
 
 

From b15a880e8eb01df4f41fb2b27e52893c07a987d4 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Fri, 22 Oct 2021 17:33:46 +0200
Subject: [PATCH 144/175] fix typo in comment in filetools test

Co-authored-by: Simon Branford <4967+branfosj@users.noreply.github.com>
---
 test/framework/filetools.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/framework/filetools.py b/test/framework/filetools.py
index 1438a5482a..e511f31eb3 100644
--- a/test/framework/filetools.py
+++ b/test/framework/filetools.py
@@ -575,7 +575,7 @@ def fake_urllib_open(url, *args, **kwargs):
             fn = os.path.basename(url)
             target_path = os.path.join(self.test_prefix, fn)
 
-            # replaceurlopen with function that raises HTTP error 403
+            # replace urlopen with function that raises HTTP error 403
             def fake_urllib_open(url, *args, **kwargs):
                 raise ft.std_urllib.HTTPError(url, 403, "Forbidden", "", StringIO())
 

From 343f4f009a382913b19cdf295aac58a0c8ef2001 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Fri, 22 Oct 2021 17:34:17 +0200
Subject: [PATCH 145/175] keep order of options alphabetically sorted by moving
 down insecure-download

---
 easybuild/tools/options.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py
index 4c2f7091d9..5ef43c952c 100644
--- a/easybuild/tools/options.py
+++ b/easybuild/tools/options.py
@@ -407,8 +407,6 @@ def override_options(self):
             'force-download': ("Force re-downloading of sources and/or patches, "
                                "even if they are available already in source path",
                                'choice', 'store_or_None', DEFAULT_FORCE_DOWNLOAD, FORCE_DOWNLOAD_CHOICES),
-            'insecure-download': ("Don't check the server certificate against the available certificate authorities.",
-                                  None, 'store_true', False),
             'generate-devel-module': ("Generate a develop module file, implies --force if disabled",
                                       None, 'store_true', True),
             'group': ("Group to be used for software installations (only verified, not set)", None, 'store', None),
@@ -428,6 +426,8 @@ def override_options(self):
             'ignore-checksums': ("Ignore failing checksum verification", None, 'store_true', False),
             'ignore-test-failure': ("Ignore a failing test step", None, 'store_true', False),
             'ignore-osdeps': ("Ignore any listed OS dependencies", None, 'store_true', False),
+            'insecure-download': ("Don't check the server certificate against the available certificate authorities.",
+                                  None, 'store_true', False),
             'install-latest-eb-release': ("Install latest known version of easybuild", None, 'store_true', False),
             'lib-lib64-symlink': ("Automatically create symlinks for lib/ pointing to lib64/ if the former is missing",
                                   None, 'store_true', True),

From 447c1da420e7ae24e73adef41cfc6e74f05c3ce1 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Mon, 25 Oct 2021 09:53:07 +0200
Subject: [PATCH 146/175] fix occasional failure in test_run_cmd_async

---
 test/framework/run.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/test/framework/run.py b/test/framework/run.py
index a6a88b638c..24128aad6b 100644
--- a/test/framework/run.py
+++ b/test/framework/run.py
@@ -616,6 +616,9 @@ def test_run_cmd_async(self):
 
         cmd_info = run_cmd(error_test_cmd, asynchronous=True)
         res = check_async_cmd(*cmd_info, fail_on_error=False)
+        # keep checking until command is fully done
+        while not res['done']:
+            res = check_async_cmd(*cmd_info, fail_on_error=False)
         self.assertEqual(res, {'done': True, 'exit_code': 123, 'output': "FAIL!\n"})
 
         # also test with a command that produces a lot of output,

From d82ade9348cce77368ea1baca5a37de9a33a6b29 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Mon, 25 Oct 2021 09:53:26 +0200
Subject: [PATCH 147/175] check early for opt-in to using experimental feature
 when --parallel-extensions-install is used

---
 easybuild/tools/options.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py
index 99ed51721b..2142a50f4a 100644
--- a/easybuild/tools/options.py
+++ b/easybuild/tools/options.py
@@ -892,6 +892,10 @@ def postprocess(self):
         # set tmpdir
         self.tmpdir = set_tmpdir(self.options.tmpdir)
 
+        # early check for opt-in to installing extensions in parallel (experimental feature)
+        if self.options.parallel_extensions_install:
+            self.log.experimental("installing extensions in parallel")
+
         # take --include options into account (unless instructed otherwise)
         if self.with_include:
             self._postprocess_include()

From 42c0bb3ad4fac2acb2d1ae6fc8601c4fe3ffa802 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Mon, 25 Oct 2021 09:57:07 +0200
Subject: [PATCH 148/175] tweak extensions progress bar label to also show 'X/Y
 done' when installing extensions in parallel

---
 easybuild/framework/easyblock.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index e4b1254d0e..2f31ea464b 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -1743,14 +1743,16 @@ def update_exts_progress_bar_helper(running_exts, progress_size):
             """Helper function to update extensions progress bar."""
             running_exts_cnt = len(running_exts)
             if running_exts_cnt > 1:
-                progress_info = "Installing %d extensions: " % running_exts_cnt
+                progress_info = "Installing %d extensions" % running_exts_cnt
             elif running_exts_cnt == 1:
                 progress_info = "Installing extension "
             else:
                 progress_info = "Not installing extensions (yet)"
 
-            progress_info += ', '.join(e.name for e in running_exts)
-            progress_info += " (%d/%d done)" % (len(installed_ext_names), exts_cnt)
+            if running_exts_cnt:
+                progress_info += " (%d/%d done): " % (len(installed_ext_names), exts_cnt)
+                progress_info += ', '.join(e.name for e in running_exts)
+
             self.update_exts_progress_bar(progress_info, progress_size=progress_size)
 
         iter_id = 0

From 283404b268bb360b138a581182b3f877a4221467 Mon Sep 17 00:00:00 2001
From: Simon Branford <s.j.branford@bham.ac.uk>
Date: Tue, 26 Oct 2021 10:48:34 +0100
Subject: [PATCH 149/175] remove '--depth 1' from git clone

---
 easybuild/tools/filetools.py | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py
index b313928df1..062d47c737 100644
--- a/easybuild/tools/filetools.py
+++ b/easybuild/tools/filetools.py
@@ -2598,11 +2598,6 @@ def get_source_tarball_from_git(filename, targetdir, git_config):
     # compose 'git clone' command, and run it
     clone_cmd = ['git', 'clone']
 
-    if not keep_git_dir:
-        # Speed up cloning by only fetching the most recent commit, not the whole history
-        # When we don't want to keep the .git folder there won't be a difference in the result
-        clone_cmd.extend(['--depth', '1'])
-
     if tag:
         clone_cmd.extend(['--branch', tag])
         if recursive:

From 0aae4d7561b5081254654c47399e281799bb26c6 Mon Sep 17 00:00:00 2001
From: Simon Branford <s.j.branford@bham.ac.uk>
Date: Tue, 26 Oct 2021 11:20:48 +0100
Subject: [PATCH 150/175] only disable when using a commit and tests

---
 easybuild/tools/filetools.py | 5 +++++
 test/framework/filetools.py  | 4 ++--
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py
index 062d47c737..148db6352a 100644
--- a/easybuild/tools/filetools.py
+++ b/easybuild/tools/filetools.py
@@ -2598,6 +2598,11 @@ def get_source_tarball_from_git(filename, targetdir, git_config):
     # compose 'git clone' command, and run it
     clone_cmd = ['git', 'clone']
 
+    if not keep_git_dir and not commit:
+        # Speed up cloning by only fetching the most recent commit, not the whole history
+        # When we don't want to keep the .git folder there won't be a difference in the result
+        clone_cmd.extend(['--depth', '1'])
+
     if tag:
         clone_cmd.extend(['--branch', tag])
         if recursive:
diff --git a/test/framework/filetools.py b/test/framework/filetools.py
index e511f31eb3..1e61f997ef 100644
--- a/test/framework/filetools.py
+++ b/test/framework/filetools.py
@@ -2750,7 +2750,7 @@ def run_check():
         del git_config['tag']
         git_config['commit'] = '8456f86'
         expected = '\n'.join([
-            r'  running command "git clone --depth 1 --no-checkout %(git_repo)s"',
+            r'  running command "git clone --no-checkout %(git_repo)s"',
             r"  \(in .*/tmp.*\)",
             r'  running command "git checkout 8456f86 && git submodule update --init --recursive"',
             r"  \(in testrepository\)",
@@ -2761,7 +2761,7 @@ def run_check():
 
         del git_config['recursive']
         expected = '\n'.join([
-            r'  running command "git clone --depth 1 --no-checkout %(git_repo)s"',
+            r'  running command "git clone --no-checkout %(git_repo)s"',
             r"  \(in .*/tmp.*\)",
             r'  running command "git checkout 8456f86"',
             r"  \(in testrepository\)",

From cb4074dc75d10511dbabf4c864cab88b08e45b36 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Tue, 26 Oct 2021 13:57:43 +0200
Subject: [PATCH 151/175] enhance test_get_source_tarball_from_git to trigger
 fixed bug w.r.t. cloning depth

---
 test/framework/filetools.py | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/test/framework/filetools.py b/test/framework/filetools.py
index e511f31eb3..cfd56c38af 100644
--- a/test/framework/filetools.py
+++ b/test/framework/filetools.py
@@ -2784,6 +2784,7 @@ def run_check():
             test_file = os.path.join(target_dir, 'test.tar.gz')
             self.assertEqual(res, test_file)
             self.assertTrue(os.path.isfile(test_file))
+            test_tar_gzs = [os.path.basename(test_file)]
             self.assertEqual(os.listdir(target_dir), ['test.tar.gz'])
             # Check that we indeed downloaded the right tag
             extracted_dir = tempfile.mkdtemp(prefix='extracted_dir')
@@ -2805,12 +2806,21 @@ def run_check():
             self.assertTrue(os.path.isfile(os.path.join(extracted_repo_dir, 'this-is-a-tag.txt')))
 
             del git_config['tag']
-            git_config['commit'] = '8456f86'
+            git_config['commit'] = '90366ea'
             res = ft.get_source_tarball_from_git('test2.tar.gz', target_dir, git_config)
             test_file = os.path.join(target_dir, 'test2.tar.gz')
             self.assertEqual(res, test_file)
             self.assertTrue(os.path.isfile(test_file))
-            self.assertEqual(sorted(os.listdir(target_dir)), ['test.tar.gz', 'test2.tar.gz'])
+            test_tar_gzs.append(os.path.basename(test_file))
+            self.assertEqual(sorted(os.listdir(target_dir)), test_tar_gzs)
+
+            git_config['keep_git_dir'] = True
+            res = ft.get_source_tarball_from_git('test3.tar.gz', target_dir, git_config)
+            test_file = os.path.join(target_dir, 'test3.tar.gz')
+            self.assertEqual(res, test_file)
+            self.assertTrue(os.path.isfile(test_file))
+            test_tar_gzs.append(os.path.basename(test_file))
+            self.assertEqual(sorted(os.listdir(target_dir)), test_tar_gzs)
 
         except EasyBuildError as err:
             if "Network is down" in str(err):

From c5d598f19cc9824ea6b945605132eb12f992fe22 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Tue, 26 Oct 2021 14:20:28 +0200
Subject: [PATCH 152/175] inject short sleep before checking status of failing
 asynchronous command

---
 test/framework/run.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/framework/run.py b/test/framework/run.py
index 24128aad6b..da4448d741 100644
--- a/test/framework/run.py
+++ b/test/framework/run.py
@@ -611,6 +611,7 @@ def test_run_cmd_async(self):
         # check asynchronous running of failing command
         error_test_cmd = "echo 'FAIL!' >&2; exit 123"
         cmd_info = run_cmd(error_test_cmd, asynchronous=True)
+        time.sleep(1)
         error_pattern = 'cmd ".*" exited with exit code 123'
         self.assertErrorRegex(EasyBuildError, error_pattern, check_async_cmd, *cmd_info)
 

From 73af4253f6d642ba8162ef1bf8e5a32d30e9ecd4 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Tue, 26 Oct 2021 14:58:40 +0200
Subject: [PATCH 153/175] drop 'parallel' argument for install_extensions, to
 avoid having to opt-in to support for installing extensions in parallel in
 various easyblocks

---
 easybuild/framework/easyblock.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index 2f31ea464b..468eddf764 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -1642,17 +1642,16 @@ def skip_extensions(self):
 
         self.ext_instances = res
 
-    def install_extensions(self, install=True, parallel=False):
+    def install_extensions(self, install=True):
         """
         Install extensions.
 
         :param install: actually install extensions, don't just prepare environment for installing
-        :param parallel: install extensions in parallel
 
         """
         self.log.debug("List of loaded modules: %s", self.modules_tool.list())
 
-        if build_option('parallel_extensions_install') and parallel:
+        if build_option('parallel_extensions_install'):
             self.log.experimental("installing extensions in parallel")
             self.install_extensions_parallel(install=install)
         else:

From 16e02eae6fce1e3d8878ac68efc135f83046c9f3 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Tue, 26 Oct 2021 15:25:16 +0200
Subject: [PATCH 154/175] add run_async method to install extension
 asynchronously

---
 easybuild/framework/easyblock.py | 2 +-
 easybuild/framework/extension.py | 8 +++++++-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index 468eddf764..ff8a561226 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -1831,7 +1831,7 @@ def update_exts_progress_bar_helper(running_exts, progress_size):
                                           rpath_filter_dirs=self.rpath_filter_dirs)
                     if install:
                         ext.prerun()
-                        ext.run(asynchronous=True)
+                        ext.run_async()
                         running_exts.append(ext)
                         self.log.info("Started installation of extension %s in the background...", ext.name)
                         update_exts_progress_bar_helper(running_exts, 0)
diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py
index f78d1c63e6..251eed6afe 100644
--- a/easybuild/framework/extension.py
+++ b/easybuild/framework/extension.py
@@ -168,10 +168,16 @@ def prerun(self):
 
     def run(self, *args, **kwargs):
         """
-        Actual installation of a extension.
+        Actual installation of an extension.
         """
         pass
 
+    def run_async(self, *args, **kwargs):
+        """
+        Asynchronous installation of an extension.
+        """
+        raise NotImplementedError
+
     def postrun(self):
         """
         Stuff to do after installing a extension.

From 60c5d1537b3bfbb0f1b3a85676331df952e154f0 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Tue, 26 Oct 2021 20:07:35 +0200
Subject: [PATCH 155/175] move printing of progress info on installing
 extensions in parallel after every iteration, and only when not showing
 progress bars

---
 easybuild/framework/easyblock.py | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index ff8a561226..58f4303aa3 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -91,7 +91,7 @@
 from easybuild.tools.modules import Lmod, curr_module_paths, invalidate_module_caches_for, get_software_root
 from easybuild.tools.modules import get_software_root_env_var_name, get_software_version_env_var_name
 from easybuild.tools.output import PROGRESS_BAR_DOWNLOAD_ALL, PROGRESS_BAR_EASYCONFIG, PROGRESS_BAR_EXTENSIONS
-from easybuild.tools.output import start_progress_bar, stop_progress_bar, update_progress_bar
+from easybuild.tools.output import show_progress_bars, start_progress_bar, stop_progress_bar, update_progress_bar
 from easybuild.tools.package.utilities import package
 from easybuild.tools.py2vs3 import extract_method_name, string_type
 from easybuild.tools.repository.repository import init_repository
@@ -1775,16 +1775,6 @@ def update_exts_progress_bar_helper(running_exts, progress_size):
                     else:
                         self.log.debug("Installation of %s is still running...", ext.name)
 
-            # print progress info every now and then
-            if iter_id % 1 == 0:
-                msg = "%d out of %d extensions installed (%d queued, %d running: %s)"
-                installed_cnt, queued_cnt, running_cnt = len(installed_ext_names), len(exts_queue), len(running_exts)
-                if running_cnt <= 3:
-                    running_ext_names = ', '.join(x.name for x in running_exts)
-                else:
-                    running_ext_names = ', '.join(x.name for x in running_exts[:3]) + ", ..."
-                print_msg(msg % (installed_cnt, exts_cnt, queued_cnt, running_cnt, running_ext_names), log=self.log)
-
             # try to start as many extension installations as we can, taking into account number of available cores,
             # but only consider first 100 extensions still in the queue
             max_iter = min(100, len(exts_queue))
@@ -1836,6 +1826,16 @@ def update_exts_progress_bar_helper(running_exts, progress_size):
                         self.log.info("Started installation of extension %s in the background...", ext.name)
                         update_exts_progress_bar_helper(running_exts, 0)
 
+            # print progress info after every iteration (unless that info is already shown via progress bar)
+            if not show_progress_bars():
+                msg = "%d out of %d extensions installed (%d queued, %d running: %s)"
+                installed_cnt, queued_cnt, running_cnt = len(installed_ext_names), len(exts_queue), len(running_exts)
+                if running_cnt <= 3:
+                    running_ext_names = ', '.join(x.name for x in running_exts)
+                else:
+                    running_ext_names = ', '.join(x.name for x in running_exts[:3]) + ", ..."
+                print_msg(msg % (installed_cnt, exts_cnt, queued_cnt, running_cnt, running_ext_names), log=self.log)
+
     #
     # MISCELLANEOUS UTILITY FUNCTIONS
     #

From 3e186fb061c7e862e9966be31e959125b6a477a1 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Tue, 26 Oct 2021 20:09:48 +0200
Subject: [PATCH 156/175] return True in Extension.async_cmd_check if
 async_cmd_info is set to False, which indicates that no asynchronous command
 was started

---
 easybuild/framework/extension.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py
index 251eed6afe..54ea39f544 100644
--- a/easybuild/framework/extension.py
+++ b/easybuild/framework/extension.py
@@ -200,6 +200,9 @@ def async_cmd_check(self):
         """
         if self.async_cmd_info is None:
             raise EasyBuildError("No installation command running asynchronously for %s", self.name)
+        elif self.async_cmd_info is False:
+            self.log.info("No asynchronous command was started for extension %s", self.name)
+            return True
         else:
             self.log.debug("Checking on installation of extension %s...", self.name)
             # use small read size, to avoid waiting for a long time until sufficient output is produced
@@ -207,6 +210,7 @@ def async_cmd_check(self):
             self.async_cmd_output += res['output']
             if res['done']:
                 self.log.info("Installation of extension %s completed!", self.name)
+                self.async_cmd_info = None
             else:
                 self.async_cmd_check_cnt += 1
                 self.log.debug("Installation of extension %s still running (checked %d times)",

From 0dd9061e65edbe7611e5b0d9a32aaf5637c942ac Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Tue, 26 Oct 2021 20:27:04 +0200
Subject: [PATCH 157/175] add test for installing extensions in parallel

---
 .../easyblocks/generic/toy_extension.py       | 54 ++++++++++++++++---
 .../sandbox/easybuild/easyblocks/t/toy.py     | 54 +++++++++++++++----
 test/framework/toy_build.py                   | 39 ++++++++++++++
 3 files changed, 131 insertions(+), 16 deletions(-)

diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py
index bbb792e7ee..603346efe0 100644
--- a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py
+++ b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py
@@ -30,7 +30,8 @@
 
 from easybuild.framework.easyconfig import CUSTOM
 from easybuild.framework.extensioneasyblock import ExtensionEasyBlock
-from easybuild.easyblocks.toy import EB_toy
+from easybuild.easyblocks.toy import EB_toy, compose_toy_build_cmd
+from easybuild.tools.build_log import EasyBuildError
 from easybuild.tools.run import run_cmd
 
 
@@ -45,20 +46,59 @@ def extra_options():
         }
         return ExtensionEasyBlock.extra_options(extra_vars=extra_vars)
 
-    def run(self):
-        """Build toy extension."""
+    @property
+    def required_deps(self):
+        """Return list of required dependencies for this extension."""
+        deps = {
+            'bar': [],
+            'barbar': ['bar'],
+            'ls': [],
+        }
+        if self.name in deps:
+            return deps[self.name]
+        else:
+            raise EasyBuildError("Dependencies for %s are unknown!", self.name)
+
+    def run(self, *args, **kwargs):
+        """
+        Install toy extension.
+        """
         if self.src:
-            super(Toy_Extension, self).run(unpack_src=True)
-            EB_toy.configure_step(self.master, name=self.name)
             EB_toy.build_step(self.master, name=self.name, buildopts=self.cfg['buildopts'])
 
             if self.cfg['toy_ext_param']:
                 run_cmd(self.cfg['toy_ext_param'])
 
-            EB_toy.install_step(self.master, name=self.name)
-
             return self.module_generator.set_environment('TOY_EXT_%s' % self.name.upper(), self.name)
 
+    def prerun(self):
+        """
+        Prepare installation of toy extension.
+        """
+        super(Toy_Extension, self).prerun()
+
+        if self.src:
+            super(Toy_Extension, self).run(unpack_src=True)
+            EB_toy.configure_step(self.master, name=self.name)
+
+    def run_async(self):
+        """
+        Install toy extension asynchronously.
+        """
+        if self.src:
+            cmd = compose_toy_build_cmd(self.cfg, self.name, self.cfg['prebuildopts'], self.cfg['buildopts'])
+            self.async_cmd_start(cmd)
+        else:
+            self.async_cmd_info = False
+
+    def postrun(self):
+        """
+        Wrap up installation of toy extension.
+        """
+        super(Toy_Extension, self).postrun()
+
+        EB_toy.install_step(self.master, name=self.name)
+
     def sanity_check_step(self, *args, **kwargs):
         """Custom sanity check for toy extensions."""
         self.log.info("Loaded modules: %s", self.modules_tool.list())
diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toy.py b/test/framework/sandbox/easybuild/easyblocks/t/toy.py
index f9a9f7a8c5..bec0e7fe42 100644
--- a/test/framework/sandbox/easybuild/easyblocks/t/toy.py
+++ b/test/framework/sandbox/easybuild/easyblocks/t/toy.py
@@ -41,6 +41,19 @@
 from easybuild.tools.run import run_cmd
 
 
+def compose_toy_build_cmd(cfg, name, prebuildopts, buildopts):
+    """
+    Compose command to build toy.
+    """
+
+    cmd = "%(prebuildopts)s gcc %(name)s.c -o %(name)s %(buildopts)s" % {
+        'name': name,
+        'prebuildopts': prebuildopts,
+        'buildopts': buildopts,
+    }
+    return cmd
+
+
 class EB_toy(ExtensionEasyBlock):
     """Support for building/installing toy."""
 
@@ -92,17 +105,13 @@ def configure_step(self, name=None):
 
     def build_step(self, name=None, buildopts=None):
         """Build toy."""
-
         if buildopts is None:
             buildopts = self.cfg['buildopts']
-
         if name is None:
             name = self.name
-        run_cmd('%(prebuildopts)s gcc %(name)s.c -o %(name)s %(buildopts)s' % {
-            'name': name,
-            'prebuildopts': self.cfg['prebuildopts'],
-            'buildopts': buildopts,
-        })
+
+        cmd = compose_toy_build_cmd(self.cfg, name, self.cfg['prebuildopts'], buildopts)
+        run_cmd(cmd)
 
     def install_step(self, name=None):
         """Install toy."""
@@ -118,11 +127,38 @@ def install_step(self, name=None):
         mkdir(libdir, parents=True)
         write_file(os.path.join(libdir, 'lib%s.a' % name), name.upper())
 
-    def run(self):
-        """Install toy as extension."""
+    @property
+    def required_deps(self):
+        """Return list of required dependencies for this extension."""
+        if self.name == 'toy':
+            return ['bar', 'barbar']
+        else:
+            raise EasyBuildError("Dependencies for %s are unknown!", self.name)
+
+    def prerun(self):
+        """
+        Prepare installation of toy as extension.
+        """
         super(EB_toy, self).run(unpack_src=True)
         self.configure_step()
+
+    def run(self):
+        """
+        Install toy as extension.
+        """
         self.build_step()
+
+    def run_async(self):
+        """
+        Asynchronous installation of toy as extension.
+        """
+        cmd = compose_toy_build_cmd(self.cfg, self.name, self.cfg['prebuildopts'], self.cfg['buildopts'])
+        self.async_cmd_start(cmd)
+
+    def postrun(self):
+        """
+        Wrap up installation of toy as extension.
+        """
         self.install_step()
 
     def make_module_step(self, fake=False):
diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py
index 1a68de2dcb..5b75116ee1 100644
--- a/test/framework/toy_build.py
+++ b/test/framework/toy_build.py
@@ -1777,6 +1777,45 @@ def test_module_only_extensions(self):
         self.eb_main([test_ec, '--module-only', '--force'], do_build=True, raise_error=True)
         self.assertTrue(os.path.exists(toy_mod))
 
+    def test_toy_exts_parallel(self):
+        """
+        Test parallel installation of extensions (--parallel-extensions-install)
+        """
+        topdir = os.path.abspath(os.path.dirname(__file__))
+        toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')
+
+        toy_mod = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0')
+        if get_module_syntax() == 'Lua':
+            toy_mod += '.lua'
+
+        test_ec = os.path.join(self.test_prefix, 'test.eb')
+        test_ec_txt = read_file(toy_ec)
+        test_ec_txt += '\n' + '\n'.join([
+            "exts_list = [",
+            "    ('ls'),",
+            "    ('bar', '0.0'),",
+            "    ('barbar', '0.0', {",
+            "        'start_dir': 'src',",
+            "    }),",
+            "    ('toy', '0.0'),",
+            "]",
+            "sanity_check_commands = ['barbar', 'toy']",
+            "sanity_check_paths = {'files': ['bin/barbar', 'bin/toy'], 'dirs': ['bin']}",
+        ])
+        write_file(test_ec, test_ec_txt)
+
+        args = ['--parallel-extensions-install', '--experimental', '--force']
+        stdout, stderr = self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args)
+        self.assertEqual(stderr, '')
+        expected_stdout = '\n'.join([
+            "== 0 out of 4 extensions installed (2 queued, 2 running: ls, bar)",
+            "== 2 out of 4 extensions installed (1 queued, 1 running: barbar)",
+            "== 3 out of 4 extensions installed (0 queued, 1 running: toy)",
+            "== 4 out of 4 extensions installed (0 queued, 0 running: )",
+            '',
+        ])
+        self.assertEqual(stdout, expected_stdout)
+
     def test_backup_modules(self):
         """Test use of backing up of modules with --module-only."""
 

From 0c7a7dbd8b1b4a5f576d03d008e8a8feffe8bf05 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Tue, 26 Oct 2021 22:51:26 +0200
Subject: [PATCH 158/175] tweak error when Extension.required_deps is not
 implemented

---
 easybuild/framework/extension.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py
index 54ea39f544..5333627c29 100644
--- a/easybuild/framework/extension.py
+++ b/easybuild/framework/extension.py
@@ -225,7 +225,7 @@ def async_cmd_check(self):
     @property
     def required_deps(self):
         """Return list of required dependencies for this extension."""
-        raise NotImplementedError("Don't know how to determine required dependencies for %s" % self.name)
+        raise NotImplementedError("Don't know how to determine required dependencies for extension '%s'" % self.name)
 
     @property
     def toolchain(self):

From 3179ddbab7cc8dfa3c0a74444dfad01b5c8cb9ac Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Tue, 26 Oct 2021 22:51:43 +0200
Subject: [PATCH 159/175] remove unused iter_id in
 EasyBlock.install_extensions_parallel

---
 easybuild/framework/easyblock.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index 58f4303aa3..d41801c3e9 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -1754,11 +1754,8 @@ def update_exts_progress_bar_helper(running_exts, progress_size):
 
             self.update_exts_progress_bar(progress_info, progress_size=progress_size)
 
-        iter_id = 0
         while exts_queue or running_exts:
 
-            iter_id += 1
-
             # always go back to original work dir to avoid running stuff from a dir that no longer exists
             change_dir(self.orig_workdir)
 

From be818bf832f4d18854b44d9eeab2f9018a7d6d83 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 27 Oct 2021 08:47:13 +0200
Subject: [PATCH 160/175] fix typo in warning message on suppressing duplicate
 paths in append_paths/prepend_paths in ModuleGenerator (+ add test to check
 for warning message)

---
 easybuild/tools/module_generator.py |  2 +-
 test/framework/module_generator.py  | 22 ++++++++++++++++++++++
 2 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py
index 59444b273e..ef34ca62d3 100644
--- a/easybuild/tools/module_generator.py
+++ b/easybuild/tools/module_generator.py
@@ -226,7 +226,7 @@ def _filter_paths(self, key, paths):
             filtered_paths = [x for x in paths if x not in added_paths and not added_paths.add(x)]
         if filtered_paths != paths:
             removed_paths = paths if filtered_paths is None else [x for x in paths if x not in filtered_paths]
-            print_warning("Supressed adding the following path(s) to $%s of the module as they were already added: %s",
+            print_warning("Suppressed adding the following path(s) to $%s of the module as they were already added: %s",
                           key, removed_paths,
                           log=self.log)
             if not filtered_paths:
diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py
index c046180883..998fbddd70 100644
--- a/test/framework/module_generator.py
+++ b/test/framework/module_generator.py
@@ -719,6 +719,17 @@ def append_paths(*args, **kwargs):
                                               "which only expects relative paths." % self.modgen.app.installdir,
                               append_paths, "key2", ["bar", "%s/foo" % self.modgen.app.installdir])
 
+        # check for warning that is printed when same path is added multiple times
+        with self.modgen.start_module_creation():
+            self.modgen.append_paths('TEST', 'path1')
+            self.mock_stderr(True)
+            self.modgen.append_paths('TEST', 'path1')
+            stderr = self.get_stderr()
+            self.mock_stderr(False)
+            expected_warning = "\nWARNING: Suppressed adding the following path(s) to $TEST of the module "
+            expected_warning += "as they were already added: path1\n\n"
+            self.assertEqual(stderr, expected_warning)
+
     def test_module_extensions(self):
         """test the extensions() for extensions"""
         # not supported for Tcl modules
@@ -798,6 +809,17 @@ def prepend_paths(*args, **kwargs):
                                               "which only expects relative paths." % self.modgen.app.installdir,
                               prepend_paths, "key2", ["bar", "%s/foo" % self.modgen.app.installdir])
 
+        # check for warning that is printed when same path is added multiple times
+        with self.modgen.start_module_creation():
+            self.modgen.prepend_paths('TEST', 'path1')
+            self.mock_stderr(True)
+            self.modgen.prepend_paths('TEST', 'path1')
+            stderr = self.get_stderr()
+            self.mock_stderr(False)
+            expected_warning = "\nWARNING: Suppressed adding the following path(s) to $TEST of the module "
+            expected_warning += "as they were already added: path1\n\n"
+            self.assertEqual(stderr, expected_warning)
+
     def test_det_user_modpath(self):
         """Test for generic det_user_modpath method."""
         # None by default

From 36f583ab25a165467d4d0dd00133b536ad549dcb Mon Sep 17 00:00:00 2001
From: Alan O'Cais <a.ocais@fz-juelich.de>
Date: Wed, 27 Oct 2021 12:09:24 +0200
Subject: [PATCH 161/175] Make sure correct include directory is used for
 FlexiBLAS

---
 easybuild/toolchains/linalg/flexiblas.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/easybuild/toolchains/linalg/flexiblas.py b/easybuild/toolchains/linalg/flexiblas.py
index 26250570b1..c77476218f 100644
--- a/easybuild/toolchains/linalg/flexiblas.py
+++ b/easybuild/toolchains/linalg/flexiblas.py
@@ -27,6 +27,7 @@
 
 :author: Kenneth Hoste (Ghent University)
 """
+import os
 import re
 
 from easybuild.tools.toolchain.linalg import LinAlg
@@ -67,10 +68,12 @@ class FlexiBLAS(LinAlg):
     """
     BLAS_MODULE_NAME = ['FlexiBLAS']
     BLAS_LIB = ['flexiblas']
+    BLAS_INCLUDE_DIR = [os.path.join('include', 'flexiblas')]
     BLAS_FAMILY = TC_CONSTANT_FLEXIBLAS
 
     LAPACK_MODULE_NAME = ['FlexiBLAS']
     LAPACK_IS_BLAS = True
+    LAPACK_INCLUDE_DIR = [os.path.join('include', 'flexiblas')]
     LAPACK_FAMILY = TC_CONSTANT_FLEXIBLAS
 
     def banned_linked_shared_libs(self):

From cd8951dfb99d816afb69d361bf55e76a834c57c4 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 27 Oct 2021 13:55:45 +0200
Subject: [PATCH 162/175] deprecate old toolchain versions: GCC(core) < 8.0,
 iccifort < 2019.0, gompi/iimpi/iompi/foss/intel/iomkl < 2019

---
 easybuild/toolchains/foss.py     |  8 +++-----
 easybuild/toolchains/gcc.py      | 13 +++++++++++++
 easybuild/toolchains/gcccore.py  | 13 +++++++++++++
 easybuild/toolchains/gompi.py    | 12 +++---------
 easybuild/toolchains/iccifort.py |  7 ++++---
 easybuild/toolchains/iimpi.py    | 11 ++++-------
 easybuild/toolchains/intel.py    |  8 +++-----
 easybuild/toolchains/iomkl.py    | 18 ++++++++++++++++++
 easybuild/toolchains/iompi.py    | 16 ++++++++++++++++
 9 files changed, 77 insertions(+), 29 deletions(-)

diff --git a/easybuild/toolchains/foss.py b/easybuild/toolchains/foss.py
index e8e093ae09..d4bfbb671b 100644
--- a/easybuild/toolchains/foss.py
+++ b/easybuild/toolchains/foss.py
@@ -46,7 +46,7 @@ def __init__(self, *args, **kwargs):
         """Toolchain constructor."""
         super(Foss, self).__init__(*args, **kwargs)
 
-        # need to transform a version like '2016a' with something that is safe to compare with '2000'
+        # need to transform a version like '2018b' with something that is safe to compare with '2019'
         # comparing subversions that include letters causes TypeErrors in Python 3
         # 'a' is assumed to be equivalent with '.01' (January), and 'b' with '.07' (June) (good enough for this purpose)
         version = self.version.replace('a', '.01').replace('b', '.07')
@@ -84,10 +84,8 @@ def banned_linked_shared_libs(self):
     def is_deprecated(self):
         """Return whether or not this toolchain is deprecated."""
 
-        # foss toolchains older than foss/2016a are deprecated
-        # take into account that foss/2016.x is always < foss/2016a according to LooseVersion;
-        # foss/2016.01 & co are not deprecated yet...
-        if self.looseversion < LooseVersion('2016.01'):
+        # foss toolchains older than foss/2019a are deprecated since EasyBuild v4.5.0;
+        if self.looseversion < LooseVersion('2019'):
             deprecated = True
         else:
             deprecated = False
diff --git a/easybuild/toolchains/gcc.py b/easybuild/toolchains/gcc.py
index 30f3891f58..ab9d58774b 100644
--- a/easybuild/toolchains/gcc.py
+++ b/easybuild/toolchains/gcc.py
@@ -27,6 +27,8 @@
 
 :author: Kenneth Hoste (Ghent University)
 """
+from distutils.version import LooseVersion
+import re
 
 from easybuild.toolchains.gcccore import GCCcore
 from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME
@@ -38,3 +40,14 @@ class GccToolchain(GCCcore):
     COMPILER_MODULE_NAME = [NAME]
     SUBTOOLCHAIN = [GCCcore.NAME, SYSTEM_TOOLCHAIN_NAME]
     OPTIONAL = False
+
+    def is_deprecated(self):
+        """Return whether or not this toolchain is deprecated."""
+        # GCC toolchains older than GCC version 8.x are deprecated since EasyBuild v4.5.0
+        # make sure a non-symbolic version (e.g., 'system') is used before making comparisons using LooseVersion
+        if re.match('^[0-9]', self.version) and LooseVersion(self.version) < LooseVersion('8.0'):
+            deprecated = True
+        else:
+            deprecated = False
+
+        return deprecated
diff --git a/easybuild/toolchains/gcccore.py b/easybuild/toolchains/gcccore.py
index a95f0dcbdb..49a190ca28 100644
--- a/easybuild/toolchains/gcccore.py
+++ b/easybuild/toolchains/gcccore.py
@@ -27,6 +27,8 @@
 
 :author: Kenneth Hoste (Ghent University)
 """
+from distutils.version import LooseVersion
+import re
 
 from easybuild.toolchains.compiler.gcc import Gcc
 from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME
@@ -41,3 +43,14 @@ class GCCcore(Gcc):
     # GCCcore is only guaranteed to be present in recent toolchains
     # for old versions of some toolchains (GCC, intel) it is not part of the hierarchy and hence optional
     OPTIONAL = True
+
+    def is_deprecated(self):
+        """Return whether or not this toolchain is deprecated."""
+        # GCC toolchains older than GCC version 8.x are deprecated since EasyBuild v4.5.0
+        # make sure a non-symbolic version (e.g., 'system') is used before making comparisons using LooseVersion
+        if re.match('^[0-9]', self.version) and LooseVersion(self.version) < LooseVersion('8.0'):
+            deprecated = True
+        else:
+            deprecated = False
+
+        return deprecated
diff --git a/easybuild/toolchains/gompi.py b/easybuild/toolchains/gompi.py
index a5bbc4c7b9..f9ed1ab1bc 100644
--- a/easybuild/toolchains/gompi.py
+++ b/easybuild/toolchains/gompi.py
@@ -41,7 +41,7 @@ class Gompi(GccToolchain, OpenMPI):
 
     def is_deprecated(self):
         """Return whether or not this toolchain is deprecated."""
-        # need to transform a version like '2016a' with something that is safe to compare with '2000'
+        # need to transform a version like '2018b' with something that is safe to compare with '2019'
         # comparing subversions that include letters causes TypeErrors in Python 3
         # 'a' is assumed to be equivalent with '.01' (January), and 'b' with '.07' (June) (good enough for this purpose)
         version = self.version.replace('a', '.01').replace('b', '.07')
@@ -50,14 +50,8 @@ def is_deprecated(self):
 
         # make sure a non-symbolic version (e.g., 'system') is used before making comparisons using LooseVersion
         if re.match('^[0-9]', version):
-            gompi_ver = LooseVersion(version)
-            # deprecate oldest gompi toolchains (versions 1.x)
-            if gompi_ver < LooseVersion('2000'):
-                deprecated = True
-            # gompi toolchains older than gompi/2016a are deprecated
-            # take into account that gompi/2016.x is always < gompi/2016a according to LooseVersion;
-            # gompi/2016.01 & co are not deprecated yet...
-            elif gompi_ver < LooseVersion('2016.01'):
+            # gompi toolchains older than gompi/2019a are deprecated since EasyBuild v4.5.0
+            if LooseVersion(version) < LooseVersion('2019'):
                 deprecated = True
 
         return deprecated
diff --git a/easybuild/toolchains/iccifort.py b/easybuild/toolchains/iccifort.py
index ee6f39846c..cb44c0408b 100644
--- a/easybuild/toolchains/iccifort.py
+++ b/easybuild/toolchains/iccifort.py
@@ -50,14 +50,15 @@ class IccIfort(IntelIccIfort):
     def is_deprecated(self):
         """Return whether or not this toolchain is deprecated."""
 
-        # need to transform a version like '2016a' with something that is safe to compare with '2016.01'
+        # need to transform a version like '2018b' with something that is safe to compare with '2019.0'
         # comparing subversions that include letters causes TypeErrors in Python 3
         # 'a' is assumed to be equivalent with '.01' (January), and 'b' with '.07' (June) (good enough for this purpose)
         version = self.version.replace('a', '.01').replace('b', '.07')
 
-        # iccifort toolchains older than iccifort/2016.1.150-* are deprecated
+        # iccifort toolchains older than iccifort/2019.0.117-* are deprecated;
+        # note: intel/2019a uses iccifort 2019.1.144;
         # make sure a non-symbolic version (e.g., 'system') is used before making comparisons using LooseVersion
-        if re.match('^[0-9]', version) and LooseVersion(version) < LooseVersion('2016.1'):
+        if re.match('^[0-9]', version) and LooseVersion(version) < LooseVersion('2019.0'):
             deprecated = True
         else:
             deprecated = False
diff --git a/easybuild/toolchains/iimpi.py b/easybuild/toolchains/iimpi.py
index 0a2104872b..f89d17cc3a 100644
--- a/easybuild/toolchains/iimpi.py
+++ b/easybuild/toolchains/iimpi.py
@@ -53,11 +53,12 @@ def __init__(self, *args, **kwargs):
 
         # make sure a non-symbolic version (e.g., 'system') is used before making comparisons using LooseVersion
         if re.match('^[0-9]', self.version):
-            # need to transform a version like '2016a' with something that is safe to compare with '8.0', '2016.01'
+            # need to transform a version like '2018b' with something that is safe to compare with '2019'
             # comparing subversions that include letters causes TypeErrors in Python 3
             # 'a' is assumed to be equivalent with '.01' (January), and 'b' with '.07' (June)
             # (good enough for this purpose)
             self.iimpi_ver = self.version.replace('a', '.01').replace('b', '.07')
+
             if LooseVersion(self.iimpi_ver) >= LooseVersion('2020.12'):
                 self.oneapi_gen = True
                 self.SUBTOOLCHAIN = IntelCompilersToolchain.NAME
@@ -77,12 +78,8 @@ def is_deprecated(self):
 
         # make sure a non-symbolic version (e.g., 'system') is used before making comparisons using LooseVersion
         if re.match('^[0-9]', str(self.iimpi_ver)):
-            loosever = LooseVersion(self.iimpi_ver)
-            # iimpi toolchains older than iimpi/2016.01 are deprecated
-            # iimpi 8.1.5 is an exception, since it used in intel/2016a (which is not deprecated yet)
-            if loosever < LooseVersion('8.0'):
-                deprecated = True
-            elif loosever > LooseVersion('2000') and loosever < LooseVersion('2016.01'):
+            # iimpi toolchains older than iimpi/2019a are deprecated since EasyBuild v4.5.0
+            if LooseVersion(self.iimpi_ver) < LooseVersion('2019'):
                 deprecated = True
 
         return deprecated
diff --git a/easybuild/toolchains/intel.py b/easybuild/toolchains/intel.py
index bb0550baee..3639f7bca1 100644
--- a/easybuild/toolchains/intel.py
+++ b/easybuild/toolchains/intel.py
@@ -48,16 +48,14 @@ class Intel(Iimpi, IntelMKL, IntelFFTW):
 
     def is_deprecated(self):
         """Return whether or not this toolchain is deprecated."""
-        # need to transform a version like '2016a' with something that is safe to compare with '2016.01'
+        # need to transform a version like '2018b' with something that is safe to compare with '2019'
         # comparing subversions that include letters causes TypeErrors in Python 3
         # 'a' is assumed to be equivalent with '.01' (January), and 'b' with '.07' (June) (good enough for this purpose)
         version = self.version.replace('a', '.01').replace('b', '.07')
 
-        # intel toolchains older than intel/2016a are deprecated
-        # take into account that intel/2016.x is always < intel/2016a according to LooseVersion;
-        # intel/2016.01 & co are not deprecated yet...
+        # intel toolchains older than intel/2019a are deprecated since EasyBuild v4.5.0
         # make sure a non-symbolic version (e.g., 'system') is used before making comparisons using LooseVersion
-        if re.match('^[0-9]', version) and LooseVersion(version) < LooseVersion('2016.01'):
+        if re.match('^[0-9]', version) and LooseVersion(version) < LooseVersion('2019'):
             deprecated = True
         else:
             deprecated = False
diff --git a/easybuild/toolchains/iomkl.py b/easybuild/toolchains/iomkl.py
index bc68ae7b24..6c0b0e01a4 100644
--- a/easybuild/toolchains/iomkl.py
+++ b/easybuild/toolchains/iomkl.py
@@ -29,6 +29,8 @@
 :author: Stijn De Weirdt (Ghent University)
 :author: Kenneth Hoste (Ghent University)
 """
+from distutils.version import LooseVersion
+import re
 
 from easybuild.toolchains.iompi import Iompi
 from easybuild.toolchains.iimkl import Iimkl
@@ -43,3 +45,19 @@ class Iomkl(Iompi, IntelMKL, IntelFFTW):
     """
     NAME = 'iomkl'
     SUBTOOLCHAIN = [Iompi.NAME, Iimkl.NAME]
+
+    def is_deprecated(self):
+        """Return whether or not this toolchain is deprecated."""
+        # need to transform a version like '2018b' with something that is safe to compare with '2019'
+        # comparing subversions that include letters causes TypeErrors in Python 3
+        # 'a' is assumed to be equivalent with '.01' (January), and 'b' with '.07' (June) (good enough for this purpose)
+        version = self.version.replace('a', '.01').replace('b', '.07')
+
+        # iomkl toolchains older than iomkl/2019a are deprecated since EasyBuild v4.5.0
+        # make sure a non-symbolic version (e.g., 'system') is used before making comparisons using LooseVersion
+        if re.match('^[0-9]', version) and LooseVersion(version) < LooseVersion('2019'):
+            deprecated = True
+        else:
+            deprecated = False
+
+        return deprecated
diff --git a/easybuild/toolchains/iompi.py b/easybuild/toolchains/iompi.py
index 05664cb484..e3c82cb907 100644
--- a/easybuild/toolchains/iompi.py
+++ b/easybuild/toolchains/iompi.py
@@ -92,3 +92,19 @@ def set_variables(self):
             IntelCompilersToolchain.set_variables(self)
         else:
             IccIfort.set_variables(self)
+
+    def is_deprecated(self):
+        """Return whether or not this toolchain is deprecated."""
+        # need to transform a version like '2018b' with something that is safe to compare with '2019'
+        # comparing subversions that include letters causes TypeErrors in Python 3
+        # 'a' is assumed to be equivalent with '.01' (January), and 'b' with '.07' (June) (good enough for this purpose)
+        version = self.version.replace('a', '.01').replace('b', '.07')
+
+        # iompi toolchains older than iompi/2019a are deprecated since EasyBuild v4.5.0
+        # make sure a non-symbolic version (e.g., 'system') is used before making comparisons using LooseVersion
+        if re.match('^[0-9]', version) and LooseVersion(version) < LooseVersion('2019'):
+            deprecated = True
+        else:
+            deprecated = False
+
+        return deprecated

From dece32de5c13642423f13b19f9229acd6a363214 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 27 Oct 2021 15:22:37 +0200
Subject: [PATCH 163/175] add --unit-testing-mode configuration option, to
 allow use of deprecated toolchain versions when running EasyBuild framework
 test suite

---
 easybuild/framework/easyconfig/easyconfig.py | 5 ++++-
 easybuild/tools/config.py                    | 1 +
 easybuild/tools/options.py                   | 1 +
 test/framework/utilities.py                  | 7 ++++++-
 4 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py
index 0cbcb11ef3..3dc255e117 100644
--- a/easybuild/framework/easyconfig/easyconfig.py
+++ b/easybuild/framework/easyconfig/easyconfig.py
@@ -837,7 +837,10 @@ def check_deprecated(self, path):
                 raise EasyBuildError("Wrong type for value of 'deprecated' easyconfig parameter: %s", type(deprecated))
 
         if self.toolchain.is_deprecated():
-            depr_msgs.append("toolchain '%(name)s/%(version)s' is marked as deprecated" % self['toolchain'])
+            # allow use of deprecated toolchains when running unit tests,
+            # because test easyconfigs/modules often use old toolchain versions (and updating them is far from trivial)
+            if not build_option('unit_testing_mode'):
+                depr_msgs.append("toolchain '%(name)s/%(version)s' is marked as deprecated" % self['toolchain'])
 
         if depr_msgs:
             depr_msg = ', '.join(depr_msgs)
diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py
index 7b669764f4..a66f688215 100644
--- a/easybuild/tools/config.py
+++ b/easybuild/tools/config.py
@@ -285,6 +285,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
         'generate_devel_module',
         'sticky_bit',
         'trace',
+        'unit_testing_mode',
         'upload_test_report',
         'update_modules_tool_cache',
         'use_ccache',
diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py
index 5ef43c952c..ea6fab61c7 100644
--- a/easybuild/tools/options.py
+++ b/easybuild/tools/options.py
@@ -489,6 +489,7 @@ def override_options(self):
                       None, 'store', None),
             'update-modules-tool-cache': ("Update modules tool cache file(s) after generating module file",
                                           None, 'store_true', False),
+            'unit-testing-mode': ("Run in unit test mode", None, 'store_true', False),
             'use-ccache': ("Enable use of ccache to speed up compilation, with specified cache dir",
                            str, 'store', False, {'metavar': "PATH"}),
             'use-f90cache': ("Enable use of f90cache to speed up compilation, with specified cache dir",
diff --git a/test/framework/utilities.py b/test/framework/utilities.py
index 14248243d3..f105b7b81f 100644
--- a/test/framework/utilities.py
+++ b/test/framework/utilities.py
@@ -289,6 +289,10 @@ def eb_main(self, args, do_build=False, return_error=False, logfile=None, verbos
         """Helper method to call EasyBuild main function."""
         cleanup()
 
+        # always run main in unit testing mode (which for example allows for using deprecated toolchains);
+        # note: don't change 'args' value, which is passed by reference!
+        main_args = args + ['--unit-testing-mode']
+
         myerr = False
         if logfile is None:
             logfile = self.logfile
@@ -306,7 +310,7 @@ def eb_main(self, args, do_build=False, return_error=False, logfile=None, verbos
                 modtool = None
             else:
                 modtool = self.modtool
-            main(args=args, logfile=logfile, do_build=do_build, testing=testing, modtool=modtool)
+            main(args=main_args, logfile=logfile, do_build=do_build, testing=testing, modtool=modtool)
         except SystemExit as err:
             if raise_systemexit:
                 raise err
@@ -476,6 +480,7 @@ def init_config(args=None, build_options=None, with_include=True):
         'local_var_naming_check': 'error',
         'silence_deprecation_warnings': eb_go.options.silence_deprecation_warnings,
         'suffix_modules_path': GENERAL_CLASS,
+        'unit_testing_mode': True,
         'valid_module_classes': module_classes(),
         'valid_stops': [x[0] for x in EasyBlock.get_steps()],
     }

From 19bf460c0cc06a23e8ae4bb7061625aba201afc5 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 27 Oct 2021 19:42:51 +0200
Subject: [PATCH 164/175] restore resolving of non-broken symlinks in copy_file

---
 easybuild/tools/filetools.py | 12 ++++++++----
 test/framework/filetools.py  | 20 +++++++++++++++-----
 2 files changed, 23 insertions(+), 9 deletions(-)

diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py
index 148db6352a..99546c329d 100644
--- a/easybuild/tools/filetools.py
+++ b/easybuild/tools/filetools.py
@@ -2365,8 +2365,10 @@ def copy_file(path, target_path, force_in_dry_run=False):
         raise EasyBuildError("Could not copy '%s' it does not exist!", path)
     else:
         try:
+            # check whether path to copy exists (we could be copying a broken symlink, which is supported)
+            path_exists = os.path.exists(path)
             target_exists = os.path.exists(target_path)
-            if target_exists and os.path.samefile(path, target_path):
+            if target_exists and path_exists and os.path.samefile(path, target_path):
                 _log.debug("Not copying %s to %s since files are identical", path, target_path)
             # if target file exists and is owned by someone else than the current user,
             # try using shutil.copyfile to just copy the file contents
@@ -2376,7 +2378,10 @@ def copy_file(path, target_path, force_in_dry_run=False):
                 _log.info("Copied contents of file %s to %s", path, target_path)
             else:
                 mkdir(os.path.dirname(target_path), parents=True)
-                if os.path.islink(path):
+                if path_exists:
+                    shutil.copy2(path, target_path)
+                    _log.info("%s copied to %s", path, target_path)
+                elif os.path.islink(path):
                     if os.path.isdir(target_path):
                         target_path = os.path.join(target_path, os.path.basename(path))
                         _log.info("target_path changed to %s", target_path)
@@ -2385,8 +2390,7 @@ def copy_file(path, target_path, force_in_dry_run=False):
                     symlink(link_target, target_path, use_abspath_source=False)
                     _log.info("created symlink %s to %s", link_target, target_path)
                 else:
-                    shutil.copy2(path, target_path)
-                    _log.info("%s copied to %s", path, target_path)
+                    raise EasyBuildError("Specified path %s is not an existing file or a symbolic link!", path)
         except (IOError, OSError, shutil.Error) as err:
             raise EasyBuildError("Failed to copy file %s to %s: %s", path, target_path, err)
 
diff --git a/test/framework/filetools.py b/test/framework/filetools.py
index 4f22b69883..8c54f5881f 100644
--- a/test/framework/filetools.py
+++ b/test/framework/filetools.py
@@ -1739,13 +1739,23 @@ def test_copy_file(self):
         # Make sure it doesn't fail if path is a symlink and target_path is a dir
         toy_link_fn = 'toy-link-0.0.eb'
         toy_link = os.path.join(self.test_prefix, toy_link_fn)
-        ft.symlink(toy_ec, toy_link)
+        ft.symlink(target_path, toy_link)
         dir_target_path = os.path.join(self.test_prefix, 'subdir')
         ft.mkdir(dir_target_path)
         ft.copy_file(toy_link, dir_target_path)
-        self.assertTrue(os.path.islink(os.path.join(dir_target_path, toy_link_fn)))
+        copied_file = os.path.join(dir_target_path, toy_link_fn)
+        # symlinks that point to an existing file are resolved on copy (symlink itself is not copied)
+        self.assertTrue(os.path.exists(copied_file), "%s should exist" % copied_file)
+        self.assertTrue(os.path.isfile(copied_file), "%s should be a file" % copied_file)
+        ft.remove_file(copied_file)
+
+        # test copying of a broken symbolic link: copy_file should not fail, but copy it!
+        ft.remove_file(target_path)
+        ft.copy_file(toy_link, dir_target_path)
+        self.assertTrue(os.path.islink(copied_file), "%s should be a broken symbolic link" % copied_file)
+        self.assertFalse(os.path.exists(copied_file), "%s should be a broken symbolic link" % copied_file)
         self.assertEqual(os.readlink(os.path.join(dir_target_path, toy_link_fn)), os.readlink(toy_link))
-        os.remove(os.path.join(dir_target_path, toy_link))
+        ft.remove_file(copied_file)
 
         # clean error when trying to copy a directory with copy_file
         src, target = os.path.dirname(toy_ec), os.path.join(self.test_prefix, 'toy')
@@ -1781,8 +1791,8 @@ def test_copy_file(self):
         }
         init_config(build_options=build_options)
 
-        # remove target file, it shouldn't get copied under dry run
-        os.remove(target_path)
+        # make sure target file is not there, it shouldn't get copied under dry run
+        self.assertFalse(os.path.exists(target_path))
 
         self.mock_stdout(True)
         ft.copy_file(toy_ec, target_path)

From bfc96cb8b50353c62881a194a966081bf68cd2a1 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 27 Oct 2021 21:43:22 +0200
Subject: [PATCH 165/175] explictly disable rebase when pulling develop branch
 to create a merge commit, since not specifying how to reconcile divergent
 branches is an error with Git 2.33.1 and newer (fixes #3873)

---
 easybuild/tools/github.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py
index 5773d39a49..43fa6f07ae 100644
--- a/easybuild/tools/github.py
+++ b/easybuild/tools/github.py
@@ -2409,7 +2409,7 @@ def sync_with_develop(git_repo, branch_name, github_account, github_repo):
     remote = create_remote(git_repo, github_account, github_repo, https=True)
 
     # fetch latest version of develop branch
-    pull_out = git_repo.git.pull(remote.name, GITHUB_DEVELOP_BRANCH)
+    pull_out = git_repo.git.pull(remote.name, GITHUB_DEVELOP_BRANCH, no_rebase=True)
     _log.debug("Output of 'git pull %s %s': %s", remote.name, GITHUB_DEVELOP_BRANCH, pull_out)
 
     # fetch to make sure we can check out the 'develop' branch

From bb1a51cf0c94b5cda640df29b132689e8c4a2212 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Thu, 28 Oct 2021 16:54:01 +0200
Subject: [PATCH 166/175] make check for running failing command asynchronously
 more robust w.r.t. obtaining full output

---
 test/framework/run.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/framework/run.py b/test/framework/run.py
index da4448d741..6c298890d5 100644
--- a/test/framework/run.py
+++ b/test/framework/run.py
@@ -619,7 +619,7 @@ def test_run_cmd_async(self):
         res = check_async_cmd(*cmd_info, fail_on_error=False)
         # keep checking until command is fully done
         while not res['done']:
-            res = check_async_cmd(*cmd_info, fail_on_error=False)
+            res = check_async_cmd(*cmd_info, fail_on_error=False, output=res['output'])
         self.assertEqual(res, {'done': True, 'exit_code': 123, 'output': "FAIL!\n"})
 
         # also test with a command that produces a lot of output,

From 8d1d8c6518b3303d18455f775d0a2dea1c916ffc Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Thu, 28 Oct 2021 18:10:31 +0200
Subject: [PATCH 167/175] correct total count for extensions progress bar after
 skipping already installed extensions

---
 easybuild/framework/easyblock.py | 5 +++--
 easybuild/tools/output.py        | 4 +++-
 test/framework/output.py         | 1 +
 3 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index d41801c3e9..f298f4189c 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -1641,6 +1641,7 @@ def skip_extensions(self):
             self.update_exts_progress_bar("skipping installed extensions (%d/%d checked)" % (idx + 1, exts_cnt))
 
         self.ext_instances = res
+        self.update_exts_progress_bar("already installed extensions filtered out", total=len(self.ext_instances))
 
     def install_extensions(self, install=True):
         """
@@ -2573,11 +2574,11 @@ def init_ext_instances(self):
             pbar_label += "(%d/%d done)" % (idx + 1, exts_cnt)
             self.update_exts_progress_bar(pbar_label)
 
-    def update_exts_progress_bar(self, info, progress_size=0):
+    def update_exts_progress_bar(self, info, progress_size=0, total=None):
         """
         Update extensions progress bar with specified info and amount of progress made
         """
-        update_progress_bar(PROGRESS_BAR_EXTENSIONS, label=info, progress_size=progress_size)
+        update_progress_bar(PROGRESS_BAR_EXTENSIONS, label=info, progress_size=progress_size, total=total)
 
     def extensions_step(self, fetch=False, install=True):
         """
diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py
index 6882af3c6b..207a658569 100644
--- a/easybuild/tools/output.py
+++ b/easybuild/tools/output.py
@@ -286,7 +286,7 @@ def start_progress_bar(bar_type, size, label=None):
         pbar.update(task_id, description=label)
 
 
-def update_progress_bar(bar_type, label=None, progress_size=1):
+def update_progress_bar(bar_type, label=None, progress_size=1, total=None):
     """
     Update progress bar of given type (if it was started), add progress of given size.
 
@@ -300,6 +300,8 @@ def update_progress_bar(bar_type, label=None, progress_size=1):
             pbar.update(task_id, description=label)
         if progress_size:
             pbar.update(task_id, advance=progress_size)
+        if total:
+            pbar.update(task_id, total=total)
 
 
 def stop_progress_bar(bar_type, visible=False):
diff --git a/test/framework/output.py b/test/framework/output.py
index ea574d043a..fdcba7038f 100644
--- a/test/framework/output.py
+++ b/test/framework/output.py
@@ -178,6 +178,7 @@ def test_get_start_update_stop_progress_bar(self):
         # also test normal cycle: start, update, stop
         start_progress_bar(PROGRESS_BAR_EXTENSIONS, 100)
         update_progress_bar(PROGRESS_BAR_EXTENSIONS)  # single step progress
+        update_progress_bar(PROGRESS_BAR_EXTENSIONS, total=50)
         update_progress_bar(PROGRESS_BAR_EXTENSIONS, label="test123", progress_size=5)
         stop_progress_bar(PROGRESS_BAR_EXTENSIONS)
 

From fe9b9b1d300d9e1ef4e5edc4fbdf2de37639e85e Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Thu, 28 Oct 2021 19:17:49 +0200
Subject: [PATCH 168/175] don't try to exclude skipped steps from total step
 count (just always update easyconfig progress bar), self.skip_step only
 really works after check_readiness_step has been run...

---
 easybuild/framework/easyblock.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index f298f4189c..608ee0ec5a 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -3745,8 +3745,7 @@ def run_all_steps(self, run_test_cases):
         # figure out how many steps will actually be run (not be skipped)
         step_cnt = 0
         for (step_name, _, _, skippable) in steps:
-            if not self.skip_step(step_name, skippable):
-                step_cnt += 1
+            step_cnt += 1
             if self.cfg['stop'] == step_name:
                 break
 
@@ -3793,7 +3792,7 @@ def run_all_steps(self, run_test_cases):
                             elif self.logdebug or build_option('trace'):
                                 print_msg("... (took < 1 sec)", log=self.log, silent=self.silent)
 
-                    update_progress_bar(PROGRESS_BAR_EASYCONFIG)
+                update_progress_bar(PROGRESS_BAR_EASYCONFIG)
 
         except StopException:
             pass

From 5db0645dbc96106639db06760bb8e73985fbca16 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Thu, 28 Oct 2021 19:59:21 +0200
Subject: [PATCH 169/175] tweak test_http_header_fields_urlpat to download from
 sources.easybuild.io rather than https://ftp.gnu.org

---
 test/framework/options.py | 23 ++++++++++++++++-------
 1 file changed, 16 insertions(+), 7 deletions(-)

diff --git a/test/framework/options.py b/test/framework/options.py
index 4e78999742..4823d16f1e 100644
--- a/test/framework/options.py
+++ b/test/framework/options.py
@@ -2756,9 +2756,14 @@ def test_http_header_fields_urlpat(self):
         """Test use of --http-header-fields-urlpat."""
         tmpdir = tempfile.mkdtemp()
         test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
-        ec_file = os.path.join(test_ecs_dir, 'g', 'gzip', 'gzip-1.6-GCC-4.9.2.eb')
+        gzip_ec = os.path.join(test_ecs_dir, 'g', 'gzip', 'gzip-1.6-GCC-4.9.2.eb')
+        gzip_ec_txt = read_file(gzip_ec)
+        regex = re.compile('^source_urls = .*', re.M)
+        test_ec_txt = regex.sub("source_urls = ['https://sources.easybuild.io/g/gzip']", gzip_ec_txt)
+        test_ec = os.path.join(self.test_prefix, 'test.eb')
+        write_file(test_ec, test_ec_txt)
         common_args = [
-            ec_file,
+            test_ec,
             '--stop=fetch',
             '--debug',
             '--force',
@@ -2800,7 +2805,7 @@ def run_and_assert(args, msg, words_expected=None, words_unexpected=None):
         # A: simple direct case (all is logged because passed directly via EasyBuild configuration options)
         args = list(common_args)
         args.extend([
-            '--http-header-fields-urlpat=gnu.org::%s:%s' % (testdohdr, testdoval),
+            '--http-header-fields-urlpat=easybuild.io::%s:%s' % (testdohdr, testdoval),
             '--http-header-fields-urlpat=nomatch.com::%s:%s' % (testdonthdr, testdontval),
         ])
         # expect to find everything passed on cmdline
@@ -2813,7 +2818,7 @@ def run_and_assert(args, msg, words_expected=None, words_unexpected=None):
 
         # B: simple file case (secrets in file are not logged)
         txt = '\n'.join([
-            'gnu.org::%s: %s' % (testdohdr, testdoval),
+            'easybuild.io::%s: %s' % (testdohdr, testdoval),
             'nomatch.com::%s: %s' % (testdonthdr, testdontval),
             '',
         ])
@@ -2825,7 +2830,7 @@ def run_and_assert(args, msg, words_expected=None, words_unexpected=None):
 
         # C: recursion one: header value is another file
         txt = '\n'.join([
-            'gnu.org::%s: %s' % (testdohdr, testincfile),
+            'easybuild.io::%s: %s' % (testdohdr, testincfile),
             'nomatch.com::%s: %s' % (testdonthdr, testexcfile),
             '',
         ])
@@ -2839,7 +2844,11 @@ def run_and_assert(args, msg, words_expected=None, words_unexpected=None):
         run_and_assert(args, "case C", expected, not_expected)
 
         # D: recursion two: header field+value is another file,
-        write_file(testcmdfile, '\n'.join(['gnu.org::%s' % (testinchdrfile), 'nomatch.com::%s' % (testexchdrfile), '']))
+        write_file(testcmdfile, '\n'.join([
+            'easybuild.io::%s' % (testinchdrfile),
+            'nomatch.com::%s' % (testexchdrfile),
+            '',
+        ]))
         write_file(testinchdrfile, '%s: %s\n' % (testdohdr, testdoval))
         write_file(testexchdrfile, '%s: %s\n' % (testdonthdr, testdontval))
         # expect to find only the header key (and the literal filename) and only for the appropriate url
@@ -2851,7 +2860,7 @@ def run_and_assert(args, msg, words_expected=None, words_unexpected=None):
         # E: recursion three: url pattern + header field + value in another file
         write_file(testcmdfile, '%s\n' % (testurlpatfile))
         txt = '\n'.join([
-            'gnu.org::%s: %s' % (testdohdr, testdoval),
+            'easybuild.io::%s: %s' % (testdohdr, testdoval),
             'nomatch.com::%s: %s' % (testdonthdr, testdontval),
             '',
         ])

From 6e33e605a67a5a64fb50b1d430e42bd1dde5294e Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Thu, 28 Oct 2021 20:01:47 +0200
Subject: [PATCH 170/175] drop unused 'skippable' local variable when
 determining step count in run_all_steps

Co-authored-by: Simon Branford <4967+branfosj@users.noreply.github.com>
---
 easybuild/framework/easyblock.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index 608ee0ec5a..59bf0b7e6a 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -3744,7 +3744,7 @@ def run_all_steps(self, run_test_cases):
 
         # figure out how many steps will actually be run (not be skipped)
         step_cnt = 0
-        for (step_name, _, _, skippable) in steps:
+        for (step_name, _, _, _) in steps:
             step_cnt += 1
             if self.cfg['stop'] == step_name:
                 break

From ec3f5f6f111aae2c53bc23e8deca3eee86cf669f Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Thu, 28 Oct 2021 21:26:29 +0200
Subject: [PATCH 171/175] also deprecate old versions of toolchains that
 include CUDA component + gimpi

---
 easybuild/toolchains/gcccuda.py | 19 +++++++++++++++++++
 easybuild/toolchains/gimpi.py   | 19 +++++++++++++++++++
 easybuild/toolchains/iimpic.py  | 19 +++++++++++++++++++
 3 files changed, 57 insertions(+)

diff --git a/easybuild/toolchains/gcccuda.py b/easybuild/toolchains/gcccuda.py
index e7a1d5b8c8..2ac83412be 100644
--- a/easybuild/toolchains/gcccuda.py
+++ b/easybuild/toolchains/gcccuda.py
@@ -27,6 +27,8 @@
 
 :author: Kenneth Hoste (Ghent University)
 """
+import re
+from distutils.version import LooseVersion
 
 from easybuild.toolchains.compiler.cuda import Cuda
 from easybuild.toolchains.gcc import GccToolchain
@@ -38,3 +40,20 @@ class GccCUDA(GccToolchain, Cuda):
 
     COMPILER_MODULE_NAME = ['GCC', 'CUDA']
     SUBTOOLCHAIN = GccToolchain.NAME
+
+    def is_deprecated(self):
+        """Return whether or not this toolchain is deprecated."""
+        # need to transform a version like '2018b' with something that is safe to compare with '2019'
+        # comparing subversions that include letters causes TypeErrors in Python 3
+        # 'a' is assumed to be equivalent with '.01' (January), and 'b' with '.07' (June) (good enough for this purpose)
+        version = self.version.replace('a', '.01').replace('b', '.07')
+
+        deprecated = False
+
+        # make sure a non-symbolic version (e.g., 'system') is used before making comparisons using LooseVersion
+        if re.match('^[0-9]', version):
+            # gompi toolchains older than gompi/2019a are deprecated since EasyBuild v4.5.0
+            if LooseVersion(version) < LooseVersion('2019'):
+                deprecated = True
+
+        return deprecated
diff --git a/easybuild/toolchains/gimpi.py b/easybuild/toolchains/gimpi.py
index ce93b4acd2..8c42e3f9cc 100644
--- a/easybuild/toolchains/gimpi.py
+++ b/easybuild/toolchains/gimpi.py
@@ -28,6 +28,8 @@
 :author: Stijn De Weirdt (Ghent University)
 :author: Kenneth Hoste (Ghent University)
 """
+import re
+from distutils.version import LooseVersion
 
 from easybuild.toolchains.gcc import GccToolchain
 from easybuild.toolchains.mpi.intelmpi import IntelMPI
@@ -37,3 +39,20 @@ class Gimpi(GccToolchain, IntelMPI):
     """Compiler toolchain with GCC and Intel MPI."""
     NAME = 'gimpi'
     SUBTOOLCHAIN = GccToolchain.NAME
+
+    def is_deprecated(self):
+        """Return whether or not this toolchain is deprecated."""
+        # need to transform a version like '2018b' with something that is safe to compare with '2019'
+        # comparing subversions that include letters causes TypeErrors in Python 3
+        # 'a' is assumed to be equivalent with '.01' (January), and 'b' with '.07' (June) (good enough for this purpose)
+        version = self.version.replace('a', '.01').replace('b', '.07')
+
+        deprecated = False
+
+        # make sure a non-symbolic version (e.g., 'system') is used before making comparisons using LooseVersion
+        if re.match('^[0-9]', version):
+            # gompi toolchains older than gompi/2019a are deprecated since EasyBuild v4.5.0
+            if LooseVersion(version) < LooseVersion('2019'):
+                deprecated = True
+
+        return deprecated
diff --git a/easybuild/toolchains/iimpic.py b/easybuild/toolchains/iimpic.py
index 1940e20a30..c50bc7e0f7 100644
--- a/easybuild/toolchains/iimpic.py
+++ b/easybuild/toolchains/iimpic.py
@@ -27,6 +27,8 @@
 
 :author: Ake Sandgren (HPC2N)
 """
+import re
+from distutils.version import LooseVersion
 
 from easybuild.toolchains.iccifortcuda import IccIfortCUDA
 from easybuild.toolchains.mpi.intelmpi import IntelMPI
@@ -36,3 +38,20 @@ class Iimpic(IccIfortCUDA, IntelMPI):
     """Compiler toolchain with Intel compilers (icc/ifort), Intel MPI and CUDA."""
     NAME = 'iimpic'
     SUBTOOLCHAIN = IccIfortCUDA.NAME
+
+    def is_deprecated(self):
+        """Return whether or not this toolchain is deprecated."""
+        # need to transform a version like '2018b' with something that is safe to compare with '2019'
+        # comparing subversions that include letters causes TypeErrors in Python 3
+        # 'a' is assumed to be equivalent with '.01' (January), and 'b' with '.07' (June) (good enough for this purpose)
+        version = self.version.replace('a', '.01').replace('b', '.07')
+
+        deprecated = False
+
+        # make sure a non-symbolic version (e.g., 'system') is used before making comparisons using LooseVersion
+        if re.match('^[0-9]', version):
+            # gompi toolchains older than gompi/2019a are deprecated since EasyBuild v4.5.0
+            if LooseVersion(version) < LooseVersion('2019'):
+                deprecated = True
+
+        return deprecated

From 5ff67516dadd8fdb3c0c917ad4aa4cb59edea886 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Thu, 28 Oct 2021 21:38:14 +0200
Subject: [PATCH 172/175] fix typo in comments for gimpi and iimpic toolchains

Co-authored-by: Simon Branford <4967+branfosj@users.noreply.github.com>
---
 easybuild/toolchains/gimpi.py  | 2 +-
 easybuild/toolchains/iimpic.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/easybuild/toolchains/gimpi.py b/easybuild/toolchains/gimpi.py
index 8c42e3f9cc..56555cc42f 100644
--- a/easybuild/toolchains/gimpi.py
+++ b/easybuild/toolchains/gimpi.py
@@ -51,7 +51,7 @@ def is_deprecated(self):
 
         # make sure a non-symbolic version (e.g., 'system') is used before making comparisons using LooseVersion
         if re.match('^[0-9]', version):
-            # gompi toolchains older than gompi/2019a are deprecated since EasyBuild v4.5.0
+            # gimpi toolchains older than gimpi/2019a are deprecated since EasyBuild v4.5.0
             if LooseVersion(version) < LooseVersion('2019'):
                 deprecated = True
 
diff --git a/easybuild/toolchains/iimpic.py b/easybuild/toolchains/iimpic.py
index c50bc7e0f7..0673ffb81b 100644
--- a/easybuild/toolchains/iimpic.py
+++ b/easybuild/toolchains/iimpic.py
@@ -50,7 +50,7 @@ def is_deprecated(self):
 
         # make sure a non-symbolic version (e.g., 'system') is used before making comparisons using LooseVersion
         if re.match('^[0-9]', version):
-            # gompi toolchains older than gompi/2019a are deprecated since EasyBuild v4.5.0
+            # iimpic toolchains older than iimpic/2019a are deprecated since EasyBuild v4.5.0
             if LooseVersion(version) < LooseVersion('2019'):
                 deprecated = True
 

From 155f88d65f0c8bfedbf2d61d874a06f3cd82d4d1 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Thu, 28 Oct 2021 21:38:58 +0200
Subject: [PATCH 173/175] fix typo in comments for gcccuda toolchain

Co-authored-by: Simon Branford <4967+branfosj@users.noreply.github.com>
---
 easybuild/toolchains/gcccuda.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/easybuild/toolchains/gcccuda.py b/easybuild/toolchains/gcccuda.py
index 2ac83412be..6d2fa190ed 100644
--- a/easybuild/toolchains/gcccuda.py
+++ b/easybuild/toolchains/gcccuda.py
@@ -52,7 +52,7 @@ def is_deprecated(self):
 
         # make sure a non-symbolic version (e.g., 'system') is used before making comparisons using LooseVersion
         if re.match('^[0-9]', version):
-            # gompi toolchains older than gompi/2019a are deprecated since EasyBuild v4.5.0
+            # gcccuda toolchains older than gcccuda/2019a are deprecated since EasyBuild v4.5.0
             if LooseVersion(version) < LooseVersion('2019'):
                 deprecated = True
 

From 923ee946bd815f5640e60afee1355a734e02da42 Mon Sep 17 00:00:00 2001
From: Miguel Dias Costa <migueldiascosta@gmail.com>
Date: Fri, 29 Oct 2021 10:15:05 +0800
Subject: [PATCH 174/175] prepare release notes for EasyBuild v4.5.0 + bump
 version to 4.5.0

---
 RELEASE_NOTES              | 47 ++++++++++++++++++++++++++++++++++++++
 easybuild/tools/version.py |  2 +-
 2 files changed, 48 insertions(+), 1 deletion(-)

diff --git a/RELEASE_NOTES b/RELEASE_NOTES
index e622dd198b..eb1979fed9 100644
--- a/RELEASE_NOTES
+++ b/RELEASE_NOTES
@@ -4,6 +4,53 @@ For more detailed information, please see the git log.
 These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
 
 
+v4.5.0 (October 29th 2021)
+--------------------------
+
+feature release
+
+- various enhancements, including:
+  - add --review-pr-max and --review-pr-filter options to limit easyconfigs shown in multi-diff + retain order of easyconfigs being compared with (#3754)
+  - use Rich (if available) to show progress bar when installing easyconfigs (#3823)
+    - expand progress bar to full screen width (#3826)
+    - add --output-style configuration option, which can be used to disable use of Rich or type of any colored output (#3833)
+    - disable progress bars when running the tests to avoid messing up test suite output (#3835)
+    - use separate different progress bars for different aspects of the installations being performed (#3844)
+    - fixes & tweaks for overall & easyconfig progress bars (#3864)
+    - make update_progress_bar a bit more robust by just doing nothing if the corresponding progress bar was not started (and making stopping of a non-started progress bar fatal) (#3867)
+    - fix easyconfig + extensions progress bars when --skip is used (#3882)
+  - add support for checking required/optional EasyBuild dependencies via 'eb --check-eb-deps' (#3829)
+  - add support for collecting GPU info (via nvidia-smi), and include it in --show-system-info and test report (#3851)
+  - added support for --insecure-download configuration option (#3859)
+  - make intelfftw toolchain component aware of imkl-FFTW module (#3861)
+  - make sure the contrib/hooks tree is included in the distribution (#3862)
+  - add check_async_cmd function to facilitate checking on asynchronously running commands (#3865)
+  - add initial/experimental support for installing extensions in parallel (#3667)
+  - filter out duplicate paths added to module files (#3770)
+- various bug fixes, including:
+  - ensure that path configuration options have absolute path values (#3832)
+  - fix broken test by taking into account changed error raised by Python 3.9.7 when copying directory via shutil.copyfile (#3840)
+  - ensure newer location of CUDA stubs is taken into account by RPATH filter (#3850)
+  - replace which by command -v to avoid dependency on which (#3852)
+  - fix copy_file so it doesn't fail when copying a symbolic link if the target path is an existing directory (#3855)
+  - refactor EasyBlock to decouple collecting of information on extension source/patch files from downloading them (#3860)
+  - fix library paths to add to $LDFLAGS for intel-compilers toolchain component (#3866)
+  - remove '--depth 1' from git clone when 'commit' specified (#3871)
+  - enhance test_get_source_tarball_from_git to trigger fixed bug w.r.t. cloning depth (#3872)
+  - fix typo in warning message on suppressing duplicate paths in append_paths/prepend_paths in ModuleGenerator (+ add test to check for warning message) (#3874)
+  - make sure correct include directory is used for FlexiBLAS (#3875)
+  - restore resolving of non-broken symlinks in copy_file (#3877)
+  - clarify error message when determining required dependencies for extension fails (#3878)
+  - explictly disable rebase when pulling develop branch to create a merge commit, since not specifying how to reconcile divergent branches is an error with Git 2.33.1 and newer (#3879)
+  - make check for running failing command asynchronously more robust w.r.t. obtaining full output (#3881)
+  - tweak test_http_header_fields_urlpat to download from sources.easybuild.io rather than https://ftp.gnu.org (#3883)
+- other changes:
+  - change copy_file function to raise an error when trying to copy non-existing file (#3836)
+  - only print the hook messages if EasyBuild is running in debug mode (#3843)
+  - deprecate old toolchain versions (pre-2019a common toolchains) (#3876)
+  - also deprecate old versions of toolchains that include CUDA component + gimpi (#3884)
+
+
 v4.4.2 (September 7th 2021)
 ---------------------------
 
diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py
index 2216e1f42d..3aeeffa234 100644
--- a/easybuild/tools/version.py
+++ b/easybuild/tools/version.py
@@ -43,7 +43,7 @@
 # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like
 #   UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0'
 # This causes problems further up the dependency chain...
-VERSION = LooseVersion('4.4.3.dev0')
+VERSION = LooseVersion('4.5.0')
 UNKNOWN = 'UNKNOWN'
 
 

From 37c4c1b5193d5356b9017c78e3fa7be2c234233b Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Fri, 29 Oct 2021 08:53:52 +0200
Subject: [PATCH 175/175] tweak release notes for v4.5.0 release

---
 RELEASE_NOTES | 40 +++++++++++++++-------------------------
 1 file changed, 15 insertions(+), 25 deletions(-)

diff --git a/RELEASE_NOTES b/RELEASE_NOTES
index eb1979fed9..5fbbbd2ef9 100644
--- a/RELEASE_NOTES
+++ b/RELEASE_NOTES
@@ -10,45 +10,35 @@ v4.5.0 (October 29th 2021)
 feature release
 
 - various enhancements, including:
-  - add --review-pr-max and --review-pr-filter options to limit easyconfigs shown in multi-diff + retain order of easyconfigs being compared with (#3754)
-  - use Rich (if available) to show progress bar when installing easyconfigs (#3823)
-    - expand progress bar to full screen width (#3826)
-    - add --output-style configuration option, which can be used to disable use of Rich or type of any colored output (#3833)
-    - disable progress bars when running the tests to avoid messing up test suite output (#3835)
-    - use separate different progress bars for different aspects of the installations being performed (#3844)
-    - fixes & tweaks for overall & easyconfig progress bars (#3864)
-    - make update_progress_bar a bit more robust by just doing nothing if the corresponding progress bar was not started (and making stopping of a non-started progress bar fatal) (#3867)
-    - fix easyconfig + extensions progress bars when --skip is used (#3882)
+  - add --review-pr-max and --review-pr-filter options to limit easyconfigs used by --review-pr + retain order of easyconfigs being compared with (#3754)
+  - use Rich (if available) to show progress bars when installing easyconfigs (#3823, #3826, #3833, #3835, #3844, #3864, #3867, #3882)
+    - see also https://docs.easybuild.io/en/latest/Progress_bars.html
   - add support for checking required/optional EasyBuild dependencies via 'eb --check-eb-deps' (#3829)
   - add support for collecting GPU info (via nvidia-smi), and include it in --show-system-info and test report (#3851)
   - added support for --insecure-download configuration option (#3859)
   - make intelfftw toolchain component aware of imkl-FFTW module (#3861)
-  - make sure the contrib/hooks tree is included in the distribution (#3862)
-  - add check_async_cmd function to facilitate checking on asynchronously running commands (#3865)
-  - add initial/experimental support for installing extensions in parallel (#3667)
-  - filter out duplicate paths added to module files (#3770)
+  - make sure the contrib/hooks tree is included in the source tarball for easybuild-framework (#3862)
+  - add check_async_cmd function to facilitate checking on asynchronously running commands (#3865, #3881)
+  - add initial/experimental support for installing extensions in parallel (#3667, #3878)
+    - see also https://docs.easybuild.io/en/latest/Installing_extensions_in_parallel.html
+  - filter out duplicate paths added to module files, and print warning when they occur (#3770, #3874)
 - various bug fixes, including:
   - ensure that path configuration options have absolute path values (#3832)
   - fix broken test by taking into account changed error raised by Python 3.9.7 when copying directory via shutil.copyfile (#3840)
   - ensure newer location of CUDA stubs is taken into account by RPATH filter (#3850)
-  - replace which by command -v to avoid dependency on which (#3852)
-  - fix copy_file so it doesn't fail when copying a symbolic link if the target path is an existing directory (#3855)
+  - replace 'which' by 'command -v' in 'eb' wrapper script to avoid dependency on 'which' command (#3852)
   - refactor EasyBlock to decouple collecting of information on extension source/patch files from downloading them (#3860)
+    - in this context, the EasyBlock.fetch_extension_sources method was deprecated, and replaced by EasyBlock.collect_exts_file_info
   - fix library paths to add to $LDFLAGS for intel-compilers toolchain component (#3866)
-  - remove '--depth 1' from git clone when 'commit' specified (#3871)
-  - enhance test_get_source_tarball_from_git to trigger fixed bug w.r.t. cloning depth (#3872)
-  - fix typo in warning message on suppressing duplicate paths in append_paths/prepend_paths in ModuleGenerator (+ add test to check for warning message) (#3874)
-  - make sure correct include directory is used for FlexiBLAS (#3875)
-  - restore resolving of non-broken symlinks in copy_file (#3877)
-  - clarify error message when determining required dependencies for extension fails (#3878)
+  - remove '--depth 1' from git clone when 'commit' specified (#3871, #3872)
+  - make sure correct include directory is used for FlexiBLAS toolchain component (#3875)
   - explictly disable rebase when pulling develop branch to create a merge commit, since not specifying how to reconcile divergent branches is an error with Git 2.33.1 and newer (#3879)
-  - make check for running failing command asynchronously more robust w.r.t. obtaining full output (#3881)
   - tweak test_http_header_fields_urlpat to download from sources.easybuild.io rather than https://ftp.gnu.org (#3883)
 - other changes:
-  - change copy_file function to raise an error when trying to copy non-existing file (#3836)
+  - change copy_file function to raise an error when trying to copy non-existing file (#3836, #3855, #3877)
   - only print the hook messages if EasyBuild is running in debug mode (#3843)
-  - deprecate old toolchain versions (pre-2019a common toolchains) (#3876)
-  - also deprecate old versions of toolchains that include CUDA component + gimpi (#3884)
+  - deprecate old toolchain versions (pre-2019a common toolchains) (#3876, #3884)
+    - see also https://docs.easybuild.io/en/latest/Deprecated-easyconfigs.html#deprecated-toolchains
 
 
 v4.4.2 (September 7th 2021)