Merge pull request #3886 from easybuilders/4.5.x

release EasyBuild v4.5.0

RELEASE_NOTES

@@ -4,6 +4,43 @@ For more detailed information, please see the git log.
 These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
 
 
+v4.5.0 (October 29th 2021)
+--------------------------
+
+feature release
+
+- various enhancements, including:
+  - add --review-pr-max and --review-pr-filter options to limit easyconfigs used by --review-pr + retain order of easyconfigs being compared with (#3754)
+  - use Rich (if available) to show progress bars when installing easyconfigs (#3823, #3826, #3833, #3835, #3844, #3864, #3867, #3882)
+    - see also https://docs.easybuild.io/en/latest/Progress_bars.html
+  - add support for checking required/optional EasyBuild dependencies via 'eb --check-eb-deps' (#3829)
+  - add support for collecting GPU info (via nvidia-smi), and include it in --show-system-info and test report (#3851)
+  - added support for --insecure-download configuration option (#3859)
+  - make intelfftw toolchain component aware of imkl-FFTW module (#3861)
+  - make sure the contrib/hooks tree is included in the source tarball for easybuild-framework (#3862)
+  - add check_async_cmd function to facilitate checking on asynchronously running commands (#3865, #3881)
+  - add initial/experimental support for installing extensions in parallel (#3667, #3878)
+    - see also https://docs.easybuild.io/en/latest/Installing_extensions_in_parallel.html
+  - filter out duplicate paths added to module files, and print warning when they occur (#3770, #3874)
+- various bug fixes, including:
+  - ensure that path configuration options have absolute path values (#3832)
+  - fix broken test by taking into account changed error raised by Python 3.9.7 when copying directory via shutil.copyfile (#3840)
+  - ensure newer location of CUDA stubs is taken into account by RPATH filter (#3850)
+  - replace 'which' by 'command -v' in 'eb' wrapper script to avoid dependency on 'which' command (#3852)
+  - refactor EasyBlock to decouple collecting of information on extension source/patch files from downloading them (#3860)
+    - in this context, the EasyBlock.fetch_extension_sources method was deprecated, and replaced by EasyBlock.collect_exts_file_info
+  - fix library paths to add to $LDFLAGS for intel-compilers toolchain component (#3866)
+  - remove '--depth 1' from git clone when 'commit' specified (#3871, #3872)
+  - make sure correct include directory is used for FlexiBLAS toolchain component (#3875)
+  - explictly disable rebase when pulling develop branch to create a merge commit, since not specifying how to reconcile divergent branches is an error with Git 2.33.1 and newer (#3879)
+  - tweak test_http_header_fields_urlpat to download from sources.easybuild.io rather than https://ftp.gnu.org (#3883)
+- other changes:
+  - change copy_file function to raise an error when trying to copy non-existing file (#3836, #3855, #3877)
+  - only print the hook messages if EasyBuild is running in debug mode (#3843)
+  - deprecate old toolchain versions (pre-2019a common toolchains) (#3876, #3884)
+    - see also https://docs.easybuild.io/en/latest/Deprecated-easyconfigs.html#deprecated-toolchains
+
+
 v4.4.2 (September 7th 2021)
 ---------------------------
 

easybuild/framework/easyconfig/easyconfig.py

@@ -773,6 +773,28 @@ def remove_false_versions(deps):
         # indicate that this is a parsed easyconfig
         self._config['parsed'] = [True, "This is a parsed easyconfig", "HIDDEN"]
 
+    def count_files(self):
+        """
+        Determine number of files (sources + patches) required for this easyconfig.
+        """
+        cnt = len(self['sources']) + len(self['patches'])
+
+        for ext in self['exts_list']:
+            if isinstance(ext, tuple) and len(ext) >= 3:
+                ext_opts = ext[2]
+                # check for 'sources' first, since that's also considered first by EasyBlock.fetch_extension_sources
+                if 'sources' in ext_opts:
+                    cnt += len(ext_opts['sources'])
+                elif 'source_tmpl' in ext_opts:
+                    cnt += 1
+                else:
+                    # assume there's always one source file;
+                    # for extensions using PythonPackage, no 'source' or 'sources' may be specified
+                    cnt += 1
+                cnt += len(ext_opts.get('patches', []))
+
+        return cnt
+
     def local_var_naming(self, local_var_naming_check):
         """Deal with local variables that do not follow the recommended naming scheme (if any)."""
 
@@ -815,7 +837,10 @@ def check_deprecated(self, path):
                 raise EasyBuildError("Wrong type for value of 'deprecated' easyconfig parameter: %s", type(deprecated))
 
         if self.toolchain.is_deprecated():
-            depr_msgs.append("toolchain '%(name)s/%(version)s' is marked as deprecated" % self['toolchain'])
+            # allow use of deprecated toolchains when running unit tests,
+            # because test easyconfigs/modules often use old toolchain versions (and updating them is far from trivial)
+            if not build_option('unit_testing_mode'):
+                depr_msgs.append("toolchain '%(name)s/%(version)s' is marked as deprecated" % self['toolchain'])
 
         if depr_msgs:
             depr_msg = ', '.join(depr_msgs)

easybuild/framework/easyconfig/tools.py

@@ -457,7 +457,7 @@ def find_related_easyconfigs(path, ec):
         toolchain_pattern = ''
 
     potential_paths = [glob.glob(ec_path) for ec_path in create_paths(path, name, '*')]
-    potential_paths = sum(potential_paths, [])  # flatten
+    potential_paths = sorted(sum(potential_paths, []), reverse=True)  # flatten
     _log.debug("found these potential paths: %s" % potential_paths)
 
     parsed_version = LooseVersion(version).version
@@ -488,10 +488,10 @@ def find_related_easyconfigs(path, ec):
         else:
             _log.debug("No related easyconfigs in potential paths using '%s'" % regex)
 
-    return sorted(res)
+    return res
 
 
-def review_pr(paths=None, pr=None, colored=True, branch='develop', testing=False):
+def review_pr(paths=None, pr=None, colored=True, branch='develop', testing=False, max_ecs=None, filter_ecs=None):
     """
     Print multi-diff overview between specified easyconfigs or PR and specified branch.
     :param pr: pull request number in easybuild-easyconfigs repo to review
@@ -526,6 +526,10 @@ def review_pr(paths=None, pr=None, colored=True, branch='develop', testing=False
             pr_msg = "new PR"
         _log.debug("File in %s %s has these related easyconfigs: %s" % (pr_msg, ec['spec'], files))
         if files:
+            if filter_ecs is not None:
+                files = [x for x in files if filter_ecs.search(x)]
+            if max_ecs is not None:
+                files = files[:max_ecs]
             lines.append(multidiff(ec['spec'], files, colored=colored))
         else:
             lines.extend(['', "(no related easyconfigs found for %s)\n" % os.path.basename(ec['spec'])])

easybuild/framework/extension.py

@@ -40,7 +40,7 @@
 from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP, template_constant_dict
 from easybuild.tools.build_log import EasyBuildError, raise_nosupport
 from easybuild.tools.filetools import change_dir
-from easybuild.tools.run import run_cmd
+from easybuild.tools.run import check_async_cmd, run_cmd
 from easybuild.tools.py2vs3 import string_type
 
 
@@ -139,6 +139,12 @@ def __init__(self, mself, ext, extra_params=None):
                                key, name, version, value)
 
         self.sanity_check_fail_msgs = []
+        self.async_cmd_info = None
+        self.async_cmd_output = None
+        self.async_cmd_check_cnt = None
+        # initial read size should be relatively small,
+        # to avoid hanging for a long time until desired output is available in async_cmd_check
+        self.async_cmd_read_size = 1024
 
     @property
     def name(self):
@@ -160,18 +166,67 @@ def prerun(self):
         """
         pass
 
-    def run(self):
+    def run(self, *args, **kwargs):
         """
-        Actual installation of a extension.
+        Actual installation of an extension.
         """
         pass
 
+    def run_async(self, *args, **kwargs):
+        """
+        Asynchronous installation of an extension.
+        """
+        raise NotImplementedError
+
     def postrun(self):
         """
         Stuff to do after installing a extension.
         """
         self.master.run_post_install_commands(commands=self.cfg.get('postinstallcmds', []))
 
+    def async_cmd_start(self, cmd, inp=None):
+        """
+        Start installation asynchronously using specified command.
+        """
+        self.async_cmd_output = ''
+        self.async_cmd_check_cnt = 0
+        self.async_cmd_info = run_cmd(cmd, log_all=True, simple=False, inp=inp, regexp=False, asynchronous=True)
+
+    def async_cmd_check(self):
+        """
+        Check progress of installation command that was started asynchronously.
+
+        :return: True if command completed, False otherwise
+        """
+        if self.async_cmd_info is None:
+            raise EasyBuildError("No installation command running asynchronously for %s", self.name)
+        elif self.async_cmd_info is False:
+            self.log.info("No asynchronous command was started for extension %s", self.name)
+            return True
+        else:
+            self.log.debug("Checking on installation of extension %s...", self.name)
+            # use small read size, to avoid waiting for a long time until sufficient output is produced
+            res = check_async_cmd(*self.async_cmd_info, output_read_size=self.async_cmd_read_size)
+            self.async_cmd_output += res['output']
+            if res['done']:
+                self.log.info("Installation of extension %s completed!", self.name)
+                self.async_cmd_info = None
+            else:
+                self.async_cmd_check_cnt += 1
+                self.log.debug("Installation of extension %s still running (checked %d times)",
+                               self.name, self.async_cmd_check_cnt)
+                # increase read size after sufficient checks,
+                # to avoid that installation hangs due to output buffer filling up...
+                if self.async_cmd_check_cnt % 10 == 0 and self.async_cmd_read_size < (1024 ** 2):
+                    self.async_cmd_read_size *= 2
+
+            return res['done']
+
+    @property
+    def required_deps(self):
+        """Return list of required dependencies for this extension."""
+        raise NotImplementedError("Don't know how to determine required dependencies for extension '%s'" % self.name)
+
     @property
     def toolchain(self):
         """

easybuild/main.py

@@ -68,12 +68,16 @@
 from easybuild.tools.hooks import START, END, load_hooks, run_hook
 from easybuild.tools.modules import modules_tool
 from easybuild.tools.options import set_up_configuration, use_color
+from easybuild.tools.output import COLOR_GREEN, COLOR_RED, STATUS_BAR, colorize, print_checks, rich_live_cm
+from easybuild.tools.output import start_progress_bar, stop_progress_bar, update_progress_bar
 from easybuild.tools.robot import check_conflicts, dry_run, missing_deps, resolve_dependencies, search_easyconfigs
 from easybuild.tools.package.utilities import check_pkg_support
 from easybuild.tools.parallelbuild import submit_jobs
 from easybuild.tools.repository.repository import init_repository
+from easybuild.tools.systemtools import check_easybuild_deps
 from easybuild.tools.testing import create_test_report, overall_test_report, regtest, session_state
 
+
 _log = None
 
 
@@ -111,8 +115,14 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True):
     # e.g. via easyconfig.handle_allowed_system_deps
     init_env = copy.deepcopy(os.environ)
 
+    start_progress_bar(STATUS_BAR, size=len(ecs))
+
     res = []
+    ec_results = []
+    failed_cnt = 0
+
     for ec in ecs:
+
         ec_res = {}
         try:
             (ec_res['success'], app_log, err) = build_and_install_one(ec, init_env)
@@ -125,6 +135,12 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True):
             ec_res['err'] = err
             ec_res['traceback'] = traceback.format_exc()
 
+        if ec_res['success']:
+            ec_results.append(ec['full_mod_name'] + ' (' + colorize('OK', COLOR_GREEN) + ')')
+        else:
+            ec_results.append(ec['full_mod_name'] + ' (' + colorize('FAILED', COLOR_RED) + ')')
+            failed_cnt += 1
+
         # keep track of success/total count
         if ec_res['success']:
             test_msg = "Successfully built %s" % ec['spec']
@@ -154,6 +170,19 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True):
 
         res.append((ec, ec_res))
 
+        if failed_cnt:
+            # if installations failed: indicate th
+            status_label = ' (%s): ' % colorize('%s failed!' % failed_cnt, COLOR_RED)
+            failed_ecs = [x for x in ec_results[::-1] if 'FAILED' in x]
+            ok_ecs = [x for x in ec_results[::-1] if x not in failed_ecs]
+            status_label += ', '.join(failed_ecs + ok_ecs)
+        else:
+            status_label = ': ' + ', '.join(ec_results[::-1])
+
+        update_progress_bar(STATUS_BAR, label=status_label)
+
+    stop_progress_bar(STATUS_BAR)
+
     return res
 
 
@@ -245,6 +274,9 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
         search_easyconfigs(search_query, short=options.search_short, filename_only=options.search_filename,
                            terse=options.terse)
 
+    if options.check_eb_deps:
+        print_checks(check_easybuild_deps(modtool))
+
     # GitHub options that warrant a silent cleanup & exit
     if options.check_github:
         check_github()
@@ -262,7 +294,8 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
         merge_pr(options.merge_pr)
 
     elif options.review_pr:
-        print(review_pr(pr=options.review_pr, colored=use_color(options.color), testing=testing))
+        print(review_pr(pr=options.review_pr, colored=use_color(options.color), testing=testing,
+                        max_ecs=options.review_pr_max, filter_ecs=options.review_pr_filter))
 
     elif options.add_pr_labels:
         add_pr_labels(options.add_pr_labels)
@@ -283,6 +316,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
     # non-verbose cleanup after handling GitHub integration stuff or printing terse info
     early_stop_options = [
         options.add_pr_labels,
+        options.check_eb_deps,
         options.check_github,
         options.create_index,
         options.install_github_token,
@@ -521,7 +555,9 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
     if not testing or (testing and do_build):
         exit_on_failure = not (options.dump_test_report or options.upload_test_report)
 
-        ecs_with_res = build_and_install_software(ordered_ecs, init_session_state, exit_on_failure=exit_on_failure)
+        with rich_live_cm():
+            ecs_with_res = build_and_install_software(ordered_ecs, init_session_state,
+                                                      exit_on_failure=exit_on_failure)
     else:
         ecs_with_res = [(ec, {}) for ec in ordered_ecs]
 

easybuild/toolchains/compiler/intel_compilers.py

@@ -47,11 +47,11 @@ def _set_compiler_vars(self):
         Compiler._set_compiler_vars(self)
 
         root = self.get_software_root(self.COMPILER_MODULE_NAME)[0]
+        version = self.get_software_version(self.COMPILER_MODULE_NAME)[0]
 
+        libbase = os.path.join('compiler', version, 'linux')
         libpaths = [
-            'lib',
-            os.path.join('lib', 'x64'),
-            os.path.join('compiler', 'lib', 'intel64_lin'),
+            os.path.join(libbase, 'compiler', 'lib', 'intel64'),
         ]
 
         self.variables.append_subdirs("LDFLAGS", root, subdirs=libpaths)

easybuild/toolchains/fft/intelfftw.py

@@ -95,6 +95,12 @@ def _set_fftw_variables(self):
         # so make sure libraries are there before FFT_LIB is set
         imklroot = get_software_root(self.FFT_MODULE_NAME[0])
         fft_lib_dirs = [os.path.join(imklroot, d) for d in self.FFT_LIB_DIR]
+        imklfftwroot = get_software_root('imkl-FFTW')
+        if imklfftwroot:
+            # only get cluster_interface_lib from seperate module imkl-FFTW, rest via libmkl_gf/libmkl_intel
+            fft_lib_dirs += [os.path.join(imklfftwroot, 'lib')]
+            fftw_libs.remove(interface_lib)
+            fftw_mt_libs.remove(interface_lib)
 
         def fftw_lib_exists(libname):
             """Helper function to check whether FFTW library with specified name exists."""

easybuild/toolchains/foss.py

@@ -46,7 +46,7 @@ def __init__(self, *args, **kwargs):
         """Toolchain constructor."""
         super(Foss, self).__init__(*args, **kwargs)
 
-        # need to transform a version like '2016a' with something that is safe to compare with '2000'
+        # need to transform a version like '2018b' with something that is safe to compare with '2019'
         # comparing subversions that include letters causes TypeErrors in Python 3
         # 'a' is assumed to be equivalent with '.01' (January), and 'b' with '.07' (June) (good enough for this purpose)
         version = self.version.replace('a', '.01').replace('b', '.07')
@@ -84,10 +84,8 @@ def banned_linked_shared_libs(self):
     def is_deprecated(self):
         """Return whether or not this toolchain is deprecated."""
 
-        # foss toolchains older than foss/2016a are deprecated
-        # take into account that foss/2016.x is always < foss/2016a according to LooseVersion;
-        # foss/2016.01 & co are not deprecated yet...
-        if self.looseversion < LooseVersion('2016.01'):
+        # foss toolchains older than foss/2019a are deprecated since EasyBuild v4.5.0;
+        if self.looseversion < LooseVersion('2019'):
             deprecated = True
         else:
             deprecated = False

easybuild/toolchains/gcc.py

@@ -27,6 +27,8 @@
 
 :author: Kenneth Hoste (Ghent University)
 """
+from distutils.version import LooseVersion
+import re
 
 from easybuild.toolchains.gcccore import GCCcore
 from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME
@@ -38,3 +40,14 @@ class GccToolchain(GCCcore):
     COMPILER_MODULE_NAME = [NAME]
     SUBTOOLCHAIN = [GCCcore.NAME, SYSTEM_TOOLCHAIN_NAME]
     OPTIONAL = False
+
+    def is_deprecated(self):
+        """Return whether or not this toolchain is deprecated."""
+        # GCC toolchains older than GCC version 8.x are deprecated since EasyBuild v4.5.0
+        # make sure a non-symbolic version (e.g., 'system') is used before making comparisons using LooseVersion
+        if re.match('^[0-9]', self.version) and LooseVersion(self.version) < LooseVersion('8.0'):
+            deprecated = True
+        else:
+            deprecated = False
+
+        return deprecated