Merge pull request #3489 from easybuilders/4.3.x

release EasyBuild v4.3.1

.github/workflows/unit_tests.yml

@@ -6,7 +6,7 @@ jobs:
     runs-on: ubuntu-18.04
     strategy:
       matrix:
-        python: [2.7, 3.5, 3.6, 3.7, 3.8]
+        python: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9]
         modules_tool: [Lmod-7.8.22, Lmod-8.2.9, modules-tcl-1.147, modules-3.2.10, modules-4.1.4]
         module_syntax: [Lua, Tcl]
         # exclude some configuration for non-Lmod modules tool:
@@ -25,24 +25,30 @@ jobs:
             python: 3.7
           - modules_tool: modules-tcl-1.147
             python: 3.8
+          - modules_tool: modules-tcl-1.147
+            python: 3.9
           - modules_tool: modules-3.2.10
             python: 3.5
           - modules_tool: modules-3.2.10
             python: 3.7
           - modules_tool: modules-3.2.10
             python: 3.8
+          - modules_tool: modules-3.2.10
+            python: 3.9
           - modules_tool: modules-4.1.4
             python: 3.5
           - modules_tool: modules-4.1.4
             python: 3.7
           - modules_tool: modules-4.1.4
             python: 3.8
+          - modules_tool: modules-4.1.4
+            python: 3.9
       fail-fast: false
     steps:
     - uses: actions/checkout@v2
 
     - name: set up Python
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@v2
       with:
         python-version: ${{matrix.python}}
         architecture: x64
@@ -110,7 +116,7 @@ jobs:
           export PREFIX=/tmp/$USER/$GITHUB_SHA
           pip install --prefix $PREFIX dist/easybuild-framework*tar.gz
 
-    - name: run test suite & test bootstrap script
+    - name: run test suite
       env:
         EB_VERBOSE: 1
         EASYBUILD_MODULE_SYNTAX: ${{matrix.module_syntax}}
@@ -152,6 +158,25 @@ jobs:
           PRINTED_MSG=$(egrep -v "${IGNORE_PATTERNS}" test_framework_suite.log | grep '\.\n*[A-Za-z]' || true)
           test "x$PRINTED_MSG" = "x" || (echo "ERROR: Found printed messages in output of test suite\n${PRINTED_MSG}" && exit 1)
 
+    - name: test bootstrap script
+      # skip testing of bootstrap script with Python 3.9,
+      # until an EasyBuild release that is compatible with Python 3.9 is available
+      if: ${{ matrix.python != 3.9 }}
+      run: |
+          # (re)initialize environment for modules tool
+          if [ -f $HOME/moduleshome ]; then export MODULESHOME=$(cat $HOME/moduleshome); fi
+          source $(cat $HOME/mod_init); type module
+          # also pick up changes to $PATH set by sourcing $HOME/mod_init
+          export PATH=$(cat $HOME/path)
+
+          # define $EASYBUILD_MODULES_TOOL only for oldest module tools
+          # (for Lmod and EnvironmentModules 4.x the bootstrap script should correctly auto-detect the modules tool)
+          if [[ ${{matrix.modules_tool}} =~ ^modules-tcl- ]]; then
+            export EASYBUILD_MODULES_TOOL=EnvironmentModulesTcl
+          elif [[ ${{matrix.modules_tool}} =~ ^modules-3 ]]; then
+            export EASYBUILD_MODULES_TOOL=EnvironmentModulesC
+          fi
+
           # version and SHA256 checksum are hardcoded below to avoid forgetting to update the version in the script along with contents
           EB_BOOTSTRAP_VERSION=$(grep '^EB_BOOTSTRAP_VERSION' easybuild/scripts/bootstrap_eb.py | sed 's/[^0-9.]//g')
           EB_BOOTSTRAP_SHA256SUM=$(sha256sum easybuild/scripts/bootstrap_eb.py | cut -f1 -d' ')

RELEASE_NOTES

@@ -3,6 +3,33 @@ For more detailed information, please see the git log.
 
 These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
 
+v4.3.1 (October 29th 2020)
+--------------------------
+
+update/bugfix release
+
+- various enhancements, including:
+  - further GCC toolchain fixes for aarch64 (#3433)
+  - take into account --include-easyblocks-from-pr when uploading test reports (#3446)
+  - add path to pkg-config files in sysroot to $PKG_CONFIG_PATH when --sysroot is specified (#3451)
+  - add support for NVHPC compiler + toolchain (based on PGI) (#3454)
+  - check for _VERSION and _PREFIX Cray environment variables with both software and module name (#3460)
+  - allow including easyblocks from multiple PRs (#3480, #3481)
+- various bug fixes, including:
+  - avoid UnicodeDecodeError in apply_regex_substitutions when patching files that include non-UTF-8 characters (#3450)
+  - avoid appending lib stubs pattern to RPATH filter over and over again (#3452)
+  - fix missing string template on error for incorrect extension 'sources' value (#3461)
+  - fix compatibility with Python 3.9 by renaming fancy root logger (#3465)
+  - also remove empty checksums list specified in easyconfig file when using --inject-checksums (#3466)
+  - avoid confusing error log message when determining impi version while trying to define value for %(mpi_cmd_prefix)s template (#3474)
+  - unset $LD_LIBRARY_PATH when checking for OS dependencies with 'rpm' & co (#3477)
+  - don't change directory in download_repo function in tools.github (#3486)
+  - take source_urls, checksums, patches into account when extension source is specified via 'sources' (#3487)
+- other changes:
+  - consider $EB_INSTALLPYTHON in 'eb' command to specify 'python' command to use for running EasyBuild (#3428)
+  - use only the sub folder name for createSubmoduleDeps script (#3464)
+
+
 v4.3.0 (September 13th 2020)
 ----------------------------
 

easybuild/base/fancylogger.py

@@ -456,7 +456,7 @@ def getLogger(name=None, fname=False, clsname=False, fancyrecord=None):
     if not is_fancyroot():
         # deliberately not calling getRootLoggerName function to determine actual root logger name,
         # because it is prohibitively expensive in some texts (even when using 'python -O')
-        nameparts.append('root')
+        nameparts.append('fancyroot')
 
     if fancyrecord is None:
         # Altough we could set it as default value in the function definition

easybuild/framework/easyblock.py

@@ -502,6 +502,8 @@ def fetch_extension_sources(self, skip_checksums=False):
         if self.dry_run:
             self.dry_run_msg("\nList of sources/patches for extensions:")
 
+        force_download = build_option('force_download') in [FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_SOURCES]
+
         for ext in exts_list:
             if (isinstance(ext, list) or isinstance(ext, tuple)) and ext:
 
@@ -539,88 +541,110 @@ def fetch_extension_sources(self, skip_checksums=False):
                     # resolve templates in extension options
                     ext_options = resolve_template(ext_options, template_values)
 
+                    source_urls = ext_options.get('source_urls', [])
                     checksums = ext_options.get('checksums', [])
 
-                    # use default template for name of source file if none is specified
-                    default_source_tmpl = resolve_template('%(name)s-%(version)s.tar.gz', template_values)
-                    fn = ext_options.get('source_tmpl', default_source_tmpl)
-
                     if ext_options.get('nosource', None):
-                        exts_sources.append(ext_src)
+                        self.log.debug("No sources for extension %s, as indicated by 'nosource'", ext_name)
 
                     elif ext_options.get('sources', None):
                         sources = ext_options['sources']
 
+                        # only a single source file is supported for extensions currently,
+                        # see https://github.com/easybuilders/easybuild-framework/issues/3463
                         if isinstance(sources, list):
                             if len(sources) == 1:
                                 source = sources[0]
                             else:
-                                error_msg = "'sources' spec for %s in exts_list must be single element list"
+                                error_msg = "'sources' spec for %s in exts_list must be single element list. Is: %s"
                                 raise EasyBuildError(error_msg, ext_name, sources)
                         else:
                             source = sources
 
+                        # always pass source spec as dict value to fetch_source method,
+                        # mostly so we can inject stuff like source URLs
+                        if isinstance(source, string_type):
+                            source = {'filename': source}
+                        elif not isinstance(source, dict):
+                            raise EasyBuildError("Incorrect value type for source of extension %s: %s",
+                                                 ext_name, source)
+
+                        # if no custom source URLs are specified in sources spec,
+                        # inject the ones specified for this extension
+                        if 'source_urls' not in source:
+                            source['source_urls'] = source_urls
+
                         src = self.fetch_source(source, checksums, extension=True)
-                        # Copy 'path' entry to 'src' for use with extensions
-                        ext_src.update({'src': src['path']})
-                        exts_sources.append(ext_src)
-                    else:
-                        source_urls = ext_options.get('source_urls', [])
-                        force_download = build_option('force_download') in [FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_SOURCES]
 
-                        src_fn = self.obtain_file(fn, extension=True, urls=source_urls, force_download=force_download)
+                        # copy 'path' entry to 'src' for use with extensions
+                        ext_src.update({'src': src['path']})
 
-                        if src_fn:
-                            ext_src.update({'src': src_fn})
+                    else:
+                        # use default template for name of source file if none is specified
+                        default_source_tmpl = resolve_template('%(name)s-%(version)s.tar.gz', template_values)
+
+                        # if no sources are specified via 'sources', fall back to 'source_tmpl'
+                        src_fn = ext_options.get('source_tmpl', default_source_tmpl)
+                        src_path = self.obtain_file(src_fn, extension=True, urls=source_urls,
+                                                    force_download=force_download)
+                        if src_path:
+                            ext_src.update({'src': src_path})
+                        else:
+                            raise EasyBuildError("Source for extension %s not found.", ext)
 
-                            if not skip_checksums:
-                                # report both MD5 and SHA256 checksums, since both are valid default checksum types
+                    # verify checksum for extension sources
+                    if 'src' in ext_src and not skip_checksums:
+                        src_path = ext_src['src']
+                        src_fn = os.path.basename(src_path)
+
+                        # report both MD5 and SHA256 checksums, since both are valid default checksum types
+                        for checksum_type in (CHECKSUM_TYPE_MD5, CHECKSUM_TYPE_SHA256):
+                            src_checksum = compute_checksum(src_path, checksum_type=checksum_type)
+                            self.log.info("%s checksum for %s: %s", checksum_type, src_path, src_checksum)
+
+                        # verify checksum (if provided)
+                        self.log.debug('Verifying checksums for extension source...')
+                        fn_checksum = self.get_checksum_for(checksums, index=0)
+                        if verify_checksum(src_path, fn_checksum):
+                            self.log.info('Checksum for extension source %s verified', src_fn)
+                        elif build_option('ignore_checksums'):
+                            print_warning("Ignoring failing checksum verification for %s" % src_fn)
+                        else:
+                            raise EasyBuildError('Checksum verification for extension source %s failed', src_fn)
+
+                    # locate extension patches (if any), and verify checksums
+                    ext_patches = self.fetch_patches(patch_specs=ext_options.get('patches', []), extension=True)
+                    if ext_patches:
+                        self.log.debug('Found patches for extension %s: %s', ext_name, ext_patches)
+                        ext_src.update({'patches': ext_patches})
+
+                        if not skip_checksums:
+                            for patch in ext_patches:
+                                patch = patch['path']
+                                # report both MD5 and SHA256 checksums,
+                                # since both are valid default checksum types
                                 for checksum_type in (CHECKSUM_TYPE_MD5, CHECKSUM_TYPE_SHA256):
-                                    src_checksum = compute_checksum(src_fn, checksum_type=checksum_type)
-                                    self.log.info("%s checksum for %s: %s", checksum_type, src_fn, src_checksum)
-
-                                # verify checksum (if provided)
-                                self.log.debug('Verifying checksums for extension source...')
-                                fn_checksum = self.get_checksum_for(checksums, index=0)
-                                if verify_checksum(src_fn, fn_checksum):
-                                    self.log.info('Checksum for extension source %s verified', fn)
+                                    checksum = compute_checksum(patch, checksum_type=checksum_type)
+                                    self.log.info("%s checksum for %s: %s", checksum_type, patch, checksum)
+
+                            # verify checksum (if provided)
+                            self.log.debug('Verifying checksums for extension patches...')
+                            for idx, patch in enumerate(ext_patches):
+                                patch = patch['path']
+                                patch_fn = os.path.basename(patch)
+
+                                checksum = self.get_checksum_for(checksums[1:], index=idx)
+                                if verify_checksum(patch, checksum):
+                                    self.log.info('Checksum for extension patch %s verified', patch_fn)
                                 elif build_option('ignore_checksums'):
-                                    print_warning("Ignoring failing checksum verification for %s" % fn)
+                                    print_warning("Ignoring failing checksum verification for %s" % patch_fn)
                                 else:
-                                    raise EasyBuildError('Checksum verification for extension source %s failed', fn)
-
-                            ext_patches = self.fetch_patches(patch_specs=ext_options.get('patches', []), extension=True)
-                            if ext_patches:
-                                self.log.debug('Found patches for extension %s: %s' % (ext_name, ext_patches))
-                                ext_src.update({'patches': ext_patches})
-
-                                if not skip_checksums:
-                                    for patch in ext_patches:
-                                        patch = patch['path']
-                                        # report both MD5 and SHA256 checksums,
-                                        # since both are valid default checksum types
-                                        for checksum_type in (CHECKSUM_TYPE_MD5, CHECKSUM_TYPE_SHA256):
-                                            checksum = compute_checksum(patch, checksum_type=checksum_type)
-                                            self.log.info("%s checksum for %s: %s", checksum_type, patch, checksum)
-
-                                    # verify checksum (if provided)
-                                    self.log.debug('Verifying checksums for extension patches...')
-                                    for idx, patch in enumerate(ext_patches):
-                                        patch = patch['path']
-                                        checksum = self.get_checksum_for(checksums[1:], index=idx)
-                                        if verify_checksum(patch, checksum):
-                                            self.log.info('Checksum for extension patch %s verified', patch)
-                                        elif build_option('ignore_checksums'):
-                                            print_warning("Ignoring failing checksum verification for %s" % patch)
-                                        else:
-                                            raise EasyBuildError('Checksum for extension patch %s failed', patch)
-                            else:
-                                self.log.debug('No patches found for extension %s.' % ext_name)
-
-                            exts_sources.append(ext_src)
+                                    raise EasyBuildError("Checksum verification for extension patch %s failed",
+                                                         patch_fn)
+                    else:
+                        self.log.debug('No patches found for extension %s.' % ext_name)
 
-                        else:
-                            raise EasyBuildError("Source for extension %s not found.", ext)
+                    exts_sources.append(ext_src)
 
             elif isinstance(ext, string_type):
                 exts_sources.append({'name': ext})
@@ -3657,11 +3681,11 @@ def make_checksum_lines(checksums, indent_level):
 
         checksums_txt = '\n'.join(checksum_lines)
 
-        # if any checksums were provided before, get rid of them
-        if app.cfg['checksums']:
+        # if 'checksums' is specified in easyconfig file, get rid of it (even if it's just an empty list)
+        checksums_regex = re.compile(r'^checksums(?:.|\n)+?\]\s*$', re.M)
+        if checksums_regex.search(ectxt):
             _log.debug("Removing existing 'checksums' easyconfig parameter definition...")
-            regex = re.compile(r'^checksums(?:.|\n)+?\]\s*$', re.M)
-            ectxt = regex.sub('', ectxt)
+            ectxt = checksums_regex.sub('', ectxt)
 
         # it is possible no sources (and hence patches) are listed, e.g. for 'bundle' easyconfigs
         if app.src:

easybuild/framework/easyconfig/easyconfig.py

@@ -1251,9 +1251,9 @@ def probe_external_module_metadata(self, mod_name, existing_metadata=None):
                 soft_name = soft_name[len(cray_prefix):]
 
         # determine software name to use in names of environment variables (upper case, '-' becomes '_')
-        soft_name_in_mod_name = convert_name(soft_name.replace('-', '_'), upper=True)
+        soft_name_env_var_infix = convert_name(soft_name.replace('-', '_'), upper=True)
 
-        var_name_pairs = [
+        var_name_pairs_templates = [
             ('CRAY_%s_PREFIX', 'CRAY_%s_VERSION'),
             ('CRAY_%s_PREFIX_DIR', 'CRAY_%s_VERSION'),
             ('CRAY_%s_DIR', 'CRAY_%s_VERSION'),
@@ -1264,10 +1264,20 @@ def probe_external_module_metadata(self, mod_name, existing_metadata=None):
             ('%s_HOME', '%s_VERSION'),
         ]
 
-        for prefix_var_name, version_var_name in var_name_pairs:
-            prefix_var_name = prefix_var_name % soft_name_in_mod_name
-            version_var_name = version_var_name % soft_name_in_mod_name
+        def mk_var_name_pair(var_name_pair, name):
+            """Complete variable name pair template using provided name."""
+            return (var_name_pair[0] % name, var_name_pair[1] % name)
+
+        var_name_pairs = [mk_var_name_pair(x, soft_name_env_var_infix) for x in var_name_pairs_templates]
+
+        # also consider name based on module name for environment variables to check
+        # for example, for the cray-netcdf-hdf5parallel module we should also check $CRAY_NETCDF_HDF5PARALLEL_VERSION
+        mod_name_env_var_infix = convert_name(mod_name.split('/')[0].replace('-', '_'), upper=True)
 
+        if mod_name_env_var_infix != soft_name_env_var_infix:
+            var_name_pairs.extend([mk_var_name_pair(x, mod_name_env_var_infix) for x in var_name_pairs_templates])
+
+        for prefix_var_name, version_var_name in var_name_pairs:
             prefix = self.modules_tool.get_setenv_value_from_modulefile(mod_name, prefix_var_name)
             version = self.modules_tool.get_setenv_value_from_modulefile(mod_name, version_var_name)