Merge pull request #3937 from easybuilders/4.5.x

release EasyBuild v4.5.2

.github/workflows/eb_command.yml

@@ -6,7 +6,7 @@ jobs:
     runs-on: ubuntu-18.04
     strategy:
       matrix:
-        python: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9]
+        python: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10']
       fail-fast: false
     steps:
     - uses: actions/checkout@v2

.github/workflows/unit_tests.yml

@@ -62,6 +62,12 @@ jobs:
           - python: 3.9
             modules_tool: ${{needs.setup.outputs.lmod8}}
             module_syntax: Tcl
+          - python: '3.10'
+            modules_tool: ${{needs.setup.outputs.lmod8}}
+            module_syntax: Lua
+          - python: '3.10'
+            modules_tool: ${{needs.setup.outputs.lmod8}}
+            module_syntax: Tcl
           # There may be encoding errors in Python 3 which are hidden when an UTF-8 encoding is set
           # Hence run the tests (again) with LC_ALL=C and Python 3.6 (or any < 3.7)
           - python: 3.6
@@ -110,13 +116,14 @@ jobs:
         GITHUB_TOKEN: ${{secrets.TEST_GITHUB_TOKEN}}
       run: |
         # don't install GitHub token when testing with Lmod 7.x or non-Lmod module tools,
+        # and only when testing with Lua as module syntax,
         # to avoid hitting GitHub rate limit;
         # tests that require a GitHub token are skipped automatically when no GitHub token is available
-        if [[ ! "${{matrix.modules_tool}}" =~ 'Lmod-7' ]] && [[ ! "${{matrix.modules_tool}}" =~ 'modules-' ]]; then
+        if [[ ! "${{matrix.modules_tool}}" =~ 'Lmod-7' ]] && [[ ! "${{matrix.modules_tool}}" =~ 'modules-' ]] && [[ "${{matrix.modules_syntax}}" == 'Lua' ]]; then
           if [ ! -z $GITHUB_TOKEN ]; then
             if [ "x${{matrix.python}}" == 'x2.6' ];
               then SET_KEYRING="keyring.set_keyring(keyring.backends.file.PlaintextKeyring())";
-              else SET_KEYRING="import keyrings; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())";
+              else SET_KEYRING="import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())";
             fi;
             python -c "import keyring; $SET_KEYRING; keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')";
           fi
@@ -193,7 +200,7 @@ jobs:
           # run test suite
           python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log
           # try and make sure output of running tests is clean (no printed messages/warnings)
-          IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.5|from cryptography.*default_backend|CryptographyDeprecationWarning: Python 2"
+          IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.5|from cryptography.*default_backend|CryptographyDeprecationWarning: Python 2|from cryptography.utils import int_from_bytes"
           # '|| true' is needed to avoid that Travis stops the job on non-zero exit of grep (i.e. when there are no matches)
           PRINTED_MSG=$(egrep -v "${IGNORE_PATTERNS}" test_framework_suite.log | grep '\.\n*[A-Za-z]' || true)
           test "x$PRINTED_MSG" = "x" || (echo "ERROR: Found printed messages in output of test suite\n${PRINTED_MSG}" && exit 1)

RELEASE_NOTES

@@ -4,6 +4,28 @@ For more detailed information, please see the git log.
 These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
 
 
+v4.5.2 (January 24th 2022)
+--------------------------
+
+update/bugfix release
+
+- various enhancements, including:
+  - automatically prepend value for env-for-shebang configuration setting with sysroot (#3919)
+  - enhance update_build_option to return original value + add support for update_build_options (#3923)
+  - fix compatibility with Python 3.10 (#3926, #3933)
+  - also take /etc/os-release into account in get_os_name and get_os_version (#3930)
+  - enhance get_cpu_architecture and get_cpu_family to be aware of RISC-V (#3931)
+- various bug fixes, including:
+  - relax pattern checks in test_toy_exts_parallel (#3921)
+  - use sources.easybuild.io as backup URL for downloading Environment Modules sources to run test suite (#3928)
+  - make intelfftw toolchain component aware of GCCcore (#3929)
+- other changes:
+  - deprecate use of (actual) patch files that don't have a filename ending with .patch (#3920)
+  - remove version restriction for keyring in requirements.txt to test with latest version (#3925)
+  - simplify requirements.txt by removing Python 2.6 support (#3927)
+  - only run GitHub tests when testing with Lua module syntax, to avoid hitting GitHub rate limit when running tests (#3938)
+
+
 v4.5.1 (December 13th 2021)
 ---------------------------
 

easybuild/base/frozendict.py

@@ -21,10 +21,10 @@
 It can be used as a drop-in replacement for dictionaries where immutability is desired.
 """
 import operator
-from collections import Mapping
 from functools import reduce
 
 from easybuild.base import fancylogger
+from easybuild.tools.py2vs3 import Mapping
 
 
 # minor adjustments:

easybuild/framework/easyblock.py

@@ -2758,14 +2758,24 @@ def package_step(self):
 
     def fix_shebang(self):
         """Fix shebang lines for specified files."""
+
+        env_for_shebang = build_option('env_for_shebang')
+        sysroot = build_option('sysroot')
+        if sysroot and not env_for_shebang.startswith(sysroot):
+            env_for_shebang = os.path.join(sysroot, env_for_shebang.lstrip('/'))
+        if os.path.exists(env_for_shebang.split(' ')[0]):
+            self.log.info("Path to 'env' command to use in patched shebang lines: %s", env_for_shebang)
+        else:
+            raise EasyBuildError("Path to 'env' command to use in shebang lines does not exist: %s", env_for_shebang)
+
         for lang in ['bash', 'perl', 'python']:
             shebang_regex = re.compile(r'^#![ ]*.*[/ ]%s.*' % lang)
             fix_shebang_for = self.cfg['fix_%s_shebang_for' % lang]
             if fix_shebang_for:
                 if isinstance(fix_shebang_for, string_type):
                     fix_shebang_for = [fix_shebang_for]
 
-                shebang = '#!%s %s' % (build_option('env_for_shebang'), lang)
+                shebang = '#!%s %s' % (env_for_shebang, lang)
                 for glob_pattern in fix_shebang_for:
                     paths = glob.glob(os.path.join(self.installdir, glob_pattern))
                     self.log.info("Fixing '%s' shebang to '%s' for files that match '%s': %s",

easybuild/scripts/install_eb_dep.sh

@@ -16,16 +16,16 @@ PKG_VERSION="${PKG##*-}"
 CONFIG_OPTIONS=
 PRECONFIG_CMD=
 
-if [ "$PKG_NAME" == 'modules' ] && [ "$PKG_VERSION" == '3.2.10' ]; then
+if [ "$PKG_NAME" == 'modules' ]; then
     PKG_URL="http://prdownloads.sourceforge.net/modules/${PKG}.tar.gz"
-    BACKUP_PKG_URL="https://easybuilders.github.io/easybuild/files/${PKG}.tar.gz"
-    export PATH="$PREFIX/Modules/$PKG_VERSION/bin:$PATH"
-    export MOD_INIT="$PREFIX/Modules/$PKG_VERSION/init/bash"
-
-elif [ "$PKG_NAME" == 'modules' ]; then
-    PKG_URL="http://prdownloads.sourceforge.net/modules/${PKG}.tar.gz"
-    export PATH="$PREFIX/bin:$PATH"
-    export MOD_INIT="$PREFIX/init/bash"
+    BACKUP_PKG_URL="https://sources.easybuild.io/e/EnvironmentModules/${PKG}.tar.gz"
+    if [ "$PKG_VERSION" == '3.2.10' ]; then
+        export PATH="$PREFIX/Modules/$PKG_VERSION/bin:$PATH"
+        export MOD_INIT="$PREFIX/Modules/$PKG_VERSION/init/bash"
+    else
+        export PATH="$PREFIX/bin:$PATH"
+        export MOD_INIT="$PREFIX/init/bash"
+    fi
 
 elif [ "$PKG_NAME" == 'lua' ]; then
     PKG_URL="http://downloads.sourceforge.net/project/lmod/${PKG}.tar.gz"

easybuild/toolchains/fft/intelfftw.py

@@ -62,7 +62,7 @@ def _set_fftw_variables(self):
             compsuff = '_intel'
         elif get_software_root('PGI'):
             compsuff = '_pgi'
-        elif get_software_root('GCC'):
+        elif get_software_root('GCC') or get_software_root('GCCcore'):
             compsuff = '_gnu'
         else:
             error_msg = "Not using Intel compilers, PGI nor GCC, don't know compiler suffix for FFTW libraries."

easybuild/tools/config.py

@@ -590,9 +590,27 @@ def update_build_option(key, value):
     """
     # BuildOptions() is a (singleton) frozen dict, so this is less straightforward that it seems...
     build_options = BuildOptions()
+    orig_value = build_options._FrozenDict__dict[key]
     build_options._FrozenDict__dict[key] = value
     _log.warning("Build option '%s' was updated to: %s", key, build_option(key))
 
+    # Return original value, so it can be restored later if needed
+    return orig_value
+
+
+def update_build_options(key_value_dict):
+    """
+    Update build options as specified by the given dictionary (where keys are assumed to be build option names).
+    Returns dictionary with original values for the updated build options.
+    """
+    orig_key_value_dict = {}
+    for key, value in key_value_dict.items():
+        orig_key_value_dict[key] = update_build_option(key, value)
+
+    # Return original key-value pairs in a dictionary.
+    # This way, they can later be restored by a single call to update_build_options(orig_key_value_dict)
+    return orig_key_value_dict
+
 
 def build_path():
     """

easybuild/tools/filetools.py

@@ -1493,6 +1493,8 @@ def create_patch_info(patch_spec):
                                  str(patch_spec))
 
     elif isinstance(patch_spec, string_type):
+        if not patch_spec.endswith('.patch'):
+            _log.deprecated("Use of patch file with filename that doesn't end with .patch: %s" % patch_spec, '5.0')
         patch_info = {'name': patch_spec}
     else:
         error_msg = "Wrong patch spec, should be string of 2-tuple with patch name + argument: %s"