Merge pull request #3443 from easybuilders/4.3.x

release EasyBuild v4.3.0
easybuilders · Sep 13, 2020 · de3532a · de3532a
2 parents 2035f5d + c5aacf3
commit de3532a
Show file tree

Hide file tree

Showing 101 changed files with 1,520 additions and 560 deletions.
diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml
@@ -16,7 +16,6 @@ jobs:
         pip install --upgrade pip
         pip install --upgrade flake8
 
-    - name: Run flake8
+    - name: Run flake8 to verify PEP8-compliance of Python code
       run: |
-        flake8 easybuild/tools
- 
+        flake8
diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml
@@ -147,7 +147,7 @@ jobs:
           # run test suite
           python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log
           # try and make sure output of running tests is clean (no printed messages/warnings)
-          IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device"
+          IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.5|from cryptography.*default_backend"
           # '|| true' is needed to avoid that Travis stops the job on non-zero exit of grep (i.e. when there are no matches)
           PRINTED_MSG=$(egrep -v "${IGNORE_PATTERNS}" test_framework_suite.log | grep '\.\n*[A-Za-z]' || true)
           test "x$PRINTED_MSG" = "x" || (echo "ERROR: Found printed messages in output of test suite\n${PRINTED_MSG}" && exit 1)
@@ -156,7 +156,7 @@ jobs:
           EB_BOOTSTRAP_VERSION=$(grep '^EB_BOOTSTRAP_VERSION' easybuild/scripts/bootstrap_eb.py | sed 's/[^0-9.]//g')
           EB_BOOTSTRAP_SHA256SUM=$(sha256sum easybuild/scripts/bootstrap_eb.py | cut -f1 -d' ')
           EB_BOOTSTRAP_FOUND="$EB_BOOTSTRAP_VERSION $EB_BOOTSTRAP_SHA256SUM"
-          EB_BOOTSTRAP_EXPECTED="20200203.01 616bf3ce812c0844bf9ea3e690f9d88b394ed48f834ddb8424a73cf45fc64ea5"
+          EB_BOOTSTRAP_EXPECTED="20200820.01 d490d229a18bd5eaa717bb8d5684d754729143d5e995e35a40c84d03ffb1de50"
           test "$EB_BOOTSTRAP_FOUND" = "$EB_BOOTSTRAP_EXPECTED" || (echo "Version check on bootstrap script failed $EB_BOOTSTRAP_FOUND" && exit 1)
 
           # test bootstrap script

diff --git a/.travis.yml b/.travis.yml
@@ -1,29 +1,10 @@
 language: python
-python: 2.7
-dist: xenial
+python: 2.6
+dist: trusty
 env:
-  matrix:
-    - LMOD_VERSION=7.8.22
-    - LMOD_VERSION=7.8.22 TEST_EASYBUILD_MODULE_SYNTAX=Tcl
-    - ENV_MOD_VERSION=3.2.10 TEST_EASYBUILD_MODULES_TOOL=EnvironmentModulesC TEST_EASYBUILD_MODULE_SYNTAX=Tcl
-    - ENV_MOD_TCL_VERSION=1.147 TEST_EASYBUILD_MODULES_TOOL=EnvironmentModulesTcl TEST_EASYBUILD_MODULE_SYNTAX=Tcl
-    - ENV_MOD_VERSION=4.1.4 TEST_EASYBUILD_MODULE_SYNTAX=Tcl TEST_EASYBUILD_MODULES_TOOL=EnvironmentModules  # Tmod 4.1.4 is used in RHEL8
+    - LMOD_VERSION=7.8.22 TEST_EASYBUILD_SILENCE_DEPRECATION_WARNINGS=Python26
 cache:
   pip: true
-matrix:
-  # mark build as finished as soon as job has failed
-  fast_finish: true
-  include:
-    # also test default configuration with Python 2.6
-    - python: 2.6
-      env: LMOD_VERSION=7.8.22 TEST_EASYBUILD_SILENCE_DEPRECATION_WARNINGS=Python26
-      dist: trusty
-    # single configuration to test with Lmod 6 and Python 2.7
-    - python: 2.7
-      env: LMOD_VERSION=6.5.1 TEST_EASYBUILD_SILENCE_DEPRECATION_WARNINGS=Lmod6
-    # also test with Python 3.6
-    - python: 3.6
-      env: LMOD_VERSION=7.8.22
 addons:
   apt:
     packages:
@@ -108,7 +89,7 @@ script:
     - EB_BOOTSTRAP_VERSION=$(grep '^EB_BOOTSTRAP_VERSION' $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py | sed 's/[^0-9.]//g')
     - EB_BOOTSTRAP_SHA256SUM=$(sha256sum $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py | cut -f1 -d' ')
     - EB_BOOTSTRAP_FOUND="$EB_BOOTSTRAP_VERSION $EB_BOOTSTRAP_SHA256SUM"
-    - EB_BOOTSTRAP_EXPECTED="20200203.01 616bf3ce812c0844bf9ea3e690f9d88b394ed48f834ddb8424a73cf45fc64ea5"
+    - EB_BOOTSTRAP_EXPECTED="20200820.01 d490d229a18bd5eaa717bb8d5684d754729143d5e995e35a40c84d03ffb1de50"
     - test "$EB_BOOTSTRAP_FOUND" = "$EB_BOOTSTRAP_EXPECTED" || (echo "Version check on bootstrap script failed $EB_BOOTSTRAP_FOUND" && exit 1)
     # test bootstrap script
     - python $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py /tmp/$TRAVIS_JOB_ID/eb_bootstrap

diff --git a/README.rst b/README.rst
@@ -1,3 +1,5 @@
+.. image:: https://github.com/easybuilders/easybuild-framework/workflows/EasyBuild%20framework%20unit%20tests/badge.svg?branch=develop
+
 .. image:: https://easybuilders.github.io/easybuild/images/easybuild_logo_small.png
    :align: center
 
@@ -31,16 +33,3 @@ Related Python packages:
     with the latest compatible versions of the easybuild-framework and easybuild-easyblocks packages
   * GitHub repository: https://github.com/easybuilders/easybuild-easyconfigs
   * PyPi: https://pypi.python.org/pypi/easybuild-easyconfigs
-
-
-*Build status overview:*
-
-* **master** branch:
-
-  .. image:: https://travis-ci.org/easybuilders/easybuild-framework.svg?branch=master
-      :target: https://travis-ci.org/easybuilders/easybuild-framework/branches
-
-* **develop** branch:
-
-  .. image:: https://travis-ci.org/easybuilders/easybuild-framework.svg?branch=develop
-      :target: https://travis-ci.org/easybuilders/easybuild-framework/branches
diff --git a/RELEASE_NOTES b/RELEASE_NOTES
@@ -3,6 +3,46 @@ For more detailed information, please see the git log.
 
 These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
 
+v4.3.0 (September 13th 2020)
+----------------------------
+
+feature release
+
+- various enhancements, including:
+  - add script to create 'sources' entries for git submodules (#3369, #3436)
+  - add templates for CUDA compute capabilities (#3382)
+    - '%(cuda_compute_capabilities)s', '%(cuda_sm_comma_sep)s', '%(cuda_sm_space_sep)s'
+  - add EasyBuild configuration option --generate-devel-module (#3388)
+    - to allow disabling generating of "devel" modules via --disable-generate-devel-module
+  - set up a minimal build environment when using system compiler (#3399)
+    - by default, $CC is set to 'gcc' and $CXX is set to 'g++' when using 'system' toolchain
+    - minimal build environment can be customized via --minimal-build-env configuration option
+  - add --sysroot configuration option to specify alternative location of system root (#3419)
+    - this is useful when installing software in a Gentoo Prefix environment (for example)
+- various bug fixes, including:
+  - allow including easyblocks from multiple locations, by combining --include-easyblocks and --include-easyblocks-from-pr (#3311)
+  - also escape backslashes in quote_py_str() (#3386)
+  - use one argument 'module swap' statements in Tcl modulefiles (required by Modules 4.2.3+) (#3397)
+  - fix copying of (non-existing) file with apply_patch (#3400)
+  - create symlink from lib64 to lib subdir in installation directories to avoid that GCC prefers /lib64 system directories (#3401)
+  - fix default value for 'lib64_fallback_sanity_check' build option (#3402)
+  - correctly determine commit status in --merge-pr (#3406)
+  - stop installing ccache wrapper for Fortran compiler (#3409)
+  - fix issues with applying PR patch in --from-pr (#3414)
+  - make RPATH wrapper script more robust by using 'python -E -s -S' to run rpath_args.py (#3422)
+  - don't inject -Wl,-rpath options when '-x c++-header' compiler option is used (#3424)
+  - fix lack of -mno-recip on aarch64 by tweaking GCC options used when 'precise' toolchain option is enabled (#3425)
+  - make sure self.start_dir is set in ExtensionEasyBlock (#3426, #3435)
+  - exclude local variables from typo check in easyconfig files (#3427)
+- other changes:
+  - stick to cryptography 2.9.2 when using Python 2.7 to avoid broken test suite in CI (#3392)
+  - automatically enable --ignore-locks with --fetch (#3404)
+  - switch to status badge based on tests run in GitHub Actions CI in README (#3415)
+  - make flake8 code style checks pass (#3416, #3417)
+  - limit Travis to only test with Python 2.6 + Lomod 7 (#3418)
+  - ignore deprecation warning raised for cryptography when using Python 3.5 produced by output of test suite (#3423)
+
+
 v4.2.2 (July 8th 2020)
 ----------------------
 

diff --git a/contrib/hooks/hpc2n_hooks.py b/contrib/hooks/hpc2n_hooks.py
@@ -11,6 +11,7 @@
 from easybuild.tools.modules import get_software_root
 from easybuild.tools.systemtools import get_shared_lib_ext
 
+
 # Add/remove dependencies and/or patches
 # Access to the raw values before templating and such.
 def parse_hook(ec, *args, **kwargs):
@@ -58,7 +59,7 @@ def add_extra_dependencies(ec, dep_type, extra_deps):
             if LooseVersion(ec.version) >= LooseVersion('3'):
                 pmix_version = '2.2.1'
             if LooseVersion(ec.version) >= LooseVersion('4'):
-                pmix_version = '3.0.2' # OpenMPI 4.0.0 is not compatible with PMIx 3.1.x
+                pmix_version = '3.0.2'  # OpenMPI 4.0.0 is not compatible with PMIx 3.1.x
 
             extra_deps.append(('PMIx', pmix_version))
             # Use of external PMIx requires external libevent

diff --git a/easybuild/base/fancylogger.py b/easybuild/base/fancylogger.py
@@ -224,6 +224,7 @@ def getLevelInt(level_name):
 
 class FancyStreamHandler(logging.StreamHandler):
     """The logging StreamHandler with uniform named arg in __init__ for selecting the stream."""
+
     def __init__(self, stream=None, stdout=None):
         """Initialize the stream (default is sys.stderr)
             - stream : a specific stream to use
@@ -245,6 +246,7 @@ class FancyLogRecord(logging.LogRecord):
     This class defines a custom log record.
     Adding extra specifiers is as simple as adding attributes to the log record
     """
+
     def __init__(self, *args, **kwargs):
         logging.LogRecord.__init__(self, *args, **kwargs)
         # modify custom specifiers here
@@ -782,7 +784,7 @@ def getAllExistingLoggers():
     """
     # not-so-well documented manager (in 2.6 and later)
     # return list of (name,logger) tuple
-    return [x for x in logging.Logger.manager.loggerDict.items()]+[(logging.root.name, logging.root)]
+    return [x for x in logging.Logger.manager.loggerDict.items()] + [(logging.root.name, logging.root)]
 
 
 def getAllNonFancyloggers():

diff --git a/easybuild/base/generaloption.py b/easybuild/base/generaloption.py
@@ -307,7 +307,7 @@ def take_action(self, action, dest, opt, value, values, parser):
                     empty = get_empty_add_flex(lvalue, self=self)
                     if empty in value:
                         ind = value.index(empty)
-                        lvalue = value[:ind] + default + value[ind+1:]
+                        lvalue = value[:ind] + default + value[ind + 1:]
                     else:
                         lvalue = value
             elif action == "regex":
@@ -1573,7 +1573,7 @@ def generate_cmd_line(self, ignore=None, add_default=None):
                                        (opt_name, default, action))
                 else:
                     if opt_value == default and ((action in ('store_true',) +
-                                                 ExtOption.EXTOPTION_LOG and default is False) or
+                                                  ExtOption.EXTOPTION_LOG and default is False) or
                                                  (action in ('store_false',) and default is True)):
                         if hasattr(self.parser.option_class, 'ENABLE') and \
                            hasattr(self.parser.option_class, 'DISABLE'):
@@ -1588,10 +1588,10 @@ def generate_cmd_line(self, ignore=None, add_default=None):
                 if default is not None:
                     if action == 'add_flex' and default:
                         for ind, elem in enumerate(opt_value):
-                            if elem == default[0] and opt_value[ind:ind+len(default)] == default:
+                            if elem == default[0] and opt_value[ind:ind + len(default)] == default:
                                 empty = get_empty_add_flex(opt_value, self=self)
                                 # TODO: this will only work for tuples and lists
-                                opt_value = opt_value[:ind] + type(opt_value)([empty]) + opt_value[ind+len(default):]
+                                opt_value = opt_value[:ind] + type(opt_value)([empty]) + opt_value[ind + len(default):]
                                 # only the first occurence
                                 break
                     elif hasattr(opt_value, '__neg__'):

diff --git a/easybuild/base/optcomplete.py b/easybuild/base/optcomplete.py
@@ -186,6 +186,7 @@ class NoneCompleter(Completer):
 
 class ListCompleter(Completer):
     """Completes by filtering using a fixed list of strings."""
+
     def __init__(self, stringlist):
         self.olist = stringlist
 
@@ -263,6 +264,7 @@ def _call(self, **kwargs):
 
 class KnownHostsCompleter(Completer):
     """Completes a list of known hostnames"""
+
     def _call(self, **kwargs):
         if SHELL == BASH:
             return "_known_hosts"
@@ -310,6 +312,7 @@ def _call(self, **kwargs):
 
 class CompleterOption(OPTION_CLASS):
     """optparse Option class with completer attribute"""
+
     def __init__(self, *args, **kwargs):
         completer = kwargs.pop('completer', None)
         OPTION_CLASS.__init__(self, *args, **kwargs)
@@ -587,7 +590,7 @@ def autocomplete(parser, arg_completer=None, opt_completer=None, subcmd_complete
             'completer_kwargs%s' % str(completer_kwargs),
             # 'completer_completions %s' % completer_completions,
             'completions %s' % completions,
-            ])
+        ])
         if isinstance(debugfn, logging.Logger):
             debugfn.debug(txt)
         else:

diff --git a/easybuild/base/rest.py b/easybuild/base/rest.py
@@ -228,6 +228,7 @@ class RequestBuilder(object):
 
     To understand the method(...) calls, check out github.client.Client.
     '''
+
     def __init__(self, client):
         """Constructor"""
         self.client = client
@@ -288,6 +289,7 @@ class RestClient(object):
     try to validate the url you feed it. On the other hand, it
     automatically supports the full API--so why should you care?
     """
+
     def __init__(self, *args, **kwargs):
         """We create a client with the given arguments"""
         self.client = Client(*args, **kwargs)

diff --git a/easybuild/base/testing.py b/easybuild/base/testing.py
@@ -60,7 +60,7 @@ def nicediff(txta, txtb, offset=5):
     res_idx = []
     # very bruteforce
     for didx in different_idx:
-        for idx in range(max(didx-offset, 0), min(didx+offset, len(diff)-1)):
+        for idx in range(max(didx - offset, 0), min(didx + offset, len(diff) - 1)):
             if idx not in res_idx:
                 res_idx.append(idx)
     res_idx.sort()

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
@@ -76,7 +76,7 @@
 from easybuild.tools.filetools import diff_files, dir_contains_files, download_file, encode_class_name, extract_file
 from easybuild.tools.filetools import find_backup_name_candidate, get_source_tarball_from_git, is_alt_pypi_url
 from easybuild.tools.filetools import is_binary, is_sha256_checksum, mkdir, move_file, move_logs, read_file, remove_dir
-from easybuild.tools.filetools import remove_file, remove_lock, verify_checksum, weld_paths, write_file
+from easybuild.tools.filetools import remove_file, remove_lock, verify_checksum, weld_paths, write_file, symlink
 from easybuild.tools.hooks import BUILD_STEP, CLEANUP_STEP, CONFIGURE_STEP, EXTENSIONS_STEP, FETCH_STEP, INSTALL_STEP
 from easybuild.tools.hooks import MODULE_STEP, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTITER_STEP, POSTPROC_STEP
 from easybuild.tools.hooks import PREPARE_STEP, READY_STEP, SANITYCHECK_STEP, SOURCE_STEP, TEST_STEP, TESTCASES_STEP
@@ -449,7 +449,7 @@ def fetch_patches(self, patch_specs=None, extension=False, checksums=None):
 
                 # this *must* be of typ int, nothing else
                 # no 'isinstance(..., int)', since that would make True/False also acceptable
-                if type(patch_spec[1]) == int:
+                if isinstance(patch_spec[1], int):
                     level = patch_spec[1]
                 elif isinstance(patch_spec[1], string_type):
                     # non-patch files are assumed to be files to copy
@@ -2029,8 +2029,7 @@ def patch_step(self, beginpath=None):
             src = os.path.abspath(weld_paths(beginpath, srcpathsuffix))
             self.log.debug("Applying patch %s in path %s", patch, src)
 
-            if not apply_patch(patch['path'], src, copy=copy_patch, level=level):
-                raise EasyBuildError("Applying patch %s failed", patch['name'])
+            apply_patch(patch['path'], src, copy=copy_patch, level=level)
 
     def prepare_step(self, start_dir=True, load_tc_deps_modules=True):
         """
@@ -2239,7 +2238,7 @@ def extensions_step(self, fetch=False, install=True):
             # always go back to original work dir to avoid running stuff from a dir that no longer exists
             change_dir(self.orig_workdir)
 
-            tup = (ext.name, ext.version or '', idx+1, exts_cnt)
+            tup = (ext.name, ext.version or '', idx + 1, exts_cnt)
             print_msg("installing extension %s %s (%d/%d)..." % tup, silent=self.silent)
 
             if self.dry_run:
@@ -2348,6 +2347,15 @@ def post_install_step(self):
                 run_cmd(cmd, simple=True, log_ok=True, log_all=True)
 
         self.fix_shebang()
+        # GCC linker searches system /lib64 path before the $LIBRARY_PATH paths.
+        # However for each <dir> in $LIBRARY_PATH (where <dir> is often <prefix>/lib) it searches <dir>/../lib64 first.
+        # So we create <prefix>/lib64 as a symlink to <prefix>/lib to make it prefer EB installed libraries.
+        # See https://github.com/easybuilders/easybuild-easyconfigs/issues/5776
+        if build_option('lib64_lib_symlink'):
+            lib_dir = os.path.join(self.installdir, 'lib')
+            lib64_dir = os.path.join(self.installdir, 'lib64')
+            if os.path.exists(lib_dir) and not os.path.exists(lib64_dir):
+                symlink(lib_dir, lib64_dir)
 
     def sanity_check_step(self, *args, **kwargs):
         """
@@ -2884,7 +2892,7 @@ def make_module_step(self, fake=False):
             mod_symlink_paths = ActiveMNS().det_module_symlink_paths(self.cfg)
             self.module_generator.create_symlinks(mod_symlink_paths, fake=fake)
 
-            if ActiveMNS().mns.det_make_devel_module() and not fake:
+            if ActiveMNS().mns.det_make_devel_module() and not fake and build_option('generate_devel_module'):
                 self.make_devel_module()
             else:
                 self.log.info("Skipping devel module...")

diff --git a/easybuild/framework/easyconfig/__init__.py b/easybuild/framework/easyconfig/__init__.py
@@ -35,4 +35,4 @@
 EASYCONFIGS_PKG_SUBDIR = 'easyconfigs'
 
 # is used in some tools
-from easybuild.framework.easyconfig.easyconfig import EasyConfig
+from easybuild.framework.easyconfig.easyconfig import EasyConfig  # noqa
diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py
@@ -87,6 +87,7 @@
     'buildopts': ['', 'Extra options passed to make step (default already has -j X)', BUILD],
     'checksums': [[], "Checksums for sources and patches", BUILD],
     'configopts': ['', 'Extra options passed to configure (default already has --prefix)', BUILD],
+    'cuda_compute_capabilities': [[], "List of CUDA compute capabilities to build with (if supported)", BUILD],
     'easyblock': [None, "EasyBlock to use for building; if set to None, an easyblock is selected "
                         "based on the software name", BUILD],
     'easybuild_version': [None, "EasyBuild-version this spec-file was written for", BUILD],