release EasyBuild v3.4.1

RELEASE_NOTES

@@ -3,6 +3,23 @@ For more detailed information, please see the git log.
 
 These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
 
+v3.4.1 (October 17th 2017)
+--------------------------
+
+bugfix release
+- various enhancements, including:
+  - improve trace output for executed commands + drop requirement for --experimental for --trace (#2306)
+  - add giolf toolchain definition: GCC, IntelMPI, OpenBLAS, (Sca)LAPACK and FFTW (#2310)
+  - add support for --force-download and --ignore-checksums (#2313, #2314)
+  - flesh out pypi_source_urls from derive_alt_pypi_url (#2319)
+- various bug fixes, including:
+  - also check for use of --rebuild next to --force to skip sanity check with --module-only (#2307)
+  - ensure $TMPDIR is set to a short path for OpenMPI v2.x (#2311)
+  - guard 'module load' commands in generated modules under --recursive-unload to avoid load storms (#2316)
+  - correctly deal with use of special characters in description & co (#2320)
+  - fix incorrect FFT_INC_DIR for Intel MKL (#2323)
+
+
 v3.4.0 (September 10th 2017)
 ----------------------------
 

easybuild/framework/easyblock.py

@@ -63,6 +63,7 @@
 from easybuild.tools.build_details import get_build_stats
 from easybuild.tools.build_log import EasyBuildError, dry_run_msg, dry_run_warning, dry_run_set_dirs
 from easybuild.tools.build_log import print_error, print_msg, print_warning
+from easybuild.tools.config import FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_PATCHES, FORCE_DOWNLOAD_SOURCES
 from easybuild.tools.config import build_option, build_path, get_log_filename, get_repository, get_repositorypath
 from easybuild.tools.config import install_path, log_path, package_path, source_paths
 from easybuild.tools.environment import restore_env, sanitize_env
@@ -364,7 +365,8 @@ def fetch_sources(self, sources=None, checksums=None):
                 raise EasyBuildError("Unexpected source spec, not a string or dict: %s", source)
 
             # check if the sources can be located
-            path = self.obtain_file(filename, download_filename=download_filename)
+            force_download = build_option('force_download') in [FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_SOURCES]
+            path = self.obtain_file(filename, download_filename=download_filename, force_download=force_download)
             if path:
                 self.log.debug('File %s found for source %s' % (path, filename))
                 self.src.append({
@@ -416,7 +418,8 @@ def fetch_patches(self, patch_specs=None, extension=False, checksums=None):
             else:
                 patch_file = patch_spec
 
-            path = self.obtain_file(patch_file, extension=extension)
+            force_download = build_option('force_download') in [FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_PATCHES]
+            path = self.obtain_file(patch_file, extension=extension, force_download=force_download)
             if path:
                 self.log.debug('File %s found for patch %s' % (path, patch_spec))
                 patchspec = {
@@ -496,7 +499,8 @@ def fetch_extension_sources(self, skip_checksums=False):
                         exts_sources.append(ext_src)
                     else:
                         source_urls = [resolve_template(url, ext_src) for url in ext_options.get('source_urls', [])]
-                        src_fn = self.obtain_file(fn, extension=True, urls=source_urls)
+                        force_download = build_option('force_download') in [FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_SOURCES]
+                        src_fn = self.obtain_file(fn, extension=True, urls=source_urls, force_download=force_download)
 
                         if src_fn:
                             ext_src.update({'src': src_fn})
@@ -510,9 +514,11 @@ def fetch_extension_sources(self, skip_checksums=False):
                                 if checksums:
                                     fn_checksum = self.get_checksum_for(checksums, filename=src_fn, index=0)
                                     if verify_checksum(src_fn, fn_checksum):
-                                        self.log.info('Checksum for ext source %s verified', fn)
+                                        self.log.info('Checksum for extension source %s verified', fn)
+                                    elif build_option('ignore_checksums'):
+                                        print_warning("Ignoring failing checksum verification for %s" % fn)
                                     else:
-                                        raise EasyBuildError('Checksum for ext source %s failed', fn)
+                                        raise EasyBuildError('Checksum verification for extension source %s failed', fn)
 
                             ext_patches = self.fetch_patches(patch_specs=ext_options.get('patches', []), extension=True)
                             if ext_patches:
@@ -533,6 +539,8 @@ def fetch_extension_sources(self, skip_checksums=False):
                                             checksum = self.get_checksum_for(checksums[1:], filename=patch, index=idx)
                                             if verify_checksum(patch, checksum):
                                                 self.log.info('Checksum for extension patch %s verified', patch)
+                                            elif build_option('ignore_checksums'):
+                                                print_warning("Ignoring failing checksum verification for %s" % patch)
                                             else:
                                                 raise EasyBuildError('Checksum for extension patch %s failed', patch)
                             else:
@@ -551,17 +559,18 @@ def fetch_extension_sources(self, skip_checksums=False):
 
         return exts_sources
 
-    def obtain_file(self, filename, extension=False, urls=None, download_filename=None):
+    def obtain_file(self, filename, extension=False, urls=None, download_filename=None, force_download=False):
         """
         Locate the file with the given name
         - searches in different subdirectories of source path
         - supports fetching file from the web if path is specified as an url (i.e. starts with "http://:")
-
         :param filename: filename of source
         :param extension: indicates whether locations for extension sources should also be considered
         :param urls: list of source URLs where this file may be available
         :param download_filename: filename with which the file should be downloaded, and then renamed to <filename>
+        :force_download: always try to download file, even if it's already available in source path
         """
+        res = None
         srcpaths = source_paths()
 
         # should we download or just try and find it?
@@ -583,13 +592,15 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
 
                 # only download when it's not there yet
                 if os.path.exists(fullpath):
-                    self.log.info("Found file %s at %s, no need to download it." % (filename, filepath))
-                    return fullpath
-
-                else:
-                    if download_file(filename, url, fullpath):
+                    if force_download:
+                        print_warning("Found file %s at %s, but re-downloading it anyway..." % (filename, filepath))
+                    else:
+                        self.log.info("Found file %s at %s, no need to download it", filename, filepath)
                         return fullpath
 
+                if download_file(filename, url, fullpath):
+                    return fullpath
+
             except IOError, err:
                 raise EasyBuildError("Downloading file %s from url %s to %s failed: %s", filename, url, fullpath, err)
 
@@ -636,13 +647,17 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
 
                     for fp in fullpaths:
                         if os.path.isfile(fp):
-                            self.log.info("Found file %s at %s" % (filename, fp))
+                            self.log.info("Found file %s at %s", filename, fp)
                             foundfile = os.path.abspath(fp)
                             break  # no need to try further
                         else:
                             failedpaths.append(fp)
 
                 if foundfile:
+                    if force_download:
+                        print_warning("Found file %s at %s, but re-downloading it anyway..." % (filename, foundfile))
+                        foundfile = None
+
                     break  # no need to try other source paths
 
             if foundfile:
@@ -1635,10 +1650,12 @@ def checksum_step(self):
                 expected_checksum = fil['checksum'] or '(none)'
                 self.dry_run_msg("* expected checksum for %s: %s", filename, expected_checksum)
             else:
-                if not verify_checksum(fil['path'], fil['checksum']):
-                    raise EasyBuildError("Checksum verification for %s using %s failed.", fil['path'], fil['checksum'])
-                else:
+                if verify_checksum(fil['path'], fil['checksum']):
                     self.log.info("Checksum verification for %s using %s passed." % (fil['path'], fil['checksum']))
+                elif build_option('ignore_checksums'):
+                    print_warning("Ignoring failing checksum verification for %s" % fil['name'])
+                else:
+                    raise EasyBuildError("Checksum verification for %s using %s failed.", fil['path'], fil['checksum'])
 
     def extract_step(self):
         """
@@ -1895,7 +1912,7 @@ def package_step(self):
 
             pkgtype = build_option('package_type')
             pkgdir_dest = os.path.abspath(package_path())
-            opt_force = build_option('force')
+            opt_force = build_option('force') or build_option('rebuild')
 
             self.log.info("Generating %s package in %s", pkgtype, pkgdir_dest)
             pkgdir_src = package(self)
@@ -2225,6 +2242,8 @@ def cleanup_step(self):
         if not build_option('cleanup_builddir'):
             self.log.info("Keeping builddir %s" % self.builddir)
 
+        self.toolchain.cleanup()
+
         env.restore_env_vars(self.cfg['unwanted_env_vars'])
 
         self.restore_iterate_opts()
@@ -2383,7 +2402,7 @@ def update_config_template_run_step(self):
     def _skip_step(self, step, skippable):
         """Dedice whether or not to skip the specified step."""
         module_only = build_option('module_only')
-        force = build_option('force')
+        force = build_option('force') or build_option('rebuild')
         skip = False
 
         # skip step if specified as individual (skippable) step

easybuild/toolchains/fft/intelfftw.py

@@ -85,7 +85,7 @@ def _set_fftw_variables(self):
         self.log.debug('fftw_libs %s' % fftw_libs.__repr__())
 
         self.FFT_LIB_DIR = self.BLAS_LIB_DIR
-        self.FFT_INCLUDE_DIR = self.BLAS_INCLUDE_DIR
+        self.FFT_INCLUDE_DIR = [os.path.join(d, 'fftw') for d in self.BLAS_INCLUDE_DIR]
 
         # building the FFTW interfaces is optional,
         # so make sure libraries are there before FFT_LIB is set

easybuild/toolchains/giolf.py

@@ -0,0 +1,40 @@
+##
+# Copyright 2013-2017 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild.  If not, see <http://www.gnu.org/licenses/>.
+##
+"""
+EasyBuild support for goolf compiler toolchain (includes GCC, IntelMPI, OpenBLAS, LAPACK, ScaLAPACK and FFTW).
+
+:author: Kenneth Hoste (Ghent University)
+"""
+
+from easybuild.toolchains.gimpi import Gimpi
+from easybuild.toolchains.fft.fftw import Fftw
+from easybuild.toolchains.linalg.openblas import OpenBLAS
+from easybuild.toolchains.linalg.scalapack import ScaLAPACK
+
+
+class Goolf(Gimpi, OpenBLAS, ScaLAPACK, Fftw):
+    """Compiler toolchain with GCC, IntelMPI, OpenBLAS, ScaLAPACK and FFTW."""
+    NAME = 'giolf'
+    SUBTOOLCHAIN = Gimpi.NAME

easybuild/toolchains/mpi/openmpi.py

@@ -28,8 +28,13 @@
 :author: Stijn De Weirdt (Ghent University)
 :author: Kenneth Hoste (Ghent University)
 """
+import os
+import shutil
+import tempfile
 from distutils.version import LooseVersion
 
+import easybuild.tools.environment as env
+from easybuild.tools.build_log import print_warning
 from easybuild.tools.toolchain.constants import COMPILER_VARIABLES, MPI_COMPILER_VARIABLES
 from easybuild.tools.toolchain.mpi import Mpi
 from easybuild.tools.toolchain.variables import CommandFlagList
@@ -57,6 +62,30 @@ class OpenMPI(Mpi):
 
     MPI_LINK_INFO_OPTION = '-showme:link'
 
+    def __init__(self, *args, **kwargs):
+        """Toolchain constructor"""
+        super(OpenMPI, self).__init__(*args, **kwargs)
+
+        self.orig_tmpdir = None
+
+    def prepare(self, *args, **kwargs):
+        """
+        Prepare for using OpenMPI library in toolchain environment
+        """
+        super(OpenMPI, self).prepare(*args, **kwargs)
+
+        # OpenMPI 2.x trips if path specified in $TMPDIR is too long
+        # see https://www.open-mpi.org/faq/?category=osx#startup-errors-with-open-mpi-2.0.x
+        self.orig_tmpdir = os.environ.get('TMPDIR')
+        ompi_ver = self.get_software_version(self.MPI_MODULE_NAME)[0]
+        if LooseVersion(ompi_ver) >= LooseVersion('2.0') and LooseVersion(ompi_ver) < LooseVersion('3.0'):
+            if len(self.orig_tmpdir) > 40:
+                tmpdir = tempfile.mkdtemp(prefix='/tmp/')
+                env.setvar('TMPDIR', tmpdir)
+                warn_msg = "Long $TMPDIR path may cause problems with OpenMPI 2.x, using shorter path: %s" % tmpdir
+                self.log.warning(warn_msg)
+                print_warning(warn_msg)
+
     def _set_mpi_compiler_variables(self):
         """Define MPI wrapper commands (depends on OpenMPI version) and add OMPI_* variables to set."""
         ompi_ver = self.get_software_version(self.MPI_MODULE_NAME)[0]
@@ -75,3 +104,16 @@ def _set_mpi_compiler_variables(self):
             self.variables.nappend('OMPI_%s' % var, str(self.variables[var].get_first()), var_class=CommandFlagList)
 
         super(OpenMPI, self)._set_mpi_compiler_variables()
+
+    def cleanup(self, *args, **kwargs):
+        """Clean up after using OpenMPI in toolchain."""
+        super(OpenMPI, self).cleanup(*args, **kwargs)
+
+        tmpdir = os.environ.get('TMPDIR')
+        if tmpdir != self.orig_tmpdir:
+            try:
+                shutil.rmtree(tmpdir)
+            except OSError as err:
+                print_warning("Failed to clean up temporary directory %s: %s", tmpdir, err)
+            env.setvar('TMPDIR', self.orig_tmpdir)
+            self.log.info("$TMPDIR restored to %s", self.orig_tmpdir)

easybuild/tools/build_log.py

@@ -37,6 +37,7 @@
 import sys
 import tempfile
 from copy import copy
+from datetime import datetime
 from vsc.utils import fancylogger
 from vsc.utils.exceptions import LoggedException
 
@@ -299,3 +300,21 @@ def print_warning(message, silent=False):
     """
     if not silent:
         sys.stderr.write("\nWARNING: %s\n\n" % message)
+
+
+def time_str_since(start_time):
+    """
+    Return string representing amount of time that has passed since specified timestamp
+
+    :param start_time: datetime value representing start time
+    :return: string value representing amount of time passed since start_time;
+             format: "[[%d hours, ]%d mins, ]%d sec(s)"
+    """
+    tot_time = datetime.now() - start_time
+    tot_secs = tot_time.seconds + tot_time.days * 24 * 3600
+    if tot_secs > 0:
+        res = datetime.utcfromtimestamp(tot_secs).strftime('%Hh%Mm%Ss')
+    else:
+        res = "< 1s"
+
+    return res

easybuild/tools/config.py

@@ -89,6 +89,13 @@
 LOADED_MODULES_ACTIONS = [ERROR, IGNORE, PURGE, UNLOAD, WARN]
 DEFAULT_ALLOW_LOADED_MODULES = ('EasyBuild',)
 
+FORCE_DOWNLOAD_ALL = 'all'
+FORCE_DOWNLOAD_PATCHES = 'patches'
+FORCE_DOWNLOAD_SOURCES = 'sources'
+FORCE_DOWNLOAD_CHOICES = [FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_PATCHES, FORCE_DOWNLOAD_SOURCES]
+DEFAULT_FORCE_DOWNLOAD = FORCE_DOWNLOAD_SOURCES
+
+
 # utility function for obtaining default paths
 def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
     """Create full path, avoid '/' at the end."""
@@ -111,6 +118,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
         'filter_env_vars',
         'hide_deps',
         'hide_toolchains',
+        'force_download',
         'from_pr',
         'git_working_dirs_path',
         'pr_branch_name',
@@ -158,6 +166,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
         'force',
         'group_writable_installdir',
         'hidden',
+        'ignore_checksums',
         'install_latest_eb_release',
         'minimal_toolchains',
         'module_only',