diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 803e5aa65f..2cae3138f3 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -43,6 +43,14 @@ jobs: python: 3.8 - modules_tool: modules-4.1.4 python: 3.9 + - modules_tool: Lmod-7.8.22 + python: 3.5 + - modules_tool: Lmod-7.8.22 + python: 3.7 + - modules_tool: Lmod-7.8.22 + python: 3.8 + - modules_tool: Lmod-7.8.22 + python: 3.9 fail-fast: false steps: - uses: actions/checkout@v2 @@ -122,6 +130,9 @@ jobs: EASYBUILD_MODULE_SYNTAX: ${{matrix.module_syntax}} TEST_EASYBUILD_MODULE_SYNTAX: ${{matrix.module_syntax}} run: | + # run tests *outside* of checked out easybuild-framework directory, + # to ensure we're testing installed version (see previous step) + cd $HOME # initialize environment for modules tool if [ -f $HOME/moduleshome ]; then export MODULESHOME=$(cat $HOME/moduleshome); fi source $(cat $HOME/mod_init); type module @@ -153,15 +164,12 @@ jobs: # run test suite python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log # try and make sure output of running tests is clean (no printed messages/warnings) - IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.5|from cryptography.*default_backend" + IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.5|from cryptography.*default_backend|CryptographyDeprecationWarning: Python 2" # '|| true' is needed to avoid that Travis stops the job on non-zero exit of grep (i.e. when there are no matches) PRINTED_MSG=$(egrep -v "${IGNORE_PATTERNS}" test_framework_suite.log | grep '\.\n*[A-Za-z]' || true) test "x$PRINTED_MSG" = "x" || (echo "ERROR: Found printed messages in output of test suite\n${PRINTED_MSG}" && exit 1) - name: test bootstrap script - # skip testing of bootstrap script with Python 3.9, - # until an EasyBuild release that is compatible with Python 3.9 is available - if: ${{ matrix.python != 3.9 }} run: | # (re)initialize environment for modules tool if [ -f $HOME/moduleshome ]; then export MODULESHOME=$(cat $HOME/moduleshome); fi diff --git a/MANIFEST.in b/MANIFEST.in index 0e89e87364..5a739b43a9 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,7 +4,7 @@ include optcomplete.bash recursive-include etc * recursive-include easybuild *py recursive-include easybuild/scripts * -recursive-include test *py *eb +recursive-include test *py *eb *yaml recursive-include test/framework/modules * recursive-include test/framework/sandbox/sources * include CONTRIBUTING.md diff --git a/RELEASE_NOTES b/RELEASE_NOTES index bd91b3a56c..10cf597707 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -3,6 +3,41 @@ For more detailed information, please see the git log. These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html. +v4.3.2 (December 10th 2020) +--------------------------- + +update/bugfix release + +- add (experimental) support for specifying easyconfig files via an "easystack" file (#3479, #3511, #3515, #3517, #3520, #3521) + - see also https://easybuild.readthedocs.io/en/latest/Easystack-files.html +- add definition for new 'gobff' toolchain using BLIS and LibFLAME (#3505) +- various enhancements, including: + - add support for toolchain options like 'extra_cxxflags' to specify extra compiler options (#2193) + - fix combination of --copy-ec and --from-pr (#3482) + - enhance copy_files function: support single file target, error on empty input list, support verbose mode (#3483) + - cache result of fetch_files_from_pr function (mainly to speed up framework test suite) (#3484) + - add locate_files function to filetools module (#3485) + - add support for %(module_name)s template value (#3497) + - clarify input format for --cuda-compute-capabilities in 'eb --help' output (#3509) + - add support for skiping unit tests (test step) via --skip-test-step (#3524) +- various bug fixes, including: + - also ignore vsc.* imports coming from from pkg_resources/__init__.py (setuptools) in fake vsc namespace (#3491) + - don't pass username in github_api_get_request when no GitHub token is available (#3494) + - also inject -rpath options for all entries in $LIBRARY_PATH in RPATH wrappers (#3495) + - avoid TypeError being raised by list_toolchains (#3499) + - check if PR is already merged in --merge-pr (#3502) + - graciously handle wrong PR id in fetch_pr_data (#3503) + - fix regression in apply_regex_substitutions: also accept list of paths to patch (#3507) + - update installation procedure for EasyBuild in generated Singularity container recipes (#3510) + - fix GitHub Actions workflow for test suite: run outside of repo checkout + also test bootstrap script with Python 3.9 (#3518) + - bump cryptography from 2.9.2 to 3.2 for Python 2 in requirements.txt (#3519) + - fix 'eb --help=rst' when running with Python 3 (#3525) +- other changes: + - exclude test configurations with Lmod 7 and Python 3, except for Python 3.6 (#3496) + - significantly speed up parsing of easyconfig files by only extracting comments from an easyconfig file when they're actually needed (#3498) + - don't include file/ldd/readelf commands run during RPATH sanity check in --trace output (#3508) + + v4.3.1 (October 29th 2020) -------------------------- diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 14489e27b9..5d43b050c4 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2139,10 +2139,11 @@ def build_step(self): def test_step(self): """Run unit tests provided by software (if any).""" - if self.cfg['runtest']: + unit_test_cmd = self.cfg['runtest'] + if unit_test_cmd: - self.log.debug("Trying to execute %s as a command for running unit tests...") - (out, _) = run_cmd(self.cfg['runtest'], log_all=True, simple=False) + self.log.debug("Trying to execute %s as a command for running unit tests...", unit_test_cmd) + (out, _) = run_cmd(unit_test_cmd, log_all=True, simple=False) return out @@ -2460,7 +2461,7 @@ def sanity_check_rpath(self, rpath_dirs=None): for path in [os.path.join(dirpath, x) for x in os.listdir(dirpath)]: self.log.debug("Sanity checking RPATH for %s", path) - out, ec = run_cmd("file %s" % path, simple=False) + out, ec = run_cmd("file %s" % path, simple=False, trace=False) if ec: fails.append("Failed to run 'file %s': %s" % (path, out)) @@ -2470,7 +2471,7 @@ def sanity_check_rpath(self, rpath_dirs=None): # ELF 64-bit LSB shared object, x86-64, version 1 (SYSV), dynamically linked, not stripped if "dynamically linked" in out: # check whether all required libraries are found via 'ldd' - out, ec = run_cmd("ldd %s" % path, simple=False) + out, ec = run_cmd("ldd %s" % path, simple=False, trace=False) if ec: fail_msg = "Failed to run 'ldd %s': %s" % (path, out) self.log.warning(fail_msg) @@ -2483,7 +2484,7 @@ def sanity_check_rpath(self, rpath_dirs=None): self.log.debug("Output of 'ldd %s' checked, looks OK", path) # check whether RPATH section in 'readelf -d' output is there - out, ec = run_cmd("readelf -d %s" % path, simple=False) + out, ec = run_cmd("readelf -d %s" % path, simple=False, trace=False) if ec: fail_msg = "Failed to run 'readelf %s': %s" % (path, out) self.log.warning(fail_msg) @@ -3310,6 +3311,10 @@ def build_and_install_one(ecdict, init_env): _log.debug("Skip set to %s" % skip) app.cfg['skip'] = skip + if build_option('skip_test_step'): + _log.debug('Adding test_step to skipped steps') + app.cfg.update('skipsteps', TEST_STEP, allow_duplicate=False) + # build easyconfig errormsg = '(no error)' # timing info diff --git a/easybuild/framework/easyconfig/format/format.py b/easybuild/framework/easyconfig/format/format.py index 89bf9ecc3e..ff9acac3d6 100644 --- a/easybuild/framework/easyconfig/format/format.py +++ b/easybuild/framework/easyconfig/format/format.py @@ -612,12 +612,17 @@ def __init__(self): raise EasyBuildError('Invalid version number %s (incorrect length)', self.VERSION) self.rawtext = None # text version of the easyconfig - self.comments = {} # comments in easyconfig file + self._comments = {} # comments in easyconfig file self.header = None # easyconfig header (e.g., format version, license, ...) self.docstring = None # easyconfig docstring (e.g., author, maintainer, ...) self.specs = {} + @property + def comments(self): + """Return comments in easyconfig file""" + return self._comments + def set_specifications(self, specs): """Set specifications.""" self.log.debug('Set copy of specs %s' % specs) diff --git a/easybuild/framework/easyconfig/format/one.py b/easybuild/framework/easyconfig/format/one.py index e28b94bd50..6d789a4c6a 100644 --- a/easybuild/framework/easyconfig/format/one.py +++ b/easybuild/framework/easyconfig/format/one.py @@ -135,7 +135,8 @@ def parse(self, txt): """ Pre-process txt to extract header, docstring and pyheader, with non-indented section markers enforced. """ - super(FormatOneZero, self).parse(txt, strict_section_markers=True) + self.rawcontent = txt + super(FormatOneZero, self).parse(self.rawcontent, strict_section_markers=True) def _reformat_line(self, param_name, param_val, outer=False, addlen=0): """ @@ -356,6 +357,16 @@ def dump(self, ecfg, default_values, templ_const, templ_val, toolchain_hierarchy return '\n'.join(dump) + @property + def comments(self): + """ + Return comments (and extract them first if needed). + """ + if not self._comments: + self.extract_comments(self.rawcontent) + + return self._comments + def extract_comments(self, rawtxt): """ Extract comments from raw content. @@ -363,7 +374,7 @@ def extract_comments(self, rawtxt): Discriminates between comment header, comments above a line (parameter definition), and inline comments. Inline comments on items of iterable values are also extracted. """ - self.comments = { + self._comments = { 'above': {}, # comments above a parameter definition 'header': [], # header comment lines 'inline': {}, # inline comments diff --git a/easybuild/framework/easyconfig/parser.py b/easybuild/framework/easyconfig/parser.py index e1294cf545..e291a2d356 100644 --- a/easybuild/framework/easyconfig/parser.py +++ b/easybuild/framework/easyconfig/parser.py @@ -110,8 +110,6 @@ def __init__(self, filename=None, format_version=None, rawcontent=None, else: raise EasyBuildError("Neither filename nor rawcontent provided to EasyConfigParser") - self._formatter.extract_comments(self.rawcontent) - def process(self, filename=None): """Create an instance""" self._read(filename=filename) diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index 47f5c3175c..9af16131d1 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -47,6 +47,7 @@ # derived from easyconfig, but not from ._config directly TEMPLATE_NAMES_EASYCONFIG = [ ('arch', "System architecture (e.g. x86_64, aarch64, ppc64le, ...)"), + ('module_name', "Module name"), ('nameletter', "First letter of software name"), ('toolchain_name', "Toolchain name"), ('toolchain_version', "Toolchain version"), @@ -72,8 +73,8 @@ ] # values taken from the EasyBlock before each step TEMPLATE_NAMES_EASYBLOCK_RUN_STEP = [ - ('installdir', "Installation directory"), ('builddir', "Build directory"), + ('installdir', "Installation directory"), ] # software names for which to define ver and shortver templates TEMPLATE_SOFTWARE_VERSIONS = [ @@ -208,6 +209,10 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None) softname = config['name'] if softname is not None: template_values['nameletter'] = softname[0] + + elif name[0] == 'module_name': + template_values['module_name'] = getattr(config, 'short_mod_name', None) + else: raise EasyBuildError("Undefined name %s from TEMPLATE_NAMES_EASYCONFIG", name) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 2a3260ae80..56a64c5588 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -53,8 +53,9 @@ from easybuild.tools.build_log import EasyBuildError, print_msg, print_warning from easybuild.tools.config import build_option from easybuild.tools.environment import restore_env -from easybuild.tools.filetools import find_easyconfigs, is_patch_file, read_file, resolve_path, which, write_file -from easybuild.tools.github import fetch_easyconfigs_from_pr, download_repo +from easybuild.tools.filetools import find_easyconfigs, is_patch_file, locate_files +from easybuild.tools.filetools import read_file, resolve_path, which, write_file +from easybuild.tools.github import fetch_easyconfigs_from_pr, fetch_files_from_pr, download_repo from easybuild.tools.multidiff import multidiff from easybuild.tools.py2vs3 import OrderedDict from easybuild.tools.toolchain.toolchain import is_system_toolchain @@ -348,44 +349,13 @@ def det_easyconfig_paths(orig_paths): ec_files = [path for path in pr_files if path.endswith('.eb')] if ec_files and robot_path: - # look for easyconfigs with relative paths in robot search path, - # unless they were found at the given relative paths - - # determine which easyconfigs files need to be found, if any - ecs_to_find = [] - for idx, ec_file in enumerate(ec_files): - if ec_file == os.path.basename(ec_file) and not os.path.exists(ec_file): - ecs_to_find.append((idx, ec_file)) - _log.debug("List of easyconfig files to find: %s" % ecs_to_find) - - # find missing easyconfigs by walking paths in robot search path - for path in robot_path: - _log.debug("Looking for missing easyconfig files (%d left) in %s..." % (len(ecs_to_find), path)) - for (subpath, dirnames, filenames) in os.walk(path, topdown=True): - for idx, orig_path in ecs_to_find[:]: - if orig_path in filenames: - full_path = os.path.join(subpath, orig_path) - _log.info("Found %s in %s: %s" % (orig_path, path, full_path)) - ec_files[idx] = full_path - # if file was found, stop looking for it (first hit wins) - ecs_to_find.remove((idx, orig_path)) - - # stop os.walk insanity as soon as we have all we need (os.walk loop) - if not ecs_to_find: - break - - # ignore subdirs specified to be ignored by replacing items in dirnames list used by os.walk - dirnames[:] = [d for d in dirnames if d not in build_option('ignore_dirs')] - - # ignore archived easyconfigs, unless specified otherwise - if not build_option('consider_archived_easyconfigs'): - dirnames[:] = [d for d in dirnames if d != EASYCONFIGS_ARCHIVE_DIR] - - # stop os.walk insanity as soon as we have all we need (outer loop) - if not ecs_to_find: - break - - return [os.path.abspath(ec_file) for ec_file in ec_files] + ignore_subdirs = build_option('ignore_dirs') + if not build_option('consider_archived_easyconfigs'): + ignore_subdirs.append(EASYCONFIGS_ARCHIVE_DIR) + + ec_files = locate_files(ec_files, robot_path, ignore_subdirs=ignore_subdirs) + + return ec_files def parse_easyconfigs(paths, validate=True): @@ -728,3 +698,65 @@ def avail_easyblocks(): easyblock_mod_name, easyblocks[easyblock_mod_name]['loc'], path) return easyblocks + + +def det_copy_ec_specs(orig_paths, from_pr): + """Determine list of paths + target directory for --copy-ec.""" + + target_path, paths = None, [] + + # if only one argument is specified, use current directory as target directory + if len(orig_paths) == 1: + target_path = os.getcwd() + paths = orig_paths[:] + + # if multiple arguments are specified, assume that last argument is target location, + # and remove that from list of paths to copy + elif orig_paths: + target_path = orig_paths[-1] + paths = orig_paths[:-1] + + # if --from-pr was used in combination with --copy-ec, some extra care must be taken + if from_pr: + # pull in the paths to all the changed files in the PR, + # which includes easyconfigs but also patch files (& maybe more); + # do this in a dedicated subdirectory of the working tmpdir, + # to avoid potential trouble with already existing files in the working tmpdir + # (note: we use a fixed subdirectory in the working tmpdir here rather than a unique random subdirectory, + # to ensure that the caching for fetch_files_from_pr works across calls for the same PR) + tmpdir = os.path.join(tempfile.gettempdir(), 'fetch_files_from_pr_%s' % from_pr) + pr_paths = fetch_files_from_pr(pr=from_pr, path=tmpdir) + + # assume that files need to be copied to current working directory for now + target_path = os.getcwd() + + if orig_paths: + last_path = orig_paths[-1] + + # check files touched by PR and see if the target directory for --copy-ec + # corresponds to the name of one of these files; + # if so we should copy the specified file(s) to the current working directory, + # since interpreting the last argument as target location is very unlikely to be correct in this case + pr_filenames = [os.path.basename(p) for p in pr_paths] + if last_path in pr_filenames: + paths = orig_paths[:] + else: + target_path = last_path + # exclude last argument that is used as target location + paths = orig_paths[:-1] + + # if list of files to copy is empty at this point, + # we simply copy *all* files touched by the PR + if not paths: + paths = pr_paths + + # replace path for files touched by PR (no need to worry about others) + for idx, path in enumerate(paths): + filename = os.path.basename(path) + pr_matches = [x for x in pr_paths if os.path.basename(x) == filename] + if len(pr_matches) == 1: + paths[idx] = pr_matches[0] + elif pr_matches: + raise EasyBuildError("Found multiple paths for %s in PR: %s", filename, pr_matches) + + return paths, target_path diff --git a/easybuild/framework/easystack.py b/easybuild/framework/easystack.py new file mode 100644 index 0000000000..8d71866c65 --- /dev/null +++ b/easybuild/framework/easystack.py @@ -0,0 +1,234 @@ +# Copyright 2020-2020 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +# +""" +Support for easybuild-ing from multiple easyconfigs based on +information obtained from provided file (easystack) with build specifications. + +:author: Denis Kristak (Inuits) +:author: Pavel Grochal (Inuits) +""" + +from easybuild.base import fancylogger +from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.filetools import read_file +from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version +from easybuild.tools.utilities import only_if_module_is_available +try: + import yaml +except ImportError: + pass +_log = fancylogger.getLogger('easystack', fname=False) + + +class EasyStack(object): + """One class instance per easystack. General options + list of all SoftwareSpecs instances""" + + def __init__(self): + self.easybuild_version = None + self.robot = False + self.software_list = [] + + def compose_ec_filenames(self): + """Returns a list of all easyconfig names""" + ec_filenames = [] + for sw in self.software_list: + full_ec_version = det_full_ec_version({ + 'toolchain': {'name': sw.toolchain_name, 'version': sw.toolchain_version}, + 'version': sw.version, + 'versionsuffix': sw.versionsuffix, + }) + ec_filename = '%s-%s.eb' % (sw.name, full_ec_version) + ec_filenames.append(ec_filename) + return ec_filenames + + # flags applicable to all sw (i.e. robot) + def get_general_options(self): + """Returns general options (flags applicable to all sw (i.e. --robot))""" + general_options = {} + # TODO add support for general_options + # general_options['robot'] = self.robot + # general_options['easybuild_version'] = self.easybuild_version + return general_options + + +class SoftwareSpecs(object): + """Contains information about every software that should be installed""" + + def __init__(self, name, version, versionsuffix, toolchain_version, toolchain_name): + self.name = name + self.version = version + self.toolchain_version = toolchain_version + self.toolchain_name = toolchain_name + self.versionsuffix = versionsuffix + + +class EasyStackParser(object): + """Parser for easystack files (in YAML syntax).""" + + @staticmethod + def parse(filepath): + """Parses YAML file and assigns obtained values to SW config instances as well as general config instance""" + yaml_txt = read_file(filepath) + easystack_raw = yaml.safe_load(yaml_txt) + easystack = EasyStack() + + try: + software = easystack_raw["software"] + except KeyError: + wrong_structure_file = "Not a valid EasyStack YAML file: no 'software' key found" + raise EasyBuildError(wrong_structure_file) + + # assign software-specific easystack attributes + for name in software: + # ensure we have a string value (YAML parser returns type = dict + # if levels under the current attribute are present) + name = str(name) + try: + toolchains = software[name]['toolchains'] + except KeyError: + raise EasyBuildError("Toolchains for software '%s' are not defined in %s", name, filepath) + for toolchain in toolchains: + toolchain = str(toolchain) + toolchain_parts = toolchain.split('-', 1) + if len(toolchain_parts) == 2: + toolchain_name, toolchain_version = toolchain_parts + elif len(toolchain_parts) == 1: + toolchain_name, toolchain_version = toolchain, '' + else: + raise EasyBuildError("Incorrect toolchain specification for '%s' in %s, too many parts: %s", + name, filepath, toolchain_parts) + + try: + # if version string containts asterisk or labels, raise error (asterisks not supported) + versions = toolchains[toolchain]['versions'] + except TypeError as err: + wrong_structure_err = "An error occurred when interpreting " + wrong_structure_err += "the data for software %s: %s" % (name, err) + raise EasyBuildError(wrong_structure_err) + if '*' in str(versions): + asterisk_err = "EasyStack specifications of '%s' in %s contain asterisk. " + asterisk_err += "Wildcard feature is not supported yet." + raise EasyBuildError(asterisk_err, name, filepath) + + # yaml versions can be in different formats in yaml file + # firstly, check if versions in yaml file are read as a dictionary. + # Example of yaml structure: + # ======================================================================== + # versions: + # 2.25: + # 2.23: + # versionsuffix: '-R-4.0.0' + # ======================================================================== + if isinstance(versions, dict): + for version in versions: + if versions[version] is not None: + version_spec = versions[version] + if 'versionsuffix' in version_spec: + versionsuffix = str(version_spec['versionsuffix']) + else: + versionsuffix = '' + if 'exclude-labels' in str(version_spec) or 'include-labels' in str(version_spec): + lab_err = "EasyStack specifications of '%s' in %s " + lab_err += "contain labels. Labels aren't supported yet." + raise EasyBuildError(lab_err, name, filepath) + else: + versionsuffix = '' + + specs = { + 'name': name, + 'toolchain_name': toolchain_name, + 'toolchain_version': toolchain_version, + 'version': version, + 'versionsuffix': versionsuffix, + } + sw = SoftwareSpecs(**specs) + + # append newly created class instance to the list in instance of EasyStack class + easystack.software_list.append(sw) + continue + + # is format read as a list of versions? + # ======================================================================== + # versions: + # [2.24, 2.51] + # ======================================================================== + elif isinstance(versions, list): + versions_list = versions + + # format = multiple lines without ':' (read as a string)? + # ======================================================================== + # versions: + # 2.24 + # 2.51 + # ======================================================================== + elif isinstance(versions, str): + versions_list = str(versions).split() + + # format read as float (containing one version only)? + # ======================================================================== + # versions: + # 2.24 + # ======================================================================== + elif isinstance(versions, float): + versions_list = [str(versions)] + + # if no version is a dictionary, versionsuffix isn't specified + versionsuffix = '' + + for version in versions_list: + sw = SoftwareSpecs( + name=name, version=version, versionsuffix=versionsuffix, + toolchain_name=toolchain_name, toolchain_version=toolchain_version) + # append newly created class instance to the list in instance of EasyStack class + easystack.software_list.append(sw) + + # assign general easystack attributes + easystack.easybuild_version = easystack_raw.get('easybuild_version', None) + easystack.robot = easystack_raw.get('robot', False) + + return easystack + + +@only_if_module_is_available('yaml', pkgname='PyYAML') +def parse_easystack(filepath): + """Parses through easystack file, returns what EC are to be installed together with their options.""" + log_msg = "Support for easybuild-ing from multiple easyconfigs based on " + log_msg += "information obtained from provided file (easystack) with build specifications." + _log.experimental(log_msg) + _log.info("Building from easystack: '%s'" % filepath) + + # class instance which contains all info about planned build + easystack = EasyStackParser.parse(filepath) + + easyconfig_names = easystack.compose_ec_filenames() + + general_options = easystack.get_general_options() + + _log.debug("EasyStack parsed. Proceeding to install these Easyconfigs: %s" % ', '.join(sorted(easyconfig_names))) + if len(general_options) != 0: + _log.debug("General options for installation are: \n%s" % str(general_options)) + else: + _log.debug("No general options were specified in easystack") + + return easyconfig_names, general_options diff --git a/easybuild/main.py b/easybuild/main.py index bd7a0b0f5e..c2030a7d68 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -47,20 +47,22 @@ from easybuild.framework.easyblock import build_and_install_one, inject_checksums from easybuild.framework.easyconfig import EASYCONFIGS_PKG_SUBDIR +from easybuild.framework.easystack import parse_easystack from easybuild.framework.easyconfig.easyconfig import clean_up_easyconfigs from easybuild.framework.easyconfig.easyconfig import fix_deprecated_easyconfigs, verify_easyconfig_filename from easybuild.framework.easyconfig.style import cmdline_easyconfigs_style_check -from easybuild.framework.easyconfig.tools import categorize_files_by_type, dep_graph +from easybuild.framework.easyconfig.tools import categorize_files_by_type, dep_graph, det_copy_ec_specs from easybuild.framework.easyconfig.tools import det_easyconfig_paths, dump_env_script, get_paths_for from easybuild.framework.easyconfig.tools import parse_easyconfigs, review_pr, run_contrib_checks, skip_available from easybuild.framework.easyconfig.tweak import obtain_ec_for, tweak from easybuild.tools.config import find_last_log, get_repository, get_repositorypath, build_option from easybuild.tools.containers.common import containerize from easybuild.tools.docs import list_software -from easybuild.tools.filetools import adjust_permissions, cleanup, copy_file, copy_files, dump_index, load_index -from easybuild.tools.filetools import read_file, register_lock_cleanup_signal_handlers, write_file -from easybuild.tools.github import check_github, close_pr, new_branch_github, find_easybuild_easyconfig -from easybuild.tools.github import install_github_token, list_prs, new_pr, new_pr_from_branch, merge_pr +from easybuild.tools.filetools import adjust_permissions, cleanup, copy_files, dump_index, load_index +from easybuild.tools.filetools import locate_files, read_file, register_lock_cleanup_signal_handlers, write_file +from easybuild.tools.github import check_github, close_pr, find_easybuild_easyconfig +from easybuild.tools.github import install_github_token, list_prs, merge_pr, new_branch_github, new_pr +from easybuild.tools.github import new_pr_from_branch from easybuild.tools.github import sync_branch_with_develop, sync_pr_with_develop, update_branch, update_pr from easybuild.tools.hooks import START, END, load_hooks, run_hook from easybuild.tools.modules import modules_tool @@ -223,6 +225,13 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): last_log = find_last_log(logfile) or '(none)' print_msg(last_log, log=_log, prefix=False) + # if easystack is provided with the command, commands with arguments from it will be executed + if options.easystack: + # TODO add general_options (i.e. robot) to build options + orig_paths, general_options = parse_easystack(options.easystack) + if general_options: + raise EasyBuildError("Specifying general configuration options in easystack file is not supported yet.") + # check whether packaging is supported when it's being used if options.package: check_pkg_support() @@ -303,12 +312,9 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): eb_file = find_easybuild_easyconfig() orig_paths.append(eb_file) - if len(orig_paths) == 1: - # if only one easyconfig file is specified, use current directory as target directory - target_path = os.getcwd() - elif orig_paths: - # last path is target when --copy-ec is used, so remove that from the list - target_path = orig_paths.pop() if options.copy_ec else None + if options.copy_ec: + # figure out list of files to copy + target location (taking into account --from-pr) + orig_paths, target_path = det_copy_ec_specs(orig_paths, options.from_pr) categorized_paths = categorize_files_by_type(orig_paths) @@ -321,17 +327,17 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): # determine paths to easyconfigs determined_paths = det_easyconfig_paths(categorized_paths['easyconfigs']) - if (options.copy_ec and not tweaked_ecs_paths) or options.fix_deprecated_easyconfigs or options.show_ec: + # only copy easyconfigs here if we're not using --try-* (that's handled below) + copy_ec = options.copy_ec and not tweaked_ecs_paths + + if copy_ec or options.fix_deprecated_easyconfigs or options.show_ec: if options.copy_ec: - if len(determined_paths) == 1: - copy_file(determined_paths[0], target_path) - print_msg("%s copied to %s" % (os.path.basename(determined_paths[0]), target_path), prefix=False) - elif len(determined_paths) > 1: - copy_files(determined_paths, target_path) - print_msg("%d file(s) copied to %s" % (len(determined_paths), target_path), prefix=False) - else: - raise EasyBuildError("One of more files to copy should be specified!") + # at this point some paths may still just be filenames rather than absolute paths, + # so try to determine full path for those too via robot search path + paths = locate_files(orig_paths, robot_path) + + copy_files(paths, target_path, target_single_file=True, allow_empty=False, verbose=True) elif options.fix_deprecated_easyconfigs: fix_deprecated_easyconfigs(determined_paths) @@ -361,7 +367,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): if options.regtest or options.aggregate_regtest: _log.info("Running regression test") # fallback: easybuild-easyconfigs install path - regtest_ok = regtest([path[0] for path in paths] or easyconfigs_pkg_paths, modtool) + regtest_ok = regtest([x for (x, _) in paths] or easyconfigs_pkg_paths, modtool) if not regtest_ok: _log.info("Regression test failed (partially)!") sys.exit(31) # exit -> 3x1t -> 31 @@ -429,8 +435,9 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): if tweaked_ecs_in_all_ecs: # Clean them, then copy them clean_up_easyconfigs(tweaked_ecs_in_all_ecs) - copy_files(tweaked_ecs_in_all_ecs, target_path) - print_msg("%d file(s) copied to %s" % (len(tweaked_ecs_in_all_ecs), target_path), prefix=False) + copy_files(tweaked_ecs_in_all_ecs, target_path, allow_empty=False, verbose=True) + + clean_exit(logfile, eb_tmpdir, testing) # creating/updating PRs if pr_options: diff --git a/easybuild/scripts/rpath_args.py b/easybuild/scripts/rpath_args.py index 66b52b8408..150cdc3c3b 100755 --- a/easybuild/scripts/rpath_args.py +++ b/easybuild/scripts/rpath_args.py @@ -81,7 +81,6 @@ add_rpath_args = False cmd_args.append(arg) - # FIXME: also consider $LIBRARY_PATH? # FIXME: support to hard inject additional library paths? # FIXME: support to specify list of path prefixes that should not be RPATH'ed into account? # FIXME skip paths in /tmp, build dir, etc.? @@ -96,14 +95,15 @@ else: lib_path = arg[2:] - if os.path.isabs(lib_path) and (rpath_filter is None or not rpath_filter.match(lib_path)): + if lib_path and os.path.isabs(lib_path) and (rpath_filter is None or not rpath_filter.match(lib_path)): # inject -rpath flag in front for every -L with an absolute path, # also retain the -L flag (without reordering!) cmd_args_rpath.append(flag_prefix + '-rpath=%s' % lib_path) cmd_args.append('-L%s' % lib_path) else: - # don't RPATH in relative paths; - # it doesn't make much sense, and it can also break the build because it may result in reordering lib paths + # don't RPATH in empty or relative paths, or paths that are filtered out; + # linking relative paths via RPATH doesn't make much sense, + # and it can also break the build because it may result in reordering lib paths cmd_args.append('-L%s' % lib_path) # replace --enable-new-dtags with --disable-new-dtags if it's used; @@ -119,6 +119,14 @@ idx += 1 +# also inject -rpath options for all entries in $LIBRARY_PATH, +# unless they are there already +for lib_path in os.getenv('LIBRARY_PATH', '').split(os.pathsep): + if lib_path and os.path.isabs(lib_path) and (rpath_filter is None or not rpath_filter.match(lib_path)): + rpath_arg = flag_prefix + '-rpath=%s' % lib_path + if rpath_arg not in cmd_args_rpath: + cmd_args_rpath.append(rpath_arg) + if add_rpath_args: # try to make sure that RUNPATH is not used by always injecting --disable-new-dtags cmd_args_rpath.insert(0, flag_prefix + '--disable-new-dtags') diff --git a/easybuild/toolchains/gobff.py b/easybuild/toolchains/gobff.py new file mode 100644 index 0000000000..be396c60ed --- /dev/null +++ b/easybuild/toolchains/gobff.py @@ -0,0 +1,41 @@ +## +# Copyright 2013-2020 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for gobff compiler toolchain (includes GCC, OpenMPI, BLIS, libFLAME, ScaLAPACK and FFTW). + +:author: Sebastian Achilles (Forschungszentrum Juelich GmbH) +""" + +from easybuild.toolchains.gompi import Gompi +from easybuild.toolchains.linalg.blis import Blis +from easybuild.toolchains.linalg.flame import Flame +from easybuild.toolchains.linalg.scalapack import ScaLAPACK +from easybuild.toolchains.fft.fftw import Fftw + + +class Gobff(Gompi, Blis, Flame, ScaLAPACK, Fftw): + """Compiler toolchain with GCC, OpenMPI, BLIS, libFLAME, ScaLAPACK and FFTW.""" + NAME = 'gobff' + SUBTOOLCHAIN = Gompi.NAME diff --git a/easybuild/toolchains/linalg/blis.py b/easybuild/toolchains/linalg/blis.py index 95b37769a4..74c916a5c9 100644 --- a/easybuild/toolchains/linalg/blis.py +++ b/easybuild/toolchains/linalg/blis.py @@ -27,7 +27,9 @@ :author: Kenneth Hoste (Ghent University) :author: Bart Oldeman (McGill University, Calcul Quebec, Compute Canada) +:author: Sebastian Achilles (Forschungszentrum Juelich GmbH) """ +from distutils.version import LooseVersion from easybuild.tools.toolchain.linalg import LinAlg @@ -42,3 +44,14 @@ class Blis(LinAlg): BLAS_MODULE_NAME = ['BLIS'] BLAS_LIB = ['blis'] BLAS_FAMILY = TC_CONSTANT_BLIS + + def _set_blas_variables(self): + """AMD's fork with version number > 2.1 names the MT library blis-mt, while vanilla BLIS doesn't.""" + + # This assumes that AMD's BLIS has ver > 2.1 and vanilla BLIS < 2.1 + + found_version = self.get_software_version(self.BLAS_MODULE_NAME)[0] + if LooseVersion(found_version) > LooseVersion('2.1'): + self.BLAS_LIB_MT = ['blis-mt'] + + super(Blis, self)._set_blas_variables() diff --git a/easybuild/toolchains/linalg/flame.py b/easybuild/toolchains/linalg/flame.py index 8d67bf2558..ce99050a65 100644 --- a/easybuild/toolchains/linalg/flame.py +++ b/easybuild/toolchains/linalg/flame.py @@ -27,6 +27,7 @@ :author: Stijn De Weirdt (Ghent University) :author: Kenneth Hoste (Ghent University) +:author: Sebastian Achilles (Forschungszentrum Juelich GmbH) """ from easybuild.toolchains.linalg.lapack import Lapack @@ -37,6 +38,6 @@ class Flame(Lapack): """Less trivial module, provides FLAME support.""" - LAPACK_MODULE_NAME = ['FLAME'] + Lapack.LAPACK_MODULE_NAME # no super() - LAPACK_LIB = ['lapack2flame', 'flame'] + Lapack.LAPACK_LIB # no super() + LAPACK_MODULE_NAME = ['libFLAME'] # + Lapack.LAPACK_MODULE_NAME # no super() + LAPACK_LIB = ['flame'] # + Lapack.LAPACK_LIB # no super() LAPACK_FAMILY = TC_CONSTANT_FLAME diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 10d5a19add..170040576e 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -101,6 +101,7 @@ DEFAULT_PKG_TOOL = PKG_TOOL_FPM DEFAULT_PKG_TYPE = PKG_TYPE_RPM DEFAULT_PNS = 'EasyBuildPNS' +DEFAULT_PR_TARGET_ACCOUNT = 'easybuilders' DEFAULT_PREFIX = os.path.join(os.path.expanduser('~'), ".local", "easybuild") DEFAULT_REPOSITORY = 'FileRepository' DEFAULT_WAIT_ON_LOCK_INTERVAL = 60 @@ -204,7 +205,6 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'pr_branch_name', 'pr_commit_msg', 'pr_descr', - 'pr_target_account', 'pr_target_repo', 'pr_title', 'rpath_filter', @@ -253,6 +253,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'sequential', 'set_gid_bit', 'skip_test_cases', + 'skip_test_step', 'generate_devel_module', 'sticky_bit', 'trace', @@ -307,6 +308,9 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): DEFAULT_PKG_TYPE: [ 'package_type', ], + DEFAULT_PR_TARGET_ACCOUNT: [ + 'pr_target_account', + ], GENERAL_CLASS: [ 'suffix_modules_path', ], diff --git a/easybuild/tools/containers/singularity.py b/easybuild/tools/containers/singularity.py index 4ecbed6990..9583f25d32 100644 --- a/easybuild/tools/containers/singularity.py +++ b/easybuild/tools/containers/singularity.py @@ -314,13 +314,6 @@ def resolve_template_data(self): "# install EasyBuild using pip", # upgrade pip "pip install -U pip", - "pip install wheel", - # EasyBuild 3.x requires setuptools as runtime dependency - "pip install -U setuptools", - # stick to previous version of vsc-install to avoid requiring mock (which causes installation problems) - # stick to previous version of vsc-base to avoid requiring 'future' (irrelevant for EasyBuild) - # this is just a temporary measure, since vsc-install & vsc-base have been ingested for EasyBuild 4.x - "pip install 'vsc-install<0.11.4' 'vsc-base<2.9.0'", "pip install easybuild", ]) diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index 9624d43983..1fd8d119e9 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -726,16 +726,16 @@ def list_software_txt(software, detailed=False): def list_toolchains(output_format=FORMAT_TXT): """Show list of known toolchains.""" _, all_tcs = search_toolchain('') - all_tcs_names = [x.NAME for x in all_tcs] - tclist = sorted(zip(all_tcs_names, all_tcs)) - tcs = dict() - for (tcname, tcc) in tclist: + # filter deprecated 'dummy' toolchain + all_tcs = [x for x in all_tcs if x.NAME != DUMMY_TOOLCHAIN_NAME] + all_tcs_names = [x.NAME for x in all_tcs] - # filter deprecated 'dummy' toolchain - if tcname == DUMMY_TOOLCHAIN_NAME: - continue + # start with dict that maps toolchain name to corresponding subclass of Toolchain + tcs = dict(zip(all_tcs_names, all_tcs)) + for tcname in sorted(tcs): + tcc = tcs[tcname] tc = tcc(version='1.2.3') # version doesn't matter here, but something needs to be there tcs[tcname] = tc.definition() diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 1c3dbf5f5b..dc44bfee3d 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -792,6 +792,60 @@ def find_easyconfigs(path, ignore_dirs=None): return files +def locate_files(files, paths, ignore_subdirs=None): + """ + Determine full path for list of files, in given list of paths (directories). + """ + # determine which files need to be found, if any + files_to_find = [] + for idx, filepath in enumerate(files): + if filepath == os.path.basename(filepath) and not os.path.exists(filepath): + files_to_find.append((idx, filepath)) + _log.debug("List of files to find: %s", files_to_find) + + # find missing easyconfigs by walking paths in robot search path + for path in paths: + + # skip non-existing paths + if not os.path.exists(path): + _log.debug("%s does not exist, skipping it...", path) + continue + + _log.debug("Looking for missing files (%d left) in %s...", len(files_to_find), path) + + # try to load index for current path, or create one + path_index = load_index(path, ignore_dirs=ignore_subdirs) + if path_index is None or build_option('ignore_index'): + _log.info("No index found for %s, creating one (in memory)...", path) + path_index = create_index(path, ignore_dirs=ignore_subdirs) + else: + _log.info("Index found for %s, so using it...", path) + + for filepath in path_index: + for idx, file_to_find in files_to_find[:]: + if os.path.basename(filepath) == file_to_find: + full_path = os.path.join(path, filepath) + _log.info("Found %s in %s: %s", file_to_find, path, full_path) + files[idx] = full_path + # if file was found, stop looking for it (first hit wins) + files_to_find.remove((idx, file_to_find)) + + # stop as soon as we have all we need (path index loop) + if not files_to_find: + break + + # stop as soon as we have all we need (paths loop) + if not files_to_find: + break + + if files_to_find: + filenames = ', '.join([f for (_, f) in files_to_find]) + paths = ', '.join(paths) + raise EasyBuildError("One or more files not found: %s (search paths: %s)", filenames, paths) + + return [os.path.abspath(f) for f in files] + + def find_glob_pattern(glob_pattern, fail_on_no_match=True): """Find unique file/dir matching glob_pattern (raises error if more than one match is found)""" if build_option('extended_dry_run'): @@ -1279,25 +1333,30 @@ def apply_patch(patch_file, dest, fn=None, copy=False, level=None, use_git_am=Fa return True -def apply_regex_substitutions(path, regex_subs, backup='.orig.eb'): +def apply_regex_substitutions(paths, regex_subs, backup='.orig.eb'): """ Apply specified list of regex substitutions. - :param path: path to file to patch + :param paths: list of paths to files to patch (or just a single filepath) :param regex_subs: list of substitutions to apply, specified as (, ) :param backup: create backup of original file with specified suffix (no backup if value evaluates to False) """ + if isinstance(paths, string_type): + paths = [paths] + # only report when in 'dry run' mode if build_option('extended_dry_run'): - dry_run_msg("applying regex substitutions to file %s" % path, silent=build_option('silent')) + paths_str = ', '.join(paths) + dry_run_msg("applying regex substitutions to file(s): %s" % paths_str, silent=build_option('silent')) for regex, subtxt in regex_subs: dry_run_msg(" * regex pattern '%s', replacement string '%s'" % (regex, subtxt)) else: - _log.info("Applying following regex substitutions to %s: %s", path, regex_subs) + _log.info("Applying following regex substitutions to %s: %s", paths, regex_subs) - for i, (regex, subtxt) in enumerate(regex_subs): - regex_subs[i] = (re.compile(regex), subtxt) + compiled_regex_subs = [] + for regex, subtxt in regex_subs: + compiled_regex_subs.append((re.compile(regex), subtxt)) if backup: backup_ext = backup @@ -1305,30 +1364,32 @@ def apply_regex_substitutions(path, regex_subs, backup='.orig.eb'): # no (persistent) backup file is created if empty string value is passed to 'backup' in fileinput.input backup_ext = '' - try: - # make sure that file can be opened in text mode; - # it's possible this fails with UnicodeDecodeError when running EasyBuild with Python 3 + for path in paths: try: - with open(path, 'r') as fp: - _ = fp.read() - except UnicodeDecodeError as err: - _log.info("Encountered UnicodeDecodeError when opening %s in text mode: %s", path, err) - path_backup = back_up_file(path) - _log.info("Editing %s to strip out non-UTF-8 characters (backup at %s)", path, path_backup) - txt = read_file(path, mode='rb') - txt_utf8 = txt.decode(encoding='utf-8', errors='replace') - write_file(path, txt_utf8) - - for line_id, line in enumerate(fileinput.input(path, inplace=1, backup=backup_ext)): - for regex, subtxt in regex_subs: - match = regex.search(line) - if match: - _log.info("Replacing line %d in %s: '%s' -> '%s'", (line_id + 1), path, match.group(0), subtxt) - line = regex.sub(subtxt, line) - sys.stdout.write(line) - - except (IOError, OSError) as err: - raise EasyBuildError("Failed to patch %s: %s", path, err) + # make sure that file can be opened in text mode; + # it's possible this fails with UnicodeDecodeError when running EasyBuild with Python 3 + try: + with open(path, 'r') as fp: + _ = fp.read() + except UnicodeDecodeError as err: + _log.info("Encountered UnicodeDecodeError when opening %s in text mode: %s", path, err) + path_backup = back_up_file(path) + _log.info("Editing %s to strip out non-UTF-8 characters (backup at %s)", path, path_backup) + txt = read_file(path, mode='rb') + txt_utf8 = txt.decode(encoding='utf-8', errors='replace') + write_file(path, txt_utf8) + + for line_id, line in enumerate(fileinput.input(path, inplace=1, backup=backup_ext)): + for regex, subtxt in compiled_regex_subs: + match = regex.search(line) + if match: + origtxt = match.group(0) + _log.info("Replacing line %d in %s: '%s' -> '%s'", (line_id + 1), path, origtxt, subtxt) + line = regex.sub(subtxt, line) + sys.stdout.write(line) + + except (IOError, OSError) as err: + raise EasyBuildError("Failed to patch %s: %s", path, err) def modify_env(old, new): @@ -2034,26 +2095,49 @@ def copy_file(path, target_path, force_in_dry_run=False): raise EasyBuildError("Failed to copy file %s to %s: %s", path, target_path, err) -def copy_files(paths, target_dir, force_in_dry_run=False): +def copy_files(paths, target_path, force_in_dry_run=False, target_single_file=False, allow_empty=True, verbose=False): """ - Copy list of files to specified target directory (which is created if it doesn't exist yet). + Copy list of files to specified target path. + Target directory is created if it doesn't exist yet. - :param filepaths: list of files to copy - :param target_dir: target directory to copy files into + :param paths: list of filepaths to copy + :param target_path: path to copy files to :param force_in_dry_run: force copying of files during dry run + :param target_single_file: if there's only a single file to copy, copy to a file at target path (not a directory) + :param allow_empty: allow empty list of paths to copy as input (if False: raise error on empty input list) + :param verbose: print a message to report copying of files """ + # dry run: just report copying, don't actually copy if not force_in_dry_run and build_option('extended_dry_run'): - dry_run_msg("copied files %s to %s" % (paths, target_dir)) - else: - if os.path.exists(target_dir): - if os.path.isdir(target_dir): - _log.info("Copying easyconfigs into existing directory %s...", target_dir) + if len(paths) == 1: + dry_run_msg("copied %s to %s" % (paths[0], target_path)) + else: + dry_run_msg("copied %d files to %s" % (len(paths), target_path)) + + # special case: single file to copy and target_single_file is True => copy to file + elif len(paths) == 1 and target_single_file: + copy_file(paths[0], target_path) + if verbose: + print_msg("%s copied to %s" % (paths[0], target_path), prefix=False) + + elif paths: + # check target path: if it exists it should be a directory; if it doesn't exist, we create it + if os.path.exists(target_path): + if os.path.isdir(target_path): + _log.info("Copying easyconfigs into existing directory %s...", target_path) else: - raise EasyBuildError("%s exists but is not a directory", target_dir) + raise EasyBuildError("%s exists but is not a directory", target_path) else: - mkdir(target_dir, parents=True) + mkdir(target_path, parents=True) + for path in paths: - copy_file(path, target_dir) + copy_file(path, target_path) + + if verbose: + print_msg("%d file(s) copied to %s" % (len(paths), target_path), prefix=False) + + elif not allow_empty: + raise EasyBuildError("One or more files to copy should be specified!") def copy_dir(path, target_path, force_in_dry_run=False, dirs_exist_ok=False, **kwargs): @@ -2275,10 +2359,11 @@ def install_fake_vsc(): ' filename, lineno = cand_filename, cand_lineno', ' break', '', - '# ignore imports from pkgutil.py (part of Python standard library),', + '# ignore imports from pkgutil.py (part of Python standard library)', + '# or from pkg_resources/__init__.py (setuptools),', '# which may happen due to a system-wide installation of vsc-base', '# even if it is not actually actively used...', - 'if os.path.basename(filename) != "pkgutil.py":', + 'if os.path.basename(filename) != "pkgutil.py" and not filename.endswith("pkg_resources/__init__.py"):', ' error_msg = "\\nERROR: Detected import from \'vsc\' namespace in %s (line %s)\\n" % (filename, lineno)', ' error_msg += "vsc-base & vsc-install were ingested into the EasyBuild framework in EasyBuild v4.0\\n"', ' error_msg += "The functionality you need may be available in the \'easybuild.base.*\' namespace.\\n"', diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 4d80344d29..3dc25eb267 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -33,6 +33,7 @@ import copy import getpass import glob +import functools import os import random import re @@ -249,6 +250,12 @@ def github_api_get_request(request_f, github_user=None, token=None, **kwargs): if token is None: token = fetch_github_token(github_user) + # if we don't have a GitHub token, don't pass username either; + # this makes sense for read-only actions like fetching files from PRs + if token is None: + _log.info("Not specifying username since no GitHub token is available for %s", github_user) + github_user = None + url = request_f(RestClient(GITHUB_API_URL, username=github_user, token=token)) try: @@ -376,17 +383,37 @@ def download_repo(repo=GITHUB_EASYCONFIGS_REPO, branch='master', account=GITHUB_ return extracted_path -def fetch_easyblocks_from_pr(pr, path=None, github_user=None): - """Fetch patched easyconfig files for a particular PR.""" - return fetch_files_from_pr(pr, path, github_user, github_repo=GITHUB_EASYBLOCKS_REPO) +def pr_files_cache(func): + """ + Decorator to cache result of fetch_files_from_pr. + """ + cache = {} + + @functools.wraps(func) + def cache_aware_func(pr, path=None, github_user=None, github_account=None, github_repo=None): + """Retrieve cached result, or fetch files from PR & cache result.""" + # cache key is combination of all function arguments (incl. optional ones) + key = (pr, github_account, github_repo, path) + + if key in cache and all(os.path.exists(x) for x in cache[key]): + _log.info("Using cached value for fetch_files_from_pr for PR #%s (account=%s, repo=%s, path=%s)", + pr, github_account, github_repo, path) + return cache[key] + else: + res = func(pr, path=path, github_user=github_user, github_account=github_account, github_repo=github_repo) + cache[key] = res + return res + # expose clear/update methods of cache + cache itself to wrapped function + cache_aware_func._cache = cache # useful in tests + cache_aware_func.clear_cache = cache.clear + cache_aware_func.update_cache = cache.update -def fetch_easyconfigs_from_pr(pr, path=None, github_user=None): - """Fetch patched easyconfig files for a particular PR.""" - return fetch_files_from_pr(pr, path, github_user, github_repo=GITHUB_EASYCONFIGS_REPO) + return cache_aware_func -def fetch_files_from_pr(pr, path=None, github_user=None, github_repo=None): +@pr_files_cache +def fetch_files_from_pr(pr, path=None, github_user=None, github_account=None, github_repo=None): """Fetch patched files for a particular PR.""" if github_user is None: @@ -409,7 +436,8 @@ def fetch_files_from_pr(pr, path=None, github_user=None, github_repo=None): # make sure path exists, create it if necessary mkdir(path, parents=True) - github_account = build_option('pr_target_account') + if github_account is None: + github_account = build_option('pr_target_account') if github_repo == GITHUB_EASYCONFIGS_REPO: easyfiles = 'easyconfigs' @@ -495,6 +523,16 @@ def fetch_files_from_pr(pr, path=None, github_user=None, github_repo=None): return files +def fetch_easyblocks_from_pr(pr, path=None, github_user=None): + """Fetch patched easyconfig files for a particular PR.""" + return fetch_files_from_pr(pr, path, github_user, github_repo=GITHUB_EASYBLOCKS_REPO) + + +def fetch_easyconfigs_from_pr(pr, path=None, github_user=None): + """Fetch patched easyconfig files for a particular PR.""" + return fetch_files_from_pr(pr, path, github_user, github_repo=GITHUB_EASYCONFIGS_REPO) + + def create_gist(txt, fn, descr=None, github_user=None, github_token=None): """Create a gist with the provided text.""" @@ -1282,7 +1320,7 @@ def merge_pr(pr): pr_target_account = build_option('pr_target_account') pr_target_repo = build_option('pr_target_repo') or GITHUB_EASYCONFIGS_REPO - pr_data, pr_url = fetch_pr_data(pr, pr_target_account, pr_target_repo, github_user, full=True) + pr_data, _ = fetch_pr_data(pr, pr_target_account, pr_target_repo, github_user, full=True) msg = "\n%s/%s PR #%s was submitted by %s, " % (pr_target_account, pr_target_repo, pr, pr_data['user']['login']) msg += "you are using GitHub account '%s'\n" % github_user @@ -1293,6 +1331,12 @@ def merge_pr(pr): force = build_option('force') dry_run = build_option('dry_run') or build_option('extended_dry_run') + if not dry_run: + if pr_data['merged']: + raise EasyBuildError("This PR is already merged.") + elif pr_data['state'] == GITHUB_STATE_CLOSED: + raise EasyBuildError("This PR is closed.") + def merge_url(gh): """Utility function to fetch merge URL for a specific PR.""" return gh.repos[pr_target_account][pr_target_repo].pulls[pr].merge @@ -2129,7 +2173,13 @@ def pr_url(gh): else: return gh.repos[pr_target_account][pr_target_repo].pulls[pr] - status, pr_data = github_api_get_request(pr_url, github_user, **parameters) + try: + status, pr_data = github_api_get_request(pr_url, github_user, **parameters) + except HTTPError as err: + raise EasyBuildError("Failed to get data for PR #%d from %s/%s (%s)\n" + "Please check PR #, account and repo.", + pr, pr_target_account, pr_target_repo, err) + if status != HTTP_STATUS_OK: raise EasyBuildError("Failed to get data for PR #%d from %s/%s (status: %d %s)", pr, pr_target_account, pr_target_repo, status, pr_data) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 38640056cf..13dbe41a91 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -64,11 +64,11 @@ from easybuild.tools.config import DEFAULT_JOB_BACKEND, DEFAULT_LOGFILE_FORMAT, DEFAULT_MAX_FAIL_RATIO_PERMS from easybuild.tools.config import DEFAULT_MINIMAL_BUILD_ENV, DEFAULT_MNS, DEFAULT_MODULE_SYNTAX, DEFAULT_MODULES_TOOL from easybuild.tools.config import DEFAULT_MODULECLASSES, DEFAULT_PATH_SUBDIRS, DEFAULT_PKG_RELEASE, DEFAULT_PKG_TOOL -from easybuild.tools.config import DEFAULT_PKG_TYPE, DEFAULT_PNS, DEFAULT_PREFIX, DEFAULT_REPOSITORY -from easybuild.tools.config import DEFAULT_WAIT_ON_LOCK_INTERVAL, DEFAULT_WAIT_ON_LOCK_LIMIT, EBROOT_ENV_VAR_ACTIONS -from easybuild.tools.config import ERROR, FORCE_DOWNLOAD_CHOICES, GENERAL_CLASS, IGNORE, JOB_DEPS_TYPE_ABORT_ON_ERROR -from easybuild.tools.config import JOB_DEPS_TYPE_ALWAYS_RUN, LOADED_MODULES_ACTIONS, LOCAL_VAR_NAMING_CHECK_WARN -from easybuild.tools.config import LOCAL_VAR_NAMING_CHECKS, WARN +from easybuild.tools.config import DEFAULT_PKG_TYPE, DEFAULT_PNS, DEFAULT_PREFIX, DEFAULT_PR_TARGET_ACCOUNT +from easybuild.tools.config import DEFAULT_REPOSITORY, DEFAULT_WAIT_ON_LOCK_INTERVAL, DEFAULT_WAIT_ON_LOCK_LIMIT +from easybuild.tools.config import EBROOT_ENV_VAR_ACTIONS, ERROR, FORCE_DOWNLOAD_CHOICES, GENERAL_CLASS, IGNORE +from easybuild.tools.config import JOB_DEPS_TYPE_ABORT_ON_ERROR, JOB_DEPS_TYPE_ALWAYS_RUN, LOADED_MODULES_ACTIONS +from easybuild.tools.config import LOCAL_VAR_NAMING_CHECK_WARN, LOCAL_VAR_NAMING_CHECKS, WARN from easybuild.tools.config import get_pretend_installpath, init, init_build_options, mk_full_default_path from easybuild.tools.configobj import ConfigObj, ConfigObjError from easybuild.tools.docs import FORMAT_TXT, FORMAT_RST @@ -78,7 +78,7 @@ from easybuild.tools.environment import restore_env, unset_env_vars from easybuild.tools.filetools import CHECKSUM_TYPE_SHA256, CHECKSUM_TYPES, expand_glob_paths, install_fake_vsc from easybuild.tools.filetools import move_file, which -from easybuild.tools.github import GITHUB_EB_MAIN, GITHUB_PR_DIRECTION_DESC, GITHUB_PR_ORDER_CREATED +from easybuild.tools.github import GITHUB_PR_DIRECTION_DESC, GITHUB_PR_ORDER_CREATED from easybuild.tools.github import GITHUB_PR_STATE_OPEN, GITHUB_PR_STATES, GITHUB_PR_ORDERS, GITHUB_PR_DIRECTIONS from easybuild.tools.github import HAVE_GITHUB_API, HAVE_KEYRING, VALID_CLOSE_PR_REASONS from easybuild.tools.github import fetch_easyblocks_from_pr, fetch_github_token @@ -356,8 +356,9 @@ def override_options(self): 'consider-archived-easyconfigs': ("Also consider archived easyconfigs", None, 'store_true', False), 'containerize': ("Generate container recipe/image", None, 'store_true', False, 'C'), 'copy-ec': ("Copy specified easyconfig(s) to specified location", None, 'store_true', False), - 'cuda-compute-capabilities': ("List of CUDA compute capabilities to use when building GPU software", - 'strlist', 'extend', None), + 'cuda-compute-capabilities': ("List of CUDA compute capabilities to use when building GPU software; " + "values should be specified as digits separated by a dot, " + "for example: 3.5,5.0,7.2", 'strlist', 'extend', None), 'debug-lmod': ("Run Lmod modules tool commands in debug module", None, 'store_true', False), 'default-opt-level': ("Specify default optimisation level", 'choice', 'store', DEFAULT_OPT_LEVEL, Compiler.COMPILER_OPT_FLAGS), @@ -438,6 +439,7 @@ def override_options(self): 'silence-deprecation-warnings': ("Silence specified deprecation warnings", 'strlist', 'extend', None), 'sticky-bit': ("Set sticky bit on newly created directories", None, 'store_true', False), 'skip-test-cases': ("Skip running test cases", None, 'store_true', False, 't'), + 'skip-test-step': ("Skip running the test step (e.g. unit tests)", None, 'store_true', False), 'generate-devel-module': ("Generate a develop module file, implies --force if disabled", None, 'store_true', True), 'sysroot': ("Location root directory of system, prefix for standard paths like /usr/lib and /usr/include", @@ -604,6 +606,8 @@ def informative_options(self): 'show-full-config': ("Show current EasyBuild configuration (all settings)", None, 'store_true', False), 'show-system-info': ("Show system information relevant to EasyBuild", None, 'store_true', False), 'terse': ("Terse output (machine-readable)", None, 'store_true', False), + 'easystack': ("Path to easystack file in YAML format, specifying details of a software stack", + None, 'store', None), }) self.log.debug("informative_options: descr %s opts %s" % (descr, opts)) @@ -642,7 +646,7 @@ def github_options(self): str, 'store', None), 'pr-commit-msg': ("Commit message for new/updated pull request created with --new-pr", str, 'store', None), 'pr-descr': ("Description for new pull request created with --new-pr", str, 'store', None), - 'pr-target-account': ("Target account for new PRs", str, 'store', GITHUB_EB_MAIN), + 'pr-target-account': ("Target account for new PRs", str, 'store', DEFAULT_PR_TARGET_ACCOUNT), 'pr-target-branch': ("Target branch for new PRs", str, 'store', DEFAULT_BRANCH), 'pr-target-repo': ("Target repository for new/updating PRs (default: auto-detect based on provided files)", str, 'store', None), diff --git a/easybuild/tools/toolchain/compiler.py b/easybuild/tools/toolchain/compiler.py index 27f255f6ec..885872e81c 100644 --- a/easybuild/tools/toolchain/compiler.py +++ b/easybuild/tools/toolchain/compiler.py @@ -93,6 +93,11 @@ class Compiler(Toolchain): 'vectorize': (None, "Enable compiler auto-vectorization, default except for noopt and lowopt"), 'packed-linker-options': (False, "Pack the linker options as comma separated list"), # ScaLAPACK mainly 'rpath': (True, "Use RPATH wrappers when --rpath is enabled in EasyBuild configuration"), + 'extra_cflags': (None, "Specify extra CFLAGS options."), + 'extra_cxxflags': (None, "Specify extra CXXFLAGS options."), + 'extra_fflags': (None, "Specify extra FFLAGS options."), + 'extra_fcflags': (None, "Specify extra FCFLAGS options."), + 'extra_f90flags': (None, "Specify extra F90FLAGS options."), } COMPILER_UNIQUE_OPTION_MAP = None @@ -110,6 +115,11 @@ class Compiler(Toolchain): 'static': 'static', 'unroll': 'unroll', 'verbose': 'v', + 'extra_cflags': '%(value)s', + 'extra_cxxflags': '%(value)s', + 'extra_fflags': '%(value)s', + 'extra_fcflags': '%(value)s', + 'extra_f90flags': '%(value)s', } COMPILER_OPTIMAL_ARCHITECTURE_OPTION = None @@ -277,15 +287,19 @@ def _set_compiler_flags(self): self.variables.nextend('PRECFLAGS', precflags[:1]) # precflags last - for var in ['CFLAGS', 'CXXFLAGS']: + for var in ['CFLAGS', 'CXXFLAGS', 'FCFLAGS', 'FFLAGS', 'F90FLAGS']: self.variables.join(var, 'OPTFLAGS', 'PRECFLAGS') self.variables.nextend(var, flags) - self.variables.nextend(var, cflags) - - for var in ['FCFLAGS', 'FFLAGS', 'F90FLAGS']: - self.variables.join(var, 'OPTFLAGS', 'PRECFLAGS') - self.variables.nextend(var, flags) - self.variables.nextend(var, fflags) + if var.startswith('C'): + self.variables.nextend(var, cflags) + else: + self.variables.nextend(var, fflags) + extra = 'extra_' + var.lower() + if self.options.get(extra): + flags = self.options.option(extra) + if not flags or flags[0] != '-': + raise EasyBuildError("toolchainopts %s: '%s' must start with a '-'." % (extra, flags)) + self.variables.nappend_el(var, flags[1:]) def _set_optimal_architecture(self, default_optarch=None): """ diff --git a/easybuild/tools/utilities.py b/easybuild/tools/utilities.py index f4e6dafe0f..931a6e1eac 100644 --- a/easybuild/tools/utilities.py +++ b/easybuild/tools/utilities.py @@ -173,18 +173,18 @@ def wrap(orig): pass if imported is None: - raise ImportError("None of the specified modules %s is available" % ', '.join(modnames)) + raise ImportError else: return orig - except ImportError as err: - # need to pass down 'err' via named argument to ensure it's in scope when using Python 3.x - def error(err=err, *args, **kwargs): - msg = "%s; required module '%s' is not available" % (err, modname) + except ImportError: + def error(*args, **kwargs): + msg = "None of the specified modules (%s) is available" % ', '.join(modnames) if pkgname: msg += " (provided by Python package %s, available from %s)" % (pkgname, url) elif url: msg += " (available from %s)" % url + msg += ", yet one of them is required!" raise EasyBuildError("ImportError: %s", msg) return error @@ -258,6 +258,10 @@ def mk_rst_table(titles, columns): """ Returns an rst table with given titles and columns (a nested list of string columns for each column) """ + # take into account that passed values may be iterators produced via 'map' + titles = list(titles) + columns = list(columns) + title_cnt, col_cnt = len(titles), len(columns) if title_cnt != col_cnt: msg = "Number of titles/columns should be equal, found %d titles and %d columns" % (title_cnt, col_cnt) diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index 1c15089e5a..ccf5fd3bfd 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.3.1') +VERSION = LooseVersion('4.3.2') UNKNOWN = 'UNKNOWN' diff --git a/requirements.txt b/requirements.txt index ebbcbaaf29..0defb13c7c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -53,7 +53,7 @@ requests archspec; python_version >= '2.7' -# cryptography 3.0 deprecates Python 2.7 -# and cryptography is not needed at all for Python 2.6 -cryptography==2.9.2; python_version == '2.7' +# cryptography 3.0 deprecates Python 2.7 (but v3.2.1 still works with Python 2.7); +# cryptography is not needed at all for Python 2.6 +cryptography==3.2.1; python_version == '2.7' cryptography; python_version >= '3.5' diff --git a/setup.py b/setup.py index 6c3fe46069..c2dc739e7c 100644 --- a/setup.py +++ b/setup.py @@ -57,7 +57,7 @@ def find_rel_test(): current = os.getcwd() os.chdir(basedir) res = [] - for subdir in ["sandbox", "easyconfigs", "modules"]: + for subdir in ["easyconfigs", "easystacks", "modules", "sandbox"]: res.extend([os.path.join(root, filename) for root, dirnames, filenames in os.walk(subdir) for filename in filenames if os.path.isfile(os.path.join(root, filename))]) diff --git a/test/framework/containers.py b/test/framework/containers.py index 3a3447e8fe..c642728b52 100644 --- a/test/framework/containers.py +++ b/test/framework/containers.py @@ -141,9 +141,7 @@ def test_end2end_singularity_recipe_config(self): self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) pip_patterns = [ - # EasyBuild and dependencies are installed with pip by default - "pip install -U setuptools", - "pip install.*vsc-base", + # EasyBuild is installed with pip by default "pip install easybuild", ] post_commands_patterns = [ diff --git a/test/framework/docs.py b/test/framework/docs.py index f6716ea171..d99344f4c5 100644 --- a/test/framework/docs.py +++ b/test/framework/docs.py @@ -256,7 +256,7 @@ def test_list_software(self): " * toy v0.0 (versionsuffix: '-deps'): system", " * toy v0.0 (versionsuffix: '-iter'): system", " * toy v0.0 (versionsuffix: '-multiple'): system", - " * toy v0.0 (versionsuffix: '-test'): gompi/2018a", + " * toy v0.0 (versionsuffix: '-test'): gompi/2018a, system", ] txt = list_software(output_format='txt', detailed=True) lines = txt.split('\n') @@ -278,7 +278,7 @@ def test_list_software(self): '``0.0`` ``-deps`` ``system`` ', '``0.0`` ``-iter`` ``system`` ', '``0.0`` ``-multiple`` ``system`` ', - '``0.0`` ``-test`` ``gompi/2018a`` ', + '``0.0`` ``-test`` ``gompi/2018a``, ``system``', '======= ============= ===========================', ] txt = list_software(output_format='rst', detailed=True) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 7a6c9af46b..2e278bbd32 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -56,7 +56,8 @@ from easybuild.framework.easyconfig.templates import template_constant_dict, to_template_str from easybuild.framework.easyconfig.style import check_easyconfigs_style from easybuild.framework.easyconfig.tools import categorize_files_by_type, check_sha256_checksums, dep_graph -from easybuild.framework.easyconfig.tools import find_related_easyconfigs, get_paths_for, parse_easyconfigs +from easybuild.framework.easyconfig.tools import det_copy_ec_specs, find_related_easyconfigs, get_paths_for +from easybuild.framework.easyconfig.tools import parse_easyconfigs from easybuild.framework.easyconfig.tweak import obtain_ec_for, tweak_one from easybuild.framework.extension import resolve_exts_filter_template from easybuild.toolchains.system import SystemToolchain @@ -994,6 +995,7 @@ def test_templating(self): 'Perl: %%(perlver)s, %%(perlmajver)s, %%(perlminver)s, %%(perlshortver)s', 'R: %%(rver)s, %%(rmajver)s, %%(rminver)s, %%(rshortver)s', ]), + 'modextrapaths = {"PI_MOD_NAME": "%%(module_name)s"}', 'license_file = HOME + "/licenses/PI/license.txt"', "github_account = 'easybuilders'", ]) % inp @@ -1028,6 +1030,7 @@ def test_templating(self): "Perl: 5.22.0, 5, 22, 5.22; " "R: 3.2.3, 3, 2, 3.2") self.assertEqual(eb['modloadmsg'], expected) + self.assertEqual(eb['modextrapaths'], {'PI_MOD_NAME': 'PI/3.04-Python-2.7.10'}) self.assertEqual(eb['license_file'], os.path.join(os.environ['HOME'], 'licenses', 'PI', 'license.txt')) # test the escaping insanity here (ie all the crap we allow in easyconfigs) @@ -2868,6 +2871,7 @@ def test_template_constant_dict(self): expected = { 'bitbucket_account': 'gzip', 'github_account': 'gzip', + 'module_name': 'gzip/1.5-foss-2018a', 'name': 'gzip', 'namelower': 'gzip', 'nameletter': 'g', @@ -2936,6 +2940,7 @@ def test_template_constant_dict(self): 'javaminver': '8', 'javashortver': '1.8', 'javaver': '1.8.0_221', + 'module_name': None, 'name': 'toy', 'namelower': 'toy', 'nameletter': 't', @@ -2976,6 +2981,7 @@ def test_template_constant_dict(self): self.assertTrue(arch_regex.match(arch), "'%s' matches with pattern '%s'" % (arch, arch_regex.pattern)) expected = { + 'module_name': None, 'name': 'foo', 'namelower': 'foo', 'nameletter': 'f', @@ -4018,6 +4024,84 @@ def test_cuda_compute_capabilities(self): self.assertEqual(ec['configopts'], 'sm_42,sm_63') self.assertEqual(ec['preconfigopts'], 'sm_42 sm_63') + def test_det_copy_ec_specs(self): + """Test det_copy_ec_specs function.""" + + cwd = os.getcwd() + + # no problems on empty list as input + paths, target_path = det_copy_ec_specs([], None) + self.assertEqual(paths, []) + self.assertEqual(target_path, None) + + # single-element list, no --from-pr => use current directory as target location + paths, target_path = det_copy_ec_specs(['test.eb'], None) + self.assertEqual(paths, ['test.eb']) + self.assertTrue(os.path.samefile(target_path, cwd)) + + # multi-element list, no --from-pr => last element is used as target location + for args in (['test.eb', 'dir'], ['test1.eb', 'test2.eb', 'dir']): + paths, target_path = det_copy_ec_specs(args, None) + self.assertEqual(paths, args[:-1]) + self.assertEqual(target_path, args[-1]) + + # use fixed PR (speeds up the test due to caching in fetch_files_from_pr; + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/8007 + from_pr = 8007 + arrow_ec_fn = 'Arrow-0.7.1-intel-2017b-Python-3.6.3.eb' + bat_ec_fn = 'bat-0.3.3-intel-2017b-Python-3.6.3.eb' + bat_patch_fn = 'bat-0.3.3-fix-pyspark.patch' + pr_files = [ + arrow_ec_fn, + bat_ec_fn, + bat_patch_fn, + ] + + # if no paths are specified, default is to copy all files touched by PR to current working directory + paths, target_path = det_copy_ec_specs([], from_pr) + self.assertEqual(len(paths), 3) + filenames = sorted([os.path.basename(x) for x in paths]) + self.assertEqual(filenames, sorted(pr_files)) + self.assertTrue(os.path.samefile(target_path, cwd)) + + # last argument is used as target directory, + # unless it corresponds to a file touched by PR + args = [bat_ec_fn, 'target_dir'] + paths, target_path = det_copy_ec_specs(args, from_pr) + self.assertEqual(len(paths), 1) + self.assertEqual(os.path.basename(paths[0]), bat_ec_fn) + self.assertEqual(target_path, 'target_dir') + + args = [bat_ec_fn] + paths, target_path = det_copy_ec_specs(args, from_pr) + self.assertEqual(len(paths), 1) + self.assertEqual(os.path.basename(paths[0]), bat_ec_fn) + self.assertTrue(os.path.samefile(target_path, cwd)) + + args = [arrow_ec_fn, bat_ec_fn] + paths, target_path = det_copy_ec_specs(args, from_pr) + self.assertEqual(len(paths), 2) + self.assertEqual(os.path.basename(paths[0]), arrow_ec_fn) + self.assertEqual(os.path.basename(paths[1]), bat_ec_fn) + self.assertTrue(os.path.samefile(target_path, cwd)) + + args = [bat_ec_fn, bat_patch_fn] + paths, target_path = det_copy_ec_specs(args, from_pr) + self.assertEqual(len(paths), 2) + self.assertEqual(os.path.basename(paths[0]), bat_ec_fn) + self.assertEqual(os.path.basename(paths[1]), bat_patch_fn) + self.assertTrue(os.path.samefile(target_path, cwd)) + + # also test with combination of local files and files from PR + args = [arrow_ec_fn, 'test.eb', 'test.patch', bat_patch_fn] + paths, target_path = det_copy_ec_specs(args, from_pr) + self.assertEqual(len(paths), 4) + self.assertEqual(os.path.basename(paths[0]), arrow_ec_fn) + self.assertEqual(paths[1], 'test.eb') + self.assertEqual(paths[2], 'test.patch') + self.assertEqual(os.path.basename(paths[3]), bat_patch_fn) + self.assertTrue(os.path.samefile(target_path, cwd)) + def suite(): """ returns all the testcases in this module """ diff --git a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-test.eb b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-test.eb new file mode 100644 index 0000000000..90cc7429d3 --- /dev/null +++ b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-test.eb @@ -0,0 +1,35 @@ +name = 'toy' +version = '0.0' +versionsuffix = '-test' + +homepage = 'https://easybuilders.github.io/easybuild' +description = "Toy C program, 100% toy." + +toolchain = SYSTEM + +sources = [SOURCE_TAR_GZ] +checksums = [[ + 'be662daa971a640e40be5c804d9d7d10', # default (MD5) + '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc', # default (SHA256) + ('adler32', '0x998410035'), + ('crc32', '0x1553842328'), + ('md5', 'be662daa971a640e40be5c804d9d7d10'), + ('sha1', 'f618096c52244539d0e89867405f573fdb0b55b0'), + ('size', 273), +]] +patches = [ + 'toy-0.0_fix-silly-typo-in-printf-statement.patch', + ('toy-extra.txt', 'toy-0.0'), +] + +sanity_check_paths = { + 'files': [('bin/yot', 'bin/toy')], + 'dirs': ['bin'], +} + +runtest = "make_test dummy_cmd" # Provide some value which is unique enough to be checked for + +postinstallcmds = ["echo TOY > %(installdir)s/README"] + +moduleclass = 'tools' +# trailing comment, leave this here, it may trigger bugs with extract_comments() diff --git a/test/framework/easystacks/test_easystack_asterisk.yaml b/test/framework/easystacks/test_easystack_asterisk.yaml new file mode 100644 index 0000000000..7f440636cd --- /dev/null +++ b/test/framework/easystacks/test_easystack_asterisk.yaml @@ -0,0 +1,6 @@ +software: + binutils: + toolchains: + GCCcore-4.9.3: + versions: + "2.11.*" \ No newline at end of file diff --git a/test/framework/easystacks/test_easystack_basic.yaml b/test/framework/easystacks/test_easystack_basic.yaml new file mode 100644 index 0000000000..4272aaf0fc --- /dev/null +++ b/test/framework/easystacks/test_easystack_basic.yaml @@ -0,0 +1,13 @@ +software: + binutils: + toolchains: + GCCcore-4.9.3: + versions: + 2.25: + 2.26: + toy: + toolchains: + gompi-2018a: + versions: + 0.0: + versionsuffix: '-test' \ No newline at end of file diff --git a/test/framework/easystacks/test_easystack_labels.yaml b/test/framework/easystacks/test_easystack_labels.yaml new file mode 100644 index 0000000000..51a113523f --- /dev/null +++ b/test/framework/easystacks/test_easystack_labels.yaml @@ -0,0 +1,7 @@ +software: + binutils: + toolchains: + GCCcore-4.9.3: + versions: + 3.11: + exclude-labels: arch:aarch64 diff --git a/test/framework/easystacks/test_easystack_wrong_structure.yaml b/test/framework/easystacks/test_easystack_wrong_structure.yaml new file mode 100644 index 0000000000..a328b5413b --- /dev/null +++ b/test/framework/easystacks/test_easystack_wrong_structure.yaml @@ -0,0 +1,6 @@ +software: + Bioconductor: + toolchains: + # foss-2020a: + versions: + 3.11 \ No newline at end of file diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 8c5a572543..7aabac47cd 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1219,6 +1219,37 @@ def test_apply_regex_substitutions(self): self.assertTrue(txt.startswith('FOO ')) self.assertTrue(txt.endswith(' bar')) + # also test apply_regex_substitutions with a *list* of paths + # cfr. https://github.com/easybuilders/easybuild-framework/issues/3493 + test_dir = os.path.join(self.test_prefix, 'test_dir') + test_file1 = os.path.join(test_dir, 'one.txt') + test_file2 = os.path.join(test_dir, 'two.txt') + ft.write_file(test_file1, "Donald is an elephant") + ft.write_file(test_file2, "2 + 2 = 5") + regexs = [ + ('Donald', 'Dumbo'), + ('= 5', '= 4'), + ] + ft.apply_regex_substitutions([test_file1, test_file2], regexs) + + # also check dry run mode + init_config(build_options={'extended_dry_run': True}) + self.mock_stderr(True) + self.mock_stdout(True) + ft.apply_regex_substitutions([test_file1, test_file2], regexs) + stderr, stdout = self.get_stderr(), self.get_stdout() + self.mock_stderr(False) + self.mock_stdout(False) + + self.assertFalse(stderr) + regex = re.compile('\n'.join([ + r"applying regex substitutions to file\(s\): .*/test_dir/one.txt, .*/test_dir/two.txt", + r" \* regex pattern 'Donald', replacement string 'Dumbo'", + r" \* regex pattern '= 5', replacement string '= 4'", + '', + ])) + self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + def test_find_flexlm_license(self): """Test find_flexlm_license function.""" lic_file1 = os.path.join(self.test_prefix, 'one.lic') @@ -1485,7 +1516,7 @@ def test_copy_file(self): else: # printing this message will make test suite fail in Travis/GitHub CI, # since we check for unexpected output produced by the tests - print("Skipping overwrite-file-owned-by-other-user copy_file test (%s is missing)", test_file_to_overwrite) + print("Skipping overwrite-file-owned-by-other-user copy_file test (%s is missing)" % test_file_to_overwrite) # also test behaviour of copy_file under --dry-run build_options = { @@ -1551,6 +1582,99 @@ def test_copy_files(self): error_pattern = "/toy-0.0.eb exists but is not a directory" self.assertErrorRegex(EasyBuildError, error_pattern, ft.copy_files, [bzip2_ec], copied_toy_ec) + # by default copy_files allows empty input list, but if allow_empty=False then an error is raised + ft.copy_files([], self.test_prefix) + error_pattern = 'One or more files to copy should be specified!' + self.assertErrorRegex(EasyBuildError, error_pattern, ft.copy_files, [], self.test_prefix, allow_empty=False) + + # test special case: copying a single file to a file target via target_single_file=True + target = os.path.join(self.test_prefix, 'target') + self.assertFalse(os.path.exists(target)) + ft.copy_files([toy_ec], target, target_single_file=True) + self.assertTrue(os.path.exists(target)) + self.assertTrue(os.path.isfile(target)) + self.assertEqual(toy_ec_txt, ft.read_file(target)) + + ft.remove_file(target) + + # also test target_single_file=True with path including a missing subdirectory + target = os.path.join(self.test_prefix, 'target_parent', 'target_subdir', 'target.txt') + self.assertFalse(os.path.exists(target)) + self.assertFalse(os.path.exists(os.path.dirname(target))) + ft.copy_files([toy_ec], target, target_single_file=True) + self.assertTrue(os.path.exists(target)) + self.assertTrue(os.path.isfile(target)) + self.assertEqual(toy_ec_txt, ft.read_file(target)) + + ft.remove_file(target) + + # default behaviour is to copy single file list to target *directory* + self.assertFalse(os.path.exists(target)) + ft.copy_files([toy_ec], target) + self.assertTrue(os.path.exists(target)) + self.assertTrue(os.path.isdir(target)) + copied_toy_ec = os.path.join(target, 'toy-0.0.eb') + self.assertTrue(os.path.exists(copied_toy_ec)) + self.assertEqual(toy_ec_txt, ft.read_file(copied_toy_ec)) + + ft.remove_dir(target) + + # test enabling verbose mode + self.mock_stderr(True) + self.mock_stdout(True) + ft.copy_files([toy_ec], target, verbose=True) + stderr, stdout = self.get_stderr(), self.get_stdout() + self.mock_stderr(False) + self.mock_stdout(False) + self.assertEqual(stderr, '') + regex = re.compile(r"^1 file\(s\) copied to .*/target") + self.assertTrue(regex.match(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + + ft.remove_dir(target) + + self.mock_stderr(True) + self.mock_stdout(True) + ft.copy_files([toy_ec], target, target_single_file=True, verbose=True) + stderr, stdout = self.get_stderr(), self.get_stdout() + self.mock_stderr(False) + self.mock_stdout(False) + self.assertEqual(stderr, '') + regex = re.compile(r"/.*/toy-0\.0\.eb copied to .*/target") + self.assertTrue(regex.match(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + + ft.remove_file(target) + + # check behaviour under -x: only printing, no actual copying + init_config(build_options={'extended_dry_run': True}) + self.assertFalse(os.path.exists(os.path.join(target, 'test.eb'))) + + self.mock_stderr(True) + self.mock_stdout(True) + ft.copy_files(['test.eb'], target) + stderr, stdout = self.get_stderr(), self.get_stdout() + self.mock_stderr(False) + self.mock_stdout(False) + + self.assertFalse(os.path.exists(os.path.join(target, 'test.eb'))) + self.assertEqual(stderr, '') + + regex = re.compile("^copied test.eb to .*/target") + self.assertTrue(regex.match(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + + self.mock_stderr(True) + self.mock_stdout(True) + ft.copy_files(['bar.eb', 'foo.eb'], target) + stderr, stdout = self.get_stderr(), self.get_stdout() + self.mock_stderr(False) + self.mock_stdout(False) + + self.assertFalse(os.path.exists(os.path.join(target, 'bar.eb'))) + self.assertFalse(os.path.exists(os.path.join(target, 'foo.eb'))) + self.assertEqual(stderr, '') + + regex = re.compile("^copied 2 files to .*/target") + self.assertTrue(regex.match(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + def test_copy_dir(self): """Test copy_dir function.""" testdir = os.path.dirname(os.path.abspath(__file__)) @@ -1889,7 +2013,7 @@ def test_index_functions(self): # test with specified path with and without trailing '/'s for path in [test_ecs, test_ecs + '/', test_ecs + '//']: index = ft.create_index(path) - self.assertEqual(len(index), 82) + self.assertEqual(len(index), 83) expected = [ os.path.join('b', 'bzip2', 'bzip2-1.0.6-GCC-4.9.2.eb'), @@ -2433,7 +2557,7 @@ def test_fake_vsc(self): regex = re.compile(r"^\nERROR: %s" % error_pattern) self.assertTrue(regex.search(stderr), "Pattern '%s' found in: %s" % (regex.pattern, stderr)) - # no error if import was detected from pkgutil.py, + # no error if import was detected from pkgutil.py or pkg_resources/__init__.py, # since that may be triggered by a system-wide vsc-base installation # (even though no code is doing 'import vsc'...) ft.move_file(test_python_mod, os.path.join(os.path.dirname(test_python_mod), 'pkgutil.py')) @@ -2441,6 +2565,15 @@ def test_fake_vsc(self): from test_fake_vsc import pkgutil self.assertTrue(pkgutil.__file__.endswith('/test_fake_vsc/pkgutil.py')) + pkg_resources_init = os.path.join(os.path.dirname(test_python_mod), 'pkg_resources', '__init__.py') + ft.write_file(pkg_resources_init, 'import vsc') + + # cleanup to force new import of 'vsc', avoid using cached import from previous attempt + del sys.modules['vsc'] + + from test_fake_vsc import pkg_resources + self.assertTrue(pkg_resources.__file__.endswith('/test_fake_vsc/pkg_resources/__init__.py')) + def test_is_generic_easyblock(self): """Test for is_generic_easyblock function.""" @@ -2680,6 +2813,79 @@ def test_locks(self): self.assertFalse(os.path.exists(lock_path)) self.assertEqual(os.listdir(locks_dir), []) + def test_locate_files(self): + """Test locate_files function.""" + + # create some files to find + one = os.path.join(self.test_prefix, '1.txt') + ft.write_file(one, 'one') + two = os.path.join(self.test_prefix, 'subdirA', '2.txt') + ft.write_file(two, 'two') + three = os.path.join(self.test_prefix, 'subdirB', '3.txt') + ft.write_file(three, 'three') + ft.mkdir(os.path.join(self.test_prefix, 'empty_subdir')) + + # empty list of files yields empty result + self.assertEqual(ft.locate_files([], []), []) + self.assertEqual(ft.locate_files([], [self.test_prefix]), []) + + # error is raised if files could not be found + error_pattern = r"One or more files not found: nosuchfile.txt \(search paths: \)" + self.assertErrorRegex(EasyBuildError, error_pattern, ft.locate_files, ['nosuchfile.txt'], []) + + # files specified via absolute path don't have to be found + res = ft.locate_files([one], []) + self.assertTrue(len(res) == 1) + self.assertTrue(os.path.samefile(res[0], one)) + + # note: don't compare file paths directly but use os.path.samefile instead, + # which is required to avoid failing tests in case temporary directory is a symbolic link (e.g. on macOS) + res = ft.locate_files(['1.txt'], [self.test_prefix]) + self.assertEqual(len(res), 1) + self.assertTrue(os.path.samefile(res[0], one)) + + res = ft.locate_files(['2.txt'], [self.test_prefix]) + self.assertEqual(len(res), 1) + self.assertTrue(os.path.samefile(res[0], two)) + + res = ft.locate_files(['1.txt', '3.txt'], [self.test_prefix]) + self.assertEqual(len(res), 2) + self.assertTrue(os.path.samefile(res[0], one)) + self.assertTrue(os.path.samefile(res[1], three)) + + # search in multiple paths + files = ['2.txt', '3.txt'] + paths = [os.path.dirname(three), os.path.dirname(two)] + res = ft.locate_files(files, paths) + self.assertEqual(len(res), 2) + self.assertTrue(os.path.samefile(res[0], two)) + self.assertTrue(os.path.samefile(res[1], three)) + + # same file specified multiple times works fine + files = ['1.txt', '2.txt', '1.txt', '3.txt', '2.txt'] + res = ft.locate_files(files, [self.test_prefix]) + self.assertEqual(len(res), 5) + for idx, expected in enumerate([one, two, one, three, two]): + self.assertTrue(os.path.samefile(res[idx], expected)) + + # only some files found yields correct warning + files = ['2.txt', '3.txt', '1.txt'] + error_pattern = r"One or more files not found: 3\.txt, 1.txt \(search paths: .*/subdirA\)" + self.assertErrorRegex(EasyBuildError, error_pattern, ft.locate_files, files, [os.path.dirname(two)]) + + # check that relative paths are found in current working dir + ft.change_dir(self.test_prefix) + rel_paths = ['subdirA/2.txt', '1.txt'] + # result is still absolute paths to those files + res = ft.locate_files(rel_paths, []) + self.assertEqual(len(res), 2) + self.assertTrue(os.path.samefile(res[0], two)) + self.assertTrue(os.path.samefile(res[1], one)) + + # no recursive search in current working dir (which would potentially be way too expensive) + error_pattern = r"One or more files not found: 2\.txt \(search paths: \)" + self.assertErrorRegex(EasyBuildError, error_pattern, ft.locate_files, ['2.txt'], []) + def suite(): """ returns all the testcases in this module """ diff --git a/test/framework/general.py b/test/framework/general.py index 07e8c36011..ecc0798146 100644 --- a/test/framework/general.py +++ b/test/framework/general.py @@ -81,14 +81,15 @@ def foo2(): def bar(): pass - err_pat = "required module 'nosuchmoduleoutthere' is not available.*package nosuchpkg.*pypi/nosuchpkg" + err_pat = r"None of the specified modules \(nosuchmoduleoutthere\) is available.*" + err_pat += r"package nosuchpkg.*pypi/nosuchpkg" self.assertErrorRegex(EasyBuildError, err_pat, bar) @only_if_module_is_available(('nosuchmodule', 'anothernosuchmodule')) def bar2(): pass - err_pat = "ImportError: None of the specified modules nosuchmodule, anothernosuchmodule is available" + err_pat = r"ImportError: None of the specified modules \(nosuchmodule, anothernosuchmodule\) is available" self.assertErrorRegex(EasyBuildError, err_pat, bar2) class Foo(): @@ -96,7 +97,8 @@ class Foo(): def foobar(self): pass - err_pat = r"required module 'thisdoesnotexist' is not available \(available from http://example.com\)" + err_pat = r"None of the specified modules \(thisdoesnotexist\) is available " + err_pat += r"\(available from http://example.com\)" self.assertErrorRegex(EasyBuildError, err_pat, Foo().foobar) def test_docstrings(self): diff --git a/test/framework/github.py b/test/framework/github.py index 80e86ac76d..c7cb90e94f 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -126,13 +126,13 @@ def test_fetch_pr_data(self): print("Skipping test_fetch_pr_data, no GitHub token available?") return - pr_data, pr_url = gh.fetch_pr_data(1, GITHUB_USER, GITHUB_REPO, GITHUB_TEST_ACCOUNT) + pr_data, _ = gh.fetch_pr_data(1, GITHUB_USER, GITHUB_REPO, GITHUB_TEST_ACCOUNT) self.assertEqual(pr_data['number'], 1) self.assertEqual(pr_data['title'], "a pr") self.assertFalse(any(key in pr_data for key in ['issue_comments', 'review', 'status_last_commit'])) - pr_data, pr_url = gh.fetch_pr_data(2, GITHUB_USER, GITHUB_REPO, GITHUB_TEST_ACCOUNT, full=True) + pr_data, _ = gh.fetch_pr_data(2, GITHUB_USER, GITHUB_REPO, GITHUB_TEST_ACCOUNT, full=True) self.assertEqual(pr_data['number'], 2) self.assertEqual(pr_data['title'], "an open pr (do not close this please)") self.assertTrue(pr_data['issue_comments']) @@ -327,6 +327,67 @@ def test_fetch_easyconfigs_from_pr(self): except URLError as err: print("Ignoring URLError '%s' in test_fetch_easyconfigs_from_pr" % err) + def test_fetch_files_from_pr_cache(self): + """Test caching for fetch_files_from_pr.""" + if self.skip_github_tests: + print("Skipping test_fetch_files_from_pr_cache, no GitHub token available?") + return + + init_config(build_options={ + 'pr_target_account': gh.GITHUB_EB_MAIN, + }) + + # clear cache first, to make sure we start with a clean slate + gh.fetch_files_from_pr.clear_cache() + self.assertFalse(gh.fetch_files_from_pr._cache) + + pr7159_filenames = [ + 'DOLFIN-2018.1.0.post1-foss-2018a-Python-3.6.4.eb', + 'OpenFOAM-5.0-20180108-foss-2018a.eb', + 'OpenFOAM-5.0-20180108-intel-2018a.eb', + 'OpenFOAM-6-foss-2018b.eb', + 'OpenFOAM-6-intel-2018a.eb', + 'OpenFOAM-v1806-foss-2018b.eb', + 'PETSc-3.9.3-foss-2018a.eb', + 'SCOTCH-6.0.6-foss-2018a.eb', + 'SCOTCH-6.0.6-foss-2018b.eb', + 'SCOTCH-6.0.6-intel-2018a.eb', + 'Trilinos-12.12.1-foss-2018a-Python-3.6.4.eb' + ] + pr7159_files = gh.fetch_easyconfigs_from_pr(7159, path=self.test_prefix, github_user=GITHUB_TEST_ACCOUNT) + self.assertEqual(sorted(pr7159_filenames), sorted(os.path.basename(f) for f in pr7159_files)) + + # check that cache has been populated for PR 7159 + self.assertEqual(len(gh.fetch_files_from_pr._cache.keys()), 1) + + # github_account value is None (results in using default 'easybuilders') + cache_key = (7159, None, 'easybuild-easyconfigs', self.test_prefix) + self.assertTrue(cache_key in gh.fetch_files_from_pr._cache.keys()) + + cache_entry = gh.fetch_files_from_pr._cache[cache_key] + self.assertEqual(sorted([os.path.basename(f) for f in cache_entry]), sorted(pr7159_filenames)) + + # same query should return result from cache entry + res = gh.fetch_easyconfigs_from_pr(7159, path=self.test_prefix, github_user=GITHUB_TEST_ACCOUNT) + self.assertEqual(res, pr7159_files) + + # inject entry in cache and check result of matching query + pr_id = 12345 + tmpdir = os.path.join(self.test_prefix, 'easyblocks-pr-12345') + pr12345_files = [ + os.path.join(tmpdir, 'foo.py'), + os.path.join(tmpdir, 'bar.py'), + ] + for fp in pr12345_files: + write_file(fp, '') + + # github_account value is None (results in using default 'easybuilders') + cache_key = (pr_id, None, 'easybuild-easyblocks', tmpdir) + gh.fetch_files_from_pr.update_cache({cache_key: pr12345_files}) + + res = gh.fetch_easyblocks_from_pr(12345, tmpdir) + self.assertEqual(sorted(pr12345_files), sorted(res)) + def test_fetch_latest_commit_sha(self): """Test fetch_latest_commit_sha function.""" if self.skip_github_tests: diff --git a/test/framework/options.py b/test/framework/options.py index 2727baa392..c3bbd35813 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -42,6 +42,7 @@ import easybuild.tools.toolchain from easybuild.base import fancylogger from easybuild.framework.easyblock import EasyBlock +from easybuild.framework.easystack import parse_easystack from easybuild.framework.easyconfig import BUILD, CUSTOM, DEPENDENCIES, EXTENSIONS, FILEMANAGEMENT, LICENSE from easybuild.framework.easyconfig import MANDATORY, MODULES, OTHER, TOOLCHAIN from easybuild.framework.easyconfig.easyconfig import EasyConfig, get_easyblock_class, robot_find_easyconfig @@ -50,8 +51,8 @@ from easybuild.tools.config import DEFAULT_MODULECLASSES from easybuild.tools.config import find_last_log, get_build_log_path, get_module_syntax, module_classes from easybuild.tools.environment import modify_env -from easybuild.tools.filetools import change_dir, copy_dir, copy_file, download_file, mkdir, read_file -from easybuild.tools.filetools import remove_dir, remove_file, which, write_file +from easybuild.tools.filetools import change_dir, copy_dir, copy_file, download_file, is_patch_file, mkdir +from easybuild.tools.filetools import read_file, remove_dir, remove_file, which, write_file from easybuild.tools.github import GITHUB_RAW, GITHUB_EB_MAIN, GITHUB_EASYCONFIGS_REPO from easybuild.tools.github import URL_SEPARATOR, fetch_github_token from easybuild.tools.modules import Lmod @@ -107,11 +108,14 @@ def setUp(self): self.orig_terminal_supports_colors = easybuild.tools.options.terminal_supports_colors self.orig_os_getuid = easybuild.main.os.getuid + self.orig_experimental = easybuild.tools.build_log.EXPERIMENTAL def tearDown(self): """Clean up after test.""" easybuild.main.os.getuid = self.orig_os_getuid easybuild.tools.options.terminal_supports_colors = self.orig_terminal_supports_colors + easybuild.tools.build_log.EXPERIMENTAL = self.orig_experimental + super(CommandLineOptionsTest, self).tearDown() def purge_environment(self): @@ -170,6 +174,26 @@ def test_help_long(self): regex = re.compile("default: True; disable with --disable-cleanup-builddir", re.M) self.assertTrue(regex.search(outtxt), "Pattern '%s' found in: %s" % (regex.pattern, outtxt)) + def test_help_rst(self): + """Test generating --help in RST output format.""" + + self.mock_stderr(True) + self.mock_stdout(True) + self.eb_main(['--help=rst'], raise_error=True) + stderr, stdout = self.get_stderr(), self.get_stdout() + self.mock_stderr(False) + self.mock_stdout(False) + + self.assertFalse(stderr) + + patterns = [ + r"^Basic options\n-------------", + r"^``--fetch``[ ]*Allow downloading sources", + ] + for pattern in patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + def test_no_args(self): """Test using no arguments.""" @@ -336,6 +360,40 @@ def test_skip(self): self.assertEqual(len(glob.glob(toy_mod_glob)), 1) + def test_skip_test_step(self): + """Test skipping testing the build (--skip-test-step).""" + + topdir = os.path.abspath(os.path.dirname(__file__)) + toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-test.eb') + + # check log message without --skip-test-step + args = [ + toy_ec, + '--extended-dry-run', + '--force', + '--debug', + ] + self.mock_stdout(True) + outtxt = self.eb_main(args, do_build=True) + self.mock_stdout(False) + found_msg = "Running method test_step part of step test" + found = re.search(found_msg, outtxt) + test_run_msg = "execute make_test dummy_cmd as a command for running unit tests" + self.assertTrue(found, "Message about test step being run is present, outtxt: %s" % outtxt) + found = re.search(test_run_msg, outtxt) + self.assertTrue(found, "Test execution command is present, outtxt: %s" % outtxt) + + # And now with the argument + args.append('--skip-test-step') + self.mock_stdout(True) + outtxt = self.eb_main(args, do_build=True) + self.mock_stdout(False) + found_msg = "Skipping test step" + found = re.search(found_msg, outtxt) + self.assertTrue(found, "Message about test step being skipped is present, outtxt: %s" % outtxt) + found = re.search(test_run_msg, outtxt) + self.assertFalse(found, "Test execution command is NOT present, outtxt: %s" % outtxt) + def test_job(self): """Test submitting build as a job.""" @@ -965,19 +1023,20 @@ def test_show_ec(self): regex = re.compile(pattern, re.M) self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) + def mocked_main(self, args): + """Run eb_main with mocked stdout/stderr.""" + self.mock_stderr(True) + self.mock_stdout(True) + self.eb_main(args, raise_error=True) + stderr, stdout = self.get_stderr(), self.get_stdout() + self.mock_stderr(False) + self.mock_stdout(False) + self.assertEqual(stderr, '') + return stdout.strip() + def test_copy_ec(self): """Test --copy-ec.""" - def mocked_main(args): - self.mock_stderr(True) - self.mock_stdout(True) - self.eb_main(args, raise_error=True) - stderr, stdout = self.get_stderr(), self.get_stdout() - self.mock_stderr(False) - self.mock_stdout(False) - self.assertEqual(stderr, '') - return stdout.strip() - topdir = os.path.dirname(os.path.abspath(__file__)) test_easyconfigs_dir = os.path.join(topdir, 'easyconfigs', 'test_ecs') @@ -987,8 +1046,9 @@ def mocked_main(args): # basic test: copying one easyconfig file to a non-existing absolute path test_ec = os.path.join(self.test_prefix, 'test.eb') args = ['--copy-ec', 'toy-0.0.eb', test_ec] - stdout = mocked_main(args) - self.assertEqual(stdout, 'toy-0.0.eb copied to %s' % test_ec) + stdout = self.mocked_main(args) + regex = re.compile(r'.*/toy-0.0.eb copied to %s' % test_ec) + self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) self.assertTrue(os.path.exists(test_ec)) self.assertEqual(toy_ec_txt, read_file(test_ec)) @@ -1001,8 +1061,9 @@ def mocked_main(args): self.assertFalse(os.path.exists(target_fn)) args = ['--copy-ec', 'toy-0.0.eb', target_fn] - stdout = mocked_main(args) - self.assertEqual(stdout, 'toy-0.0.eb copied to test.eb') + stdout = self.mocked_main(args) + regex = re.compile(r'.*/toy-0.0.eb copied to test.eb') + self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) change_dir(cwd) @@ -1013,8 +1074,9 @@ def mocked_main(args): test_target_dir = os.path.join(self.test_prefix, 'test_target_dir') mkdir(test_target_dir) args = ['--copy-ec', 'toy-0.0.eb', test_target_dir] - stdout = mocked_main(args) - self.assertEqual(stdout, 'toy-0.0.eb copied to %s' % test_target_dir) + stdout = self.mocked_main(args) + regex = re.compile(r'.*/toy-0.0.eb copied to %s' % test_target_dir) + self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) copied_toy_ec = os.path.join(test_target_dir, 'toy-0.0.eb') self.assertTrue(os.path.exists(copied_toy_ec)) @@ -1035,7 +1097,7 @@ def check_copied_files(): # copying multiple easyconfig files to a non-existing target directory (which is created automatically) args = ['--copy-ec', 'toy-0.0.eb', 'bzip2-1.0.6-GCC-4.9.2.eb', test_target_dir] - stdout = mocked_main(args) + stdout = self.mocked_main(args) self.assertEqual(stdout, '2 file(s) copied to %s' % test_target_dir) check_copied_files() @@ -1047,12 +1109,12 @@ def check_copied_files(): args[-1] = os.path.basename(test_target_dir) self.assertFalse(os.path.exists(args[-1])) - stdout = mocked_main(args) + stdout = self.mocked_main(args) self.assertEqual(stdout, '2 file(s) copied to test_target_dir') check_copied_files() - # copying multiple easyconfig to an existing target file resuts in an error + # copying multiple easyconfig to an existing target file results in an error target = os.path.join(self.test_prefix, 'test.eb') self.assertTrue(os.path.isfile(target)) args = ['--copy-ec', 'toy-0.0.eb', 'bzip2-1.0.6-GCC-4.9.2.eb', target] @@ -1065,8 +1127,8 @@ def check_copied_files(): change_dir(test_working_dir) self.assertEqual(len(os.listdir(os.getcwd())), 0) args = ['--copy-ec', 'toy-0.0.eb'] - stdout = mocked_main(args) - regex = re.compile('toy-0.0.eb copied to .*/%s' % os.path.basename(test_working_dir)) + stdout = self.mocked_main(args) + regex = re.compile('.*/toy-0.0.eb copied to .*/%s' % os.path.basename(test_working_dir)) self.assertTrue(regex.match(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) copied_toy_cwd = os.path.join(test_working_dir, 'toy-0.0.eb') self.assertTrue(os.path.exists(copied_toy_cwd)) @@ -1074,9 +1136,128 @@ def check_copied_files(): # --copy-ec without arguments results in a proper error args = ['--copy-ec'] - error_pattern = "One of more files to copy should be specified!" + error_pattern = "One or more files to copy should be specified!" self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, raise_error=True) + def test_copy_ec_from_pr(self): + """Test combination of --copy-ec with --from-pr.""" + if self.github_token is None: + print("Skipping test_copy_ec_from_pr, no GitHub token available?") + return + + test_working_dir = os.path.join(self.test_prefix, 'test_working_dir') + mkdir(test_working_dir) + test_target_dir = os.path.join(self.test_prefix, 'test_target_dir') + # Make sure the test target directory doesn't exist + remove_dir(test_target_dir) + + all_files_pr8007 = [ + 'Arrow-0.7.1-intel-2017b-Python-3.6.3.eb', + 'bat-0.3.3-fix-pyspark.patch', + 'bat-0.3.3-intel-2017b-Python-3.6.3.eb', + ] + + # test use of --copy-ec with --from-pr to the current working directory + cwd = change_dir(test_working_dir) + args = ['--copy-ec', '--from-pr', '8007'] + stdout = self.mocked_main(args) + + regex = re.compile(r"3 file\(s\) copied to .*/%s" % os.path.basename(test_working_dir)) + self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + + # check that the files exist + for pr_file in all_files_pr8007: + self.assertTrue(os.path.exists(os.path.join(test_working_dir, pr_file))) + remove_file(os.path.join(test_working_dir, pr_file)) + + # copying all files touched by PR to a non-existing target directory (which is created automatically) + self.assertFalse(os.path.exists(test_target_dir)) + args = ['--copy-ec', '--from-pr', '8007', test_target_dir] + stdout = self.mocked_main(args) + + regex = re.compile(r"3 file\(s\) copied to .*/%s" % os.path.basename(test_target_dir)) + self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + + for pr_file in all_files_pr8007: + self.assertTrue(os.path.exists(os.path.join(test_target_dir, pr_file))) + remove_dir(test_target_dir) + + # test where we select a single easyconfig file from a PR + mkdir(test_target_dir) + ec_filename = 'bat-0.3.3-intel-2017b-Python-3.6.3.eb' + args = ['--copy-ec', '--from-pr', '8007', ec_filename, test_target_dir] + stdout = self.mocked_main(args) + + regex = re.compile(r"%s copied to .*/%s" % (ec_filename, os.path.basename(test_target_dir))) + self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + + self.assertEqual(os.listdir(test_target_dir), [ec_filename]) + self.assertTrue("name = 'bat'" in read_file(os.path.join(test_target_dir, ec_filename))) + remove_dir(test_target_dir) + + # test copying of a single easyconfig file from a PR to a non-existing path + bat_ec = os.path.join(self.test_prefix, 'bat.eb') + args[-1] = bat_ec + stdout = self.mocked_main(args) + + regex = re.compile(r"%s copied to .*/bat.eb" % ec_filename) + self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + + self.assertTrue(os.path.exists(bat_ec)) + self.assertTrue("name = 'bat'" in read_file(bat_ec)) + + change_dir(cwd) + remove_dir(test_working_dir) + mkdir(test_working_dir) + change_dir(test_working_dir) + + # test copying of a patch file from a PR via --copy-ec to current directory + patch_fn = 'bat-0.3.3-fix-pyspark.patch' + args = ['--copy-ec', '--from-pr', '8007', patch_fn, '.'] + stdout = self.mocked_main(args) + + self.assertEqual(os.listdir(test_working_dir), [patch_fn]) + patch_path = os.path.join(test_working_dir, patch_fn) + self.assertTrue(os.path.exists(patch_path)) + self.assertTrue(is_patch_file(patch_path)) + remove_file(patch_path) + + # test the same thing but where we don't provide a target location + change_dir(test_working_dir) + args = ['--copy-ec', '--from-pr', '8007', ec_filename] + stdout = self.mocked_main(args) + + regex = re.compile(r"%s copied to .*/%s" % (ec_filename, os.path.basename(test_working_dir))) + self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + + self.assertEqual(os.listdir(test_working_dir), [ec_filename]) + self.assertTrue("name = 'bat'" in read_file(os.path.join(test_working_dir, ec_filename))) + + # also test copying of patch file to current directory (without specifying target location) + change_dir(test_working_dir) + args = ['--copy-ec', '--from-pr', '8007', patch_fn] + stdout = self.mocked_main(args) + + regex = re.compile(r"%s copied to .*/%s" % (patch_fn, os.path.basename(test_working_dir))) + self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + + self.assertEqual(sorted(os.listdir(test_working_dir)), sorted([ec_filename, patch_fn])) + self.assertTrue(is_patch_file(os.path.join(test_working_dir, patch_fn))) + + change_dir(cwd) + remove_dir(test_working_dir) + + # test with only one ec in the PR (final argument is taken as a filename) + test_ec = os.path.join(self.test_prefix, 'test.eb') + args = ['--copy-ec', '--from-pr', '11521', test_ec] + ec_pr11521 = "ExifTool-12.00-GCCcore-9.3.0.eb" + stdout = self.mocked_main(args) + regex = re.compile(r'.*/%s copied to %s' % (ec_pr11521, test_ec)) + self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) + self.assertTrue(os.path.exists(test_ec)) + self.assertTrue("name = 'ExifTool'" in read_file(test_ec)) + remove_file(test_ec) + def test_dry_run(self): """Test dry run (long format).""" fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') @@ -2519,7 +2700,8 @@ def test_robot_path_check(self): # different error when a non-existing easyconfig file is specified to --robot args = ['--dry-run', '--robot', 'no_such_easyconfig_file_in_robot_search_path.eb'] - self.assertErrorRegex(EasyBuildError, "Can't find path", self.eb_main, args, raise_error=True) + error_pattern = "One or more files not found: no_such_easyconfig_file_in_robot_search_path.eb" + self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, raise_error=True) for robot in ['-r%s' % self.test_prefix, '--robot=%s' % self.test_prefix]: args = ['toy-0.0.eb', '--dry-run', robot] @@ -3985,6 +4167,29 @@ def test_merge_pr(self): print("Skipping test_merge_pr, no GitHub token available?") return + # start by making sure --merge-pr without dry-run errors out for a closed PR + args = [ + '--merge-pr', + '11753', # closed PR + '--github-user=%s' % GITHUB_TEST_ACCOUNT, + ] + error_msg = r"This PR is closed." + self.mock_stdout(True) + self.assertErrorRegex(EasyBuildError, error_msg, self.eb_main, args, raise_error=True) + self.mock_stdout(False) + + # and also for an already merged PR + args = [ + '--merge-pr', + '11769', # already merged PR + '--github-user=%s' % GITHUB_TEST_ACCOUNT, + ] + error_msg = r"This PR is already merged." + self.mock_stdout(True) + self.assertErrorRegex(EasyBuildError, error_msg, self.eb_main, args, raise_error=True) + self.mock_stdout(False) + + # merged PR for EasyBuild-3.3.0.eb, is missing approved review args = [ '--merge-pr', '4781', # PR for easyconfig for EasyBuild-3.3.0.eb @@ -3993,7 +4198,6 @@ def test_merge_pr(self): '--pr-target-branch=some_branch', ] - # merged PR for EasyBuild-3.3.0.eb, is missing approved review stdout, stderr = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False) expected_stdout = '\n'.join([ @@ -5454,6 +5658,69 @@ def test_sysroot(self): os.environ['EASYBUILD_SYSROOT'] = doesnotexist self.assertErrorRegex(EasyBuildError, error_pattern, self._run_mock_eb, ['--show-config'], raise_error=True) + # end-to-end testing of unknown filename + def test_easystack_wrong_read(self): + """Test for --easystack when wrong name is provided""" + topdir = os.path.dirname(os.path.abspath(__file__)) + toy_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_nonexistent.yaml') + args = ['--easystack', toy_easystack, '--experimental'] + expected_err = "No such file or directory: '%s'" % toy_easystack + self.assertErrorRegex(EasyBuildError, expected_err, self.eb_main, args, raise_error=True) + + # testing basics - end-to-end + # expecting successful build + def test_easystack_basic(self): + """Test for --easystack -> success case""" + topdir = os.path.dirname(os.path.abspath(__file__)) + toy_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_basic.yaml') + + args = ['--easystack', toy_easystack, '--debug', '--experimental', '--dry-run'] + stdout = self.eb_main(args, do_build=True, raise_error=True) + patterns = [ + r"[\S\s]*INFO Building from easystack:[\S\s]*", + r"[\S\s]*DEBUG EasyStack parsed\. Proceeding to install these Easyconfigs: " + r"binutils-2.25-GCCcore-4.9.3.eb, binutils-2.26-GCCcore-4.9.3.eb, toy-0.0-gompi-2018a-test.eb", + r"\* \[ \] .*/test_ecs/b/binutils/binutils-2.25-GCCcore-4.9.3.eb \(module: binutils/2.25-GCCcore-4.9.3\)", + r"\* \[ \] .*/test_ecs/b/binutils/binutils-2.26-GCCcore-4.9.3.eb \(module: binutils/2.26-GCCcore-4.9.3\)", + r"\* \[ \] .*/test_ecs/t/toy/toy-0.0-gompi-2018a-test.eb \(module: toy/0.0-gompi-2018a-test\)", + ] + for pattern in patterns: + regex = re.compile(pattern) + self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + + def test_easystack_wrong_structure(self): + """Test for --easystack when yaml easystack has wrong structure""" + easybuild.tools.build_log.EXPERIMENTAL = True + topdir = os.path.dirname(os.path.abspath(__file__)) + toy_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_wrong_structure.yaml') + + expected_err = r"[\S\s]*An error occurred when interpreting the data for software Bioconductor:" + expected_err += r"( 'float' object is not subscriptable[\S\s]*" + expected_err += r"| 'float' object is unsubscriptable" + expected_err += r"| 'float' object has no attribute '__getitem__'[\S\s]*)" + self.assertErrorRegex(EasyBuildError, expected_err, parse_easystack, toy_easystack) + + def test_easystack_asterisk(self): + """Test for --easystack when yaml easystack contains asterisk (wildcard)""" + easybuild.tools.build_log.EXPERIMENTAL = True + topdir = os.path.dirname(os.path.abspath(__file__)) + toy_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_asterisk.yaml') + + expected_err = "EasyStack specifications of 'binutils' in .*/test_easystack_asterisk.yaml contain asterisk. " + expected_err += "Wildcard feature is not supported yet." + + self.assertErrorRegex(EasyBuildError, expected_err, parse_easystack, toy_easystack) + + def test_easystack_labels(self): + """Test for --easystack when yaml easystack contains exclude-labels / include-labels""" + easybuild.tools.build_log.EXPERIMENTAL = True + topdir = os.path.dirname(os.path.abspath(__file__)) + toy_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_labels.yaml') + + error_msg = "EasyStack specifications of 'binutils' in .*/test_easystack_labels.yaml contain labels. " + error_msg += "Labels aren't supported yet." + self.assertErrorRegex(EasyBuildError, error_msg, parse_easystack, toy_easystack) + def suite(): """ returns all the testcases in this module """ diff --git a/test/framework/robot.py b/test/framework/robot.py index e50f570a37..dc540674ec 100644 --- a/test/framework/robot.py +++ b/test/framework/robot.py @@ -620,12 +620,19 @@ def test_det_easyconfig_paths(self): test_ec = 'toy-0.0-deps.eb' shutil.copy2(os.path.join(test_ecs_path, 't', 'toy', test_ec), self.test_prefix) + # copy hwloc easyconfig to h/hwloc subdir in robot search path, + # to trigger bug fixed in det_easyconfig_paths (.extend rather than .append for '__archive'__ to ignore_subdirs) + hwloc_ec = 'hwloc-1.11.8-GCC-6.4.0-2.28.eb' + subdir_hwloc = os.path.join(self.test_prefix, 'h', 'hwloc') + mkdir(subdir_hwloc, parents=True) + shutil.copy2(os.path.join(test_ecs_path, 'h', 'hwloc', hwloc_ec), subdir_hwloc) shutil.copy2(os.path.join(test_ecs_path, 'i', 'intel', 'intel-2018a.eb'), self.test_prefix) self.assertFalse(os.path.exists(test_ec)) args = [ os.path.join(test_ecs_path, 't', 'toy', 'toy-0.0.eb'), test_ec, # relative path, should be resolved via robot search path + hwloc_ec, '--dry-run', '--debug', '--robot', @@ -640,6 +647,7 @@ def test_det_easyconfig_paths(self): (test_ecs_path, 'toy/0.0'), # specified easyconfigs, available at given location (self.test_prefix, 'intel/2018a'), # dependency, found in robot search path (self.test_prefix, 'toy/0.0-deps'), # specified easyconfig, found in robot search path + (self.test_prefix, 'hwloc/1.11.8-GCC-6.4.0-2.28'), # specified easyconfig, found in robot search path ] for path_prefix, module in modules: ec_fn = "%s.eb" % '-'.join(module.split('/')) @@ -654,7 +662,8 @@ def test_det_easyconfig_paths(self): '--robot', '--unittest-file=%s' % self.logfile, ] - self.assertErrorRegex(EasyBuildError, "Can't find", self.eb_main, args, logfile=dummylogfn, raise_error=True) + error_pattern = "One or more files not found: intel-2012a.eb" + self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, logfile=dummylogfn, raise_error=True) args.append('--consider-archived-easyconfigs') outtxt = self.eb_main(args, logfile=dummylogfn, raise_error=True) diff --git a/test/framework/sandbox/sources/t/toy/extensions/bar-0.0-local.tar.gz b/test/framework/sandbox/sources/t/toy/extensions/bar-0.0-local.tar.gz new file mode 100644 index 0000000000..98f5271678 Binary files /dev/null and b/test/framework/sandbox/sources/t/toy/extensions/bar-0.0-local.tar.gz differ diff --git a/test/framework/sandbox/sources/t/toy/exts-git.tar.gz b/test/framework/sandbox/sources/t/toy/exts-git.tar.gz new file mode 100644 index 0000000000..afac3590b1 Binary files /dev/null and b/test/framework/sandbox/sources/t/toy/exts-git.tar.gz differ diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 5e33ad3608..e9ce1d3b14 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -621,6 +621,15 @@ def test_misc_flags_shared(self): self.assertTrue(flag not in flags, "%s: False means no %s in %s" % (opt, flag, flags)) self.modtool.purge() + value = '--see-if-this-propagates' + for var in flag_vars: + opt = 'extra_' + var.lower() + tc = self.get_toolchain('foss', version='2018a') + tc.set_options({opt: value}) + tc.prepare() + self.assertTrue(tc.get_variable(var).endswith(' ' + value)) + self.modtool.purge() + def test_misc_flags_unique(self): """Test whether unique compiler flags are set correctly.""" @@ -1729,6 +1738,11 @@ def test_compiler_cache(self): def test_rpath_args_script(self): """Test rpath_args.py script""" + + # $LIBRARY_PATH affects result of rpath_args.py, so make sure it's not set + if 'LIBRARY_PATH' in os.environ: + del os.environ['LIBRARY_PATH'] + script = find_eb_script('rpath_args.py') rpath_inc = ','.join([ @@ -2013,6 +2027,63 @@ def test_rpath_args_script(self): cmd_args = ["'foo.c'", "'-O2'"] + ["'%s'" % x for x in extra_args.split(' ')] self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + # check whether $LIBRARY_PATH is taken into account + test_cmd_gcc = "%s gcc '' '%s' -c foo.c" % (script, rpath_inc) + pre_cmd_args_gcc = [ + "'-Wl,-rpath=%s/lib'" % self.test_prefix, + "'-Wl,-rpath=%s/lib64'" % self.test_prefix, + "'-Wl,-rpath=$ORIGIN'", + "'-Wl,-rpath=$ORIGIN/../lib'", + "'-Wl,-rpath=$ORIGIN/../lib64'", + "'-Wl,--disable-new-dtags'", + ] + post_cmd_args_gcc = [ + "'-c'", + "'foo.c'", + ] + + test_cmd_ld = "%s ld '' '%s' -L/foo foo.o -L/lib64 -lfoo -lbar -L/usr/lib -L/bar" % (script, rpath_inc) + pre_cmd_args_ld = [ + "'-rpath=%s/lib'" % self.test_prefix, + "'-rpath=%s/lib64'" % self.test_prefix, + "'-rpath=$ORIGIN'", + "'-rpath=$ORIGIN/../lib'", + "'-rpath=$ORIGIN/../lib64'", + "'--disable-new-dtags'", + "'-rpath=/foo'", + "'-rpath=/lib64'", + "'-rpath=/usr/lib'", + "'-rpath=/bar'", + ] + post_cmd_args_ld = [ + "'-L/foo'", + "'foo.o'", + "'-L/lib64'", + "'-lfoo'", + "'-lbar'", + "'-L/usr/lib'", + "'-L/bar'", + ] + + library_paths = [ + ('',), # special case: empty value + ('/path/to/lib',), + ('/path/to/lib', '/another/path/to/lib64'), + ('/path/to/lib', '/another/path/to/lib64', '/yet-another/path/to/libraries'), + ] + for library_path in library_paths: + os.environ['LIBRARY_PATH'] = ':'.join(library_path) + + out, ec = run_cmd(test_cmd_gcc, simple=False) + self.assertEqual(ec, 0) + cmd_args = pre_cmd_args_gcc + ["'-Wl,-rpath=%s'" % x for x in library_path if x] + post_cmd_args_gcc + self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + + out, ec = run_cmd(test_cmd_ld, simple=False) + self.assertEqual(ec, 0) + cmd_args = pre_cmd_args_ld + ["'-rpath=%s'" % x for x in library_path if x] + post_cmd_args_ld + self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + def test_toolchain_prepare_rpath(self): """Test toolchain.prepare under --rpath"""