diff --git a/.github/workflows/eb_command.yml b/.github/workflows/eb_command.yml new file mode 100644 index 0000000000..d0b72e3079 --- /dev/null +++ b/.github/workflows/eb_command.yml @@ -0,0 +1,88 @@ +# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions +name: Tests for the 'eb' command +on: [push, pull_request] +jobs: + test-eb: + runs-on: ubuntu-18.04 + strategy: + matrix: + python: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9] + fail-fast: false + steps: + - uses: actions/checkout@v2 + + - name: set up Python + uses: actions/setup-python@v2 + with: + python-version: ${{matrix.python}} + architecture: x64 + + - name: install OS & Python packages + run: | + # check Python version + python -V + # update to latest pip, check version + pip install --upgrade pip + pip --version + # install packages required for modules tool + sudo apt-get install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev + # fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082 + # needed for Ubuntu 18.04, but not for Ubuntu 20.04, so skipping symlinking if posix.so already exists + if [ ! -e /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so ] ; then + sudo ln -s /usr/lib/x86_64-linux-gnu/lua/5.2/posix_c.so /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so + fi + + - name: install modules tool + run: | + # avoid downloading modules tool sources into easybuild-framework dir + cd $HOME + export INSTALL_DEP=$GITHUB_WORKSPACE/easybuild/scripts/install_eb_dep.sh + # install Lmod + source $INSTALL_DEP Lmod-8.4.26 $HOME + # changes in environment are not passed to other steps, so need to create files... + echo $MOD_INIT > mod_init + echo $PATH > path + if [ ! -z $MODULESHOME ]; then echo $MODULESHOME > moduleshome; fi + + - name: install EasyBuild framework + run: | + # install from source distribution tarball, to test release as published on PyPI + python setup.py sdist + ls dist + export PREFIX=/tmp/$USER/$GITHUB_SHA + pip install --prefix $PREFIX dist/easybuild-framework*tar.gz + + - name: run tests for 'eb' command + env: + EB_VERBOSE: 1 + run: | + # run tests *outside* of checked out easybuild-framework directory, + # to ensure we're testing installed version (see previous step) + cd $HOME + # initialize environment for modules tool + if [ -f $HOME/moduleshome ]; then export MODULESHOME=$(cat $HOME/moduleshome); fi + source $(cat $HOME/mod_init); type module + # make sure 'eb' is available via $PATH, and that $PYTHONPATH is set (some tests expect that); + # also pick up changes to $PATH set by sourcing $MOD_INIT + export PREFIX=/tmp/$USER/$GITHUB_SHA + export PATH=$PREFIX/bin:$(cat $HOME/path) + export PYTHONPATH=$PREFIX/lib/python${{matrix.python}}/site-packages:$PYTHONPATH + # run --version, capture (verbose) output + eb --version | tee eb_version.out 2>&1 + # determine active Python version + pymajver=$(python -c 'import sys; print(sys.version_info[0])') + pymajminver=$(python -c 'import sys; print(".".join(str(x) for x in sys.version_info[:2]))') + # check patterns in verbose output + for pattern in "^>> Considering .python.\.\.\." "^>> .python. version: ${pymajminver}\.[0-9]\+, which matches Python ${pymajver} version requirement" "^>> 'python' is able to import 'easybuild.main', so retaining it" "^>> Selected Python command: python \(.*/bin/python\)" "^This is EasyBuild 4\.[0-9.]\+"; do + echo "Looking for pattern \"${pattern}\" in eb_version.out..." + grep "$pattern" eb_version.out + done + # also check when specifying Python command via $EB_PYTHON + for eb_python in "python${pymajver}" "python${pymajminver}"; do + export EB_PYTHON="${eb_python}" + eb --version | tee eb_version.out 2>&1 + for pattern in "^>> Considering .${eb_python}.\.\.\." "^>> .${eb_python}. version: ${pymajminver}\.[0-9]\+, which matches Python ${pymajver} version requirement" "^>> '${eb_python}' is able to import 'easybuild.main', so retaining it" "^>> Selected Python command: ${eb_python} \(.*/bin/${eb_python}\)" "^This is EasyBuild 4\.[0-9.]\+"; do + echo "Looking for pattern \"${pattern}\" in eb_version.out..." + grep "$pattern" eb_version.out + done + done diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 7375e80ed5..1f31dd395a 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -2,61 +2,70 @@ name: EasyBuild framework unit tests on: [push, pull_request] jobs: + setup: + runs-on: ubuntu-latest + outputs: + lmod7: Lmod-7.8.22 + lmod8: Lmod-8.4.27 + modulesTcl: modules-tcl-1.147 + modules3: modules-3.2.10 + modules4: modules-4.1.4 + steps: + - run: "true" build: + needs: setup runs-on: ubuntu-18.04 strategy: matrix: - python: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9] - modules_tool: [Lmod-7.8.22, Lmod-8.2.9, modules-tcl-1.147, modules-3.2.10, modules-4.1.4] + python: [2.7, 3.6] + modules_tool: + # use variables defined by 'setup' job above, see also + # https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#needs-context + - ${{needs.setup.outputs.lmod7}} + - ${{needs.setup.outputs.lmod8}} + - ${{needs.setup.outputs.modulesTcl}} + - ${{needs.setup.outputs.modules3}} + - ${{needs.setup.outputs.modules4}} module_syntax: [Lua, Tcl] lc_all: [""] - # exclude some configuration for non-Lmod modules tool: - # - don't test with Lua module syntax (only supported in Lmod) - # - exclude Python 3.x versions other than 3.6, to limit test configurations + # don't test with Lua module syntax (only supported in Lmod) exclude: - - modules_tool: modules-tcl-1.147 + - modules_tool: ${{needs.setup.outputs.modulesTcl}} module_syntax: Lua - - modules_tool: modules-3.2.10 + - modules_tool: ${{needs.setup.outputs.modules3}} module_syntax: Lua - - modules_tool: modules-4.1.4 + - modules_tool: ${{needs.setup.outputs.modules4}} module_syntax: Lua - - modules_tool: modules-tcl-1.147 - python: 3.5 - - modules_tool: modules-tcl-1.147 - python: 3.7 - - modules_tool: modules-tcl-1.147 - python: 3.8 - - modules_tool: modules-tcl-1.147 - python: 3.9 - - modules_tool: modules-3.2.10 - python: 3.5 - - modules_tool: modules-3.2.10 - python: 3.7 - - modules_tool: modules-3.2.10 - python: 3.8 - - modules_tool: modules-3.2.10 - python: 3.9 - - modules_tool: modules-4.1.4 - python: 3.5 - - modules_tool: modules-4.1.4 - python: 3.7 - - modules_tool: modules-4.1.4 - python: 3.8 - - modules_tool: modules-4.1.4 - python: 3.9 - - modules_tool: Lmod-7.8.22 - python: 3.5 - - modules_tool: Lmod-7.8.22 - python: 3.7 - - modules_tool: Lmod-7.8.22 - python: 3.8 - - modules_tool: Lmod-7.8.22 - python: 3.9 - # There may be encoding errors in Python 3 which are hidden when an UTF-8 encoding is set - # Hence run the tests (again) with LC_ALL=C and Python 3.6 (or any < 3.7) include: + # Test different Python 3 versions with Lmod 8.x (with both Lua and Tcl module syntax) + - python: 3.5 + modules_tool: ${{needs.setup.outputs.lmod8}} + module_syntax: Lua + - python: 3.5 + modules_tool: ${{needs.setup.outputs.lmod8}} + module_syntax: Tcl + - python: 3.7 + modules_tool: ${{needs.setup.outputs.lmod8}} + module_syntax: Lua + - python: 3.7 + modules_tool: ${{needs.setup.outputs.lmod8}} + module_syntax: Tcl + - python: 3.8 + modules_tool: ${{needs.setup.outputs.lmod8}} + module_syntax: Lua + - python: 3.8 + modules_tool: ${{needs.setup.outputs.lmod8}} + module_syntax: Tcl + - python: 3.9 + modules_tool: ${{needs.setup.outputs.lmod8}} + module_syntax: Lua + - python: 3.9 + modules_tool: ${{needs.setup.outputs.lmod8}} + module_syntax: Tcl + # There may be encoding errors in Python 3 which are hidden when an UTF-8 encoding is set + # Hence run the tests (again) with LC_ALL=C and Python 3.6 (or any < 3.7) - python: 3.6 - modules_tool: Lmod-8.2.9 + modules_tool: ${{needs.setup.outputs.lmod8}} module_syntax: Lua lc_all: C fail-fast: false diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 1d4cadbecb..21d810f690 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -4,6 +4,40 @@ For more detailed information, please see the git log. These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html. +v4.3.4 (April 9th 2021) +----------------------- + +update/bugfix release + +- various enhancements, including: + - add support for filtering dependencies by using False as version (#3506) + - add create_unused_dir function to create a directory which does not yet exist (#3551) + - avoid running expensive 'module use' and 'module unuse' commands when using Lmod as modules tool, update $MODULEPATH directly instead (#3557, #3633) + - create CUDA cache (for JIT compiled PTX code) in build dir instead of $HOME (#3569) + - add "Citing" section to module files (#3596) + - add support for using fallback 'arch=*' key in dependency version specified as arch->version mapping (#3600) + - also check for pending change requests and mergeable_state in check_pr_eligible_to_merge (#3604) + - ignore undismissed 'changes requested' review if there is an 'approved' review by the same user (#3607, #3608) + - sort output of 'eb --search' in natural order (respecting numbers) (#3609) + - enhance 'eb' command to ensure that easybuild.main can be imported before settling on python* command to use (#3610) + - add --env-for-shebang configuration option to define the env command to use for shebangs (#3613) + - add templates for architecture independent Python wheels (#3618) + - mention easyblocks PR in gist when uploading test report for it + fix clean_gists.py script (#3622) + - also accept regular expression value for --accept-eula-for (#3630) + - update validate_github_token function to accept GitHub token in new format (#3632) +- various bug fixes, including: + - fix $BLAS_LIB_MT for OpenBLAS, ensure -lpthread is included (#3584) + - use '--opt=val' for passing settings from config file to option parser to avoid error for values starting with '-' or '--' (#3594) + - avoid raised exception when getting output from interactive command in run_cmd_qa (#3599) + - add option to write file from file-like object and use in download_file (#3614) + - make sure that path to eb is always found by tests (#3617) +- other changes: + - add pick_default_branch function to clean up duplicate code in tools/github.py (#3592) + - refactor the CI configuration to use inclusion instead of exclusion (#3616) + - use develop branch when testing push access in --check-github (#3629) + - deprecate --accept-eula, rename to --accept-eula-for (#3630) + + v4.3.3 (February 23rd 2021) --------------------------- @@ -17,7 +51,7 @@ update/bugfix release - detect 'SYSTEM' toolchain as special case in easystack files (#3543) - enhance extract_cmd function to use 'cp -a' for shell scripts (.sh) (#3545) - allow use of alternate envvar(s) to $HOME for user modules (#3558) - - use https://sources/easybuild.io as fallback source URL (#3572, #3576) + - use https://sources.easybuild.io as fallback source URL (#3572, #3576) - add toolchain definition for iibff toolchain (#3574) - add %(cuda_cc_space_sep)s and %(cuda_cc_semicolon_sep)s templates (#3578) - add support for intel-compiler toolchain (>= 2021.x versions, oneAPI) (#3581, #3582) @@ -34,6 +68,7 @@ update/bugfix release - other changes: - rename EasyBlock._skip_step to EasyBlock.skip_step, to make it part of the public API (#3561) - make symlinking of posix_c.so to posix.so in test suite configuration conditional (#3570) + - use 'main' rather than 'master' branch in GitHub integration functionality (#3589) v4.3.2 (December 10th 2020) diff --git a/easybuild/base/generaloption.py b/easybuild/base/generaloption.py index 79fa8a85f8..b74110cdbd 100644 --- a/easybuild/base/generaloption.py +++ b/easybuild/base/generaloption.py @@ -1376,8 +1376,7 @@ def parseconfigfiles(self): configfile_values[opt_dest] = newval else: configfile_cmdline_dest.append(opt_dest) - configfile_cmdline.append("--%s" % opt_name) - configfile_cmdline.append(val) + configfile_cmdline.append("--%s=%s" % (opt_name, val)) # reparse self.log.debug('parseconfigfiles: going to parse options through cmdline %s' % configfile_cmdline) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 5fb593aee0..77e0df72a5 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1034,6 +1034,27 @@ def make_dir(self, dir_name, clean, dontcreateinstalldir=False): mkdir(dir_name, parents=True) + def set_up_cuda_cache(self): + """Set up CUDA PTX cache.""" + + cuda_cache_maxsize = build_option('cuda_cache_maxsize') + if cuda_cache_maxsize is None: + cuda_cache_maxsize = 1 * 1024 # 1 GiB default value + else: + cuda_cache_maxsize = int(cuda_cache_maxsize) + + if cuda_cache_maxsize == 0: + self.log.info("Disabling CUDA PTX cache since cache size was set to zero") + env.setvar('CUDA_CACHE_DISABLE', '1') + else: + cuda_cache_dir = build_option('cuda_cache_dir') + if not cuda_cache_dir: + cuda_cache_dir = os.path.join(self.builddir, 'eb-cuda-cache') + self.log.info("Enabling CUDA PTX cache of size %s MiB at %s", cuda_cache_maxsize, cuda_cache_dir) + env.setvar('CUDA_CACHE_DISABLE', '0') + env.setvar('CUDA_CACHE_PATH', cuda_cache_dir) + env.setvar('CUDA_CACHE_MAXSIZE', str(cuda_cache_maxsize * 1024 * 1024)) + # # MODULE UTILITY FUNCTIONS # @@ -1655,8 +1676,8 @@ def check_accepted_eula(self, name=None, more_info=None): if name is None: name = self.name - accepted_eulas = build_option('accept_eula') or [] - if self.cfg['accept_eula'] or name in accepted_eulas: + accepted_eulas = build_option('accept_eula_for') or [] + if self.cfg['accept_eula'] or name in accepted_eulas or any(re.match(x, name) for x in accepted_eulas): self.log.info("EULA for %s is accepted", name) else: error_lines = [ @@ -1667,7 +1688,7 @@ def check_accepted_eula(self, name=None, more_info=None): error_lines.extend([ "You should either:", - "- add --accept-eula=%(name)s to the 'eb' command;", + "- add --accept-eula-for=%(name)s to the 'eb' command;", "- update your EasyBuild configuration to always accept the EULA for %(name)s;", "- add 'accept_eula = True' to the easyconfig file you are using;", '', @@ -2163,6 +2184,10 @@ def prepare_step(self, start_dir=True, load_tc_deps_modules=True): self.log.info("Loading extra modules: %s", extra_modules) self.modules_tool.load(extra_modules) + # Setup CUDA cache if required. If we don't do this, CUDA will use the $HOME for its cache files + if get_software_root('CUDA') or get_software_root('CUDAcore'): + self.set_up_cuda_cache() + # guess directory to start configure/build/install process in, and move there if start_dir: self.guess_start_dir() @@ -2367,7 +2392,7 @@ def fix_shebang(self): if isinstance(fix_shebang_for, string_type): fix_shebang_for = [fix_shebang_for] - shebang = '#!/usr/bin/env %s' % lang + shebang = '#!%s %s' % (build_option('env_for_shebang'), lang) for glob_pattern in fix_shebang_for: paths = glob.glob(os.path.join(self.installdir, glob_pattern)) self.log.info("Fixing '%s' shebang to '%s' for files that match '%s': %s", diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py index 99e3c8309f..317f862b8c 100644 --- a/easybuild/framework/easyconfig/default.py +++ b/easybuild/framework/easyconfig/default.py @@ -194,6 +194,7 @@ # MODULES documentation easyconfig parameters # (docurls is part of MANDATORY) + 'citing': [None, "Free-form text that describes how the software should be cited in publications", MODULES], 'docpaths': [None, "List of paths for documentation relative to installation directory", MODULES], 'examples': [None, "Free-form text with examples on using the software", MODULES], 'site_contacts': [None, "String/list of strings with site contacts for the software", MODULES], diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 27cf331c41..c57f55c476 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -702,10 +702,13 @@ def parse(self): # parse dependency specifications # it's important that templating is still disabled at this stage! self.log.info("Parsing dependency specifications...") - self['dependencies'] = [self._parse_dependency(dep) for dep in self['dependencies']] - self['hiddendependencies'] = [ - self._parse_dependency(dep, hidden=True) for dep in self['hiddendependencies'] - ] + + def remove_false_versions(deps): + return [dep for dep in deps if not (isinstance(dep, dict) and dep['version'] is False)] + + self['dependencies'] = remove_false_versions(self._parse_dependency(dep) for dep in self['dependencies']) + self['hiddendependencies'] = remove_false_versions(self._parse_dependency(dep, hidden=True) for dep in + self['hiddendependencies']) # need to take into account that builddependencies may need to be iterated over, # i.e. when the value is a list of lists of tuples @@ -715,7 +718,7 @@ def parse(self): builddeps = [[self._parse_dependency(dep, build_only=True) for dep in x] for x in builddeps] else: builddeps = [self._parse_dependency(dep, build_only=True) for dep in builddeps] - self['builddependencies'] = builddeps + self['builddependencies'] = remove_false_versions(builddeps) # keep track of parsed multi deps, they'll come in handy during sanity check & module steps... self.multi_deps = self.get_parsed_multi_deps() diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index a1ef4d42b7..6b3ba8763d 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -155,6 +155,19 @@ ('SOURCE_%s' % suffix, '%(name)s-%(version)s.' + ext, "Source .%s bundle" % ext), ('SOURCELOWER_%s' % suffix, '%(namelower)s-%(version)s.' + ext, "Source .%s bundle with lowercase name" % ext), ] +for pyver in ('py2.py3', 'py2', 'py3'): + if pyver == 'py2.py3': + desc = 'Python 2 & Python 3' + name_infix = '' + else: + desc = 'Python ' + pyver[-1] + name_infix = pyver.upper() + '_' + TEMPLATE_CONSTANTS += [ + ('SOURCE_%sWHL' % name_infix, '%%(name)s-%%(version)s-%s-none-any.whl' % pyver, + 'Generic (non-compiled) %s wheel package' % desc), + ('SOURCELOWER_%sWHL' % name_infix, '%%(namelower)s-%%(version)s-%s-none-any.whl' % pyver, + 'Generic (non-compiled) %s wheel package with lowercase name' % desc), + ] # TODO derived config templates # versionmajor, versionminor, versionmajorminor (eg '.'.join(version.split('.')[:2])) ) diff --git a/easybuild/scripts/clean_gists.py b/easybuild/scripts/clean_gists.py index b5761713be..a290de3f42 100644 --- a/easybuild/scripts/clean_gists.py +++ b/easybuild/scripts/clean_gists.py @@ -32,9 +32,10 @@ from easybuild.base.generaloption import simple_option from easybuild.base.rest import RestClient from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.github import GITHUB_API_URL, HTTP_STATUS_OK, GITHUB_EASYCONFIGS_REPO +from easybuild.tools.github import GITHUB_API_URL, HTTP_STATUS_OK, GITHUB_EASYCONFIGS_REPO, GITHUB_EASYBLOCKS_REPO from easybuild.tools.github import GITHUB_EB_MAIN, fetch_github_token from easybuild.tools.options import EasyBuildOptions +from easybuild.tools.py2vs3 import HTTPError, URLError HTTP_DELETE_OK = 204 @@ -49,6 +50,7 @@ def main(): 'closed-pr': ('Delete all gists from closed pull-requests', None, 'store_true', True, 'p'), 'all': ('Delete all gists from Easybuild ', None, 'store_true', False, 'a'), 'orphans': ('Delete all gists without a pull-request', None, 'store_true', False, 'o'), + 'dry-run': ("Only show which gists will be deleted but don't actually delete them", None, 'store_true', False), } go = simple_option(options) @@ -58,6 +60,7 @@ def main(): raise EasyBuildError("Please tell me what to do?") if go.options.github_user is None: + EasyBuildOptions.DEFAULT_LOGLEVEL = None # Don't overwrite log level eb_go = EasyBuildOptions(envvar_prefix='EASYBUILD', go_args=[]) username = eb_go.options.github_user log.debug("Fetch github username from easybuild, found: %s", username) @@ -88,7 +91,8 @@ def main(): break log.info("Found %s gists", len(all_gists)) - regex = re.compile(r"(EasyBuild test report|EasyBuild log for failed build).*?(?:PR #(?P[0-9]+))?\)?$") + re_eb_gist = re.compile(r"(EasyBuild test report|EasyBuild log for failed build)(.*?)$") + re_pr_nr = re.compile(r"(EB )?PR #([0-9]+)") pr_cache = {} num_deleted = 0 @@ -96,43 +100,79 @@ def main(): for gist in all_gists: if not gist["description"]: continue - re_pr_num = regex.search(gist["description"]) - delete_gist = False - - if re_pr_num: - log.debug("Found a Easybuild gist (id=%s)", gist["id"]) - pr_num = re_pr_num.group("PR") - if go.options.all: - delete_gist = True - elif pr_num and go.options.closed_pr: - log.debug("Found Easybuild test report for PR #%s", pr_num) - - if pr_num not in pr_cache: - status, pr = gh.repos[GITHUB_EB_MAIN][GITHUB_EASYCONFIGS_REPO].pulls[pr_num].get() + + gist_match = re_eb_gist.search(gist["description"]) + + if not gist_match: + log.debug("Found a non-Easybuild gist (id=%s)", gist["id"]) + continue + + log.debug("Found an Easybuild gist (id=%s)", gist["id"]) + + pr_data = gist_match.group(2) + + pr_nrs_matches = re_pr_nr.findall(pr_data) + + if go.options.all: + delete_gist = True + elif not pr_nrs_matches: + log.debug("Found Easybuild test report without PR (id=%s).", gist["id"]) + delete_gist = go.options.orphans + elif go.options.closed_pr: + # All PRs must be closed + delete_gist = True + for pr_nr_match in pr_nrs_matches: + eb_str, pr_num = pr_nr_match + if eb_str or GITHUB_EASYBLOCKS_REPO in pr_data: + repo = GITHUB_EASYBLOCKS_REPO + else: + repo = GITHUB_EASYCONFIGS_REPO + + cache_key = "%s-%s" % (repo, pr_num) + + if cache_key not in pr_cache: + try: + status, pr = gh.repos[GITHUB_EB_MAIN][repo].pulls[pr_num].get() + except HTTPError as e: + status, pr = e.code, e.msg if status != HTTP_STATUS_OK: raise EasyBuildError("Failed to get pull-request #%s: error code %s, message = %s", pr_num, status, pr) - pr_cache[pr_num] = pr["state"] - - if pr_cache[pr_num] == "closed": - log.debug("Found report from closed PR #%s (id=%s)", pr_num, gist["id"]) - delete_gist = True - - elif not pr_num and go.options.orphans: - log.debug("Found Easybuild test report without PR (id=%s)", gist["id"]) - delete_gist = True + pr_cache[cache_key] = pr["state"] + + if pr_cache[cache_key] == "closed": + log.debug("Found report from closed %s PR #%s (id=%s)", repo, pr_num, gist["id"]) + elif delete_gist: + if len(pr_nrs_matches) > 1: + log.debug("Found at least 1 PR, that is not closed yet: %s/%s (id=%s)", + repo, pr_num, gist["id"]) + delete_gist = False + else: + delete_gist = True if delete_gist: - status, del_gist = gh.gists[gist["id"]].delete() + if go.options.dry_run: + log.info("DRY-RUN: Delete gist with id=%s", gist["id"]) + num_deleted += 1 + continue + try: + status, del_gist = gh.gists[gist["id"]].delete() + except HTTPError as e: + status, del_gist = e.code, e.msg + except URLError as e: + status, del_gist = None, e.reason if status != HTTP_DELETE_OK: - raise EasyBuildError("Unable to remove gist (id=%s): error code %s, message = %s", - gist["id"], status, del_gist) + log.warning("Unable to remove gist (id=%s): error code %s, message = %s", + gist["id"], status, del_gist) else: - log.info("Delete gist with id=%s", gist["id"]) + log.info("Deleted gist with id=%s", gist["id"]) num_deleted += 1 - log.info("Deleted %s gists", num_deleted) + if go.options.dry_run: + log.info("DRY-RUN: Would delete %s gists", num_deleted) + else: + log.info("Deleted %s gists", num_deleted) if __name__ == '__main__': diff --git a/easybuild/toolchains/goblf.py b/easybuild/toolchains/goblf.py index 75c782b862..3bf2d50e81 100644 --- a/easybuild/toolchains/goblf.py +++ b/easybuild/toolchains/goblf.py @@ -23,7 +23,7 @@ # along with EasyBuild. If not, see . ## """ -EasyBuild support for foss compiler toolchain (includes GCC, OpenMPI, BLIS, LAPACK, ScaLAPACK and FFTW). +EasyBuild support for goblf compiler toolchain (includes GCC, OpenMPI, BLIS, LAPACK, ScaLAPACK and FFTW). :author: Kenneth Hoste (Ghent University) :author: Bart Oldeman (McGill University, Calcul Quebec, Compute Canada) diff --git a/easybuild/toolchains/linalg/openblas.py b/easybuild/toolchains/linalg/openblas.py index 7cfd042a94..4d57f51ba9 100644 --- a/easybuild/toolchains/linalg/openblas.py +++ b/easybuild/toolchains/linalg/openblas.py @@ -40,6 +40,7 @@ class OpenBLAS(LinAlg): """ BLAS_MODULE_NAME = ['OpenBLAS'] BLAS_LIB = ['openblas'] + BLAS_LIB_MT = ['openblas'] BLAS_FAMILY = TC_CONSTANT_OPENBLAS LAPACK_MODULE_NAME = ['OpenBLAS'] diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 1a7ff08c88..b491b0cde7 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -79,6 +79,7 @@ DEFAULT_CONT_TYPE = CONT_TYPE_SINGULARITY DEFAULT_BRANCH = 'develop' +DEFAULT_ENV_FOR_SHEBANG = '/usr/bin/env' DEFAULT_ENVVAR_USERS_MODULES = 'HOME' DEFAULT_INDEX_MAX_AGE = 7 * 24 * 60 * 60 # 1 week (in seconds) DEFAULT_JOB_BACKEND = 'GC3Pie' @@ -162,7 +163,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): # build options that have a perfectly matching command line option, listed by default value BUILD_OPTIONS_CMDLINE = { None: [ - 'accept_eula', + 'accept_eula_for', 'aggregate_regtest', 'backup_modules', 'container_config', @@ -170,6 +171,8 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'container_image_name', 'container_template_recipe', 'container_tmpdir', + 'cuda_cache_dir', + 'cuda_cache_maxsize', 'cuda_compute_capabilities', 'download_timeout', 'dump_test_report', @@ -295,6 +298,9 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): DEFAULT_BRANCH: [ 'pr_target_branch', ], + DEFAULT_ENV_FOR_SHEBANG: [ + 'env_for_shebang', + ], DEFAULT_INDEX_MAX_AGE: [ 'index_max_age', ], @@ -500,6 +506,10 @@ def init_build_options(build_options=None, cmdline_options=None): _log.info("Auto-enabling ignoring of OS dependencies") cmdline_options.ignore_osdeps = True + if not cmdline_options.accept_eula_for and cmdline_options.accept_eula: + _log.deprecated("Use accept-eula-for configuration setting rather than accept-eula.", '5.0') + cmdline_options.accept_eula_for = cmdline_options.accept_eula + cmdline_build_option_names = [k for ks in BUILD_OPTIONS_CMDLINE.values() for k in ks] active_build_options.update(dict([(key, getattr(cmdline_options, key)) for key in cmdline_build_option_names])) # other options which can be derived but have no perfectly matching cmdline option @@ -533,6 +543,9 @@ def build_option(key, **kwargs): build_options = BuildOptions() if key in build_options: return build_options[key] + elif key == 'accept_eula': + _log.deprecated("Use accept_eula_for build option rather than accept_eula.", '5.0') + return build_options['accept_eula_for'] elif 'default' in kwargs: return kwargs['default'] else: diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 2ac341a622..a2e2709917 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -60,7 +60,7 @@ from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_msg, print_warning from easybuild.tools.config import DEFAULT_WAIT_ON_LOCK_INTERVAL, GENERIC_EASYBLOCK_PKG, build_option, install_path from easybuild.tools.py2vs3 import HTMLParser, std_urllib, string_type -from easybuild.tools.utilities import nub, remove_unwanted_chars +from easybuild.tools.utilities import natural_keys, nub, remove_unwanted_chars try: import requests @@ -217,7 +217,7 @@ def write_file(path, data, append=False, forced=False, backup=False, always_over overwrites current file contents without backup by default! :param path: location of file - :param data: contents to write to file + :param data: contents to write to file. Can be a file-like object of binary data :param append: append to existing file rather than overwrite :param forced: force actually writing file in (extended) dry run mode :param backup: back up existing file before overwriting or modifying it @@ -246,15 +246,21 @@ def write_file(path, data, append=False, forced=False, backup=False, always_over # cfr. https://docs.python.org/3/library/functions.html#open mode = 'a' if append else 'w' + data_is_file_obj = hasattr(data, 'read') + # special care must be taken with binary data in Python 3 - if sys.version_info[0] >= 3 and isinstance(data, bytes): + if sys.version_info[0] >= 3 and (isinstance(data, bytes) or data_is_file_obj): mode += 'b' # note: we can't use try-except-finally, because Python 2.4 doesn't support it as a single block try: mkdir(os.path.dirname(path), parents=True) with open_file(path, mode) as fh: - fh.write(data) + if data_is_file_obj: + # if a file-like object was provided, use copyfileobj (which reads the file in chunks) + shutil.copyfileobj(data, fh) + else: + fh.write(data) except IOError as err: raise EasyBuildError("Failed to write to %s: %s", path, err) @@ -710,7 +716,11 @@ def download_file(filename, url, path, forced=False): url_fd = response.raw url_fd.decode_content = True _log.debug('response code for given url %s: %s' % (url, status_code)) - write_file(path, url_fd.read(), forced=forced, backup=True) + # note: we pass the file object to write_file rather than reading the file first, + # to ensure the data is read in chunks (which prevents problems in Python 3.9+); + # cfr. https://github.com/easybuilders/easybuild-framework/issues/3455 + # and https://bugs.python.org/issue42853 + write_file(path, url_fd, forced=forced, backup=True) _log.info("Downloaded file %s from url %s to %s" % (filename, url, path)) downloaded = True url_fd.close() @@ -1000,8 +1010,11 @@ def search_file(paths, query, short=False, ignore_dirs=None, silent=False, filen if not terse: print_msg("Searching (case-insensitive) for '%s' in %s " % (query.pattern, path), log=_log, silent=silent) - path_index = load_index(path, ignore_dirs=ignore_dirs) - if path_index is None or build_option('ignore_index'): + if build_option('ignore_index'): + path_index = None + else: + path_index = load_index(path, ignore_dirs=ignore_dirs) + if path_index is None: if os.path.exists(path): _log.info("No index found for %s, creating one...", path) path_index = create_index(path, ignore_dirs=ignore_dirs) @@ -1021,15 +1034,17 @@ def search_file(paths, query, short=False, ignore_dirs=None, silent=False, filen else: path_hits.append(os.path.join(path, filepath)) - path_hits = sorted(path_hits) + path_hits = sorted(path_hits, key=natural_keys) if path_hits: - common_prefix = det_common_path_prefix(path_hits) - if not terse and short and common_prefix is not None and len(common_prefix) > len(var) * 2: - var_defs.append((var, common_prefix)) - hits.extend([os.path.join('$%s' % var, fn[len(common_prefix) + 1:]) for fn in path_hits]) - else: - hits.extend(path_hits) + if not terse and short: + common_prefix = det_common_path_prefix(path_hits) + if common_prefix is not None and len(common_prefix) > len(var) * 2: + var_defs.append((var, common_prefix)) + var_spec = '$' + var + # Replace the common prefix by var_spec + path_hits = (var_spec + fn[len(common_prefix):] for fn in path_hits) + hits.extend(path_hits) return var_defs, hits @@ -1652,6 +1667,25 @@ def patch_perl_script_autoflush(path): write_file(path, newtxt) +def set_gid_sticky_bits(path, set_gid=None, sticky=None, recursive=False): + """Set GID/sticky bits on specified path.""" + if set_gid is None: + set_gid = build_option('set_gid_bit') + if sticky is None: + sticky = build_option('sticky_bit') + + bits = 0 + if set_gid: + bits |= stat.S_ISGID + if sticky: + bits |= stat.S_ISVTX + if bits: + try: + adjust_permissions(path, bits, add=True, relative=True, recursive=recursive, onlydirs=True) + except OSError as err: + raise EasyBuildError("Failed to set groud ID/sticky bit: %s", err) + + def mkdir(path, parents=False, set_gid=None, sticky=None): """ Create a directory @@ -1687,18 +1721,9 @@ def mkdir(path, parents=False, set_gid=None, sticky=None): raise EasyBuildError("Failed to create directory %s: %s", path, err) # set group ID and sticky bits, if desired - bits = 0 - if set_gid: - bits |= stat.S_ISGID - if sticky: - bits |= stat.S_ISVTX - if bits: - try: - new_subdir = path[len(existing_parent_path):].lstrip(os.path.sep) - new_path = os.path.join(existing_parent_path, new_subdir.split(os.path.sep)[0]) - adjust_permissions(new_path, bits, add=True, relative=True, recursive=True, onlydirs=True) - except OSError as err: - raise EasyBuildError("Failed to set groud ID/sticky bit: %s", err) + new_subdir = path[len(existing_parent_path):].lstrip(os.path.sep) + new_path = os.path.join(existing_parent_path, new_subdir.split(os.path.sep)[0]) + set_gid_sticky_bits(new_path, set_gid, sticky, recursive=True) else: _log.debug("Not creating existing path %s" % path) @@ -2573,3 +2598,32 @@ def copy_framework_files(paths, target_dir): raise EasyBuildError("Couldn't find parent folder of updated file: %s", path) return file_info + + +def create_unused_dir(parent_folder, name): + """ + Create a new folder in parent_folder using name as the name. + When a folder of that name already exists, '_0' is appended which is retried for increasing numbers until + an unused name was found + """ + if not os.path.isabs(parent_folder): + parent_folder = os.path.abspath(parent_folder) + + start_path = os.path.join(parent_folder, name) + for number in range(-1, 10000): # Start with no suffix and limit the number of attempts + if number < 0: + path = start_path + else: + path = start_path + '_' + str(number) + try: + os.mkdir(path) + break + except OSError as err: + # Distinguish between error due to existing folder and anything else + if not os.path.exists(path): + raise EasyBuildError("Failed to create directory %s: %s", path, err) + + # set group ID and sticky bits, if desired + set_gid_sticky_bits(path, recursive=True) + + return path diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 71f4718e5c..9c7d04d31c 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -85,6 +85,8 @@ GITHUB_URL = 'https://github.com' GITHUB_API_URL = 'https://api.github.com' +GITHUB_BRANCH_MAIN = 'main' +GITHUB_BRANCH_MASTER = 'master' GITHUB_DIR_TYPE = u'dir' GITHUB_EB_MAIN = 'easybuilders' GITHUB_EASYBLOCKS_REPO = 'easybuild-easyblocks' @@ -120,6 +122,18 @@ } +def pick_default_branch(github_owner): + """Determine default name to use.""" + # use 'main' as default branch for 'easybuilders' organisation, + # otherwise use 'master' + if github_owner == GITHUB_EB_MAIN: + branch = GITHUB_BRANCH_MAIN + else: + branch = GITHUB_BRANCH_MASTER + + return branch + + class Githubfs(object): """This class implements some higher level functionality on top of the Github api""" @@ -133,10 +147,7 @@ def __init__(self, githubuser, reponame, branchname=None, username=None, passwor :param token: (optional) a github api token. """ if branchname is None: - if githubuser == GITHUB_EB_MAIN: - branchname = 'main' - else: - branchname = 'master' + branchname = pick_default_branch(githubuser) if token is None: token = fetch_github_token(username) @@ -318,12 +329,7 @@ def fetch_latest_commit_sha(repo, account, branch=None, github_user=None, token= :return: latest SHA1 """ if branch is None: - # use 'main' as default branch for 'easybuilders' organisation, - # otherwise use 'master' - if account == GITHUB_EB_MAIN: - branch = 'main' - else: - branch = 'master' + branch = pick_default_branch(account) status, data = github_api_get_request(lambda x: x.repos[account][repo].branches, github_user=github_user, token=token, per_page=GITHUB_MAX_PER_PAGE) @@ -356,12 +362,7 @@ def download_repo(repo=GITHUB_EASYCONFIGS_REPO, branch=None, account=GITHUB_EB_M :param github_user: name of GitHub user to use """ if branch is None: - # use 'main' as default branch for 'easybuilders' organisation, - # otherwise use 'master' - if account == GITHUB_EB_MAIN: - branch = 'main' - else: - branch = 'master' + branch = pick_default_branch(account) # make sure path exists, create it if necessary if path is None: @@ -1141,6 +1142,19 @@ def not_eligible(msg): if review['state'] == 'APPROVED': approved_review_by.append(review['user']['login']) + # check for requested changes + changes_requested_by = [] + for review in pr_data['reviews']: + if review['state'] == 'CHANGES_REQUESTED': + if review['user']['login'] not in approved_review_by + changes_requested_by: + changes_requested_by.append(review['user']['login']) + + msg_tmpl = "* no pending change requests: %s" + if changes_requested_by: + res = not_eligible(msg_tmpl % 'FAILED (changes requested by %s)' % ', '.join(changes_requested_by)) + else: + print_msg(msg_tmpl % 'OK', prefix=False) + msg_tmpl = "* approved review: %s" if approved_review_by: print_msg(msg_tmpl % 'OK (by %s)' % ', '.join(approved_review_by), prefix=False) @@ -1154,6 +1168,16 @@ def not_eligible(msg): else: res = not_eligible(msg_tmpl % 'no milestone found') + # check github mergeable state + msg_tmpl = "* mergeable state is clean: %s" + if pr_data['merged']: + print_msg(msg_tmpl % "PR is already merged", prefix=False) + elif pr_data['mergeable_state'] == GITHUB_MERGEABLE_STATE_CLEAN: + print_msg(msg_tmpl % "OK", prefix=False) + else: + reason = "FAILED (mergeable state is '%s')" % pr_data['mergeable_state'] + res = not_eligible(msg_tmpl % reason) + return res @@ -1433,7 +1457,7 @@ def post_pr_labels(pr, labels): return True -def add_pr_labels(pr, branch='develop'): +def add_pr_labels(pr, branch=GITHUB_DEVELOP_BRANCH): """ Try to determine and add labels to PR. :param pr: pull request number in easybuild-easyconfigs repo @@ -1962,7 +1986,7 @@ def check_github(): branch_name = 'test_branch_%s' % ''.join(random.choice(ascii_letters) for _ in range(5)) try: git_repo = init_repo(git_working_dir, GITHUB_EASYCONFIGS_REPO, silent=not debug) - remote_name = setup_repo(git_repo, github_account, GITHUB_EASYCONFIGS_REPO, 'main', + remote_name = setup_repo(git_repo, github_account, GITHUB_EASYCONFIGS_REPO, GITHUB_DEVELOP_BRANCH, silent=not debug, git_only=True) git_repo.create_head(branch_name) res = getattr(git_repo.remotes, remote_name).push(branch_name) @@ -2132,22 +2156,32 @@ def validate_github_token(token, github_user): * see if it conforms expectations (only [a-f]+[0-9] characters, length of 40) * see if it can be used for authenticated access """ - sha_regex = re.compile('^[0-9a-f]{40}') + # cfr. https://github.blog/2021-04-05-behind-githubs-new-authentication-token-formats/ + token_regex = re.compile('^ghp_[a-zA-Z0-9]{36}$') + token_regex_old_format = re.compile('^[0-9a-f]{40}$') # token should be 40 characters long, and only contain characters in [0-9a-f] - sanity_check = bool(sha_regex.match(token)) + sanity_check = bool(token_regex.match(token)) if sanity_check: _log.info("Sanity check on token passed") else: - _log.warning("Sanity check on token failed; token doesn't match pattern '%s'", sha_regex.pattern) + _log.warning("Sanity check on token failed; token doesn't match pattern '%s'", token_regex.pattern) + sanity_check = bool(token_regex_old_format.match(token)) + if sanity_check: + _log.info("Sanity check on token (old format) passed") + else: + _log.warning("Sanity check on token failed; token doesn't match pattern '%s'", + token_regex_old_format.pattern) # try and determine sha of latest commit in easybuilders/easybuild-easyconfigs repo through authenticated access sha = None try: - sha = fetch_latest_commit_sha(GITHUB_EASYCONFIGS_REPO, GITHUB_EB_MAIN, github_user=github_user, token=token) + sha = fetch_latest_commit_sha(GITHUB_EASYCONFIGS_REPO, GITHUB_EB_MAIN, + branch=GITHUB_DEVELOP_BRANCH, github_user=github_user, token=token) except Exception as err: _log.warning("An exception occurred when trying to use token for authenticated GitHub access: %s", err) + sha_regex = re.compile('^[0-9a-f]{40}$') token_test = bool(sha_regex.match(sha or '')) if token_test: _log.info("GitHub token can be used for authenticated GitHub access, validation passed") @@ -2161,7 +2195,8 @@ def find_easybuild_easyconfig(github_user=None): :param github_user: name of GitHub user to use when querying GitHub """ - dev_repo = download_repo(GITHUB_EASYCONFIGS_REPO, branch='develop', account=GITHUB_EB_MAIN, github_user=github_user) + dev_repo = download_repo(GITHUB_EASYCONFIGS_REPO, branch=GITHUB_DEVELOP_BRANCH, + account=GITHUB_EB_MAIN, github_user=github_user) eb_parent_path = os.path.join(dev_repo, 'easybuild', 'easyconfigs', 'e', 'EasyBuild') files = os.listdir(eb_parent_path) diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index 53a6c25916..6cb2716a7c 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -559,6 +559,9 @@ def _generate_help_text(self): # Examples (optional) lines.extend(self._generate_section('Examples', self.app.cfg['examples'], strip=True)) + # Citing (optional) + lines.extend(self._generate_section('Citing', self.app.cfg['citing'], strip=True)) + # Additional information: homepage + (if available) doc paths/urls, upstream/site contact lines.extend(self._generate_section("More information", " - Homepage: %s" % self.app.cfg['homepage'])) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 1741a27759..3e3f004dc5 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1428,7 +1428,35 @@ def use(self, path, priority=None): if priority: self.run_module(['use', '--priority', str(priority), path]) else: - self.run_module(['use', path]) + # LMod allows modifying MODULEPATH directly. So do that to avoid the costly module use + # unless priorities are in use already + if os.environ.get('__LMOD_Priority_MODULEPATH'): + self.run_module(['use', path]) + else: + cur_mod_path = os.environ.get('MODULEPATH') + if cur_mod_path is None: + new_mod_path = path + else: + new_mod_path = [path] + [p for p in cur_mod_path.split(':') if p != path] + new_mod_path = ':'.join(new_mod_path) + self.log.debug('Changing MODULEPATH from %s to %s' % + ('' if cur_mod_path is None else cur_mod_path, new_mod_path)) + os.environ['MODULEPATH'] = new_mod_path + + def unuse(self, path): + """Remove a module path""" + # We can simply remove the path from MODULEPATH to avoid the costly module call + cur_mod_path = os.environ.get('MODULEPATH') + if cur_mod_path is not None: + # Removing the last entry unsets the variable + if cur_mod_path == path: + self.log.debug('Changing MODULEPATH from %s to ' % cur_mod_path) + del os.environ['MODULEPATH'] + else: + new_mod_path = ':'.join(p for p in cur_mod_path.split(':') if p != path) + if new_mod_path != cur_mod_path: + self.log.debug('Changing MODULEPATH from %s to %s' % (cur_mod_path, new_mod_path)) + os.environ['MODULEPATH'] = new_mod_path def prepend_module_path(self, path, set_mod_paths=True, priority=None): """ diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 605ed46ab9..504b561eeb 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -60,8 +60,8 @@ from easybuild.tools.build_log import DEVEL_LOG_LEVEL, EasyBuildError from easybuild.tools.build_log import init_logging, log_start, print_msg, print_warning, raise_easybuilderror from easybuild.tools.config import CONT_IMAGE_FORMATS, CONT_TYPES, DEFAULT_CONT_TYPE, DEFAULT_ALLOW_LOADED_MODULES -from easybuild.tools.config import DEFAULT_BRANCH, DEFAULT_ENVVAR_USERS_MODULES, DEFAULT_FORCE_DOWNLOAD -from easybuild.tools.config import DEFAULT_INDEX_MAX_AGE +from easybuild.tools.config import DEFAULT_BRANCH, DEFAULT_ENV_FOR_SHEBANG, DEFAULT_ENVVAR_USERS_MODULES +from easybuild.tools.config import DEFAULT_FORCE_DOWNLOAD, DEFAULT_INDEX_MAX_AGE from easybuild.tools.config import DEFAULT_JOB_BACKEND, DEFAULT_LOGFILE_FORMAT, DEFAULT_MAX_FAIL_RATIO_PERMS from easybuild.tools.config import DEFAULT_MINIMAL_BUILD_ENV, DEFAULT_MNS, DEFAULT_MODULE_SYNTAX, DEFAULT_MODULES_TOOL from easybuild.tools.config import DEFAULT_MODULECLASSES, DEFAULT_PATH_SUBDIRS, DEFAULT_PKG_RELEASE, DEFAULT_PKG_TOOL @@ -334,7 +334,9 @@ def override_options(self): descr = ("Override options", "Override default EasyBuild behavior.") opts = OrderedDict({ - 'accept-eula': ("Accept EULA for specified software", 'strlist', 'store', []), + 'accept-eula': ("Accept EULA for specified software [DEPRECATED, use --accept-eula-for instead!]", + 'strlist', 'store', []), + 'accept-eula-for': ("Accept EULA for specified software", 'strlist', 'store', []), 'add-dummy-to-minimal-toolchains': ("Include dummy toolchain in minimal toolchain searches " "[DEPRECATED, use --add-system-to-minimal-toolchains instead!)", None, 'store_true', False), @@ -358,6 +360,11 @@ def override_options(self): 'consider-archived-easyconfigs': ("Also consider archived easyconfigs", None, 'store_true', False), 'containerize': ("Generate container recipe/image", None, 'store_true', False, 'C'), 'copy-ec': ("Copy specified easyconfig(s) to specified location", None, 'store_true', False), + 'cuda-cache-dir': ("Path to CUDA cache dir to use if enabled. Defaults to a path inside the build dir.", + str, 'store', None, {'metavar': "PATH"}), + 'cuda-cache-maxsize': ("Maximum size of the CUDA cache (in MiB) used for JIT compilation of PTX code. " + "Leave value empty to let EasyBuild choose a value or '0' to disable the cache", + int, 'store_or_None', None), 'cuda-compute-capabilities': ("List of CUDA compute capabilities to use when building GPU software; " "values should be specified as digits separated by a dot, " "for example: 3.5,5.0,7.2", 'strlist', 'extend', None), @@ -375,6 +382,8 @@ def override_options(self): None, 'store', None, 'e', {'metavar': 'CLASS'}), 'enforce-checksums': ("Enforce availability of checksums for all sources/patches, so they can be verified", None, 'store_true', False), + 'env-for-shebang': ("Define the env command to use when fixing shebangs", None, 'store', + DEFAULT_ENV_FOR_SHEBANG), 'experimental': ("Allow experimental code (with behaviour that can be changed/removed at any given time).", None, 'store_true', False), 'extra-modules': ("List of extra modules to load after setting up the build environment", diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 67977156b0..9c77edf06a 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -107,7 +107,10 @@ def get_output_from_process(proc, read_size=None, asynchronous=False): """ if asynchronous: - output = asyncprocess.recv_some(proc) + # e=False is set to avoid raising an exception when command has completed; + # that's needed to ensure we get all output, + # see https://github.com/easybuilders/easybuild-framework/issues/3593 + output = asyncprocess.recv_some(proc, e=False) elif read_size: output = proc.stdout.read(read_size) else: @@ -411,6 +414,7 @@ def check_answers_list(answers): # - otherwise the stdout/stderr buffer gets filled and it all stops working try: out = get_output_from_process(proc, asynchronous=True) + if cmd_log: cmd_log.write(out) stdout_err += out diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index d58cfc0abf..6a3aa83861 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -73,6 +73,8 @@ POWER = 'POWER' X86_64 = 'x86_64' +ARCH_KEY_PREFIX = 'arch=' + # Vendor constants AMD = 'AMD' APM = 'Applied Micro' @@ -921,20 +923,29 @@ def pick_dep_version(dep_version): result = None elif isinstance(dep_version, dict): - # figure out matches based on dict keys (after splitting on '=') - my_arch_key = 'arch=%s' % get_cpu_architecture() - arch_keys = [x for x in dep_version.keys() if x.startswith('arch=')] + arch_keys = [x for x in dep_version.keys() if x.startswith(ARCH_KEY_PREFIX)] other_keys = [x for x in dep_version.keys() if x not in arch_keys] if other_keys: - raise EasyBuildError("Unexpected keys in version: %s. Only 'arch=' keys are supported", other_keys) + other_keys = ','.join(sorted(other_keys)) + raise EasyBuildError("Unexpected keys in version: %s (only 'arch=' keys are supported)", other_keys) if arch_keys: - if my_arch_key in dep_version: - result = dep_version[my_arch_key] - _log.info("Version selected from %s using key %s: %s", dep_version, my_arch_key, result) + host_arch_key = ARCH_KEY_PREFIX + get_cpu_architecture() + star_arch_key = ARCH_KEY_PREFIX + '*' + # check for specific 'arch=' key first + if host_arch_key in dep_version: + result = dep_version[host_arch_key] + _log.info("Version selected from %s using key %s: %s", dep_version, host_arch_key, result) + # fall back to 'arch=*' + elif star_arch_key in dep_version: + result = dep_version[star_arch_key] + _log.info("Version selected for %s using fallback key %s: %s", dep_version, star_arch_key, result) else: - raise EasyBuildError("No matches for version in %s (looking for %s)", dep_version, my_arch_key) + raise EasyBuildError("No matches for version in %s (looking for %s)", dep_version, host_arch_key) + else: + raise EasyBuildError("Found empty dict as version!") else: - raise EasyBuildError("Unknown value type for version: %s", dep_version) + typ = type(dep_version) + raise EasyBuildError("Unknown value type for version: %s (%s), should be string value", typ, dep_version) return result diff --git a/easybuild/tools/testing.py b/easybuild/tools/testing.py index cf93034570..93f62babb1 100644 --- a/easybuild/tools/testing.py +++ b/easybuild/tools/testing.py @@ -138,22 +138,30 @@ def session_state(): } -def create_test_report(msg, ecs_with_res, init_session_state, pr_nr=None, gist_log=False): +def create_test_report(msg, ecs_with_res, init_session_state, pr_nr=None, gist_log=False, easyblock_pr_nrs=None): """Create test report for easyconfigs PR, in Markdown format.""" github_user = build_option('github_user') pr_target_account = build_option('pr_target_account') - pr_target_repo = build_option('pr_target_repo') or GITHUB_EASYCONFIGS_REPO + pr_target_repo = build_option('pr_target_repo') end_time = gmtime() # create a gist with a full test report test_report = [] if pr_nr is not None: + repo = pr_target_repo or GITHUB_EASYCONFIGS_REPO test_report.extend([ - "Test report for https://github.com/%s/%s/pull/%s" % (pr_target_account, pr_target_repo, pr_nr), + "Test report for https://github.com/%s/%s/pull/%s" % (pr_target_account, repo, pr_nr), "", ]) + if easyblock_pr_nrs: + repo = pr_target_repo or GITHUB_EASYBLOCKS_REPO + test_report.extend([ + "Test report for https://github.com/%s/%s/pull/%s" % (pr_target_account, repo, nr) + for nr in easyblock_pr_nrs + ]) + test_report.append("") test_report.extend([ "#### Test result", "%s" % msg, @@ -184,6 +192,8 @@ def create_test_report(msg, ecs_with_res, init_session_state, pr_nr=None, gist_l descr = "(partial) EasyBuild log for failed build of %s" % ec['spec'] if pr_nr is not None: descr += " (PR #%s)" % pr_nr + if easyblock_pr_nrs: + descr += "".join(" (easyblock PR #%s)" % nr for nr in easyblock_pr_nrs) fn = '%s_partial.log' % os.path.basename(ec['spec'])[:-3] gist_url = create_gist(partial_log_txt, fn, descr=descr, github_user=github_user) test_log = "(partial log available at %s)" % gist_url @@ -318,20 +328,21 @@ def overall_test_report(ecs_with_res, orig_cnt, success, msg, init_session_state """ dump_path = build_option('dump_test_report') pr_nr = build_option('from_pr') - eb_pr_nrs = build_option('include_easyblocks_from_pr') + easyblock_pr_nrs = build_option('include_easyblocks_from_pr') upload = build_option('upload_test_report') if upload: msg = msg + " (%d easyconfigs in total)" % orig_cnt - test_report = create_test_report(msg, ecs_with_res, init_session_state, pr_nr=pr_nr, gist_log=True) + test_report = create_test_report(msg, ecs_with_res, init_session_state, pr_nr=pr_nr, gist_log=True, + easyblock_pr_nrs=easyblock_pr_nrs) if pr_nr: # upload test report to gist and issue a comment in the PR to notify txt = post_pr_test_report(pr_nr, GITHUB_EASYCONFIGS_REPO, test_report, msg, init_session_state, success) - elif eb_pr_nrs: + elif easyblock_pr_nrs: # upload test report to gist and issue a comment in the easyblocks PR to notify - for eb_pr_nr in map(int, eb_pr_nrs): - txt = post_pr_test_report(eb_pr_nr, GITHUB_EASYBLOCKS_REPO, test_report, msg, init_session_state, - success) + for easyblock_pr_nr in map(int, easyblock_pr_nrs): + txt = post_pr_test_report(easyblock_pr_nr, GITHUB_EASYBLOCKS_REPO, test_report, msg, + init_session_state, success) else: # only upload test report as a gist gist_url = upload_test_report_as_gist(test_report['full']) diff --git a/easybuild/tools/utilities.py b/easybuild/tools/utilities.py index e1b19b2985..15c501de8b 100644 --- a/easybuild/tools/utilities.py +++ b/easybuild/tools/utilities.py @@ -316,3 +316,10 @@ def time2str(delta): res = '%d %s %d min %d sec' % (hours, hours_str, mins, secs) return res + + +def natural_keys(key): + """Can be used as the sort key in list.sort(key=natural_keys) to sort in natural order (i.e. respecting numbers)""" + def try_to_int(key_part): + return int(key_part) if key_part.isdigit() else key_part + return [try_to_int(key_part) for key_part in re.split(r'(\d+)', key)] diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index 29a940011b..74ab8d79d8 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.3.3') +VERSION = LooseVersion('4.3.4') UNKNOWN = 'UNKNOWN' diff --git a/eb b/eb index 8968892dbe..592b873432 100755 --- a/eb +++ b/eb @@ -37,6 +37,7 @@ REQ_MIN_PY2VER=6 REQ_MIN_PY3VER=5 +EASYBUILD_MAIN='easybuild.main' function verbose() { if [ ! -z ${EB_VERBOSE} ]; then echo ">> $1"; fi @@ -52,6 +53,8 @@ for python_cmd in ${EB_PYTHON} ${EB_INSTALLPYTHON} 'python' 'python3' 'python2'; verbose "Considering '$python_cmd'..." + # check whether python* command being considered is available + # (using 'command -v', since 'which' implies an extra dependency) command -v $python_cmd &> /dev/null if [ $? -eq 0 ]; then @@ -63,10 +66,25 @@ for python_cmd in ${EB_PYTHON} ${EB_INSTALLPYTHON} 'python' 'python3' 'python2'; if [ $pyver_maj -eq 2 ] && [ $pyver_min -ge $REQ_MIN_PY2VER ]; then verbose "'$python_cmd' version: $pyver, which matches Python 2 version requirement (>= 2.$REQ_MIN_PY2VER)" PYTHON=$python_cmd - break elif [ $pyver_maj -eq 3 ] && [ $pyver_min -ge $REQ_MIN_PY3VER ]; then verbose "'$python_cmd' version: $pyver, which matches Python 3 version requirement (>= 3.$REQ_MIN_PY3VER)" PYTHON=$python_cmd + fi + + if [ ! -z $PYTHON ]; then + # check whether easybuild.main is available for selected python command + $PYTHON -c "import $EASYBUILD_MAIN" 2> /dev/null + if [ $? -eq 0 ]; then + verbose "'$python_cmd' is able to import '$EASYBUILD_MAIN', so retaining it" + else + # if easybuild.main is not available, don't use this python command, keep searching... + verbose "'$python_cmd' is NOT able to import '$EASYBUILD_MAIN', so NOT retaining it" + unset PYTHON + fi + fi + + # break out of for loop if we've found a valid python command + if [ ! -z $PYTHON ]; then break fi else @@ -97,5 +115,5 @@ fi export EB_SCRIPT_PATH=$0 -verbose "$PYTHON -m easybuild.main `echo \"$@\"`" -$PYTHON -m easybuild.main "$@" +verbose "$PYTHON -m $EASYBUILD_MAIN `echo \"$@\"`" +$PYTHON -m $EASYBUILD_MAIN "$@" diff --git a/test/framework/config.py b/test/framework/config.py index 0b17a4e876..08a0594825 100644 --- a/test/framework/config.py +++ b/test/framework/config.py @@ -244,6 +244,9 @@ def test_generaloption_config_file(self): cfgtxt = '\n'.join([ '[config]', 'installpath = %s' % testpath2, + # special case: configuration option to a value starting with '--' + '[override]', + 'optarch = --test', ]) write_file(config_file, cfgtxt) @@ -261,6 +264,8 @@ def test_generaloption_config_file(self): self.assertEqual(install_path(), installpath_software) # via cmdline arg self.assertEqual(install_path('mod'), os.path.join(testpath2, 'modules')) # via config file + self.assertEqual(options.optarch, '--test') # via config file + # copy test easyconfigs to easybuild/easyconfigs subdirectory of temp directory # to check whether easyconfigs install path is auto-included in robot path tmpdir = tempfile.mkdtemp(prefix='easybuild-easyconfigs-pkg-install-path') diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index d72097a990..35f617d939 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -34,7 +34,6 @@ import sys import tempfile from inspect import cleandoc -from datetime import datetime from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config from unittest import TextTestRunner @@ -50,7 +49,6 @@ from easybuild.tools.filetools import verify_checksum, write_file from easybuild.tools.module_generator import module_generator from easybuild.tools.modules import reset_module_caches -from easybuild.tools.utilities import time2str from easybuild.tools.version import get_git_revision, this_is_easybuild from easybuild.tools.py2vs3 import string_type @@ -1896,6 +1894,60 @@ def test_prepare_step_hmns(self): self.assertEqual(len(loaded_modules), 1) self.assertEqual(loaded_modules[0]['mod_name'], 'GCC/6.4.0-2.28') + def test_prepare_step_cuda_cache(self): + """Test handling cuda-cache-* options.""" + + init_config(build_options={'cuda_cache_maxsize': None}) # Automatic mode + + test_ecs = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') + toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') + ec = process_easyconfig(toy_ec)[0] + eb = EasyBlock(ec['ec']) + eb.silent = True + eb.make_builddir() + + eb.prepare_step(start_dir=False) + logtxt = read_file(eb.logfile) + self.assertNotIn('Disabling CUDA PTX cache', logtxt) + self.assertNotIn('Enabling CUDA PTX cache', logtxt) + + # Now with CUDA + test_ec = os.path.join(self.test_prefix, 'test.eb') + test_ectxt = re.sub('^toolchain = .*', "toolchain = {'name': 'gcccuda', 'version': '2018a'}", + read_file(toy_ec), flags=re.M) + write_file(test_ec, test_ectxt) + ec = process_easyconfig(test_ec)[0] + eb = EasyBlock(ec['ec']) + eb.silent = True + eb.make_builddir() + + write_file(eb.logfile, '') + eb.prepare_step(start_dir=False) + logtxt = read_file(eb.logfile) + self.assertNotIn('Disabling CUDA PTX cache', logtxt) + self.assertIn('Enabling CUDA PTX cache', logtxt) + self.assertEqual(os.environ['CUDA_CACHE_DISABLE'], '0') + + init_config(build_options={'cuda_cache_maxsize': 0}) # Disable + write_file(eb.logfile, '') + eb.prepare_step(start_dir=False) + logtxt = read_file(eb.logfile) + self.assertIn('Disabling CUDA PTX cache', logtxt) + self.assertNotIn('Enabling CUDA PTX cache', logtxt) + self.assertEqual(os.environ['CUDA_CACHE_DISABLE'], '1') + + # Specified size and location + cuda_cache_dir = os.path.join(self.test_prefix, 'custom-cuda-cache') + init_config(build_options={'cuda_cache_maxsize': 1234, 'cuda_cache_dir': cuda_cache_dir}) + write_file(eb.logfile, '') + eb.prepare_step(start_dir=False) + logtxt = read_file(eb.logfile) + self.assertNotIn('Disabling CUDA PTX cache', logtxt) + self.assertIn('Enabling CUDA PTX cache', logtxt) + self.assertEqual(os.environ['CUDA_CACHE_DISABLE'], '0') + self.assertEqual(os.environ['CUDA_CACHE_MAXSIZE'], str(1234 * 1024 * 1024)) + self.assertEqual(os.environ['CUDA_CACHE_PATH'], cuda_cache_dir) + def test_checksum_step(self): """Test checksum step""" testdir = os.path.abspath(os.path.dirname(__file__)) @@ -2108,34 +2160,6 @@ def test_avail_easyblocks(self): self.assertEqual(hpl['class'], 'EB_HPL') self.assertTrue(hpl['loc'].endswith('sandbox/easybuild/easyblocks/h/hpl.py')) - def test_time2str(self): - """Test time2str function.""" - - start = datetime(2019, 7, 30, 5, 14, 23) - - test_cases = [ - (start, "0 sec"), - (datetime(2019, 7, 30, 5, 14, 37), "14 sec"), - (datetime(2019, 7, 30, 5, 15, 22), "59 sec"), - (datetime(2019, 7, 30, 5, 15, 23), "1 min 0 sec"), - (datetime(2019, 7, 30, 5, 16, 22), "1 min 59 sec"), - (datetime(2019, 7, 30, 5, 37, 26), "23 min 3 sec"), - (datetime(2019, 7, 30, 6, 14, 22), "59 min 59 sec"), - (datetime(2019, 7, 30, 6, 14, 23), "1 hour 0 min 0 sec"), - (datetime(2019, 7, 30, 6, 49, 14), "1 hour 34 min 51 sec"), - (datetime(2019, 7, 30, 7, 14, 23), "2 hours 0 min 0 sec"), - (datetime(2019, 7, 30, 8, 35, 59), "3 hours 21 min 36 sec"), - (datetime(2019, 7, 30, 16, 29, 24), "11 hours 15 min 1 sec"), - (datetime(2019, 7, 31, 5, 14, 22), "23 hours 59 min 59 sec"), - (datetime(2019, 7, 31, 5, 14, 23), "24 hours 0 min 0 sec"), - (datetime(2019, 8, 5, 20, 39, 44), "159 hours 25 min 21 sec"), - ] - for end, expected in test_cases: - self.assertEqual(time2str(end - start), expected) - - error_pattern = "Incorrect value type provided to time2str, should be datetime.timedelta: <.* 'int'>" - self.assertErrorRegex(EasyBuildError, error_pattern, time2str, 123) - def test_sanity_check_paths_verification(self): """Test verification of sanity_check_paths w.r.t. keys & values.""" diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index d4011f124f..f1a3b62efc 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -37,6 +37,7 @@ import stat import sys import tempfile +import textwrap from distutils.version import LooseVersion from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config from unittest import TextTestRunner @@ -72,7 +73,7 @@ from easybuild.tools.options import parse_external_modules_metadata from easybuild.tools.py2vs3 import OrderedDict, reload from easybuild.tools.robot import resolve_dependencies -from easybuild.tools.systemtools import get_shared_lib_ext +from easybuild.tools.systemtools import AARCH64, POWER, X86_64, get_cpu_architecture, get_shared_lib_ext from easybuild.tools.toolchain.utilities import search_toolchain from easybuild.tools.utilities import quote_str, quote_py_str from test.framework.utilities import find_full_path @@ -104,6 +105,7 @@ class EasyConfigTest(EnhancedTestCase): def setUp(self): """Set up everything for running a unit test.""" super(EasyConfigTest, self).setUp() + self.orig_get_cpu_architecture = st.get_cpu_architecture self.cwd = os.getcwd() self.all_stops = [x[0] for x in EasyBlock.get_steps()] @@ -122,6 +124,7 @@ def prep(self): def tearDown(self): """ make sure to remove the temporary file """ + st.get_cpu_architecture = self.orig_get_cpu_architecture super(EasyConfigTest, self).tearDown() if os.path.exists(self.eb_file): os.remove(self.eb_file) @@ -305,6 +308,69 @@ def test_dependency(self): self.assertErrorRegex(EasyBuildError, err_msg, eb._parse_dependency, (EXTERNAL_MODULE_MARKER,)) self.assertErrorRegex(EasyBuildError, err_msg, eb._parse_dependency, ('foo', '1.2.3', EXTERNAL_MODULE_MARKER)) + def test_false_dep_version(self): + """ + Test use False as dependency version via dict using 'arch=' keys, + which should result in filtering the dependency. + """ + # silence warnings about missing easyconfigs for dependencies, we don't care + init_config(build_options={'silent': True}) + + arch = get_cpu_architecture() + + self.contents = '\n'.join([ + 'easyblock = "ConfigureMake"', + 'name = "pi"', + 'version = "3.14"', + 'versionsuffix = "-test"', + 'homepage = "http://example.com"', + 'description = "test easyconfig"', + 'toolchain = {"name":"GCC", "version": "4.6.3"}', + 'builddependencies = [', + ' ("first_build", {"arch=%s": False}),' % arch, + ' ("second_build", "2.0"),', + ']', + 'dependencies = [' + ' ("first", "1.0"),', + ' ("second", {"arch=%s": False}),' % arch, + ']', + ]) + self.prep() + eb = EasyConfig(self.eb_file) + deps = eb.dependencies() + self.assertEqual(len(deps), 2) + self.assertEqual(deps[0]['name'], 'second_build') + self.assertEqual(deps[1]['name'], 'first') + + # more realistic example: only filter dep for POWER + self.contents = '\n'.join([ + 'easyblock = "ConfigureMake"', + 'name = "pi"', + 'version = "3.14"', + 'versionsuffix = "-test"', + 'homepage = "http://example.com"', + 'description = "test easyconfig"', + 'toolchain = {"name":"GCC", "version": "4.6.3"}', + 'dependencies = [' + ' ("not_on_power", {"arch=*": "1.2.3", "arch=POWER": False}),', + ']', + ]) + self.prep() + + # only non-POWER arch, dependency is retained + for arch in (AARCH64, X86_64): + st.get_cpu_architecture = lambda: arch + eb = EasyConfig(self.eb_file) + deps = eb.dependencies() + self.assertEqual(len(deps), 1) + self.assertEqual(deps[0]['name'], 'not_on_power') + + # only power, dependency gets filtered + st.get_cpu_architecture = lambda: POWER + eb = EasyConfig(self.eb_file) + deps = eb.dependencies() + self.assertEqual(deps, []) + def test_extra_options(self): """ extra_options should allow other variables to be stored """ init_config(build_options={'silent': True}) @@ -1118,6 +1184,37 @@ def test_java_wrapper_templating(self): self.assertEqual(eb['modloadmsg'], "Java: 11, 11, 11") + def test_python_whl_templating(self): + """test templating for Python wheels""" + + self.contents = textwrap.dedent(""" + easyblock = "ConfigureMake" + name = "Pi" + version = "3.14" + homepage = "https://example.com" + description = "test easyconfig" + toolchain = {"name":"GCC", "version": "4.6.3"} + sources = [ + SOURCE_WHL, + SOURCELOWER_WHL, + SOURCE_PY2_WHL, + SOURCELOWER_PY2_WHL, + SOURCE_PY3_WHL, + SOURCELOWER_PY3_WHL, + ] + """) + self.prep() + ec = EasyConfig(self.eb_file) + + sources = ec['sources'] + + self.assertEqual(sources[0], 'Pi-3.14-py2.py3-none-any.whl') + self.assertEqual(sources[1], 'pi-3.14-py2.py3-none-any.whl') + self.assertEqual(sources[2], 'Pi-3.14-py2-none-any.whl') + self.assertEqual(sources[3], 'pi-3.14-py2-none-any.whl') + self.assertEqual(sources[4], 'Pi-3.14-py3-none-any.whl') + self.assertEqual(sources[5], 'pi-3.14-py3-none-any.whl') + def test_templating_doc(self): """test templating documentation""" doc = avail_easyconfig_templates() @@ -3309,6 +3406,11 @@ def test_get_paths_for(self): self.mock_stderr(False) self.assertTrue(os.path.samefile(test_ecs, res[0])) + # Can't have EB_SCRIPT_PATH set (for some of) these tests + env_eb_script_path = os.getenv('EB_SCRIPT_PATH') + if env_eb_script_path: + del os.environ['EB_SCRIPT_PATH'] + # easyconfigs location can also be derived from location of 'eb' write_file(os.path.join(self.test_prefix, 'bin', 'eb'), "#!/bin/bash; echo 'This is a fake eb'") adjust_permissions(os.path.join(self.test_prefix, 'bin', 'eb'), stat.S_IXUSR) @@ -3325,6 +3427,10 @@ def test_get_paths_for(self): res = get_paths_for(subdir='easyconfigs', robot_path=None) self.assertTrue(os.path.samefile(test_ecs, res[-1])) + # Restore (temporarily) EB_SCRIPT_PATH value if set originally + if env_eb_script_path: + os.environ['EB_SCRIPT_PATH'] = env_eb_script_path + # also locations in sys.path are considered os.environ['PATH'] = orig_path sys.path.insert(0, self.test_prefix) @@ -3375,6 +3481,10 @@ def test_get_paths_for(self): self.assertTrue(os.path.exists(res[0])) self.assertTrue(os.path.samefile(res[0], os.path.join(someprefix, 'easybuild', 'easyconfigs'))) + # Finally restore EB_SCRIPT_PATH value if set + if env_eb_script_path: + os.environ['EB_SCRIPT_PATH'] = env_eb_script_path + def test_is_generic_easyblock(self): """Test for is_generic_easyblock function.""" diff --git a/test/framework/filetools.py b/test/framework/filetools.py index c948477aae..fb9c2440e4 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -705,6 +705,26 @@ def test_read_write_file(self): # test use of 'mode' in read_file self.assertEqual(ft.read_file(foo, mode='rb'), b'bar') + def test_write_file_obj(self): + """Test writing from a file-like object directly""" + # Write a text file + fp = os.path.join(self.test_prefix, 'test.txt') + fp_out = os.path.join(self.test_prefix, 'test_out.txt') + ft.write_file(fp, b'Hyphen: \xe2\x80\x93\nEuro sign: \xe2\x82\xac\na with dots: \xc3\xa4') + + with ft.open_file(fp, 'rb') as fh: + ft.write_file(fp_out, fh) + self.assertEqual(ft.read_file(fp_out), ft.read_file(fp)) + + # Write a binary file + fp = os.path.join(self.test_prefix, 'test.bin') + fp_out = os.path.join(self.test_prefix, 'test_out.bin') + ft.write_file(fp, b'\x00\x01'+os.urandom(42)+b'\x02\x03') + + with ft.open_file(fp, 'rb') as fh: + ft.write_file(fp_out, fh) + self.assertEqual(ft.read_file(fp_out, mode='rb'), ft.read_file(fp, mode='rb')) + def test_is_binary(self): """Test is_binary function.""" @@ -2158,11 +2178,11 @@ def test_search_file(self): self.assertEqual(var_defs, []) self.assertEqual(len(hits), 5) self.assertTrue(all(os.path.exists(p) for p in hits)) - self.assertTrue(hits[0].endswith('/hwloc-1.11.8-GCC-4.6.4.eb')) - self.assertTrue(hits[1].endswith('/hwloc-1.11.8-GCC-6.4.0-2.28.eb')) - self.assertTrue(hits[2].endswith('/hwloc-1.11.8-GCC-7.3.0-2.30.eb')) - self.assertTrue(hits[3].endswith('/hwloc-1.6.2-GCC-4.9.3-2.26.eb')) - self.assertTrue(hits[4].endswith('/hwloc-1.8-gcccuda-2018a.eb')) + self.assertTrue(hits[0].endswith('/hwloc-1.6.2-GCC-4.9.3-2.26.eb')) + self.assertTrue(hits[1].endswith('/hwloc-1.8-gcccuda-2018a.eb')) + self.assertTrue(hits[2].endswith('/hwloc-1.11.8-GCC-4.6.4.eb')) + self.assertTrue(hits[3].endswith('/hwloc-1.11.8-GCC-6.4.0-2.28.eb')) + self.assertTrue(hits[4].endswith('/hwloc-1.11.8-GCC-7.3.0-2.30.eb')) # also test case-sensitive searching var_defs, hits_bis = ft.search_file([test_ecs], 'HWLOC', silent=True, case_sensitive=True) @@ -2176,9 +2196,12 @@ def test_search_file(self): # check filename-only mode var_defs, hits = ft.search_file([test_ecs], 'HWLOC', silent=True, filename_only=True) self.assertEqual(var_defs, []) - self.assertEqual(hits, ['hwloc-1.11.8-GCC-4.6.4.eb', 'hwloc-1.11.8-GCC-6.4.0-2.28.eb', - 'hwloc-1.11.8-GCC-7.3.0-2.30.eb', 'hwloc-1.6.2-GCC-4.9.3-2.26.eb', - 'hwloc-1.8-gcccuda-2018a.eb']) + self.assertEqual(hits, ['hwloc-1.6.2-GCC-4.9.3-2.26.eb', + 'hwloc-1.8-gcccuda-2018a.eb', + 'hwloc-1.11.8-GCC-4.6.4.eb', + 'hwloc-1.11.8-GCC-6.4.0-2.28.eb', + 'hwloc-1.11.8-GCC-7.3.0-2.30.eb', + ]) # check specifying of ignored dirs var_defs, hits = ft.search_file([test_ecs], 'HWLOC', silent=True, ignore_dirs=['hwloc']) @@ -2187,28 +2210,34 @@ def test_search_file(self): # check short mode var_defs, hits = ft.search_file([test_ecs], 'HWLOC', silent=True, short=True) self.assertEqual(var_defs, [('CFGS1', os.path.join(test_ecs, 'h', 'hwloc'))]) - self.assertEqual(hits, ['$CFGS1/hwloc-1.11.8-GCC-4.6.4.eb', '$CFGS1/hwloc-1.11.8-GCC-6.4.0-2.28.eb', - '$CFGS1/hwloc-1.11.8-GCC-7.3.0-2.30.eb', '$CFGS1/hwloc-1.6.2-GCC-4.9.3-2.26.eb', - '$CFGS1/hwloc-1.8-gcccuda-2018a.eb']) + self.assertEqual(hits, ['$CFGS1/hwloc-1.6.2-GCC-4.9.3-2.26.eb', + '$CFGS1/hwloc-1.8-gcccuda-2018a.eb', + '$CFGS1/hwloc-1.11.8-GCC-4.6.4.eb', + '$CFGS1/hwloc-1.11.8-GCC-6.4.0-2.28.eb', + '$CFGS1/hwloc-1.11.8-GCC-7.3.0-2.30.eb' + ]) # check terse mode (implies 'silent', overrides 'short') var_defs, hits = ft.search_file([test_ecs], 'HWLOC', terse=True, short=True) self.assertEqual(var_defs, []) expected = [ + os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.6.2-GCC-4.9.3-2.26.eb'), + os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.8-gcccuda-2018a.eb'), os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.11.8-GCC-4.6.4.eb'), os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.11.8-GCC-6.4.0-2.28.eb'), os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.11.8-GCC-7.3.0-2.30.eb'), - os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.6.2-GCC-4.9.3-2.26.eb'), - os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.8-gcccuda-2018a.eb'), ] self.assertEqual(hits, expected) # check combo of terse and filename-only var_defs, hits = ft.search_file([test_ecs], 'HWLOC', terse=True, filename_only=True) self.assertEqual(var_defs, []) - self.assertEqual(hits, ['hwloc-1.11.8-GCC-4.6.4.eb', 'hwloc-1.11.8-GCC-6.4.0-2.28.eb', - 'hwloc-1.11.8-GCC-7.3.0-2.30.eb', 'hwloc-1.6.2-GCC-4.9.3-2.26.eb', - 'hwloc-1.8-gcccuda-2018a.eb']) + self.assertEqual(hits, ['hwloc-1.6.2-GCC-4.9.3-2.26.eb', + 'hwloc-1.8-gcccuda-2018a.eb', + 'hwloc-1.11.8-GCC-4.6.4.eb', + 'hwloc-1.11.8-GCC-6.4.0-2.28.eb', + 'hwloc-1.11.8-GCC-7.3.0-2.30.eb', + ]) # patterns that include special characters + (or ++) shouldn't cause trouble # cfr. https://github.com/easybuilders/easybuild-framework/issues/2966 @@ -2918,6 +2947,110 @@ def test_locate_files(self): error_pattern = r"One or more files not found: 2\.txt \(search paths: \)" self.assertErrorRegex(EasyBuildError, error_pattern, ft.locate_files, ['2.txt'], []) + def test_set_gid_sticky_bits(self): + """Test for set_gid_sticky_bits function.""" + test_dir = os.path.join(self.test_prefix, 'test_dir') + test_subdir = os.path.join(test_dir, 'subdir') + + ft.mkdir(test_subdir, parents=True) + dir_perms = os.lstat(test_dir)[stat.ST_MODE] + self.assertEqual(dir_perms & stat.S_ISGID, 0) + self.assertEqual(dir_perms & stat.S_ISVTX, 0) + dir_perms = os.lstat(test_subdir)[stat.ST_MODE] + self.assertEqual(dir_perms & stat.S_ISGID, 0) + self.assertEqual(dir_perms & stat.S_ISVTX, 0) + + # by default, GID & sticky bits are not set + ft.set_gid_sticky_bits(test_dir) + dir_perms = os.lstat(test_dir)[stat.ST_MODE] + self.assertEqual(dir_perms & stat.S_ISGID, 0) + self.assertEqual(dir_perms & stat.S_ISVTX, 0) + + ft.set_gid_sticky_bits(test_dir, set_gid=True) + dir_perms = os.lstat(test_dir)[stat.ST_MODE] + self.assertEqual(dir_perms & stat.S_ISGID, stat.S_ISGID) + self.assertEqual(dir_perms & stat.S_ISVTX, 0) + ft.remove_dir(test_dir) + ft.mkdir(test_subdir, parents=True) + + ft.set_gid_sticky_bits(test_dir, sticky=True) + dir_perms = os.lstat(test_dir)[stat.ST_MODE] + self.assertEqual(dir_perms & stat.S_ISGID, 0) + self.assertEqual(dir_perms & stat.S_ISVTX, stat.S_ISVTX) + ft.remove_dir(test_dir) + ft.mkdir(test_subdir, parents=True) + + ft.set_gid_sticky_bits(test_dir, set_gid=True, sticky=True) + dir_perms = os.lstat(test_dir)[stat.ST_MODE] + self.assertEqual(dir_perms & stat.S_ISGID, stat.S_ISGID) + self.assertEqual(dir_perms & stat.S_ISVTX, stat.S_ISVTX) + # no recursion by default + dir_perms = os.lstat(test_subdir)[stat.ST_MODE] + self.assertEqual(dir_perms & stat.S_ISGID, 0) + self.assertEqual(dir_perms & stat.S_ISVTX, 0) + + ft.remove_dir(test_dir) + ft.mkdir(test_subdir, parents=True) + + ft.set_gid_sticky_bits(test_dir, set_gid=True, sticky=True, recursive=True) + dir_perms = os.lstat(test_dir)[stat.ST_MODE] + self.assertEqual(dir_perms & stat.S_ISGID, stat.S_ISGID) + self.assertEqual(dir_perms & stat.S_ISVTX, stat.S_ISVTX) + dir_perms = os.lstat(test_subdir)[stat.ST_MODE] + self.assertEqual(dir_perms & stat.S_ISGID, stat.S_ISGID) + self.assertEqual(dir_perms & stat.S_ISVTX, stat.S_ISVTX) + + ft.remove_dir(test_dir) + ft.mkdir(test_subdir, parents=True) + + # set_gid_sticky_bits honors relevant build options + init_config(build_options={'set_gid_bit': True, 'sticky_bit': True}) + ft.set_gid_sticky_bits(test_dir, recursive=True) + dir_perms = os.lstat(test_dir)[stat.ST_MODE] + self.assertEqual(dir_perms & stat.S_ISGID, stat.S_ISGID) + self.assertEqual(dir_perms & stat.S_ISVTX, stat.S_ISVTX) + dir_perms = os.lstat(test_subdir)[stat.ST_MODE] + self.assertEqual(dir_perms & stat.S_ISGID, stat.S_ISGID) + self.assertEqual(dir_perms & stat.S_ISVTX, stat.S_ISVTX) + + def test_create_unused_dir(self): + """Test create_unused_dir function.""" + path = ft.create_unused_dir(self.test_prefix, 'folder') + self.assertEqual(path, os.path.join(self.test_prefix, 'folder')) + self.assertTrue(os.path.exists(path)) + + # Repeat with existing folder(s) should create new ones + for i in range(10): + path = ft.create_unused_dir(self.test_prefix, 'folder') + self.assertEqual(path, os.path.join(self.test_prefix, 'folder_%s' % i)) + self.assertTrue(os.path.exists(path)) + + # Not influenced by similar folder + path = ft.create_unused_dir(self.test_prefix, 'folder2') + self.assertEqual(path, os.path.join(self.test_prefix, 'folder2')) + self.assertTrue(os.path.exists(path)) + for i in range(10): + path = ft.create_unused_dir(self.test_prefix, 'folder2') + self.assertEqual(path, os.path.join(self.test_prefix, 'folder2_%s' % i)) + self.assertTrue(os.path.exists(path)) + + # Fail cleanly if passed a readonly folder + readonly_dir = os.path.join(self.test_prefix, 'ro_folder') + ft.mkdir(readonly_dir) + old_perms = os.lstat(readonly_dir)[stat.ST_MODE] + ft.adjust_permissions(readonly_dir, stat.S_IREAD | stat.S_IEXEC, relative=False) + try: + self.assertErrorRegex(EasyBuildError, 'Failed to create directory', + ft.create_unused_dir, readonly_dir, 'new_folder') + finally: + ft.adjust_permissions(readonly_dir, old_perms, relative=False) + + # Ignore files same as folders. So first just create a file with no contents + ft.write_file(os.path.join(self.test_prefix, 'file'), '') + path = ft.create_unused_dir(self.test_prefix, 'file') + self.assertEqual(path, os.path.join(self.test_prefix, 'file_0')) + self.assertTrue(os.path.exists(path)) + def suite(): """ returns all the testcases in this module """ diff --git a/test/framework/github.py b/test/framework/github.py index 8a11ef4dc2..dbae54fbf6 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -42,7 +42,9 @@ from easybuild.tools.config import build_option, module_classes from easybuild.tools.configobj import ConfigObj from easybuild.tools.filetools import read_file, write_file -from easybuild.tools.github import GITHUB_EASYCONFIGS_REPO, GITHUB_EASYBLOCKS_REPO, VALID_CLOSE_PR_REASONS +from easybuild.tools.github import GITHUB_EASYCONFIGS_REPO, GITHUB_EASYBLOCKS_REPO, GITHUB_MERGEABLE_STATE_CLEAN +from easybuild.tools.github import VALID_CLOSE_PR_REASONS +from easybuild.tools.github import pick_default_branch from easybuild.tools.testing import post_pr_test_report, session_state from easybuild.tools.py2vs3 import HTTPError, URLError, ascii_letters import easybuild.tools.github as gh @@ -83,6 +85,12 @@ def setUp(self): self.skip_github_tests = self.github_token is None and os.getenv('FORCE_EB_GITHUB_TESTS') is None + def test_pick_default_branch(self): + """Test pick_default_branch function.""" + + self.assertEqual(pick_default_branch('easybuilders'), 'main') + self.assertEqual(pick_default_branch('foobar'), 'master') + def test_walk(self): """test the gitubfs walk function""" if self.skip_github_tests: @@ -539,6 +547,11 @@ def test_validate_github_token(self): self.assertTrue(gh.validate_github_token(self.github_token, GITHUB_TEST_ACCOUNT)) + # if a token in the old format is available, test with that too + token_old_format = os.getenv('TEST_GITHUB_TOKEN_OLD_FORMAT') + if token_old_format: + self.assertTrue(gh.validate_github_token(token_old_format, GITHUB_TEST_ACCOUNT)) + def test_find_easybuild_easyconfig(self): """Test for find_easybuild_easyconfig function""" if self.skip_github_tests: @@ -647,7 +660,11 @@ def run_check(expected_result=False): 'issue_comments': [], 'milestone': None, 'number': '1234', - 'reviews': [], + 'merged': False, + 'mergeable_state': 'unknown', + 'reviews': [{'state': 'CHANGES_REQUESTED', 'user': {'login': 'boegel'}}, + # to check that duplicates are filtered + {'state': 'CHANGES_REQUESTED', 'user': {'login': 'boegel'}}], } test_result_warning_template = "* test suite passes: %s => not eligible for merging!" @@ -707,11 +724,21 @@ def run_check(expected_result=False): pr_data['issue_comments'].insert(2, {'body': 'lgtm'}) run_check() - pr_data['reviews'].append({'state': 'CHANGES_REQUESTED', 'user': {'login': 'boegel'}}) + expected_warning = "* no pending change requests: FAILED (changes requested by boegel)" + expected_warning += " => not eligible for merging!" + run_check() + + # if PR is approved by a different user that requested changes and that request has not been dismissed, + # the PR is still not mergeable + pr_data['reviews'].append({'state': 'APPROVED', 'user': {'login': 'not_boegel'}}) + expected_stdout_saved = expected_stdout + expected_stdout += "* approved review: OK (by not_boegel)\n" run_check() + # if the user that requested changes approves the PR, it's mergeable pr_data['reviews'].append({'state': 'APPROVED', 'user': {'login': 'boegel'}}) - expected_stdout += "* approved review: OK (by boegel)\n" + expected_stdout = expected_stdout_saved + "* no pending change requests: OK\n" + expected_stdout += "* approved review: OK (by not_boegel, boegel)\n" expected_warning = '' run_check() @@ -722,6 +749,13 @@ def run_check(expected_result=False): pr_data['milestone'] = {'title': '3.3.1'} expected_stdout += "* milestone is set: OK (3.3.1)\n" + # mergeable state must be clean + expected_warning = "* mergeable state is clean: FAILED (mergeable state is 'unknown')" + run_check() + + pr_data['mergeable_state'] = GITHUB_MERGEABLE_STATE_CLEAN + expected_stdout += "* mergeable state is clean: OK\n" + # all checks pass, PR is eligible for merging expected_warning = '' self.assertEqual(run_check(True), '') diff --git a/test/framework/modules.py b/test/framework/modules.py index e370b1a88f..a2f4633787 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -54,7 +54,7 @@ # number of modules included for testing purposes -TEST_MODULES_COUNT = 81 +TEST_MODULES_COUNT = 82 class ModulesTest(EnhancedTestCase): @@ -1142,24 +1142,98 @@ def test_module_caches(self): self.assertEqual(mod.MODULE_AVAIL_CACHE, {}) self.assertEqual(mod.MODULE_SHOW_CACHE, {}) - def test_module_use(self): - """Test 'module use'.""" + def test_module_use_unuse(self): + """Test 'module use' and 'module unuse'.""" test_dir1 = os.path.join(self.test_prefix, 'one') test_dir2 = os.path.join(self.test_prefix, 'two') test_dir3 = os.path.join(self.test_prefix, 'three') + for subdir in ('one', 'two', 'three'): + modtxt = '\n'.join([ + '#%Module', + "setenv TEST123 %s" % subdir, + ]) + write_file(os.path.join(self.test_prefix, subdir, 'test'), modtxt) + self.assertFalse(test_dir1 in os.environ.get('MODULEPATH', '')) self.modtool.use(test_dir1) - self.assertTrue(os.environ.get('MODULEPATH', '').startswith('%s:' % test_dir1)) + self.assertTrue(os.environ['MODULEPATH'].startswith('%s:' % test_dir1)) + self.modtool.use(test_dir2) + self.assertTrue(os.environ['MODULEPATH'].startswith('%s:' % test_dir2)) + self.modtool.use(test_dir3) + self.assertTrue(os.environ['MODULEPATH'].startswith('%s:' % test_dir3)) + + # make sure the right test module is loaded + self.modtool.load(['test']) + self.assertEqual(os.getenv('TEST123'), 'three') + self.modtool.unload(['test']) + + self.modtool.unuse(test_dir3) + self.assertFalse(test_dir3 in os.environ.get('MODULEPATH', '')) + + self.modtool.load(['test']) + self.assertEqual(os.getenv('TEST123'), 'two') + self.modtool.unload(['test']) + + self.modtool.unuse(test_dir2) + self.assertFalse(test_dir2 in os.environ.get('MODULEPATH', '')) + + self.modtool.load(['test']) + self.assertEqual(os.getenv('TEST123'), 'one') + self.modtool.unload(['test']) + + self.modtool.unuse(test_dir1) + self.assertFalse(test_dir1 in os.environ.get('MODULEPATH', '')) # also test use with high priority self.modtool.use(test_dir2, priority=10000) self.assertTrue(os.environ['MODULEPATH'].startswith('%s:' % test_dir2)) - # check whether prepend with priority actually works (only for Lmod) + self.modtool.load(['test']) + self.assertEqual(os.getenv('TEST123'), 'two') + self.modtool.unload(['test']) + + # Tests for Lmod only if isinstance(self.modtool, Lmod): + # check whether prepend with priority actually works (priority is specific to Lmod) + self.modtool.use(test_dir1, priority=100) self.modtool.use(test_dir3) - self.assertTrue(os.environ['MODULEPATH'].startswith('%s:%s:' % (test_dir2, test_dir3))) + self.assertTrue(os.environ['MODULEPATH'].startswith('%s:%s:%s:' % (test_dir2, test_dir1, test_dir3))) + self.modtool.load(['test']) + self.assertEqual(os.getenv('TEST123'), 'two') + self.modtool.unload(['test']) + + self.modtool.unuse(test_dir2) + self.modtool.load(['test']) + self.assertEqual(os.getenv('TEST123'), 'one') + self.modtool.unload(['test']) + + self.modtool.unuse(test_dir1) + self.modtool.load(['test']) + self.assertEqual(os.getenv('TEST123'), 'three') + self.modtool.unload(['test']) + + # Check load and unload for a single path when it is the only one + # Only for Lmod as we have some shortcuts for avoiding the module call there + old_module_path = os.environ['MODULEPATH'] + del os.environ['MODULEPATH'] + self.modtool.use(test_dir1) + self.assertEqual(os.environ['MODULEPATH'], test_dir1) + self.modtool.unuse(test_dir1) + self.assertFalse('MODULEPATH' in os.environ) + os.environ['MODULEPATH'] = old_module_path # Restore + + # Using an empty path still works (technically) (Lmod only, ignored by Tcl) + old_module_path = os.environ['MODULEPATH'] + self.modtool.use('') + self.assertEqual(os.environ['MODULEPATH'], ':' + old_module_path) + self.modtool.unuse('') + self.assertEqual(os.environ['MODULEPATH'], old_module_path) + # Even works when the whole path is empty + os.environ['MODULEPATH'] = '' + self.modtool.unuse('') + self.assertFalse('MODULEPATH' in os.environ) + os.environ['MODULEPATH'] = old_module_path # Restore def test_module_use_bash(self): """Test whether effect of 'module use' is preserved when a new bash session is started.""" diff --git a/test/framework/modules/gcccuda/2018a b/test/framework/modules/gcccuda/2018a new file mode 100644 index 0000000000..f9779f1be5 --- /dev/null +++ b/test/framework/modules/gcccuda/2018a @@ -0,0 +1,26 @@ +#%Module + +proc ModulesHelp { } { + puts stderr { GCC based compiler toolchain with CUDA support, and including + OpenMPI for MPI support, OpenBLAS (BLAS and LAPACK support), FFTW and ScaLAPACK. - Homepage: (none) +} +} + +module-whatis {GNU Compiler Collection (GCC) based compiler toolchain, along with CUDA toolkit. - Homepage: (none)} + +set root /prefix/software/gcccuda/2018a + +conflict gcccuda + +if { ![is-loaded GCC/6.4.0-2.28] } { + module load GCC/6.4.0-2.28 +} + +if { ![is-loaded CUDA/9.1.85] } { + module load CUDA/9.1.85 +} + + +setenv EBROOTGCCCUDA "$root" +setenv EBVERSIONGCCCUDA "2018a" +setenv EBDEVELGCCCUDA "$root/easybuild/gcccuda-2018a-easybuild-devel" diff --git a/test/framework/options.py b/test/framework/options.py index 3a8539014c..f3a0a29fed 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -988,7 +988,7 @@ def test_ignore_index(self): toy_ec = os.path.join(test_ecs_dir, 'test_ecs', 't', 'toy', 'toy-0.0.eb') copy_file(toy_ec, self.test_prefix) - toy_ec_list = ['toy-0.0.eb', 'toy-1.2.3.eb', 'toy-4.5.6.eb'] + toy_ec_list = ['toy-0.0.eb', 'toy-1.2.3.eb', 'toy-4.5.6.eb', 'toy-11.5.6.eb'] # install index that list more files than are actually available, # so we can check whether it's used @@ -998,15 +998,16 @@ def test_ignore_index(self): args = [ '--search=toy', '--robot-paths=%s' % self.test_prefix, + '--terse', ] self.mock_stdout(True) self.eb_main(args, testing=False, raise_error=True) stdout = self.get_stdout() self.mock_stdout(False) - for toy_ec_fn in toy_ec_list: - regex = re.compile(re.escape(os.path.join(self.test_prefix, toy_ec_fn)), re.M) - self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + # Also checks for ordering: 11.x comes last! + expected_output = '\n'.join(os.path.join(self.test_prefix, ec) for ec in toy_ec_list) + '\n' + self.assertEqual(stdout, expected_output) args.append('--ignore-index') self.mock_stdout(True) @@ -1014,11 +1015,8 @@ def test_ignore_index(self): stdout = self.get_stdout() self.mock_stdout(False) - regex = re.compile(re.escape(os.path.join(self.test_prefix, 'toy-0.0.eb')), re.M) - self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) - for toy_ec_fn in ['toy-1.2.3.eb', 'toy-4.5.6.eb']: - regex = re.compile(re.escape(os.path.join(self.test_prefix, toy_ec_fn)), re.M) - self.assertFalse(regex.search(stdout), "Pattern '%s' should not be found in: %s" % (regex.pattern, stdout)) + # This should be the only EC found + self.assertEqual(stdout, os.path.join(self.test_prefix, 'toy-0.0.eb') + '\n') def test_search_archived(self): "Test searching for archived easyconfigs" @@ -4435,7 +4433,9 @@ def test_merge_pr(self): "Checking eligibility of easybuilders/easybuild-easyconfigs PR #4781 for merging...", "* test suite passes: OK", "* last test report is successful: OK", + "* no pending change requests: OK", "* milestone is set: OK (3.3.1)", + "* mergeable state is clean: PR is already merged", ]) expected_stderr = '\n'.join([ "* targets some_branch branch: FAILED; found 'develop' => not eligible for merging!", @@ -4457,8 +4457,10 @@ def test_merge_pr(self): "* targets develop branch: OK", "* test suite passes: OK", "* last test report is successful: OK", + "* no pending change requests: OK", "* approved review: OK (by wpoely86)", "* milestone is set: OK (3.3.1)", + "* mergeable state is clean: PR is already merged", '', "Review OK, merging pull request!", '', @@ -4483,8 +4485,10 @@ def test_merge_pr(self): "Checking eligibility of easybuilders/easybuild-easyblocks PR #1206 for merging...", "* targets develop branch: OK", "* test suite passes: OK", + "* no pending change requests: OK", "* approved review: OK (by migueldiascosta)", "* milestone is set: OK (3.3.1)", + "* mergeable state is clean: PR is already merged", '', "Review OK, merging pull request!", ]) @@ -4662,7 +4666,7 @@ def test_modules_tool_vs_syntax_check(self): regex = re.compile(pattern, re.M) self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) - def test_prefix(self): + def test_prefix_option(self): """Test which configuration settings are affected by --prefix.""" txt, _ = self._run_mock_eb(['--show-full-config', '--prefix=%s' % self.test_prefix], raise_error=True) @@ -5890,7 +5894,7 @@ def test_sysroot(self): self.assertErrorRegex(EasyBuildError, error_pattern, self._run_mock_eb, ['--show-config'], raise_error=True) def test_accept_eula_for(self): - """Test --accept-eula configuration option.""" + """Test --accept-eula-for configuration option.""" # use toy-0.0.eb easyconfig file that comes with the tests topdir = os.path.abspath(os.path.dirname(__file__)) @@ -5907,27 +5911,56 @@ def test_accept_eula_for(self): args = [test_ec, '--force'] error_pattern = r"The End User License Argreement \(EULA\) for toy is currently not accepted!" self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, do_build=True, raise_error=True) - - # installation proceeds if EasyBuild is configured to accept EULA for specified software via --accept-eula - self.eb_main(args + ['--accept-eula=foo,toy,bar'], do_build=True, raise_error=True) - toy_modfile = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0') if get_module_syntax() == 'Lua': toy_modfile += '.lua' - self.assertTrue(os.path.exists(toy_modfile)) + + # installation proceeds if EasyBuild is configured to accept EULA for specified software via --accept-eula-for + for val in ('foo,toy,bar', '.*', 't.y'): + self.eb_main(args + ['--accept-eula-for=' + val], do_build=True, raise_error=True) + + self.assertTrue(os.path.exists(toy_modfile)) + + remove_dir(self.test_installpath) + self.assertFalse(os.path.exists(toy_modfile)) + + # also check use of $EASYBUILD_ACCEPT_EULA to accept EULA for specified software + os.environ['EASYBUILD_ACCEPT_EULA_FOR'] = val + self.eb_main(args, do_build=True, raise_error=True) + self.assertTrue(os.path.exists(toy_modfile)) + + remove_dir(self.test_installpath) + self.assertFalse(os.path.exists(toy_modfile)) + + del os.environ['EASYBUILD_ACCEPT_EULA_FOR'] + + # also check deprecated --accept-eula configuration option + self.allow_deprecated_behaviour() + + self.mock_stderr(True) + self.eb_main(args + ['--accept-eula=foo,toy,bar'], do_build=True, raise_error=True) + stderr = self.get_stderr() + self.mock_stderr(False) + self.assertTrue("Use accept-eula-for configuration setting rather than accept-eula" in stderr) remove_dir(self.test_installpath) self.assertFalse(os.path.exists(toy_modfile)) - # also check use of $EASYBUILD_ACCEPT_EULA to accept EULA for specified software + # also via $EASYBUILD_ACCEPT_EULA + self.mock_stderr(True) os.environ['EASYBUILD_ACCEPT_EULA'] = 'toy' self.eb_main(args, do_build=True, raise_error=True) + stderr = self.get_stderr() + self.mock_stderr(False) + self.assertTrue(os.path.exists(toy_modfile)) + self.assertTrue("Use accept-eula-for configuration setting rather than accept-eula" in stderr) remove_dir(self.test_installpath) self.assertFalse(os.path.exists(toy_modfile)) # also check accepting EULA via 'accept_eula = True' in easyconfig file + self.disallow_deprecated_behaviour() del os.environ['EASYBUILD_ACCEPT_EULA'] write_file(test_ec, test_ec_txt + '\naccept_eula = True') self.eb_main(args, do_build=True, raise_error=True) diff --git a/test/framework/robot.py b/test/framework/robot.py index 9c29cb25cd..0f44bf34e3 100644 --- a/test/framework/robot.py +++ b/test/framework/robot.py @@ -1513,10 +1513,10 @@ def test_search_easyconfigs(self): paths = search_easyconfigs('8-gcc', consider_extra_paths=False, print_result=False) ref_paths = [ + os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.8-gcccuda-2018a.eb'), os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.11.8-GCC-4.6.4.eb'), os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.11.8-GCC-6.4.0-2.28.eb'), os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.11.8-GCC-7.3.0-2.30.eb'), - os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.8-gcccuda-2018a.eb'), os.path.join(test_ecs, 'o', 'OpenBLAS', 'OpenBLAS-0.2.8-GCC-4.8.2-LAPACK-3.4.2.eb') ] self.assertEqual(paths, ref_paths) diff --git a/test/framework/run.py b/test/framework/run.py index b6ceaf8127..8486119e24 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -351,9 +351,27 @@ def test_run_cmd_qa_buffering(self): cmd += 'echo "Pick a number: "; read number; echo "Picked number: $number"' (out, ec) = run_cmd_qa(cmd, {'Pick a number: ': '42'}, log_all=True, maxhits=5) + self.assertEqual(ec, 0) regex = re.compile("Picked number: 42$") self.assertTrue(regex.search(out), "Pattern '%s' found in: %s" % (regex.pattern, out)) + # also test with script run as interactive command that quickly exits with non-zero exit code; + # see https://github.com/easybuilders/easybuild-framework/issues/3593 + script_txt = '\n'.join([ + "#/bin/bash", + "echo 'Hello, I am about to exit'", + "echo 'ERROR: I failed' >&2", + "exit 1", + ]) + script = os.path.join(self.test_prefix, 'test.sh') + write_file(script, script_txt) + adjust_permissions(script, stat.S_IXUSR) + + out, ec = run_cmd_qa(script, {}, log_ok=False) + + self.assertEqual(ec, 1) + self.assertEqual(out, "Hello, I am about to exit\nERROR: I failed\n") + def test_run_cmd_qa_log_all(self): """Test run_cmd_qa with log_output enabled""" (out, ec) = run_cmd_qa("echo 'n: '; read n; seq 1 $n", {'n: ': '5'}, log_all=True) diff --git a/test/framework/suite.py b/test/framework/suite.py index 466401aa4e..41c13d188f 100755 --- a/test/framework/suite.py +++ b/test/framework/suite.py @@ -77,6 +77,7 @@ import test.framework.toy_build as t import test.framework.type_checking as et import test.framework.tweak as tw +import test.framework.utilities_test as u import test.framework.variables as v import test.framework.yeb as y @@ -118,7 +119,7 @@ # call suite() for each module and then run them all # note: make sure the options unit tests run first, to avoid running some of them with a readily initialized config tests = [gen, bl, o, r, ef, ev, ebco, ep, e, mg, m, mt, f, run, a, robot, b, v, g, tcv, tc, t, c, s, lic, f_c, - tw, p, i, pkg, d, env, et, y, st, h, ct, lib] + tw, p, i, pkg, d, env, et, y, st, h, ct, lib, u] SUITE = unittest.TestSuite([x.suite() for x in tests]) res = unittest.TextTestRunner().run(SUITE) diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index eba1625a17..3654882086 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -916,6 +916,24 @@ def test_pick_dep_version(self): error_pattern = "Unknown value type for version" self.assertErrorRegex(EasyBuildError, error_pattern, pick_dep_version, ('1.2.3', '4.5.6')) + # check support for using 'arch=*' as fallback key + dep_ver_dict = { + 'arch=*': '1.2.3', + 'arch=foo': '1.2.3-foo', + 'arch=POWER': '1.2.3-ppc64le', + } + self.assertEqual(pick_dep_version(dep_ver_dict), '1.2.3-ppc64le') + + del dep_ver_dict['arch=POWER'] + self.assertEqual(pick_dep_version(dep_ver_dict), '1.2.3') + + # check how faulty input is handled + self.assertErrorRegex(EasyBuildError, "Found empty dict as version!", pick_dep_version, {}) + error_pattern = r"Unexpected keys in version: bar,foo \(only 'arch=' keys are supported\)" + self.assertErrorRegex(EasyBuildError, error_pattern, pick_dep_version, {'foo': '1.2', 'bar': '2.3'}) + error_pattern = r"Unknown value type for version: .* \(1.23\), should be string value" + self.assertErrorRegex(EasyBuildError, error_pattern, pick_dep_version, 1.23) + def test_check_os_dependency(self): """Test check_os_dependency.""" diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 24affe177b..28013b468a 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -1449,7 +1449,7 @@ def test_old_new_iccifort(self): libscalack_intel4 = "-lmkl_scalapack_lp64 -lmkl_blacs_intelmpi_lp64 -lmkl_intel_lp64 -lmkl_sequential " libscalack_intel4 += "-lmkl_core" - libblas_mt_fosscuda = "-lopenblas -lgfortran" + libblas_mt_fosscuda = "-lopenblas -lgfortran -lpthread" libscalack_fosscuda = "-lscalapack -lopenblas -lgfortran" libfft_mt_fosscuda = "-lfftw3_omp -lfftw3 -lpthread" diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 9809e5b999..f3c0abda5a 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -270,6 +270,7 @@ def test_toy_tweaked(self): "modluafooter = 'io.stderr:write(\"oh hai!\")'", # ignored when module syntax is Tcl "usage = 'This toy is easy to use, 100%!'", "examples = 'No example available, 0% complete'", + "citing = 'If you use this package, please cite our paper https://ieeexplore.ieee.org/document/6495863'", "docpaths = ['share/doc/toy/readme.txt', 'share/doc/toy/html/index.html']", "docurls = ['https://easybuilders.github.io/easybuild/toy/docs.html']", "upstream_contacts = 'support@toy.org'", @@ -1386,6 +1387,11 @@ def test_toy_module_fulltxt(self): r'No example available, 0% complete', r'', r'', + r'Citing', + r'======', + r'If you use this package, please cite our paper https://ieeexplore.ieee.org/document/6495863', + r'', + r'', r'More information', r'================', r' - Homepage: https://easybuilders.github.io/easybuild', @@ -2854,6 +2860,45 @@ def test_fix_shebang(self): self.assertTrue(bash_shebang_regex.match(bashbin_txt), "Pattern '%s' found in %s: %s" % (bash_shebang_regex.pattern, bashbin_path, bashbin_txt)) + # now test with a custom env command + extra_args = ['--env-for-shebang=/usr/bin/env -S'] + self.test_toy_build(ec_file=test_ec, extra_args=extra_args, raise_error=True) + + toy_bindir = os.path.join(self.test_installpath, 'software', 'toy', '0.0', 'bin') + + # bin/toy and bin/toy2 should *not* be patched, since they're binary files + toy_txt = read_file(os.path.join(toy_bindir, 'toy'), mode='rb') + for fn in ['toy.perl', 'toy.python']: + fn_txt = read_file(os.path.join(toy_bindir, fn), mode='rb') + # no shebang added + self.assertFalse(fn_txt.startswith(b"#!/")) + # exact same file as original binary (untouched) + self.assertEqual(toy_txt, fn_txt) + + # no re.M, this should match at start of file! + py_shebang_regex = re.compile(r'^#!/usr/bin/env -S python\n# test$') + for pybin in ['t1.py', 't2.py', 't3.py', 't4.py', 't5.py', 't6.py', 't7.py']: + pybin_path = os.path.join(toy_bindir, pybin) + pybin_txt = read_file(pybin_path) + self.assertTrue(py_shebang_regex.match(pybin_txt), + "Pattern '%s' found in %s: %s" % (py_shebang_regex.pattern, pybin_path, pybin_txt)) + + # no re.M, this should match at start of file! + perl_shebang_regex = re.compile(r'^#!/usr/bin/env -S perl\n# test$') + for perlbin in ['t1.pl', 't2.pl', 't3.pl', 't4.pl', 't5.pl', 't6.pl', 't7.pl']: + perlbin_path = os.path.join(toy_bindir, perlbin) + perlbin_txt = read_file(perlbin_path) + self.assertTrue(perl_shebang_regex.match(perlbin_txt), + "Pattern '%s' found in %s: %s" % (perl_shebang_regex.pattern, perlbin_path, perlbin_txt)) + + # There are 2 bash files which shouldn't be influenced by fix_shebang + bash_shebang_regex = re.compile(r'^#!/usr/bin/env bash\n# test$') + for bashbin in ['b1.sh', 'b2.sh']: + bashbin_path = os.path.join(toy_bindir, bashbin) + bashbin_txt = read_file(bashbin_path) + self.assertTrue(bash_shebang_regex.match(bashbin_txt), + "Pattern '%s' found in %s: %s" % (bash_shebang_regex.pattern, bashbin_path, bashbin_txt)) + def test_toy_system_toolchain_alias(self): """Test use of 'system' toolchain alias.""" toy_ec = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') diff --git a/test/framework/utilities.py b/test/framework/utilities.py index 57fafc3dbc..39f98384ff 100644 --- a/test/framework/utilities.py +++ b/test/framework/utilities.py @@ -50,7 +50,7 @@ from easybuild.tools.config import GENERAL_CLASS, Singleton, module_classes from easybuild.tools.configobj import ConfigObj from easybuild.tools.environment import modify_env -from easybuild.tools.filetools import copy_dir, mkdir, read_file +from easybuild.tools.filetools import copy_dir, mkdir, read_file, which from easybuild.tools.modules import curr_module_paths, modules_tool, reset_module_caches from easybuild.tools.options import CONFIG_ENV_VAR_PREFIX, EasyBuildOptions, set_tmpdir from easybuild.tools.py2vs3 import reload @@ -124,6 +124,12 @@ def setUp(self): # make sure that the tests only pick up easyconfigs provided with the tests os.environ['EASYBUILD_ROBOT_PATHS'] = os.path.join(testdir, 'easyconfigs', 'test_ecs') + # make sure that the EasyBuild installation is still known even if we purge an EB module + if os.getenv('EB_SCRIPT_PATH') is None: + eb_path = which('eb') + if eb_path is not None: + os.environ['EB_SCRIPT_PATH'] = eb_path + # make sure no deprecated behaviour is being triggered (unless intended by the test) self.orig_current_version = eb_build_log.CURRENT_VERSION self.disallow_deprecated_behaviour() diff --git a/test/framework/utilities_test.py b/test/framework/utilities_test.py new file mode 100644 index 0000000000..efe9fb7fd3 --- /dev/null +++ b/test/framework/utilities_test.py @@ -0,0 +1,109 @@ +## +# Copyright 2012-2021 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Unit tests for utilities.py + +@author: Jens Timmerman (Ghent University) +@author: Kenneth Hoste (Ghent University) +@author: Alexander Grund (TU Dresden) +""" +import os +import random +import sys +import tempfile +from datetime import datetime +from unittest import TextTestRunner + +from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered +from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.utilities import time2str, natural_keys + + +class UtilitiesTest(EnhancedTestCase): + """Class for utilities testcases """ + + def setUp(self): + """ setup """ + super(UtilitiesTest, self).setUp() + + self.test_tmp_logdir = tempfile.mkdtemp() + os.environ['EASYBUILD_TMP_LOGDIR'] = self.test_tmp_logdir + + def test_time2str(self): + """Test time2str function.""" + + start = datetime(2019, 7, 30, 5, 14, 23) + + test_cases = [ + (start, "0 sec"), + (datetime(2019, 7, 30, 5, 14, 37), "14 sec"), + (datetime(2019, 7, 30, 5, 15, 22), "59 sec"), + (datetime(2019, 7, 30, 5, 15, 23), "1 min 0 sec"), + (datetime(2019, 7, 30, 5, 16, 22), "1 min 59 sec"), + (datetime(2019, 7, 30, 5, 37, 26), "23 min 3 sec"), + (datetime(2019, 7, 30, 6, 14, 22), "59 min 59 sec"), + (datetime(2019, 7, 30, 6, 14, 23), "1 hour 0 min 0 sec"), + (datetime(2019, 7, 30, 6, 49, 14), "1 hour 34 min 51 sec"), + (datetime(2019, 7, 30, 7, 14, 23), "2 hours 0 min 0 sec"), + (datetime(2019, 7, 30, 8, 35, 59), "3 hours 21 min 36 sec"), + (datetime(2019, 7, 30, 16, 29, 24), "11 hours 15 min 1 sec"), + (datetime(2019, 7, 31, 5, 14, 22), "23 hours 59 min 59 sec"), + (datetime(2019, 7, 31, 5, 14, 23), "24 hours 0 min 0 sec"), + (datetime(2019, 8, 5, 20, 39, 44), "159 hours 25 min 21 sec"), + ] + for end, expected in test_cases: + self.assertEqual(time2str(end - start), expected) + + error_pattern = "Incorrect value type provided to time2str, should be datetime.timedelta: <.* 'int'>" + self.assertErrorRegex(EasyBuildError, error_pattern, time2str, 123) + + def test_natural_keys(self): + """Test the natural_keys function""" + sorted_items = [ + 'ACoolSw-1.0', + 'ACoolSw-2.1', + 'ACoolSw-11.0', + 'ACoolSw-23.0', + 'ACoolSw-30.0', + 'ACoolSw-30.1', + 'BigNumber-1234567890', + 'BigNumber-1234567891', + 'NoNumbers', + 'VeryLastEntry-10' + ] + shuffled_items = sorted_items[:] + random.shuffle(shuffled_items) + shuffled_items.sort(key=natural_keys) + self.assertEqual(shuffled_items, sorted_items) + + +def suite(): + """ return all the tests in this file """ + return TestLoaderFiltered().loadTestsFromTestCase(UtilitiesTest, sys.argv[1:]) + + +if __name__ == '__main__': + res = TextTestRunner(verbosity=1).run(suite()) + sys.exit(len(res.failures))