diff --git a/.github/workflows/bootstrap_script.yml b/.github/workflows/bootstrap_script.yml index fad4cb11b7..d8e4f52f95 100644 --- a/.github/workflows/bootstrap_script.yml +++ b/.github/workflows/bootstrap_script.yml @@ -2,6 +2,9 @@ name: test EasyBuild bootstrap script on: [push, pull_request] +permissions: + contents: read # to fetch code (actions/checkout) + concurrency: group: ${{format('{0}:{1}:{2}', github.repository, github.ref, github.workflow)}} cancel-in-progress: true @@ -69,11 +72,17 @@ jobs: - name: install OS & Python packages run: | - # disable apt update, we don't really need it, - # and it does more harm than good (it's fairly expensive, and it results in flaky test runs) - # sudo apt update # for modules tool - sudo apt install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev + APT_PKGS="lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev" + + # Avoid apt-get update, as we don't really need it, + # and it does more harm than good (it's fairly expensive, and it results in flaky test runs) + if ! sudo apt-get install $APT_PKGS; then + # Try to update cache, then try again to resolve 404s of old packages + sudo apt-get update -yqq || true + sudo apt-get install $APT_PKGS + fi + # fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082 # needed for Ubuntu 18.04, but not for Ubuntu 20.04, so skipping symlinking if posix.so already exists if [ ! -e /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so ] ; then diff --git a/.github/workflows/container_tests.yml b/.github/workflows/container_tests.yml index 2e8f84c7d8..14c4037754 100644 --- a/.github/workflows/container_tests.yml +++ b/.github/workflows/container_tests.yml @@ -2,6 +2,9 @@ name: Tests for container support on: [push, pull_request] +permissions: + contents: read # to fetch code (actions/checkout) + concurrency: group: ${{format('{0}:{1}:{2}', github.repository, github.ref, github.workflow)}} cancel-in-progress: true @@ -25,13 +28,19 @@ jobs: - name: install OS & Python packages run: | - # ensure package list is up to date to avoid 404's for old packages - sudo apt update -yqq - # for building Singularity images - sudo apt install rpm - sudo apt install yum # for modules tool - sudo apt install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev + APT_PKGS="lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev" + # for building Singularity images + APT_PKGS+=" rpm yum" + + # Avoid apt-get update, as we don't really need it, + # and it does more harm than good (it's fairly expensive, and it results in flaky test runs) + if ! sudo apt-get install $APT_PKGS; then + # Try to update cache, then try again to resolve 404s of old packages + sudo apt-get update -yqq || true + sudo apt-get install $APT_PKGS + fi + # fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082 # needed for Ubuntu 18.04, but not for Ubuntu 20.04, so skipping symlinking if posix.so already exists if [ ! -e /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so ] ; then @@ -53,15 +62,16 @@ jobs: # see https://github.com/apptainer/singularity/issues/5390#issuecomment-899111181 - name: install Singularity run: | + sudo apt-get update # install alien, which can be used to convert RPMs to Debian packages - sudo apt install alien + sudo apt-get install alien alien --version # determine latest version of Singularity available in EPEL, and download RPM singularity_rpm=$(curl -sL https://dl.fedoraproject.org/pub/archive/epel/8.5/Everything/x86_64/Packages/s/ | grep singularity | sed 's/.*singularity/singularity/g' | sed 's/rpm.*/rpm/g') curl -OL https://dl.fedoraproject.org/pub/archive/epel/8.5/Everything/x86_64/Packages/s/${singularity_rpm} # convert Singularity RPM to Debian package, and install it sudo alien -d ${singularity_rpm} - sudo apt install ./singularity*.deb + sudo apt-get install ./singularity*.deb singularity --version - name: install sources diff --git a/.github/workflows/eb_command.yml b/.github/workflows/eb_command.yml index bb0869e69a..2e20bbd4cc 100644 --- a/.github/workflows/eb_command.yml +++ b/.github/workflows/eb_command.yml @@ -2,6 +2,9 @@ name: Tests for the 'eb' command on: [push, pull_request] +permissions: + contents: read # to fetch code (actions/checkout) + concurrency: group: ${{format('{0}:{1}:{2}', github.repository, github.ref, github.workflow)}} cancel-in-progress: true @@ -29,8 +32,18 @@ jobs: # update to latest pip, check version pip install --upgrade pip pip --version - # install packages required for modules tool - sudo apt install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev + + # for modules tool + APT_PKGS="lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev" + + # Avoid apt-get update, as we don't really need it, + # and it does more harm than good (it's fairly expensive, and it results in flaky test runs) + if ! sudo apt-get install $APT_PKGS; then + # Try to update cache, then try again to resolve 404s of old packages + sudo apt-get update -yqq || true + sudo apt-get install $APT_PKGS + fi + # fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082 # needed for Ubuntu 18.04, but not for Ubuntu 20.04, so skipping symlinking if posix.so already exists if [ ! -e /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so ] ; then diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 5003aad15e..14d736fe80 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -1,6 +1,9 @@ name: Static Analysis on: [push, pull_request] +permissions: + contents: read # to fetch code (actions/checkout) + concurrency: group: ${{format('{0}:{1}:{2}', github.repository, github.ref, github.workflow)}} cancel-in-progress: true diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index f017a51dda..5475952dd2 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -2,6 +2,9 @@ name: EasyBuild framework unit tests on: [push, pull_request] +permissions: + contents: read # to fetch code (actions/checkout) + concurrency: group: ${{format('{0}:{1}:{2}', github.repository, github.ref, github.workflow)}} cancel-in-progress: true @@ -85,20 +88,26 @@ jobs: - name: install OS & Python packages run: | - # disable apt update, we don't really need it, - # and it does more harm than good (it's fairly expensive, and it results in flaky test runs) - # sudo apt update # for modules tool - sudo apt install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev + APT_PKGS="lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev" + # for GitPython, python-hglib + APT_PKGS+=" git mercurial" + # dep for GC3Pie + APT_PKGS+=" time" + + # Avoid apt-get update, as we don't really need it, + # and it does more harm than good (it's fairly expensive, and it results in flaky test runs) + if ! sudo apt-get install $APT_PKGS; then + # Try to update cache, then try again to resolve 404s of old packages + sudo apt-get update -yqq || true + sudo apt-get install $APT_PKGS + fi + # fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082 # needed for Ubuntu 18.04, but not for Ubuntu 20.04, so skipping symlinking if posix.so already exists if [ ! -e /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so ] ; then sudo ln -s /usr/lib/x86_64-linux-gnu/lua/5.2/posix_c.so /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so fi - # for GitPython, python-hglib - sudo apt install git mercurial - # dep for GC3Pie - sudo apt install time # Python packages pip --version pip install --upgrade pip @@ -203,7 +212,7 @@ jobs: # run test suite python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log # try and make sure output of running tests is clean (no printed messages/warnings) - IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.[56]|from cryptography.*default_backend|CryptographyDeprecationWarning: Python 2|from cryptography.utils import int_from_bytes|Blowfish|GC3Pie not available, skipping test" + IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.[56]|from cryptography.* import |CryptographyDeprecationWarning: Python 2|Blowfish|GC3Pie not available, skipping test" # '|| true' is needed to avoid that Travis stops the job on non-zero exit of grep (i.e. when there are no matches) PRINTED_MSG=$(egrep -v "${IGNORE_PATTERNS}" test_framework_suite.log | grep '\.\n*[A-Za-z]' || true) - test "x$PRINTED_MSG" = "x" || (echo "ERROR: Found printed messages in output of test suite\n${PRINTED_MSG}" && exit 1) + test "x$PRINTED_MSG" = "x" || (echo "ERROR: Found printed messages in output of test suite" && echo "${PRINTED_MSG}" && exit 1) diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index defa5709fc..0000000000 --- a/.travis.yml +++ /dev/null @@ -1,103 +0,0 @@ -language: python -python: 2.6 -dist: trusty -env: - - LMOD_VERSION=7.8.22 TEST_EASYBUILD_SILENCE_DEPRECATION_WARNINGS=Python26 -cache: - pip: true -addons: - apt: - packages: - # for environment modules/Lmod - - tcl8.5 - # for GitPython, python-hglib - - git - - mercurial - # for GC3Pie (optional dep for EasyBuild) - - time -before_install: - - pip --version - - pip install --upgrade pip - - pip --version - # coveralls doesn't support Python 2.6 anymore, so don't try to install it when testing with Python 2.6 - - if [ "x$TRAVIS_PYTHON_VERSION" != 'x2.6' ]; then pip install coveralls; fi - - pip install -r requirements.txt - # git config is required to make actual git commits (cfr. tests for GitRepository) - - git config --global user.name "Travis CI" - - git config --global user.email "travis@travis-ci.org" -install: - # install environment modules or Lmod - - export INSTALL_DEP=$TRAVIS_BUILD_DIR/easybuild/scripts/install_eb_dep.sh - - cd $HOME # avoid downloading into easybuild-framework dir - - if [ ! -z $ENV_MOD_VERSION ]; then source $INSTALL_DEP modules-${ENV_MOD_VERSION} $HOME; fi - - if [ ! -z $LMOD_VERSION ]; then source $INSTALL_DEP lua-5.1.4.8 $HOME; fi - - if [ ! -z $LMOD_VERSION ]; then source $INSTALL_DEP Lmod-${LMOD_VERSION} $HOME; fi - - if [ ! -z $ENV_MOD_TCL_VERSION ]; then source $INSTALL_DEP modules-tcl-${ENV_MOD_TCL_VERSION} $HOME; fi -script: - # make sure 'ml' alias is defined, otherwise sourcing the init script fails (silently) for Lmod (< 5.9.3) - - if [ ! -z $MOD_INIT ] && [ ! -z $LMOD_VERSION ]; then alias ml=foobar; fi - # set up environment for modules tool (if $MOD_INIT is defined) - - if [ ! -z $MOD_INIT ]; then source $MOD_INIT; type module; fi - # install GitHub token; - # unset $GITHUB_TOKEN environment variable after installing token, - # to avoid that it is included in environment dump that is included in EasyBuild debug logs, - # which causes test_from_pr_token_log to fail... - - if [ ! -z $GITHUB_TOKEN ]; then - if [ "x$TRAVIS_PYTHON_VERSION" == 'x2.6' ]; - then SET_KEYRING="keyring.set_keyring(keyring.backends.file.PlaintextKeyring())"; - else SET_KEYRING="import keyrings; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())"; - fi; - python -c "import keyring; $SET_KEYRING; keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')"; - fi; - unset GITHUB_TOKEN; - - if [ ! -z $TEST_EASYBUILD_MODULES_TOOL ]; then export EASYBUILD_MODULES_TOOL=$TEST_EASYBUILD_MODULES_TOOL; fi - - if [ ! -z $TEST_EASYBUILD_MODULE_SYNTAX ]; then export EASYBUILD_MODULE_SYNTAX=$TEST_EASYBUILD_MODULE_SYNTAX; fi - # create 'source distribution' tarball, like we do when publishing a release to PyPI - - cd $TRAVIS_BUILD_DIR; python setup.py sdist - - ls -l $TRAVIS_BUILD_DIR/dist/ - # set up environment for EasyBuild framework tests - - export PATH=/tmp/$TRAVIS_JOB_ID/bin:$PATH - - export PYTHONPATH=/tmp/$TRAVIS_JOB_ID/lib/python$TRAVIS_PYTHON_VERSION/site-packages - # install easybuild-framework to unique temporary location using prepared sdist tarball - - mkdir -p $PYTHONPATH; easy_install --prefix /tmp/$TRAVIS_JOB_ID $TRAVIS_BUILD_DIR/dist/easybuild-framework*tar.gz - # make sure there are no (top-level) "import setuptools" or "import pkg_resources" statements, - # since EasyBuild should not have a runtime requirement on setuptools - - SETUPTOOLS_IMPORTS=$(egrep -RI '^(from|import)[ ]*pkg_resources|^(from|import)[ ]*setuptools' easybuild/ || true) - - test "x$SETUPTOOLS_IMPORTS" = "x" || (echo "Found setuptools and/or pkg_resources imports in easybuild/:\n${SETUPTOOLS_IMPORTS}" && exit 1) - # move outside of checkout of easybuild-framework repository, - # to run tests on an *installed* version of the EasyBuild framework; - # this is done to catch possible packaging issues - - cd $HOME - # eb --version with verbose output from 'eb' command (to show selection of 'python' command to use) - - EB_VERBOSE=1 eb --version - # check GitHub configuration - - eb --check-github --github-user=easybuild_test - # create file owned by root but writable by anyone (used by test_copy_file) - - sudo touch /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt - - sudo chmod o+w /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt - # run coverage on all travis builds except for Python 2.6 - - if [ "x$TRAVIS_PYTHON_VERSION" != 'x2.6' ]; then coverage run -m test.framework.suite 2>&1 | tee test_framework_suite.log; coverage report -m --ignore-errors; fi - # invoke the regression test for Python 2.6 the original way without coverage - - if [ "x$TRAVIS_PYTHON_VERSION" == 'x2.6' ]; then python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log; fi - # try and make sure output of running tests is clean (no printed messages/warnings) - - IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|lib/python2.6/site-packages|requires Lmod as modules tool" - # '|| true' is needed to avoid that Travis stops the job on non-zero exit of grep (i.e. when there are no matches) - - PRINTED_MSG=$(egrep -v "${IGNORE_PATTERNS}" test_framework_suite.log | grep '\.\n*[A-Za-z]' || true) - - test "x$PRINTED_MSG" = "x" || (echo "Found printed messages in output of test suite\n${PRINTED_MSG}" && exit 1) - # check bootstrap script version - # version and SHA256 checksum are hardcoded below to avoid forgetting to update the version in the script along with contents - - EB_BOOTSTRAP_VERSION=$(grep '^EB_BOOTSTRAP_VERSION' $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py | sed 's/[^0-9.]//g') - - EB_BOOTSTRAP_SHA256SUM=$(sha256sum $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py | cut -f1 -d' ') - - EB_BOOTSTRAP_FOUND="$EB_BOOTSTRAP_VERSION $EB_BOOTSTRAP_SHA256SUM" - - EB_BOOTSTRAP_EXPECTED="20210106.01 c2d93de0dd91123eb4f51cfc16d1f5efb80f1d238b3d6cd100994086887a1ae0" - - test "$EB_BOOTSTRAP_FOUND" = "$EB_BOOTSTRAP_EXPECTED" || (echo "Version check on bootstrap script failed $EB_BOOTSTRAP_FOUND" && exit 1) - # test bootstrap script - - python $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py /tmp/$TRAVIS_JOB_ID/eb_bootstrap - # unset $PYTHONPATH to avoid mixing two EasyBuild 'installations' when testing bootstrapped EasyBuild module - - unset PYTHONPATH - # simply sanity check on bootstrapped EasyBuild module - - module use /tmp/$TRAVIS_JOB_ID/eb_bootstrap/modules/all - - module load EasyBuild; eb --version -after_success: - - if [ "x$TRAVIS_PYTHON_VERSION" != 'x2.6' ]; then coveralls; fi - diff --git a/README.rst b/README.rst index 3ea89fff2c..36dc86697f 100644 --- a/README.rst +++ b/README.rst @@ -4,7 +4,7 @@ .. image:: https://github.com/easybuilders/easybuild-framework/workflows/EasyBuild%20framework%20unit%20tests/badge.svg?branch=develop -`EasyBuild `_ is a software build +`EasyBuild `_ is a software build and installation framework that allows you to manage (scientific) software on High Performance Computing (HPC) systems in an efficient way. @@ -13,7 +13,7 @@ supports the implementation and use of so-called easyblocks which implement the software install procedure for a particular (group of) software package(s). -The EasyBuild documentation is available at http://easybuild.readthedocs.org/. +The EasyBuild documentation is available at http://docs.easybuild.io/. The EasyBuild framework source code is hosted on GitHub, along with an issue tracker for bug reports and feature requests, see diff --git a/RELEASE_NOTES b/RELEASE_NOTES index cc9d77d6fd..e42ea5dc90 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -4,6 +4,48 @@ For more detailed information, please see the git log. These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html. +v4.7.1 (March 20th 2023) +------------------------ + +update/bugfix release + +- various enhancements, including: + - add option to make sanity_check_paths arch dependent (#3845) + - add support for %(start_dir)s easyconfig template (#4073) + - add Ubuntu friendly package naming scheme EasyBuildDebFriendlyPNS (#4115) + - allow to directly import constants from easybuild.framework.easyconfig.constants (#4144) + - accept single source as dict and single checksum in check_checksums_for (#4180) + - add pre/post extension hook that is triggered before/after individual extension installations (#4193) + - enforce absolute paths as start dir of extensions (#4196) + - print warning when non-existing `start_dir` is specified for extension (#4201) + - add support to silence deprecation warning for easyconfigs and toolchains (#4207) + - skip directories when fixing shebangs (#4209) + - add support for 'modunloadmsg' easyconfig parameter (#4223) +- various bug fixes, including: + - fix use of `locate` in `check_os_dependency` (#4166) + - add VERSION to easybuild.easyblocks namespace in test sandbox (#4171) + - also apply filter to asyncprocess test subsuite (#4172) + - don't use deprecated SafeConfigParser when running with Python 3.x (#4173) + - update CI workflows to run 'apt-get update' if installation of packages via 'apt-get install' failed, likely due to stale apt cache (#4176) + - fixes for findPythonDeps script (#4179) + - check type of versionsuffix value in det_full_ec_version, and raise useful error message if it's not a string (#4184) + - avoid GC3Pie deprecation warning (#4185) + - correctly restore sys.path in tests (#4186) + - suppress output of --skip-test-step test (#4187) + - fix flaky test_search_file due to accidental matches for `.*C++` search pattern (#4190) + - silence distutils deprecation warnings (#4204) + - improve handling of start_dir and add tests for cases where either ext_dir or initial start_dir or both are unset or None (#4206) + - restore initial environment before processing each easystack item (#4213) + - make test_help_long work on small terminals (#4218) +- other changes: + - use better test assertions by replacing use of assertFalse/assertTrue (#4170, #4205) + - remove Travis CI configuration file (#4174) + - filter out deprecation warnings for platform.dist in get_os_version and platform.linux_distribution in get_os_name (#4175) + - only give read permissions in GitHub Actions workflows (#4182) + - fix website/docs links in README (#4199) + - only print "default: " for configuration option of strlist type if default is not empty (#4220) + + v4.7.0 (January 9th 2023) ------------------------- diff --git a/easybuild/base/generaloption.py b/easybuild/base/generaloption.py index 644da562f4..51f7daafe3 100644 --- a/easybuild/base/generaloption.py +++ b/easybuild/base/generaloption.py @@ -45,7 +45,7 @@ from easybuild.base.fancylogger import getLogger, setroot, setLogLevel, getDetailsLogLevels from easybuild.base.optcomplete import autocomplete, CompleterOption -from easybuild.tools.py2vs3 import StringIO, configparser, string_type, subprocess_popen_text +from easybuild.tools.py2vs3 import StringIO, configparser, ConfigParser, string_type, subprocess_popen_text from easybuild.tools.utilities import mk_md_table, mk_rst_table, nub, shell_quote try: @@ -898,7 +898,6 @@ class GeneralOption(object): CONFIGFILES_INIT = [] # initial list of defaults, overwritten by go_configfiles options CONFIGFILES_IGNORE = [] CONFIGFILES_MAIN_SECTION = 'MAIN' # sectionname that contains the non-grouped/non-prefixed options - CONFIGFILE_PARSER = configparser.SafeConfigParser CONFIGFILE_CASESENSITIVE = True METAVAR_DEFAULT = True # generate a default metavar @@ -1150,7 +1149,8 @@ def add_group_parser(self, opt_dict, description, prefix=None, otherdefaults=Non if len(str(default)) == 0: extra_help.append("default: ''") # empty string elif typ in ExtOption.TYPE_STRLIST: - extra_help.append("default: %s" % sep.join(default)) + if default: + extra_help.append("default: %s" % sep.join(default)) else: extra_help.append("default: %s" % default) @@ -1274,7 +1274,7 @@ def configfile_parser_init(self, initenv=None): the 2nd level key,value is the key=value. All section names, keys and values are converted to strings. """ - self.configfile_parser = self.CONFIGFILE_PARSER() + self.configfile_parser = ConfigParser() # make case sensitive if self.CONFIGFILE_CASESENSITIVE: diff --git a/easybuild/base/testing.py b/easybuild/base/testing.py index cda4e36d1b..01b64d84ef 100644 --- a/easybuild/base/testing.py +++ b/easybuild/base/testing.py @@ -34,6 +34,7 @@ * Kenneth Hoste (Ghent University) """ import difflib +import os import pprint import re import sys @@ -117,6 +118,18 @@ def assertEqual(self, a, b, msg=None): raise AssertionError("%s:\nDIFF%s:\n%s" % (msg, limit, ''.join(diff[:self.ASSERT_MAX_DIFF]))) + def assertExists(self, path, msg=None): + """Assert the given path exists""" + if msg is None: + msg = "'%s' should exist" % path + self.assertTrue(os.path.exists(path), msg) + + def assertNotExists(self, path, msg=None): + """Assert the given path exists""" + if msg is None: + msg = "'%s' should not exist" % path + self.assertFalse(os.path.exists(path), msg) + def setUp(self): """Prepare test case.""" super(TestCase, self).setUp() @@ -157,7 +170,7 @@ def assertErrorRegex(self, error, regex, call, *args, **kwargs): call(*args, **kwargs) str_kwargs = ['='.join([k, str(v)]) for (k, v) in kwargs.items()] str_args = ', '.join(list(map(str, args)) + str_kwargs) - self.assertTrue(False, "Expected errors with %s(%s) call should occur" % (call.__name__, str_args)) + self.fail("Expected errors with %s(%s) call should occur" % (call.__name__, str_args)) except error as err: msg = self.convert_exception_to_str(err) if self.is_string(regex): diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index f1d84cbc06..1d177eafb9 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -84,9 +84,9 @@ from easybuild.tools.filetools import is_binary, is_sha256_checksum, mkdir, move_file, move_logs, read_file, remove_dir from easybuild.tools.filetools import remove_file, remove_lock, verify_checksum, weld_paths, write_file, symlink from easybuild.tools.hooks import BUILD_STEP, CLEANUP_STEP, CONFIGURE_STEP, EXTENSIONS_STEP, FETCH_STEP, INSTALL_STEP -from easybuild.tools.hooks import MODULE_STEP, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTITER_STEP, POSTPROC_STEP -from easybuild.tools.hooks import PREPARE_STEP, READY_STEP, SANITYCHECK_STEP, SOURCE_STEP, TEST_STEP, TESTCASES_STEP -from easybuild.tools.hooks import MODULE_WRITE, load_hooks, run_hook +from easybuild.tools.hooks import MODULE_STEP, MODULE_WRITE, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTITER_STEP +from easybuild.tools.hooks import POSTPROC_STEP, PREPARE_STEP, READY_STEP, SANITYCHECK_STEP, SOURCE_STEP +from easybuild.tools.hooks import SINGLE_EXTENSION, TEST_STEP, TESTCASES_STEP, load_hooks, run_hook from easybuild.tools.run import check_async_cmd, run_cmd from easybuild.tools.jenkins import write_to_xml from easybuild.tools.module_generator import ModuleGeneratorLua, ModuleGeneratorTcl, module_generator, dependencies_for @@ -100,7 +100,7 @@ from easybuild.tools.py2vs3 import extract_method_name, string_type from easybuild.tools.repository.repository import init_repository from easybuild.tools.systemtools import check_linked_shared_libs, det_parallelism, get_linked_libs_raw -from easybuild.tools.systemtools import get_shared_lib_ext, use_group +from easybuild.tools.systemtools import get_shared_lib_ext, pick_system_specific_value, use_group from easybuild.tools.utilities import INDENT_4SPACES, get_class_for, nub, quote_str from easybuild.tools.utilities import remove_unwanted_chars, time2str, trace_msg from easybuild.tools.version import this_is_easybuild, VERBOSE_VERSION, VERSION @@ -1412,6 +1412,13 @@ def make_module_extra(self, altroot=None, altversion=None): modloadmsg += '\n' lines.append(self.module_generator.msg_on_load(modloadmsg)) + modunloadmsg = self.cfg['modunloadmsg'] + if modunloadmsg: + # add trailing newline to prevent that shell prompt is 'glued' to module unload/purge message + if not modunloadmsg.endswith('\n'): + modunloadmsg += '\n' + lines.append(self.module_generator.msg_on_unload(modunloadmsg)) + for (key, value) in self.cfg['modaliases'].items(): lines.append(self.module_generator.set_alias(key, value)) @@ -1559,8 +1566,8 @@ def make_module_req(self): paths = sum((glob.glob(path) if path else [path] for path in reqs), []) # sum flattens to list # If lib64 is just a symlink to lib we fixup the paths to avoid duplicates - lib64_is_symlink = (all(os.path.isdir(path) for path in ['lib', 'lib64']) - and os.path.samefile('lib', 'lib64')) + lib64_is_symlink = (all(os.path.isdir(path) for path in ['lib', 'lib64']) and + os.path.samefile('lib', 'lib64')) if lib64_is_symlink: fixed_paths = [] for path in paths: @@ -1851,6 +1858,8 @@ def install_extensions_sequential(self, install=True): self.log.info("Starting extension %s", ext.name) + run_hook(SINGLE_EXTENSION, self.hooks, pre_step_hook=True, args=[ext]) + # always go back to original work dir to avoid running stuff from a dir that no longer exists change_dir(self.orig_workdir) @@ -1897,6 +1906,8 @@ def install_extensions_sequential(self, install=True): self.update_exts_progress_bar(progress_info, progress_size=1) + run_hook(SINGLE_EXTENSION, self.hooks, post_step_hook=True, args=[ext]) + def install_extensions_parallel(self, install=True): """ Install extensions in parallel. @@ -2418,6 +2429,11 @@ def check_checksums_for(self, ent, sub='', source_cnt=None): sources = ent.get('sources', []) patches = ent.get('patches', []) checksums = ent.get('checksums', []) + # Single source should be re-wrapped as a list, and checksums with it + if isinstance(sources, dict): + sources = [sources] + if isinstance(checksums, string_type): + checksums = [checksums] if not checksums: checksums_from_json = self.get_checksums_from_json() @@ -2902,6 +2918,10 @@ def fix_shebang(self): for path in paths: # check whether file should be patched by checking whether it has a shebang we want to tweak; # this also helps to skip binary files we may be hitting (but only with Python 3) + if os.path.isdir(path): + self.log.debug("Skipping shebang fix for directory '%s'", path) + continue + try: contents = read_file(path, mode='r') should_patch = shebang_regex.match(contents) @@ -3318,6 +3338,15 @@ def _sanity_check_step_common(self, custom_paths, custom_commands): error_msg += "values should be lists (at least one non-empty)." raise EasyBuildError(error_msg % ', '.join("'%s'" % k for k in known_keys)) + # Resolve arch specific entries + for values in paths.values(): + new_values = [] + for value in values: + value = pick_system_specific_value('sanity_check_paths', value, allow_none=True) + if value is not None: + new_values.append(value) + values[:] = new_values + # if enhance_sanity_check is not enabled, only sanity_check_commands specified in the easyconfig file are used, # the ones provided by the easyblock (via custom_commands) are ignored if ec_commands and not enhance_sanity_check: diff --git a/easybuild/framework/easyconfig/constants.py b/easybuild/framework/easyconfig/constants.py index 7c4ae76281..2e62580db4 100644 --- a/easybuild/framework/easyconfig/constants.py +++ b/easybuild/framework/easyconfig/constants.py @@ -84,3 +84,7 @@ def _get_arch_constant(): 'OS_PKG_PAM_DEV': (('pam-devel', 'libpam0g-dev'), "OS packages providing Pluggable Authentication Module (PAM) development support"), } + +# Add EasyConfig constants to export list +globals().update({name: value for name, (value, _) in EASYCONFIG_CONSTANTS.items()}) +__all__ = ['EXTERNAL_MODULE_MARKER', 'EASYCONFIG_CONSTANTS'] + list(EASYCONFIG_CONSTANTS.keys()) diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py index 680a56d84e..b37c3660f0 100644 --- a/easybuild/framework/easyconfig/default.py +++ b/easybuild/framework/easyconfig/default.py @@ -195,6 +195,7 @@ 'modextrapaths': [{}, "Extra paths to be prepended in module file", MODULES], 'modextravars': [{}, "Extra environment variables to be added to module file", MODULES], 'modloadmsg': [{}, "Message that should be printed when generated module is loaded", MODULES], + 'modunloadmsg': [{}, "Message that should be printed when generated module is unloaded", MODULES], 'modluafooter': ["", "Footer to include in generated module file (Lua syntax)", MODULES], 'modaltsoftname': [None, "Module name to use (rather than using software name", MODULES], 'modtclfooter': ["", "Footer to include in generated module file (Tcl syntax)", MODULES], diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 4d8fc2d782..f4c4be464e 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -834,14 +834,16 @@ def check_deprecated(self, path): deprecated = self['deprecated'] if deprecated: if isinstance(deprecated, string_type): - depr_msgs.append("easyconfig file '%s' is marked as deprecated:\n%s\n" % (path, deprecated)) + if 'easyconfig' not in build_option('silence_deprecation_warnings'): + depr_msgs.append("easyconfig file '%s' is marked as deprecated:\n%s\n" % (path, deprecated)) else: raise EasyBuildError("Wrong type for value of 'deprecated' easyconfig parameter: %s", type(deprecated)) if self.toolchain.is_deprecated(): # allow use of deprecated toolchains when running unit tests, # because test easyconfigs/modules often use old toolchain versions (and updating them is far from trivial) - if not build_option('unit_testing_mode'): + if (not build_option('unit_testing_mode') + and 'toolchain' not in build_option('silence_deprecation_warnings')): depr_msgs.append("toolchain '%(name)s/%(version)s' is marked as deprecated" % self['toolchain']) if depr_msgs: diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index 8897a64ba3..603423c20b 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -75,6 +75,7 @@ TEMPLATE_NAMES_EASYBLOCK_RUN_STEP = [ ('builddir', "Build directory"), ('installdir', "Installation directory"), + ('start_dir', "Directory in which the build process begins"), ] # software names for which to define ver and shortver templates TEMPLATE_SOFTWARE_VERSIONS = [ diff --git a/easybuild/framework/easyconfig/types.py b/easybuild/framework/easyconfig/types.py index e07863ddaa..622479f480 100644 --- a/easybuild/framework/easyconfig/types.py +++ b/easybuild/framework/easyconfig/types.py @@ -377,6 +377,42 @@ def to_list_of_strings_and_tuples_and_dicts(spec): return str_tup_list +def to_sanity_check_paths_entry(spec): + """ + Convert a 'list of lists and strings' to a 'list of tuples and strings' while allowing dicts of lists or strings + + Example: + ['foo', ['bar', 'baz'], {'f42': ['a', 'b']}] + to + ['foo', ('bar', 'baz'), {'f42': ('a', 'b')}] + """ + result = [] + + if not isinstance(spec, (list, tuple)): + raise EasyBuildError("Expected value to be a list, found %s (%s)", spec, type(spec)) + + for elem in spec: + if isinstance(elem, (string_type, tuple)): + result.append(elem) + elif isinstance(elem, list): + result.append(tuple(elem)) + elif isinstance(elem, dict): + for key, value in elem.items(): + if not isinstance(key, string_type): + raise EasyBuildError("Expected keys to be of type string, got %s (%s)", key, type(key)) + elif isinstance(value, list): + elem[key] = tuple(value) + elif not isinstance(value, (string_type, tuple)): + raise EasyBuildError("Expected elements to be of type string, tuple or list, got %s (%s)", + value, type(value)) + result.append(elem) + else: + raise EasyBuildError("Expected elements to be of type string, tuple/list or dict, got %s (%s)", + elem, type(elem)) + + return result + + def to_sanity_check_paths_dict(spec): """ Convert a sanity_check_paths dict as received by yaml (a dict with list values that contain either lists or strings) @@ -391,7 +427,7 @@ def to_sanity_check_paths_dict(spec): sanity_check_dict = {} for key in spec: - sanity_check_dict[key] = to_list_of_strings_and_tuples(spec[key]) + sanity_check_dict[key] = to_sanity_check_paths_entry(spec[key]) return sanity_check_dict @@ -554,11 +590,18 @@ def ensure_iterable_license_specs(specs): 'key_types': [str], } )) +STRING_OR_TUPLE_DICT = (dict, as_hashable( + { + 'elem_types': [str], + 'key_types': [str, TUPLE_OF_STRINGS], + } +)) STRING_OR_TUPLE_OR_DICT_LIST = (list, as_hashable({'elem_types': [str, TUPLE_OF_STRINGS, STRING_DICT]})) +SANITY_CHECK_PATHS_ENTRY = (list, as_hashable({'elem_types': [str, TUPLE_OF_STRINGS, STRING_OR_TUPLE_DICT]})) SANITY_CHECK_PATHS_DICT = (dict, as_hashable({ 'elem_types': { - SANITY_CHECK_PATHS_FILES: [STRING_OR_TUPLE_LIST], - SANITY_CHECK_PATHS_DIRS: [STRING_OR_TUPLE_LIST], + SANITY_CHECK_PATHS_FILES: [SANITY_CHECK_PATHS_ENTRY], + SANITY_CHECK_PATHS_DIRS: [SANITY_CHECK_PATHS_ENTRY], }, 'opt_keys': [], 'req_keys': [SANITY_CHECK_PATHS_FILES, SANITY_CHECK_PATHS_DIRS], @@ -573,8 +616,8 @@ def ensure_iterable_license_specs(specs): CHECKSUMS = (list, as_hashable({'elem_types': [str, tuple, STRING_DICT, CHECKSUM_LIST]})) CHECKABLE_TYPES = [CHECKSUM_LIST, CHECKSUMS, DEPENDENCIES, DEPENDENCY_DICT, LIST_OF_STRINGS, - SANITY_CHECK_PATHS_DICT, STRING_DICT, STRING_OR_TUPLE_LIST, STRING_OR_TUPLE_OR_DICT_LIST, - TOOLCHAIN_DICT, TUPLE_OF_STRINGS] + SANITY_CHECK_PATHS_DICT, SANITY_CHECK_PATHS_ENTRY, STRING_DICT, STRING_OR_TUPLE_LIST, + STRING_OR_TUPLE_DICT, STRING_OR_TUPLE_OR_DICT_LIST, TOOLCHAIN_DICT, TUPLE_OF_STRINGS] # easy types, that can be verified with isinstance EASY_TYPES = [string_type, bool, dict, int, list, str, tuple] diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index b7db5b8172..ed90f15e6d 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -119,6 +119,12 @@ def __init__(self, mself, ext, extra_params=None): for key in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP: self.cfg.template_values[key[0]] = str(getattr(self.master, key[0], None)) + # We can't inherit the 'start_dir' value from the parent (which will be set, and will most likely be wrong). + # It should be specified for the extension specifically, or be empty (so it is auto-derived). + self.cfg['start_dir'] = self.ext.get('options', {}).get('start_dir', None) + # Also clear the template + del self.cfg.template_values['start_dir'] + # list of source/patch files: we use an empty list as default value like in EasyBlock self.src = resolve_template(self.ext.get('src', []), self.cfg.template_values) self.src_extract_cmd = self.ext.get('extract_cmd', None) diff --git a/easybuild/framework/extensioneasyblock.py b/easybuild/framework/extensioneasyblock.py index c9aa1eeacb..6ad116b20b 100644 --- a/easybuild/framework/extensioneasyblock.py +++ b/easybuild/framework/extensioneasyblock.py @@ -33,7 +33,8 @@ from easybuild.framework.easyblock import EasyBlock from easybuild.framework.easyconfig import CUSTOM from easybuild.framework.extension import Extension -from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.build_log import EasyBuildError, print_warning +from easybuild.tools.config import build_option from easybuild.tools.filetools import change_dir, extract_file from easybuild.tools.utilities import remove_unwanted_chars, trace_msg @@ -83,9 +84,7 @@ def __init__(self, *args, **kwargs): # name and version properties of EasyBlock are used, so make sure name and version are correct self.cfg['name'] = self.ext.get('name', None) self.cfg['version'] = self.ext.get('version', None) - # We can't inherit the 'start_dir' value from the parent (which will be set, and will most likely be wrong). - # It should be specified for the extension specifically, or be empty (so it is auto-derived). - self.cfg['start_dir'] = self.ext.get('options', {}).get('start_dir', None) + self.builddir = self.master.builddir self.installdir = self.master.installdir self.modules_tool = self.master.modules_tool @@ -100,18 +99,39 @@ def __init__(self, *args, **kwargs): self.ext_dir = None # dir where extension source was unpacked def _set_start_dir(self): - """Set value for self.start_dir - - Uses existing value of self.start_dir if it is already set and exists - otherwise self.ext_dir (path to extracted source) if that is set and exists, similar to guess_start_dir + """Set absolute path of self.start_dir similarly to EasyBlock.guess_start_dir + + Uses existing value of self.start_dir defaulting to self.ext_dir. + If self.ext_dir (path to extracted source) is set, it is used as the base dir for relative paths. + Otherwise otherwise self.builddir is used as the base. + When neither start_dir nor ext_dir are set or when the computed start_dir does not exist + the start dir is not changed. + The computed start dir will not end in path separators """ - possible_dirs = (self.start_dir, self.ext_dir) - for possible_dir in possible_dirs: - if possible_dir and os.path.isdir(possible_dir): - self.cfg['start_dir'] = possible_dir - self.log.debug("Using start_dir: %s", self.start_dir) - return - self.log.debug("Unable to determine start_dir as none of these paths is set and exists: %s", possible_dirs) + ext_start_dir = self.start_dir + if self.ext_dir: + if not os.path.isabs(self.ext_dir): + raise EasyBuildError("ext_dir must be an absolute path. Is: '%s'", self.ext_dir) + ext_start_dir = os.path.join(self.ext_dir, ext_start_dir or '') + elif ext_start_dir is not None: + if not os.path.isabs(self.builddir): + raise EasyBuildError("builddir must be an absolute path. Is: '%s'", self.builddir) + ext_start_dir = os.path.join(self.builddir, ext_start_dir) + + if ext_start_dir and os.path.isdir(ext_start_dir): + ext_start_dir = ext_start_dir.rstrip(os.sep) or os.sep + self.log.debug("Using extension start dir: %s", ext_start_dir) + self.cfg['start_dir'] = ext_start_dir + self.cfg.template_values['start_dir'] = ext_start_dir + elif ext_start_dir is None: + # This may be on purpose, e.g. for Python WHL files which do not get extracted + self.log.debug("Start dir is not set.") + else: + # non-existing start dir means wrong input from user + warn_msg = "Provided start dir (%s) for extension %s does not exist: %s" % (self.start_dir, self.name, + ext_start_dir) + self.log.warning(warn_msg) + print_warning(warn_msg, silent=build_option('silent')) def run(self, unpack_src=False): """Common operations for extensions: unpacking sources, patching, ...""" diff --git a/easybuild/main.py b/easybuild/main.py index 6b2dfd62f0..319c80b5ae 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -62,6 +62,7 @@ from easybuild.tools.config import find_last_log, get_repository, get_repositorypath, build_option from easybuild.tools.containers.common import containerize from easybuild.tools.docs import list_software +from easybuild.tools.environment import restore_env from easybuild.tools.filetools import adjust_permissions, cleanup, copy_files, dump_index, load_index from easybuild.tools.filetools import locate_files, read_file, register_lock_cleanup_signal_handlers, write_file from easybuild.tools.github import check_github, close_pr, find_easybuild_easyconfig @@ -232,6 +233,9 @@ def process_easystack(easystack_path, args, logfile, testing, init_session_state """ easystack = parse_easystack(easystack_path) + # keep copy of original environment, so we can restore it for every easystack entry + init_env = copy.deepcopy(os.environ) + global _log # TODO: insert fast loop that validates if all command line options are valid. If there are errors in options, @@ -244,10 +248,14 @@ def process_easystack(easystack_path, args, logfile, testing, init_session_state do_cleanup = True for (path, ec_opts) in easystack.ec_opt_tuples: _log.debug("Starting build for %s" % path) + # wipe easyconfig caches easyconfig._easyconfigs_cache.clear() easyconfig._easyconfig_files_cache.clear() + # restore environment + restore_env(init_env) + # If EasyConfig specific arguments were supplied in EasyStack file # merge arguments with original command line args if ec_opts is not None: @@ -595,8 +603,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): eb_go, cfg_settings = set_up_configuration(args=args, logfile=logfile, testing=testing) options, orig_paths = eb_go.options, eb_go.args - silence_deprecation_warnings = build_option('silence_deprecation_warnings') or [] - if 'python2' not in silence_deprecation_warnings: + if 'python2' not in build_option('silence_deprecation_warnings'): python2_is_deprecated() global _log diff --git a/easybuild/scripts/findPythonDeps.py b/easybuild/scripts/findPythonDeps.py index 0d3e34ed25..663926a9d6 100755 --- a/easybuild/scripts/findPythonDeps.py +++ b/easybuild/scripts/findPythonDeps.py @@ -119,7 +119,7 @@ def print_deps(package, verbose): installed_modules = {mod.project_name for mod in pkg_resources.working_set} if verbose: - print("Installed modules: %s" % installed_modules) + print("Installed modules: " + ', '.join(sorted(installed_modules))) # iterate over deps in reverse order, get rid of duplicates along the way # also filter out Python packages that are already installed in current environment @@ -187,7 +187,7 @@ def print_deps(package, verbose): run_cmd(['eb', args.ec, '--dump-env', '--force'], action_desc='Dump build environment') os.chdir(old_dir) - cmd = 'source %s/*.env && %s %s "%s"' % (tmp_dir, sys.executable, sys.argv[0], args.package) + cmd = "source %s/*.env && python %s '%s'" % (tmp_dir, sys.argv[0], args.package) if args.verbose: cmd += ' --verbose' print('Restarting script in new build environment') diff --git a/easybuild/toolchains/iimkl.py b/easybuild/toolchains/iimkl.py index 64b4b8b3af..9d3d6a97f3 100644 --- a/easybuild/toolchains/iimkl.py +++ b/easybuild/toolchains/iimkl.py @@ -32,13 +32,13 @@ * Kenneth Hoste (Ghent University) * Bart Oldeman (McGill University, Calcul Quebec, Compute Canada) """ -from distutils.version import LooseVersion import re from easybuild.toolchains.iccifort import IccIfort from easybuild.toolchains.intel_compilers import IntelCompilersToolchain from easybuild.toolchains.fft.intelfftw import IntelFFTW from easybuild.toolchains.linalg.intelmkl import IntelMKL +from easybuild.tools import LooseVersion class Iimkl(IccIfort, IntelCompilersToolchain, IntelMKL, IntelFFTW): diff --git a/easybuild/tools/__init__.py b/easybuild/tools/__init__.py index 3cda3dd99b..dee4fc0d12 100644 --- a/easybuild/tools/__init__.py +++ b/easybuild/tools/__init__.py @@ -36,4 +36,15 @@ """ __path__ = __import__('pkgutil').extend_path(__path__, __name__) + +import distutils.version +import warnings from easybuild.tools.loose_version import LooseVersion # noqa(F401) + + +class StrictVersion(distutils.version.StrictVersion): + """Temporary wrapper over distuitls StrictVersion that silences the deprecation warning""" + def __init__(self, *args, **kwargs): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=DeprecationWarning) + distutils.version.StrictVersion.__init__(self, *args, **kwargs) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 113addd7e6..43344541e7 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -254,7 +254,6 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'rpath_filter', 'rpath_override_dirs', 'required_linked_shared_libs', - 'silence_deprecation_warnings', 'skip', 'stop', 'subdir_user_modules', @@ -335,6 +334,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'include_easyblocks_from_pr', 'robot', 'search_paths', + 'silence_deprecation_warnings', ], WARN: [ 'check_ebroot_env_vars', diff --git a/easybuild/tools/configobj.py b/easybuild/tools/configobj.py index 433238b95f..48c7dd348d 100644 --- a/easybuild/tools/configobj.py +++ b/easybuild/tools/configobj.py @@ -2034,8 +2034,8 @@ def write(self, outfile=None, section=None): # Turn the list to a string, joined with correct newlines newline = self.newlines or os.linesep - if (getattr(outfile, 'mode', None) is not None and outfile.mode == 'w' - and sys.platform == 'win32' and newline == '\r\n'): + if (getattr(outfile, 'mode', None) is not None and outfile.mode == 'w' and + sys.platform == 'win32' and newline == '\r\n'): # Windows specific hack to avoid writing '\r\r\n' newline = '\n' output = self._a_to_u(newline).join(out) diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index bde8cc6df7..1fcc6ece83 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -315,7 +315,7 @@ def avail_easyconfig_params_md(title, grouped_params): values = [grouped_params[grpname][key] for key in keys] table_values = [ ['`%s`' % name for name in keys], # parameter name - [x[0] for x in values], # description + [x[0].replace('<', '<').replace('>', '>') for x in values], # description ['`' + str(quote_str(x[1])) + '`' for x in values] # default value ] @@ -559,7 +559,7 @@ def avail_easyconfig_templates_md(): for name, pref in TEMPLATE_SOFTWARE_VERSIONS: ver.append('``%%(%sshortver)s``' % pref) ver.append('``%%(%sver)s``' % pref) - ver_desc.append('short version for %s (.)' % name) + ver_desc.append('short version for %s (``.``)' % name) ver_desc.append('full version for %s' % name) table_values = [ver, ver_desc] doc.extend(md_title_and_table(title, table_titles, table_values, title_level=2)) diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py index 2861579393..1f95bacadf 100644 --- a/easybuild/tools/hooks.py +++ b/easybuild/tools/hooks.py @@ -61,6 +61,7 @@ START = 'start' PARSE = 'parse' +SINGLE_EXTENSION = 'single_extension' MODULE_WRITE = 'module_write' END = 'end' @@ -73,7 +74,31 @@ INSTALL_STEP, EXTENSIONS_STEP, POSTPROC_STEP, SANITYCHECK_STEP, CLEANUP_STEP, MODULE_STEP, PERMISSIONS_STEP, PACKAGE_STEP, TESTCASES_STEP] -HOOK_NAMES = [START, PARSE, MODULE_WRITE] + [p + s for s in STEP_NAMES for p in [PRE_PREF, POST_PREF]] + [END] +# hook names (in order of being triggered) +HOOK_NAMES = [ + START, + PARSE, +] + [p + x for x in STEP_NAMES[:STEP_NAMES.index(EXTENSIONS_STEP)] + for p in [PRE_PREF, POST_PREF]] + [ + # pre-extensions hook is triggered before starting installation of extensions, + # pre/post extension (singular) hook is triggered when installing individual extensions, + # post-extensions hook is triggered after installation of extensions + PRE_PREF + EXTENSIONS_STEP, + PRE_PREF + SINGLE_EXTENSION, + POST_PREF + SINGLE_EXTENSION, + POST_PREF + EXTENSIONS_STEP, +] + [p + x for x in STEP_NAMES[STEP_NAMES.index(EXTENSIONS_STEP)+1:STEP_NAMES.index(MODULE_STEP)] + for p in [PRE_PREF, POST_PREF]] + [ + # pre-module hook hook is triggered before starting module step which creates module file, + # module_write hook is triggered when module file has been written, + # post-module hook hook is triggered before after running module step + PRE_PREF + MODULE_STEP, + MODULE_WRITE, + POST_PREF + MODULE_STEP, +] + [p + x for x in STEP_NAMES[STEP_NAMES.index(MODULE_STEP)+1:] + for p in [PRE_PREF, POST_PREF]] + [ + END, +] KNOWN_HOOKS = [h + HOOK_SUFF for h in HOOK_NAMES] diff --git a/easybuild/tools/job/gc3pie.py b/easybuild/tools/job/gc3pie.py index aa05ec8668..005cff2e90 100644 --- a/easybuild/tools/job/gc3pie.py +++ b/easybuild/tools/job/gc3pie.py @@ -34,6 +34,7 @@ import os from time import gmtime, strftime import time +import warnings from easybuild.base import fancylogger from easybuild.tools import LooseVersion @@ -47,7 +48,10 @@ try: - import gc3libs + with warnings.catch_warnings(): + # Workaround https://github.com/gc3pie/gc3pie/issues/670 + warnings.simplefilter("ignore", category=DeprecationWarning) + import gc3libs import gc3libs.exceptions from gc3libs import Application, Run, create_engine from gc3libs.quantity import hours as hr @@ -225,7 +229,10 @@ def complete(self): """ # create an instance of `Engine` using the list of configuration files try: - self._engine = create_engine(*self.config_files, resource_errors_are_fatal=True) + with warnings.catch_warnings(): + # Workaround https://github.com/gc3pie/gc3pie/issues/670 + warnings.simplefilter("ignore", category=DeprecationWarning) + self._engine = create_engine(*self.config_files, resource_errors_are_fatal=True) except gc3libs.exceptions.Error as err: raise EasyBuildError("Failed to create GC3Pie engine: %s", err) diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index 6b3c292584..33ff4659d4 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -502,6 +502,12 @@ def msg_on_load(self, msg): """ raise NotImplementedError + def msg_on_unload(self, msg): + """ + Add a message that should be printed when unloading the module. + """ + raise NotImplementedError + def set_alias(self, key, value): """ Generate set-alias statement in modulefile for the given key/value pair. @@ -951,6 +957,15 @@ def msg_on_load(self, msg): print_cmd = "puts stderr %s" % quote_str(msg, tcl=True) return '\n'.join(['', self.conditional_statement("module-info mode load", print_cmd, indent=False)]) + def msg_on_unload(self, msg): + """ + Add a message that should be printed when unloading the module. + """ + # escape any (non-escaped) characters with special meaning by prefixing them with a backslash + msg = re.sub(r'((?= 7.7.38) diff --git a/easybuild/tools/module_naming_scheme/utilities.py b/easybuild/tools/module_naming_scheme/utilities.py index 2380ef2af4..8e8f3e919e 100644 --- a/easybuild/tools/module_naming_scheme/utilities.py +++ b/easybuild/tools/module_naming_scheme/utilities.py @@ -38,6 +38,7 @@ import string from easybuild.base import fancylogger +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.module_naming_scheme.mns import ModuleNamingScheme from easybuild.tools.py2vs3 import string_type from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME, is_system_toolchain @@ -62,7 +63,17 @@ def det_full_ec_version(ec): ecver = "%s-%s-%s" % (ec['version'], toolchain['name'], toolchain['version']) # prepend/append version prefix/suffix - ecver = ''.join([x for x in [ec.get('versionprefix', ''), ecver, ec.get('versionsuffix', '')] if x]) + versionprefix = ec.get('versionprefix', '') + if not isinstance(versionprefix, string_type): + raise EasyBuildError("versionprefix value should be a string, found '%s': %s (full spec: %s)", + type(versionprefix).__name__, versionprefix, ec) + + versionsuffix = ec.get('versionsuffix', '') + if not isinstance(versionsuffix, string_type): + raise EasyBuildError("versionsuffix value should be a string, found '%s': %s (full spec: %s)", + type(versionsuffix).__name__, versionsuffix, ec) + + ecver = ''.join([x for x in [versionprefix, ecver, versionsuffix] if x]) return ecver diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index b9421f0fae..386643e706 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -40,9 +40,9 @@ import os import re import shlex -from distutils.version import StrictVersion from easybuild.base import fancylogger +from easybuild.tools import StrictVersion from easybuild.tools.build_log import EasyBuildError, print_warning from easybuild.tools.config import ERROR, IGNORE, PURGE, UNLOAD, UNSET from easybuild.tools.config import EBROOT_ENV_VAR_ACTIONS, LOADED_MODULES_ACTIONS @@ -274,9 +274,7 @@ def set_and_check_version(self): depr_msg = "Support for %s version < %s is deprecated, " % (self.NAME, self.DEPR_VERSION) depr_msg += "found version %s" % self.version - silence_deprecation_warnings = build_option('silence_deprecation_warnings') or [] - - if self.version.startswith('6') and 'Lmod6' in silence_deprecation_warnings: + if self.version.startswith('6') and 'Lmod6' in build_option('silence_deprecation_warnings'): self.log.warning(depr_msg) else: self.log.deprecated(depr_msg, '5.0') diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 08d726116f..57ac2b89ef 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -339,12 +339,14 @@ def override_options(self): # override options descr = ("Override options", "Override default EasyBuild behavior.") + all_deprecations = ('python2', 'Lmod6', 'easyconfig', 'toolchain') + opts = OrderedDict({ 'accept-eula': ("Accept EULA for specified software [DEPRECATED, use --accept-eula-for instead!]", 'strlist', 'store', []), 'accept-eula-for': ("Accept EULA for specified software", 'strlist', 'store', []), 'add-dummy-to-minimal-toolchains': ("Include dummy toolchain in minimal toolchain searches " - "[DEPRECATED, use --add-system-to-minimal-toolchains instead!)", + "[DEPRECATED, use --add-system-to-minimal-toolchains instead!]", None, 'store_true', False), 'add-system-to-minimal-toolchains': ("Include system toolchain in minimal toolchain searches", None, 'store_true', False), @@ -494,7 +496,9 @@ def override_options(self): 'set-default-module': ("Set the generated module as default", None, 'store_true', False), 'set-gid-bit': ("Set group ID bit on newly created directories", None, 'store_true', False), 'show-progress-bar': ("Show progress bar in terminal output", None, 'store_true', True), - 'silence-deprecation-warnings': ("Silence specified deprecation warnings", 'strlist', 'extend', None), + 'silence-deprecation-warnings': ( + "Silence specified deprecation warnings out of (%s)" % ', '.join(all_deprecations), + 'strlist', 'extend', []), 'skip-extensions': ("Skip installation of extensions", None, 'store_true', False), 'skip-test-cases': ("Skip running test cases", None, 'store_true', False, 't'), 'skip-test-step': ("Skip running the test step (e.g. unit tests)", None, 'store_true', False), diff --git a/easybuild/tools/package/package_naming_scheme/easybuild_deb_friendly_pns.py b/easybuild/tools/package/package_naming_scheme/easybuild_deb_friendly_pns.py new file mode 100644 index 0000000000..9ae9b37419 --- /dev/null +++ b/easybuild/tools/package/package_naming_scheme/easybuild_deb_friendly_pns.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- vim: set fileencoding=utf-8 +## +# Copyright 2015-2022 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Implementation of the EasyBuild deb friendly packaging naming scheme + +:author: Robert Schmidt (Ottawa Hospital Research Institute) +:author: Kenneth Hoste (Ghent University) +:author: Martin Budsjö (VolvoCars) +""" +from easybuild.tools.package.package_naming_scheme.easybuild_pns import EasyBuildPNS +from easybuild.tools.version import VERSION as EASYBUILD_VERSION + + +class EasyBuildDebFriendlyPNS(EasyBuildPNS): + """Class implmenting the Deb friendly EasyBuild packaging naming scheme.""" + + def version(self, ec): + """Determine package version: EasyBuild version used to build & install.""" + ebver = str(EASYBUILD_VERSION) + if ebver.endswith('dev'): + # try and make sure that 'dev' EasyBuild version is not considered newer just because it's longer + # (e.g., 2.2.0 vs 2.2.0dev) + # cfr. http://rpm.org/ticket/56, + # https://debian-handbook.info/browse/stable/sect.manipulating-packages-with-dpkg.html (see box in 5.4.3) + ebver.replace('dev', '~dev') + + # Make sure to add a `0` to the ebver if it doesn't start with a number + if not ebver[0].isdigit(): + ebver = '0'+ebver + + # + # Postfix `-eb` to ebver instead of prefixing it to comply with + # https://www.debian.org/doc/debian-policy/ch-controlfields.html#version + return '%s-eb' % ebver diff --git a/easybuild/tools/py2vs3/py2.py b/easybuild/tools/py2vs3/py2.py index 56189373f4..bd7e164e39 100644 --- a/easybuild/tools/py2vs3/py2.py +++ b/easybuild/tools/py2vs3/py2.py @@ -45,6 +45,9 @@ from urllib import urlencode # noqa from urllib2 import HTTPError, HTTPSHandler, Request, URLError, build_opener, urlopen # noqa +# Use the safe version. In Python 3.2+ this is the default already +ConfigParser = configparser.SafeConfigParser + # reload function (built-in in Python 2) reload = reload diff --git a/easybuild/tools/py2vs3/py3.py b/easybuild/tools/py2vs3/py3.py index 52eb4d91ed..91a5342075 100644 --- a/easybuild/tools/py2vs3/py3.py +++ b/easybuild/tools/py2vs3/py3.py @@ -46,6 +46,7 @@ from string import ascii_letters, ascii_lowercase # noqa from urllib.request import HTTPError, HTTPSHandler, Request, URLError, build_opener, urlopen # noqa from urllib.parse import urlencode # noqa +from configparser import ConfigParser # noqa # reload function (no longer a built-in in Python 3) # importlib only works with Python 3.4 & newer diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index 769f729b2a..627da42bb6 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -41,6 +41,7 @@ import struct import sys import termios +import warnings from ctypes.util import find_library from socket import gethostname from easybuild.tools.py2vs3 import subprocess_popen_text @@ -734,7 +735,10 @@ def get_os_name(): if hasattr(platform, 'linux_distribution'): # platform.linux_distribution is more useful, but only available since Python 2.6 # this allows to differentiate between Fedora, CentOS, RHEL and Scientific Linux (Rocks is just CentOS) - os_name = platform.linux_distribution()[0].strip() + with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=PendingDeprecationWarning) + warnings.simplefilter("ignore", category=DeprecationWarning) + os_name = platform.linux_distribution()[0].strip() # take into account that on some OSs, platform.distribution returns an empty string as OS name, # for example on OpenSUSE Leap 15.2 @@ -771,7 +775,10 @@ def get_os_version(): # platform.dist was removed in Python 3.8 if hasattr(platform, 'dist'): - os_version = platform.dist()[1] + with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=PendingDeprecationWarning) + warnings.simplefilter("ignore", category=DeprecationWarning) + os_version = platform.dist()[1] # take into account that on some OSs, platform.dist returns an empty string as OS version, # for example on OpenSUSE Leap 15.2 @@ -873,9 +880,14 @@ def check_os_dependency(dep): # try locate if it's available if not found and which('locate'): - cmd = 'locate --regexp "/%s$"' % dep - found = run_cmd(cmd, simple=True, log_all=False, log_ok=False, force_in_dry_run=True, trace=False, - stream_output=False) + cmd = 'locate -c --regexp "/%s$"' % dep + out, ec = run_cmd(cmd, simple=False, log_all=False, log_ok=False, force_in_dry_run=True, trace=False, + stream_output=False) + try: + found = (ec == 0 and int(out.strip()) > 0) + except ValueError: + # Returned something else than an int -> Error + found = False return found @@ -1245,6 +1257,46 @@ def check_python_version(): return (python_maj_ver, python_min_ver) +def pick_system_specific_value(description, options_or_value, allow_none=False): + """Pick an entry for the current system when the input has multiple options + + :param description: Descriptive string about the value to be retrieved. Used for logging. + :param options_or_value: Either a dictionary with options to choose from or a value of any other type + :param allow_none: When True and no matching arch key was found, return None instead of an error + + :return options_or_value when it is not a dictionary or the matching entry (if existing) + """ + result = options_or_value + if isinstance(options_or_value, dict): + if not options_or_value: + raise EasyBuildError("Found empty dict as %s!", description) + other_keys = [x for x in options_or_value.keys() if not x.startswith(ARCH_KEY_PREFIX)] + if other_keys: + other_keys = ','.join(sorted(other_keys)) + raise EasyBuildError("Unexpected keys in %s: %s (only '%s' keys are supported)", + description, other_keys, ARCH_KEY_PREFIX) + host_arch_key = ARCH_KEY_PREFIX + get_cpu_architecture() + star_arch_key = ARCH_KEY_PREFIX + '*' + # check for specific 'arch=' key first + try: + result = options_or_value[host_arch_key] + _log.info("Selected %s from %s for %s (using key %s)", + result, options_or_value, description, host_arch_key) + except KeyError: + # fall back to 'arch=*' + try: + result = options_or_value[star_arch_key] + _log.info("Selected %s from %s for %s (using fallback key %s)", + result, options_or_value, description, star_arch_key) + except KeyError: + if allow_none: + result = None + else: + raise EasyBuildError("No matches for %s in %s (looking for %s)", + description, options_or_value, host_arch_key) + return result + + def pick_dep_version(dep_version): """ Pick the correct dependency version to use for this system. @@ -1252,41 +1304,16 @@ def pick_dep_version(dep_version): * a string value (or None) * a dict with options to choose from - Return value is the version to use. + Return value is the version to use or False to skip this dependency. """ - if isinstance(dep_version, string_type): - _log.debug("Version is already a string ('%s'), OK", dep_version) - result = dep_version - - elif dep_version is None: + if dep_version is None: _log.debug("Version is None, OK") result = None - - elif isinstance(dep_version, dict): - arch_keys = [x for x in dep_version.keys() if x.startswith(ARCH_KEY_PREFIX)] - other_keys = [x for x in dep_version.keys() if x not in arch_keys] - if other_keys: - other_keys = ','.join(sorted(other_keys)) - raise EasyBuildError("Unexpected keys in version: %s (only 'arch=' keys are supported)", other_keys) - if arch_keys: - host_arch_key = ARCH_KEY_PREFIX + get_cpu_architecture() - star_arch_key = ARCH_KEY_PREFIX + '*' - # check for specific 'arch=' key first - if host_arch_key in dep_version: - result = dep_version[host_arch_key] - _log.info("Version selected from %s using key %s: %s", dep_version, host_arch_key, result) - # fall back to 'arch=*' - elif star_arch_key in dep_version: - result = dep_version[star_arch_key] - _log.info("Version selected for %s using fallback key %s: %s", dep_version, star_arch_key, result) - else: - raise EasyBuildError("No matches for version in %s (looking for %s)", dep_version, host_arch_key) - else: - raise EasyBuildError("Found empty dict as version!") - else: - typ = type(dep_version) - raise EasyBuildError("Unknown value type for version: %s (%s), should be string value", typ, dep_version) + result = pick_system_specific_value("version", dep_version) + if not isinstance(result, string_type) and result is not False: + typ = type(dep_version) + raise EasyBuildError("Unknown value type for version: %s (%s), should be string value", typ, dep_version) return result diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index 6b2244dbd4..c0ff386f78 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -45,7 +45,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.7.0') +VERSION = LooseVersion('4.7.1') UNKNOWN = 'UNKNOWN' diff --git a/test/framework/asyncprocess.py b/test/framework/asyncprocess.py index fd5f340422..f8db1927d8 100644 --- a/test/framework/asyncprocess.py +++ b/test/framework/asyncprocess.py @@ -30,8 +30,8 @@ import sys import time -from test.framework.utilities import EnhancedTestCase -from unittest import TextTestRunner, TestSuite +from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered +from unittest import TextTestRunner import easybuild.tools.asyncprocess as p from easybuild.tools.asyncprocess import Popen @@ -46,7 +46,7 @@ def setUp(self): super(AsyncProcessTest, self).setUp() self.shell = Popen('sh', stdin=p.PIPE, stdout=p.PIPE, shell=True, executable='/bin/bash') - def runTest(self): + def test_echo_between_process(self): """ try echoing some text and see if it comes back out """ p.send_all(self.shell, "echo hello\n") time.sleep(0.1) @@ -69,7 +69,7 @@ def tearDown(self): def suite(): """ returns all the testcases in this module """ - return TestSuite([AsyncProcessTest()]) + return TestLoaderFiltered().loadTestsFromTestCase(AsyncProcessTest, sys.argv[1:]) if __name__ == '__main__': diff --git a/test/framework/build_log.py b/test/framework/build_log.py index 491d331a64..a1792b998f 100644 --- a/test/framework/build_log.py +++ b/test/framework/build_log.py @@ -264,7 +264,7 @@ def run_check(args, silent=False, expected_stderr='', **kwargs): expected = "\nWARNING: Test log message with a logger involved.\n\n" run_check(["Test log message with a logger involved."], expected_stderr=expected, log=logger) log_txt = read_file(tmp_logfile) - self.assertTrue("WARNING Test log message with a logger involved." in log_txt) + self.assertIn("WARNING Test log message with a logger involved.", log_txt) def test_print_error(self): """Test print_error""" @@ -388,26 +388,26 @@ def test_init_logging(self): tmp_logfile = os.path.join(self.test_prefix, 'test.log') log, logfile = init_logging(tmp_logfile, silent=True) self.assertEqual(logfile, tmp_logfile) - self.assertTrue(os.path.exists(logfile)) - self.assertTrue(isinstance(log, EasyBuildLog)) + self.assertExists(logfile) + self.assertIsInstance(log, EasyBuildLog) stop_logging(logfile) # no log provided, so create one (should be file in $TMPDIR) log, logfile = init_logging(None, silent=True) - self.assertTrue(os.path.exists(logfile)) + self.assertExists(logfile) self.assertEqual(os.path.dirname(logfile), tmpdir) - self.assertTrue(isinstance(log, EasyBuildLog)) + self.assertIsInstance(log, EasyBuildLog) stop_logging(logfile) # no problem with specifying a different directory to put log file in (even if it doesn't exist yet) tmp_logdir = os.path.join(self.test_prefix, 'tmp_logs') - self.assertFalse(os.path.exists(tmp_logdir)) + self.assertNotExists(tmp_logdir) log, logfile = init_logging(None, silent=True, tmp_logdir=tmp_logdir) self.assertEqual(os.path.dirname(logfile), tmp_logdir) - self.assertTrue(isinstance(log, EasyBuildLog)) + self.assertIsInstance(log, EasyBuildLog) stop_logging(logfile) @@ -416,9 +416,9 @@ def test_init_logging(self): log, logfile = init_logging(None) stdout = self.get_stdout() self.mock_stdout(False) - self.assertTrue(os.path.exists(logfile)) + self.assertExists(logfile) self.assertEqual(os.path.dirname(logfile), tmpdir) - self.assertTrue(isinstance(log, EasyBuildLog)) + self.assertIsInstance(log, EasyBuildLog) self.assertTrue(stdout.startswith("== Temporary log file in case of crash")) stop_logging(logfile) @@ -428,7 +428,7 @@ def test_init_logging(self): log, logfile = init_logging(None, logtostdout=True) self.mock_stdout(False) self.assertEqual(logfile, None) - self.assertTrue(isinstance(log, EasyBuildLog)) + self.assertIsInstance(log, EasyBuildLog) stop_logging(logfile, logtostdout=True) diff --git a/test/framework/config.py b/test/framework/config.py index 62feb80d8d..4b42672cfc 100644 --- a/test/framework/config.py +++ b/test/framework/config.py @@ -151,7 +151,7 @@ def test_generaloption_config(self): self.assertEqual(install_path(typ='mod'), os.path.join(install, 'modules')) self.assertEqual(options.installpath, install) - self.assertTrue(config_file in options.configfiles) + self.assertIn(config_file, options.configfiles) # check mixed command line/env var configuration prefix = os.path.join(self.tmpdir, 'test3') @@ -341,8 +341,8 @@ def test_configuration_variables(self): cv1 = ConfigurationVariables() cv2 = ConfigurationVariables() cv3 = ConfigurationVariables({'foo': 'bar'}) # note: argument is ignored, an instance is already available - self.assertTrue(cv1 is cv2) - self.assertTrue(cv1 is cv3) + self.assertIs(cv1, cv2) + self.assertIs(cv1, cv3) def test_build_options(self): """Test usage of BuildOptions.""" @@ -353,8 +353,8 @@ def test_build_options(self): bo1 = BuildOptions() bo2 = BuildOptions() bo3 = BuildOptions({'foo': 'bar'}) # note: argument is ignored, an instance is already available - self.assertTrue(bo1 is bo2) - self.assertTrue(bo1 is bo3) + self.assertIs(bo1, bo2) + self.assertIs(bo1, bo3) # test basic functionality BuildOptions.__class__._instances.clear() @@ -362,7 +362,7 @@ def test_build_options(self): 'debug': False, 'force': True }) - self.assertTrue(not bo['debug']) + self.assertFalse(bo['debug']) self.assertTrue(bo['force']) # updating is impossible (methods are not even available) @@ -394,7 +394,7 @@ def test_build_options(self): # there should be only one BuildOptions instance bo2 = BuildOptions() - self.assertTrue(bo is bo2) + self.assertIs(bo, bo2) def test_XDG_CONFIG_env_vars(self): """Test effect of XDG_CONFIG* environment variables on default configuration.""" @@ -499,7 +499,9 @@ def test_flex_robot_paths(self): # prepend path to test easyconfigs into Python search path, so it gets picked up as --robot-paths default orig_sys_path = sys.path[:] - sys.path = [tmpdir] + [p for p in sys.path if not os.path.exists(os.path.join(p, 'easybuild', 'easyconfigs'))] + sys.path[:] = [tmpdir] + [ + p for p in sys.path if not os.path.exists(os.path.join(p, 'easybuild', 'easyconfigs')) + ] # default: only pick up installed easyconfigs via sys.path eb_go = eboptions.parse_options(args=[]) @@ -618,7 +620,7 @@ def test_get_log_filename(self): res = get_log_filename('foo', '1.2.3', date='19700101', timestamp='094651', add_salt=True) regex = re.compile(os.path.join(tmpdir, r'easybuild-foo-1\.2\.3-19700101\.094651\.[a-zA-Z]{5}\.log$')) self.assertTrue(regex.match(res), "Pattern '%s' matches '%s'" % (regex.pattern, res)) - self.assertTrue(res not in prev_log_filenames) + self.assertNotIn(res, prev_log_filenames) prev_log_filenames.append(res) def test_log_file_format(self): diff --git a/test/framework/containers.py b/test/framework/containers.py index d9754cd2e0..bd6dfde7f6 100644 --- a/test/framework/containers.py +++ b/test/framework/containers.py @@ -212,7 +212,7 @@ def test_end2end_singularity_recipe_config(self): self.assertTrue(txt.startswith(expected), "Container recipe starts with '%s':\n\n%s" % (expected, txt)) # no OS packages are installed by default when starting from an existing image - self.assertFalse("yum install" in txt) + self.assertNotIn("yum install", txt) for pattern in pip_patterns + post_commands_patterns + [eb_pattern]: regex = re.compile('^' + pattern, re.M) @@ -308,7 +308,7 @@ def test_end2end_singularity_image(self): ] self.check_regexs(regexs, stdout) - self.assertTrue(os.path.exists(os.path.join(containerpath, 'toy-0.0.%s' % ext))) + self.assertExists(os.path.join(containerpath, 'toy-0.0.%s' % ext)) remove_file(os.path.join(containerpath, 'Singularity.toy-0.0')) @@ -330,7 +330,7 @@ def test_end2end_singularity_image(self): self.check_regexs(regexs, stdout) cont_img = os.path.join(containerpath, 'foo-bar.img') - self.assertTrue(os.path.exists(cont_img)) + self.assertExists(cont_img) remove_file(os.path.join(containerpath, 'Singularity.foo-bar')) @@ -348,7 +348,7 @@ def test_end2end_singularity_image(self): "WARNING: overwriting existing container image at %s due to --force" % cont_img, ]) self.check_regexs(regexs, stdout) - self.assertTrue(os.path.exists(cont_img)) + self.assertExists(cont_img) # also check behaviour under --extended-dry-run args.append('--extended-dry-run') diff --git a/test/framework/docs.py b/test/framework/docs.py index 7f6cd813b2..146d05b24c 100644 --- a/test/framework/docs.py +++ b/test/framework/docs.py @@ -465,7 +465,7 @@ def test_gen_easyblocks_overview(self): "==================== ================================================================", ]) - self.assertTrue(check_configuremake in ebdoc, "Found '%s' in: %s" % (check_configuremake, ebdoc)) + self.assertIn(check_configuremake, ebdoc) names = [] for mod in modules: @@ -473,7 +473,7 @@ def test_gen_easyblocks_overview(self): eb_class = getattr(mod, name) # skip imported classes that are not easyblocks if eb_class.__module__.startswith(gen_easyblocks_pkg): - self.assertTrue(name in ebdoc) + self.assertIn(name, ebdoc) names.append(name) toc = [":ref:`" + n + "`" for n in sorted(set(names))] @@ -511,7 +511,7 @@ def test_gen_easyblocks_overview(self): "installopts |Extra options for installation", ]) - self.assertTrue(check_configuremake in ebdoc, "Found '%s' in: %s" % (check_configuremake, ebdoc)) + self.assertIn(check_configuremake, ebdoc) names = [] for mod in modules: @@ -519,7 +519,7 @@ def test_gen_easyblocks_overview(self): eb_class = getattr(mod, name) # skip imported classes that are not easyblocks if eb_class.__module__.startswith(gen_easyblocks_pkg): - self.assertTrue(name in ebdoc) + self.assertIn(name, ebdoc) names.append(name) toc = ["\\[" + n + "\\]\\(#" + n.lower() + "\\)" for n in sorted(set(names))] @@ -531,7 +531,7 @@ def test_license_docs(self): """Test license_documentation function.""" lic_docs = avail_easyconfig_licenses(output_format='txt') gplv3 = "GPLv3: The GNU General Public License" - self.assertTrue(gplv3 in lic_docs, "%s found in: %s" % (gplv3, lic_docs)) + self.assertIn(gplv3, lic_docs) lic_docs = avail_easyconfig_licenses(output_format='rst') regex = re.compile(r"^``GPLv3``\s*The GNU General Public License", re.M) @@ -804,7 +804,7 @@ def test_avail_easyconfig_templates(self): r"^## Template names/values derived from easyconfig instance", r"^``%\(version_major\)s``\s+|Major version", r"^## Template names/values for \(short\) software versions", - r"^``%\(pyshortver\)s``\s+|short version for Python \(\.\)", + r"^``%\(pyshortver\)s``\s+|short version for Python \(``\.``\)", r"^## Template constants that can be used in easyconfigs", r"^``SOURCE_TAR_GZ``\s+|Source \.tar\.gz bundle \(%\(name\)s-%\(version\)s.tar.gz\)", ] diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 3950170a0c..018c3e293a 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -38,6 +38,7 @@ from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config from unittest import TextTestRunner +import easybuild.tools.systemtools as st from easybuild.framework.easyblock import EasyBlock, get_easyblock_instance from easybuild.framework.easyconfig import CUSTOM from easybuild.framework.easyconfig.easyconfig import EasyConfig @@ -46,7 +47,7 @@ from easybuild.tools import LooseVersion, config from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import get_module_syntax, update_build_option -from easybuild.tools.filetools import change_dir, copy_dir, copy_file, mkdir, read_file, remove_file +from easybuild.tools.filetools import change_dir, copy_dir, copy_file, mkdir, read_file, remove_dir, remove_file from easybuild.tools.filetools import verify_checksum, write_file from easybuild.tools.module_generator import module_generator from easybuild.tools.modules import EnvironmentModules, Lmod, reset_module_caches @@ -85,13 +86,13 @@ def check_extra_options_format(extra_options): """Make sure extra_options value is of correct format.""" # EasyBuild v2.0: dict with keys and values # (breaks backward compatibility compared to v1.x) - self.assertTrue(isinstance(extra_options, dict)) # conversion to a dict works + self.assertIsInstance(extra_options, dict) # conversion to a dict works extra_options.items() extra_options.keys() extra_options.values() for key in extra_options.keys(): - self.assertTrue(isinstance(extra_options[key], list)) - self.assertTrue(len(extra_options[key]), 3) + self.assertIsInstance(extra_options[key], list) + self.assertEqual(len(extra_options[key]), 3) name = "pi" version = "3.14" @@ -127,7 +128,7 @@ def check_extra_options_format(extra_options): self.assertEqual(exeb1.cfg['name'], 'foo') extra_options = exeb1.extra_options() check_extra_options_format(extra_options) - self.assertTrue('options' in extra_options) + self.assertIn('options', extra_options) # Reporting test failure should work also for the extension EB self.assertRaises(EasyBuildError, exeb1.report_test_failure, "Fails") @@ -137,7 +138,7 @@ def check_extra_options_format(extra_options): self.assertEqual(exeb2.cfg['version'], '3.14') extra_options = exeb2.extra_options() check_extra_options_format(extra_options) - self.assertTrue('options' in extra_options) + self.assertIn('options', extra_options) # Reporting test failure should work also for the extension EB self.assertRaises(EasyBuildError, exeb2.report_test_failure, "Fails") @@ -149,7 +150,7 @@ def extra_options(): self.assertEqual(texeb.cfg['name'], 'bar') extra_options = texeb.extra_options() check_extra_options_format(extra_options) - self.assertTrue('options' in extra_options) + self.assertIn('options', extra_options) self.assertEqual(extra_options['extra_param'], [None, "help", CUSTOM]) # cleanup @@ -201,7 +202,7 @@ def test_load_module(self): # we expect $TMPDIR to be tweaked by the prepare step (OpenMPI 2.x doesn't like long $TMPDIR values) tweaked_tmpdir = os.environ.get('TMPDIR') - self.assertTrue(tweaked_tmpdir != orig_tmpdir) + self.assertNotEqual(tweaked_tmpdir, orig_tmpdir) eb.make_module_step() eb.load_module() @@ -233,7 +234,7 @@ def test_fake_module_load(self): if get_module_syntax() == 'Lua': pi_modfile += '.lua' - self.assertTrue(os.path.exists(pi_modfile)) + self.assertExists(pi_modfile) # check whether temporary module file is marked as default if get_module_syntax() == 'Lua': @@ -241,7 +242,7 @@ def test_fake_module_load(self): self.assertTrue(os.path.samefile(default_symlink, pi_modfile)) else: dot_version_txt = read_file(os.path.join(fake_mod_data[0], 'pi', '.version')) - self.assertTrue("set ModulesVersion 3.14" in dot_version_txt) + self.assertIn("set ModulesVersion 3.14", dot_version_txt) eb.clean_up_fake_module(fake_mod_data) @@ -304,7 +305,7 @@ def test_make_module_extend_modpath(self): r'\s+prepend_path\("MODULEPATH", pathJoin\(%s, %s\)\)' % (home, pj_usermodsdir), ]) else: - self.assertTrue(False, "Unknown module syntax: %s" % module_syntax) + self.fail("Unknown module syntax: %s" % module_syntax) for regex in regexs: regex = re.compile(regex, re.M) @@ -349,7 +350,7 @@ def test_make_module_extend_modpath(self): r'\s+prepend_path\("MODULEPATH", pathJoin\(%s, %s\)\)' % (module_envvar, pj_usermodsdir), ]) else: - self.assertTrue(False, "Unknown module syntax: %s" % module_syntax) + self.fail("Unknown module syntax: %s" % module_syntax) for regex in regexs: regex = re.compile(regex, re.M) @@ -386,7 +387,7 @@ def test_make_module_extend_modpath(self): mkdir(os.path.join(config.install_path(), "existing_dir", usermodsdir_extension), parents=True) change_dir(os.path.join(config.install_path(), "existing_dir")) self.modtool.run_module('load', 'mytest') - self.assertFalse(usermodsdir_extension in os.environ['MODULEPATH']) + self.assertNotIn(usermodsdir_extension, os.environ['MODULEPATH']) self.modtool.run_module('unload', 'mytest') change_dir(cwd) @@ -396,14 +397,14 @@ def test_make_module_extend_modpath(self): # Check MODULEPATH when neither directories exist self.modtool.run_module('load', 'mytest') - self.assertFalse(site_modules in os.environ['MODULEPATH']) - self.assertFalse(user_modules in os.environ['MODULEPATH']) + self.assertNotIn(site_modules, os.environ['MODULEPATH']) + self.assertNotIn(user_modules, os.environ['MODULEPATH']) self.modtool.run_module('unload', 'mytest') # Now create the directory for site modules mkdir(site_modules, parents=True) self.modtool.run_module('load', 'mytest') self.assertTrue(os.environ['MODULEPATH'].startswith(site_modules)) - self.assertFalse(user_modules in os.environ['MODULEPATH']) + self.assertNotIn(user_modules, os.environ['MODULEPATH']) self.modtool.run_module('unload', 'mytest') # Now create the directory for user modules mkdir(user_modules, parents=True) @@ -453,14 +454,14 @@ def test_make_module_req(self): self.assertTrue(re.search(r'^prepend_path\("CLASSPATH", pathJoin\(root, "bla.jar"\)\)$', guess, re.M)) self.assertTrue(re.search(r'^prepend_path\("CLASSPATH", pathJoin\(root, "foo.jar"\)\)$', guess, re.M)) self.assertTrue(re.search(r'^prepend_path\("MANPATH", pathJoin\(root, "share/man"\)\)$', guess, re.M)) - self.assertTrue('prepend_path("CMAKE_PREFIX_PATH", root)' in guess) + self.assertIn('prepend_path("CMAKE_PREFIX_PATH", root)', guess) # bin/ is not added to $PATH if it doesn't include files self.assertFalse(re.search(r'^prepend_path\("PATH", pathJoin\(root, "bin"\)\)$', guess, re.M)) self.assertFalse(re.search(r'^prepend_path\("PATH", pathJoin\(root, "sbin"\)\)$', guess, re.M)) # no include/ subdirectory, so no $CPATH update statement self.assertFalse(re.search(r'^prepend_path\("CPATH", .*\)$', guess, re.M)) else: - self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) + self.fail("Unknown module syntax: %s" % get_module_syntax()) # check that bin is only added to PATH if there are files in there write_file(os.path.join(eb.installdir, 'bin', 'test'), 'test') @@ -473,7 +474,7 @@ def test_make_module_req(self): self.assertTrue(re.search(r'^prepend_path\("PATH", pathJoin\(root, "bin"\)\)$', guess, re.M)) self.assertFalse(re.search(r'^prepend_path\("PATH", pathJoin\(root, "sbin"\)\)$', guess, re.M)) else: - self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) + self.fail("Unknown module syntax: %s" % get_module_syntax()) # Check that lib64 is only added to CMAKE_LIBRARY_PATH if there are files in there # but only if it is not a symlink to lib @@ -481,7 +482,7 @@ def test_make_module_req(self): if get_module_syntax() == 'Tcl': self.assertFalse(re.search(r"^prepend-path\s+CMAKE_LIBRARY_PATH\s+\$root/lib64$", guess, re.M)) elif get_module_syntax() == 'Lua': - self.assertFalse('prepend_path("CMAKE_LIBRARY_PATH", pathJoin(root, "lib64"))' in guess) + self.assertNotIn('prepend_path("CMAKE_LIBRARY_PATH", pathJoin(root, "lib64"))', guess) # -- With files write_file(os.path.join(eb.installdir, 'lib64', 'libfoo.so'), 'test') with eb.module_generator.start_module_creation(): @@ -489,7 +490,7 @@ def test_make_module_req(self): if get_module_syntax() == 'Tcl': self.assertTrue(re.search(r"^prepend-path\s+CMAKE_LIBRARY_PATH\s+\$root/lib64$", guess, re.M)) elif get_module_syntax() == 'Lua': - self.assertTrue('prepend_path("CMAKE_LIBRARY_PATH", pathJoin(root, "lib64"))' in guess) + self.assertIn('prepend_path("CMAKE_LIBRARY_PATH", pathJoin(root, "lib64"))', guess) # -- With files in lib and lib64 symlinks to lib write_file(os.path.join(eb.installdir, 'lib', 'libfoo.so'), 'test') shutil.rmtree(os.path.join(eb.installdir, 'lib64')) @@ -499,7 +500,7 @@ def test_make_module_req(self): if get_module_syntax() == 'Tcl': self.assertFalse(re.search(r"^prepend-path\s+CMAKE_LIBRARY_PATH\s+\$root/lib64$", guess, re.M)) elif get_module_syntax() == 'Lua': - self.assertFalse('prepend_path("CMAKE_LIBRARY_PATH", pathJoin(root, "lib64"))' in guess) + self.assertNotIn('prepend_path("CMAKE_LIBRARY_PATH", pathJoin(root, "lib64"))', guess) # With files in /lib and /lib64 symlinked to /lib there should be exactly 1 entry for (LD_)LIBRARY_PATH # pointing to /lib @@ -521,7 +522,7 @@ def test_make_module_req(self): elif get_module_syntax() == 'Lua': self.assertTrue(re.match(r'^\nprepend_path\("PATH", pathJoin\(root, "bin"\)\)\n$', txt, re.M)) else: - self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) + self.fail("Unknown module syntax: %s" % get_module_syntax()) # check for correct behaviour if empty string is specified as one of the values # prepend-path statements should be included for both the 'bin' subdir and the install root @@ -535,7 +536,7 @@ def test_make_module_req(self): self.assertTrue(re.search(r'\nprepend_path\("PATH", pathJoin\(root, "bin"\)\)\n', txt, re.M)) self.assertTrue(re.search(r'\nprepend_path\("PATH", root\)\n', txt, re.M)) else: - self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) + self.fail("Unknown module syntax: %s" % get_module_syntax()) # check for correct order of prepend statements when providing a list (and that no duplicates are allowed) eb.make_module_req_guess = lambda: {'LD_LIBRARY_PATH': ['lib/pathC', 'lib/pathA', 'lib/pathB', 'lib/pathA']} @@ -561,7 +562,7 @@ def test_make_module_req(self): r'prepend_path\("LD_LIBRARY_PATH", pathJoin\(root, "lib/pathA"\)\)\n', txt, re.M)) else: - self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) + self.fail("Unknown module syntax: %s" % get_module_syntax()) # If PATH or LD_LIBRARY_PATH contain only folders, do not add an entry sub_lib_path = os.path.join('lib', 'path_folders') @@ -629,7 +630,7 @@ def test_make_module_extra(self): r'setenv\("EBDEVELPI", pathJoin\(root, "easybuild/pi-3.14-gompi-2018a-easybuild-devel"\)\)', ])) else: - self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) + self.fail("Unknown module syntax: %s" % get_module_syntax()) defaulttxt = eb.make_module_extra().strip() self.assertTrue(expected_default.match(defaulttxt), @@ -667,7 +668,7 @@ def test_make_module_extra(self): if get_module_syntax() == 'Lua': modpath += '.lua' - self.assertTrue(os.path.exists(modpath), "%s exists" % modpath) + self.assertExists(modpath) txt = read_file(modpath) patterns = [ r"^prepend[-_]path.*TEST_PATH_VAR.*root.*foo", @@ -714,7 +715,7 @@ def test_make_module_deppaths(self): 'end', ]) else: - self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) + self.fail("Unknown module syntax: %s" % get_module_syntax()) expected = use_load self.assertEqual(eb.make_module_deppaths().strip(), expected) @@ -776,7 +777,7 @@ def test_make_module_dep(self): 'end', ]) else: - self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) + self.fail("Unknown module syntax: %s" % get_module_syntax()) expected = tc_load + '\n\n' + fftw_load + '\n\n' + lapack_load self.assertEqual(eb.make_module_dep().strip(), expected) @@ -801,7 +802,7 @@ def test_make_module_dep(self): 'end', ]) else: - self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) + self.fail("Unknown module syntax: %s" % get_module_syntax()) expected = tc_load + '\n\n' + fftw_load + '\n\n' + lapack_load self.assertEqual(eb.make_module_dep(unload_info=unload_info).strip(), expected) @@ -1134,14 +1135,12 @@ def test_skip_extensions_step(self): # 'ext1' should be in eb.ext_instances eb_exts = [x.name for x in eb.ext_instances] - self.assertTrue('ext1' in eb_exts) + self.assertIn('ext1', eb_exts) # 'EXT-2' should not - self.assertFalse('EXT-2' in eb_exts) - self.assertFalse('EXT_2' in eb_exts) - self.assertFalse('ext-2' in eb_exts) - self.assertFalse('ext_2' in eb_exts) + self.assertNotIn('EXT-2', eb_exts) + self.assertNotIn('ext_2', eb_exts) # 'ext3' should not - self.assertFalse('ext3' in eb_exts) + self.assertNotIn('ext3', eb_exts) # cleanup eb.close_log() @@ -1204,7 +1203,7 @@ def test_make_module_step(self): modpath = os.path.join(eb.make_module_step(), name, version) if get_module_syntax() == 'Lua': modpath += '.lua' - self.assertTrue(os.path.exists(modpath), "%s exists" % modpath) + self.assertExists(modpath) # verify contents of module txt = read_file(modpath) @@ -1223,7 +1222,7 @@ def test_make_module_step(self): self.assertTrue(re.search(r'^setenv\("EBVERSION%s", "%s"\)$' % (name.upper(), version), txt, re.M)) else: - self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) + self.fail("Unknown module syntax: %s" % get_module_syntax()) for (key, val) in modextravars.items(): pushenv = False @@ -1238,7 +1237,7 @@ def test_make_module_step(self): elif get_module_syntax() == 'Lua': regex = re.compile(r'^%s\("%s", "%s"\)$' % (env_setter, key, val), re.M) else: - self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) + self.fail("Unknown module syntax: %s" % get_module_syntax()) self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt)) for (key, vals) in modextrapaths.items(): @@ -1250,7 +1249,7 @@ def test_make_module_step(self): elif get_module_syntax() == 'Lua': regex = re.compile(r'^prepend_path\("%s", pathJoin\(root, "%s"\)\)$' % (key, val), re.M) else: - self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) + self.fail("Unknown module syntax: %s" % get_module_syntax()) self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt)) # Check for duplicates num_prepends = len(regex.findall(txt)) @@ -1262,7 +1261,7 @@ def test_make_module_step(self): elif get_module_syntax() == 'Lua': regex = re.compile(r'^\s*load\("%s"\)$' % os.path.join(name, ver), re.M) else: - self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) + self.fail("Unknown module syntax: %s" % get_module_syntax()) self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt)) for (name, ver) in [('test', '1.2.3')]: @@ -1271,7 +1270,7 @@ def test_make_module_step(self): elif get_module_syntax() == 'Lua': regex = re.compile(r'^\s*load\("%s/.%s"\)$' % (name, ver), re.M) else: - self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) + self.fail("Unknown module syntax: %s" % get_module_syntax()) self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt)) for (name, ver) in [('OpenMPI', '2.1.2-GCC-6.4.0-2.28')]: @@ -1280,7 +1279,7 @@ def test_make_module_step(self): elif get_module_syntax() == 'Lua': regex = re.compile(r'^\s*load\("%s/.?%s"\)$' % (name, ver), re.M) else: - self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) + self.fail("Unknown module syntax: %s" % get_module_syntax()) self.assertFalse(regex.search(txt), "Pattern '%s' *not* found in %s" % (regex.pattern, txt)) os.environ['TEST_PUSHENV'] = '0' @@ -1375,13 +1374,13 @@ def test_make_builddir(self): builddir = eb.builddir testfile = os.path.join(builddir, 'test123', 'foobar.txt') write_file(testfile, 'test123') - self.assertTrue(os.path.exists(testfile)) + self.assertExists(testfile) eb.make_builddir() self.assertEqual(builddir, eb.builddir) # file is gone because directory was removed and re-created - self.assertFalse(os.path.exists(testfile)) - self.assertFalse(os.path.exists(os.path.dirname(testfile))) + self.assertNotExists(testfile) + self.assertNotExists(os.path.dirname(testfile)) self.assertEqual(os.listdir(eb.builddir), []) # make sure that build directory does *not* get re-created when we're building in installation directory @@ -1393,7 +1392,7 @@ def test_make_builddir(self): builddir = eb.builddir testfile = os.path.join(builddir, 'test123', 'foobar.txt') write_file(testfile, 'test123') - self.assertTrue(os.path.exists(testfile)) + self.assertExists(testfile) self.assertEqual(os.listdir(eb.builddir), ['test123']) self.assertEqual(os.listdir(os.path.join(eb.builddir, 'test123')), ['foobar.txt']) @@ -1402,7 +1401,7 @@ def test_make_builddir(self): eb.make_builddir() eb.make_installdir() self.assertEqual(builddir, eb.builddir) - self.assertTrue(os.path.exists(testfile)) + self.assertExists(testfile) self.assertEqual(os.listdir(eb.builddir), ['test123']) self.assertEqual(os.listdir(os.path.join(eb.builddir, 'test123')), ['foobar.txt']) @@ -1411,8 +1410,8 @@ def test_make_builddir(self): eb.make_builddir() eb.make_installdir() self.assertEqual(builddir, eb.builddir) - self.assertFalse(os.path.exists(testfile)) - self.assertFalse(os.path.exists(os.path.dirname(testfile))) + self.assertNotExists(testfile) + self.assertNotExists(os.path.dirname(testfile)) self.assertEqual(os.listdir(eb.builddir), []) def test_get_easyblock_instance(self): @@ -1422,7 +1421,7 @@ def test_get_easyblock_instance(self): ec = process_easyconfig(os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb'))[0] eb = get_easyblock_instance(ec) - self.assertTrue(isinstance(eb, EB_toy)) + self.assertIsInstance(eb, EB_toy) # check whether 'This is easyblock' log message is there tup = ('EB_toy', 'easybuild.easyblocks.toy', '.*test/framework/sandbox/easybuild/easyblocks/t/toy.pyc*') @@ -1484,9 +1483,9 @@ def test_fetch_sources(self): self.assertEqual(len(eb.src), 3) for idx in range(3): self.assertEqual(eb.src[idx]['name'], expected_sources[idx]) - self.assertTrue(os.path.exists(eb.src[idx]['path'])) + self.assertExists(eb.src[idx]['path']) source_loc = os.path.join(toy_source_dir, expected_sources[idx]) - self.assertTrue(os.path.exists(source_loc)) + self.assertExists(source_loc) self.assertTrue(os.path.samefile(eb.src[idx]['path'], source_loc)) self.assertEqual(eb.src[0]['cmd'], None) self.assertEqual(eb.src[1]['cmd'], "gunzip %s") @@ -1637,7 +1636,7 @@ def test_fetch_patches(self): eb.fetch_patches() self.assertEqual(len(eb.patches), 2) self.assertEqual(eb.patches[0]['name'], toy_patch) - self.assertFalse('level' in eb.patches[0]) + self.assertNotIn('level', eb.patches[0]) # reset eb.patches = [] @@ -1752,7 +1751,7 @@ def test_obtain_file(self): # toy tarball was indeed re-downloaded to tmpdir self.assertEqual(res, os.path.join(tmpdir, 't', 'toy', toy_tarball)) - self.assertTrue(os.path.exists(os.path.join(tmpdir, 't', 'toy', toy_tarball))) + self.assertExists(os.path.join(tmpdir, 't', 'toy', toy_tarball)) # obtain_file yields error for non-existing files fn = 'thisisclearlyanonexistingfile' @@ -1785,7 +1784,7 @@ def test_obtain_file(self): if res is not None: loc = os.path.join(tmpdir, 't', 'toy', fn) self.assertEqual(res, loc) - self.assertTrue(os.path.exists(loc), "%s file is found at %s" % (fn, loc)) + self.assertExists(loc) txt = read_file(loc) eb_regex = re.compile("EasyBuild: building software with ease") self.assertTrue(eb_regex.search(txt), "Pattern '%s' found in: %s" % (eb_regex.pattern, txt)) @@ -1835,7 +1834,7 @@ def test_collect_exts_file_info(self): exts_file_info = toy_eb.collect_exts_file_info() - self.assertTrue(isinstance(exts_file_info, list)) + self.assertIsInstance(exts_file_info, list) self.assertEqual(len(exts_file_info), 4) self.assertEqual(exts_file_info[0], {'name': 'ls'}) @@ -1851,34 +1850,34 @@ def test_collect_exts_file_info(self): self.assertEqual(exts_file_info[2]['name'], 'barbar') self.assertEqual(exts_file_info[2]['src'], os.path.join(toy_ext_sources, 'barbar-0.0.tar.gz')) - self.assertFalse('patches' in exts_file_info[2]) + self.assertNotIn('patches', exts_file_info[2]) self.assertEqual(exts_file_info[3]['name'], 'toy') self.assertEqual(exts_file_info[3]['src'], os.path.join(toy_sources, 'toy-0.0.tar.gz')) - self.assertFalse('patches' in exts_file_info[3]) + self.assertNotIn('patches', exts_file_info[3]) # location of files is missing when fetch_files is set to False exts_file_info = toy_eb.collect_exts_file_info(fetch_files=False, verify_checksums=False) - self.assertTrue(isinstance(exts_file_info, list)) + self.assertIsInstance(exts_file_info, list) self.assertEqual(len(exts_file_info), 4) self.assertEqual(exts_file_info[0], {'name': 'ls'}) self.assertEqual(exts_file_info[1]['name'], 'bar') - self.assertFalse('src' in exts_file_info[1]) + self.assertNotIn('src', exts_file_info[1]) self.assertEqual(exts_file_info[1]['patches'][0]['name'], bar_patch1) - self.assertFalse('path' in exts_file_info[1]['patches'][0]) + self.assertNotIn('path', exts_file_info[1]['patches'][0]) self.assertEqual(exts_file_info[1]['patches'][1]['name'], bar_patch2) - self.assertFalse('path' in exts_file_info[1]['patches'][1]) + self.assertNotIn('path', exts_file_info[1]['patches'][1]) self.assertEqual(exts_file_info[2]['name'], 'barbar') - self.assertFalse('src' in exts_file_info[2]) - self.assertFalse('patches' in exts_file_info[2]) + self.assertNotIn('src', exts_file_info[2]) + self.assertNotIn('patches', exts_file_info[2]) self.assertEqual(exts_file_info[3]['name'], 'toy') - self.assertFalse('src' in exts_file_info[3]) - self.assertFalse('patches' in exts_file_info[3]) + self.assertNotIn('src', exts_file_info[3]) + self.assertNotIn('patches', exts_file_info[3]) error_msg = "Can't verify checksums for extension files if they are not being fetched" self.assertErrorRegex(EasyBuildError, error_msg, toy_eb.collect_exts_file_info, fetch_files=False) @@ -1899,7 +1898,7 @@ def test_obtain_file_extension(self): ext = ExtensionEasyBlock(toy_eb, test_ext) ext_src_path = ext.obtain_file(test_ext_src_fn) self.assertEqual(os.path.basename(ext_src_path), 'toy-0.0.tar.gz') - self.assertTrue(os.path.exists(ext_src_path)) + self.assertExists(ext_src_path) def test_check_readiness(self): """Test check_readiness method.""" @@ -1977,13 +1976,13 @@ def test_exclude_path_to_top_of_module_tree(self): elif get_module_syntax() == 'Lua': self.assertFalse(re.search('load("%s")' % dep, modtxt), failmsg) else: - self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) + self.fail("Unknown module syntax: %s" % get_module_syntax()) # modpath_extensions_for should spit out correct result, even if modules are loaded icc_mod = 'icc/%s' % intel_ver impi_mod = 'impi/5.1.2.150' self.modtool.load([icc_mod]) - self.assertTrue(impi_modfile_path in self.modtool.show(impi_mod)) + self.assertIn(impi_modfile_path, self.modtool.show(impi_mod)) self.modtool.load([impi_mod]) expected = { icc_mod: [os.path.join(modpath, 'Compiler', 'intel', intel_ver)], @@ -2019,7 +2018,7 @@ def test_patch_step(self): # verify that patches were applied toydir = os.path.join(eb.builddir, 'toy-0.0') self.assertEqual(sorted(os.listdir(toydir)), ['toy-extra.txt', 'toy.source']) - self.assertTrue("and very proud of it" in read_file(os.path.join(toydir, 'toy.source'))) + self.assertIn("and very proud of it", read_file(os.path.join(toydir, 'toy.source'))) self.assertEqual(read_file(os.path.join(toydir, 'toy-extra.txt')), 'moar!\n') # check again with backup of patched files enabled @@ -2031,8 +2030,8 @@ def test_patch_step(self): # verify that patches were applied toydir = os.path.join(eb.builddir, 'toy-0.0') self.assertEqual(sorted(os.listdir(toydir)), ['toy-extra.txt', 'toy.source', 'toy.source.orig']) - self.assertTrue("and very proud of it" in read_file(os.path.join(toydir, 'toy.source'))) - self.assertFalse("and very proud of it" in read_file(os.path.join(toydir, 'toy.source.orig'))) + self.assertIn("and very proud of it", read_file(os.path.join(toydir, 'toy.source'))) + self.assertNotIn("and very proud of it", read_file(os.path.join(toydir, 'toy.source.orig'))) self.assertEqual(read_file(os.path.join(toydir, 'toy-extra.txt')), 'moar!\n') def test_extensions_sanity_check(self): @@ -2044,6 +2043,7 @@ def test_extensions_sanity_check(self): # Do this before loading the easyblock to check the non-translated output below os.environ['LC_ALL'] = 'C' + os.environ['LANG'] = 'C' # this import only works here, since EB_toy is a test easyblock from easybuild.easyblocks.toy import EB_toy @@ -2059,7 +2059,7 @@ def test_extensions_sanity_check(self): eb.silent = True error_pattern = r"Sanity check failed: extensions sanity check failed for 1 extensions: toy\n" error_pattern += r"failing sanity check for 'toy' extension: " - error_pattern += r'command "thisshouldfail" failed; output:\n/bin/bash: thisshouldfail: command not found' + error_pattern += r'command "thisshouldfail" failed; output:\n/bin/bash:.* thisshouldfail: command not found' self.assertErrorRegex(EasyBuildError, error_pattern, eb.run_all_steps, True) # purposely put sanity check command in place that breaks the build, @@ -2136,7 +2136,7 @@ def test_guess_start_dir(self): ec = process_easyconfig(os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb'))[0] cwd = os.getcwd() - self.assertTrue(os.path.exists(cwd)) + self.assertExists(cwd) def check_start_dir(expected_start_dir): """Check start dir.""" @@ -2165,6 +2165,105 @@ def check_start_dir(expected_start_dir): err_pattern = "Specified start dir .*/toy-0.0/thisstartdirisnotthere does not exist" self.assertErrorRegex(EasyBuildError, err_pattern, check_start_dir, 'whatever') + def test_extension_set_start_dir(self): + """Test start dir with extensions.""" + test_easyconfigs = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') + ec = process_easyconfig(os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb'))[0] + + cwd = os.getcwd() + self.assertExists(cwd) + + def check_ext_start_dir(expected_start_dir, unpack_src=True): + """Check start dir.""" + # make sure we're in an existing directory at the start + change_dir(cwd) + eb = EasyBlock(ec['ec']) + eb.extensions_step(fetch=True, install=False) + # extract sources of the extension + ext = eb.ext_instances[-1] + ext.run(unpack_src=unpack_src) + + if expected_start_dir is None: + self.assertIsNone(ext.start_dir) + else: + self.assertTrue(os.path.isabs(ext.start_dir)) + if ext.start_dir != os.sep: + self.assertFalse(ext.start_dir.endswith(os.sep)) + if os.path.isabs(expected_start_dir): + abs_expected_start_dir = expected_start_dir + else: + abs_expected_start_dir = os.path.join(eb.builddir, expected_start_dir) + self.assertEqual(ext.start_dir, abs_expected_start_dir) + if not os.path.exists(eb.builddir): + eb.make_builddir() # Required to exist for samefile + self.assertTrue(os.path.samefile(ext.start_dir, abs_expected_start_dir)) + if unpack_src: + self.assertTrue(os.path.samefile(os.getcwd(), abs_expected_start_dir)) + else: + # When not unpacking we don't change the CWD + self.assertEqual(os.getcwd(), cwd) + remove_dir(eb.builddir) + + ec['ec']['exts_defaultclass'] = 'DummyExtension' + + # default (no start_dir specified): use unpacked dir as start dir + ec['ec']['exts_list'] = [ + ('barbar', '0.0', {}), + ] + with self.mocked_stdout_stderr(): + check_ext_start_dir('barbar/barbar-0.0') + check_ext_start_dir(None, unpack_src=False) + self.assertFalse(self.get_stderr()) + + # use start dir defined in extension + ec['ec']['exts_list'] = [ + ('barbar', '0.0', { + 'start_dir': 'src'}), + ] + with self.mocked_stdout_stderr(): + check_ext_start_dir('barbar/barbar-0.0/src') + self.assertFalse(self.get_stderr()) + + # clean error when specified start dir does not exist + ec['ec']['exts_list'] = [ + ('barbar', '0.0', { + 'start_dir': 'nonexistingdir'}), + ] + with self.mocked_stdout_stderr(): + err_pattern = "Failed to change from .*barbar/barbar-0.0 to nonexistingdir.*" + self.assertErrorRegex(EasyBuildError, err_pattern, check_ext_start_dir, 'whatever') + stderr = self.get_stderr() + warning_pattern = "WARNING: Provided start dir (nonexistingdir) for extension barbar does not exist" + self.assertIn(warning_pattern, stderr) + + # No error when using relative path in non-extracted source for some reason + ec['ec']['exts_list'] = [ + ('barbar', '0.0', { + 'start_dir': '.'}), # The build directory which does exist + ] + with self.mocked_stdout_stderr(): + check_ext_start_dir('.', unpack_src=False) + self.assertFalse(self.get_stderr()) + + # Keep absolute path in start_dir + assert os.path.isabs(self.test_prefix) + ec['ec']['exts_list'] = [ + ('barbar', '0.0', { + 'start_dir': self.test_prefix}), + ] + with self.mocked_stdout_stderr(): + check_ext_start_dir(self.test_prefix, unpack_src=False) + self.assertFalse(self.get_stderr()) + + # Support / (absolute path) if explicitely requested + ec['ec']['exts_list'] = [ + ('barbar', '0.0', { + 'start_dir': os.sep}), + ] + with self.mocked_stdout_stderr(): + check_ext_start_dir(os.sep, unpack_src=False) + self.assertFalse(self.get_stderr()) + def test_prepare_step(self): """Test prepare step (setting up build environment).""" test_easyconfigs = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') @@ -2238,7 +2337,7 @@ def test_prepare_step_hmns(self): # see also https://github.com/easybuilders/easybuild-framework/issues/2186 self.setup_hierarchical_modules() - self.assertTrue('GCC/6.4.0-2.28' in self.modtool.available()) + self.assertIn('GCC/6.4.0-2.28', self.modtool.available()) self.reset_modulepath([]) self.assertEqual(os.environ.get('MODULEPATH'), None) @@ -2369,7 +2468,7 @@ def test_checksum_step(self): elif isinstance(ext, tuple): self.assertEqual(ext[2].get('checksums', []), []) else: - self.assertTrue(False, "Incorrect extension type: %s" % type(ext)) + self.fail("Incorrect extension type: %s" % type(ext)) # put checksums.json in place next to easyconfig file being used for the tests toy_checksums_json = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'checksums.json') @@ -2530,6 +2629,15 @@ def run_checks(): # sources can also have dict entries eb.cfg['sources'] = [{'filename': 'toy-0.0.tar.gz', 'download_fileame': 'toy.tar.gz'}] self.assertEqual(eb.check_checksums(), []) + # Same in extensions: Single source as dict, checksum as string + eb.cfg['exts_list'] = [( + 'toy-ext', '42', + { + 'sources': {'filename': 'toy-ext.tar.gz'}, + 'checksums': '81a3accc894592152f81814fbf133d39afad52885ab52c25018722c7bda92487', + } + )] + self.assertEqual(eb.check_checksums(), []) # no checksums in easyconfig, then picked up from checksums.json next to easyconfig file test_ec = os.path.join(self.test_prefix, 'test.eb') @@ -2551,8 +2659,8 @@ def run_checks(): def test_this_is_easybuild(self): """Test 'this_is_easybuild' function (and get_git_revision function used by it).""" # make sure both return a non-Unicode string - self.assertTrue(isinstance(get_git_revision(), str)) - self.assertTrue(isinstance(this_is_easybuild(), str)) + self.assertIsInstance(get_git_revision(), str) + self.assertIsInstance(this_is_easybuild(), str) def test_stale_module_caches(self): """Test whether module caches are reset between builds.""" @@ -2615,7 +2723,7 @@ def test_avail_easyblocks(self): self.assertTrue(all(key.startswith('easybuild.easyblocks') for key in easyblocks)) for modname in ['foo', 'generic.bar', 'toy', 'gcc', 'hpl']: - self.assertTrue('easybuild.easyblocks.%s' % modname in easyblocks) + self.assertIn('easybuild.easyblocks.%s' % modname, easyblocks) foo = easyblocks['easybuild.easyblocks.foo'] self.assertEqual(foo['class'], 'EB_foo') @@ -2637,6 +2745,32 @@ def test_avail_easyblocks(self): self.assertEqual(hpl['class'], 'EB_HPL') self.assertTrue(hpl['loc'].endswith('sandbox/easybuild/easyblocks/h/hpl.py')) + def test_arch_specific_sanity_check(self): + """Tests that the correct version is chosen for this architecture""" + + my_arch = st.get_cpu_architecture() + + self.contents = '\n'.join([ + 'easyblock = "ConfigureMake"', + 'name = "test"', + 'version = "0.2"', + 'homepage = "https://example.com"', + 'description = "test"', + 'toolchain = SYSTEM', + 'sanity_check_paths = {', + " 'files': [{'arch=%s': 'correct.a'}, 'default.a']," % my_arch, + " 'dirs': [{'arch=%s': ('correct', 'alternative')}, {'arch=no-arch': 'not-used'}]," % my_arch, + '}', + ]) + self.writeEC() + ec = EasyConfig(self.eb_file) + eb = EasyBlock(ec) + paths, _, _ = eb._sanity_check_step_common(None, None) + + self.assertEqual(set(paths.keys()), set(('files', 'dirs'))) + self.assertEqual(paths['files'], ['correct.a', 'default.a']) + self.assertEqual(paths['dirs'], [('correct', 'alternative')]) + def test_sanity_check_paths_verification(self): """Test verification of sanity_check_paths w.r.t. keys & values.""" diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index c111b9e5a2..f09a0896a6 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -75,7 +75,8 @@ from easybuild.tools.py2vs3 import OrderedDict, reload from easybuild.tools.robot import resolve_dependencies from easybuild.tools.systemtools import AARCH64, KNOWN_ARCH_CONSTANTS, POWER, X86_64 -from easybuild.tools.systemtools import get_cpu_architecture, get_shared_lib_ext +from easybuild.tools.systemtools import get_cpu_architecture, get_shared_lib_ext, get_os_name, get_os_version + from easybuild.tools.toolchain.utilities import search_toolchain from easybuild.tools.utilities import quote_str, quote_py_str from test.framework.github import GITHUB_TEST_ACCOUNT @@ -231,7 +232,7 @@ def test_system_toolchain_constant(self): self.prep() eb = EasyConfig(self.eb_file) self.assertEqual(eb['toolchain'], {'name': 'system', 'version': 'system'}) - self.assertTrue(isinstance(eb.toolchain, SystemToolchain)) + self.assertIsInstance(eb.toolchain, SystemToolchain) def test_shlib_ext(self): """ inside easyconfigs shared_lib_ext should be set """ @@ -593,8 +594,8 @@ def test_extensions_templates(self): self.assertEqual(toy_ext.cfg['prebuildopts'], expected_prebuildopts) # check whether files expected to be installed for 'toy' extension are in place - self.assertTrue(os.path.exists(os.path.join(pi_installdir, 'bin', 'toy'))) - self.assertTrue(os.path.exists(os.path.join(pi_installdir, 'lib', 'libtoy.a'))) + self.assertExists(os.path.join(pi_installdir, 'bin', 'toy')) + self.assertExists(os.path.join(pi_installdir, 'lib', 'libtoy.a')) def test_suggestions(self): """ If a typo is present, suggestions should be provided (if possible) """ @@ -688,9 +689,9 @@ def test_tweaking(self): self.assertEqual(eb['versionsuffix'], versuff) self.assertEqual(eb['toolchain']['version'], tcver) self.assertEqual(eb['patches'], new_patches) - self.assertTrue(eb['runtest'] is False) - self.assertTrue(eb['hidden'] is True) - self.assertTrue(eb['parallel'] is None) + self.assertIs(eb['runtest'], False) + self.assertIs(eb['hidden'], True) + self.assertIsNone(eb['parallel']) self.assertEqual(eb['test_none'], 'False') self.assertEqual(eb['test_bool'], 'True') self.assertEqual(eb['test_123'], 'None') @@ -760,6 +761,15 @@ def test_installversion(self): # only version key is strictly needed self.assertEqual(det_full_ec_version({'version': '1.2.3'}), '1.2.3') + # check how faulty dep spec is handled + faulty_dep_spec = { + 'name': 'test', + 'version': '1.2.3', + 'versionsuffix': {'name': 'system', 'version': 'system'}, + } + error_pattern = "versionsuffix value should be a string, found 'dict'" + self.assertErrorRegex(EasyBuildError, error_pattern, det_full_ec_version, faulty_dep_spec) + def test_obtain_easyconfig(self): """test obtaining an easyconfig file given certain specifications""" init_config(build_options={'silent': True}) @@ -1090,6 +1100,13 @@ def test_templating_constants(self): 'Perl: %%(perlver)s, %%(perlmajver)s, %%(perlminver)s, %%(perlshortver)s', 'R: %%(rver)s, %%(rmajver)s, %%(rminver)s, %%(rshortver)s', ]), + 'modunloadmsg = "%s"' % '; '.join([ + 'CUDA: %%(cudaver)s, %%(cudamajver)s, %%(cudaminver)s, %%(cudashortver)s', + 'Java: %%(javaver)s, %%(javamajver)s, %%(javaminver)s, %%(javashortver)s', + 'Python: %%(pyver)s, %%(pymajver)s, %%(pyminver)s, %%(pyshortver)s', + 'Perl: %%(perlver)s, %%(perlmajver)s, %%(perlminver)s, %%(perlshortver)s', + 'R: %%(rver)s, %%(rmajver)s, %%(rminver)s, %%(rshortver)s', + ]), 'modextrapaths = {"PI_MOD_NAME": "%%(module_name)s"}', 'license_file = HOME + "/licenses/PI/license.txt"', "github_account = 'easybuilders'", @@ -1129,6 +1146,7 @@ def test_templating_constants(self): "Perl: 5.22.0, 5, 22, 5.22; " "R: 3.2.3, 3, 2, 3.2") self.assertEqual(ec['modloadmsg'], expected) + self.assertEqual(ec['modunloadmsg'], expected) self.assertEqual(ec['modextrapaths'], {'PI_MOD_NAME': 'PI/3.04-Python-2.7.10'}) self.assertEqual(ec['license_file'], os.path.join(os.environ['HOME'], 'licenses', 'PI', 'license.txt')) @@ -1215,6 +1233,7 @@ def test_java_wrapper_templating(self): 'toolchain = {"name":"GCC", "version": "4.6.3"}', 'dependencies = [("Java", "11", "", SYSTEM)]', 'modloadmsg = "Java: %(javaver)s, %(javamajver)s, %(javashortver)s"', + 'modunloadmsg = "Java: %(javaver)s, %(javamajver)s, %(javashortver)s"', ]) self.prep() eb = EasyConfig(self.eb_file) @@ -1223,9 +1242,10 @@ def test_java_wrapper_templating(self): self.assertEqual(eb.template_values['javaver'], '11') self.assertEqual(eb.template_values['javamajver'], '11') self.assertEqual(eb.template_values['javashortver'], '11') - self.assertFalse('javaminver' in eb.template_values) + self.assertNotIn('javaminver', eb.template_values) self.assertEqual(eb['modloadmsg'], "Java: 11, 11, 11") + self.assertEqual(eb['modunloadmsg'], "Java: 11, 11, 11") def test_python_whl_templating(self): """test templating for Python wheels""" @@ -1274,6 +1294,52 @@ def test_templating_doc(self): self.assertEqual(len(doc.split('\n')), sum([2 * len(temps) - 1] + [len(x) for x in temps])) + def test_start_dir_template(self): + """Test the %(startdir)s template""" + + test_easyconfigs = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') + ec = process_easyconfig(os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb'))[0] + + self.contents = textwrap.dedent(""" + name = 'toy' + version = '0.0' + + homepage = 'https://easybuilders.github.io/easybuild' + description = 'Toy C program, 100% toy.' + + toolchain = SYSTEM + + sources = [SOURCE_TAR_GZ] + + preconfigopts = 'echo start_dir in configure is %(start_dir)s && ' + prebuildopts = 'echo start_dir in build is %(start_dir)s && ' + + exts_defaultclass = 'EB_Toy' + exts_list = [ + ('bar', '0.0', { + 'sources': ['bar-0.0-local.tar.gz'], + 'preconfigopts': 'echo start_dir in extension configure is %(start_dir)s && ', + 'prebuildopts': 'echo start_dir in extension build is %(start_dir)s && ', + }), + ] + + moduleclass = 'tools' + """) + self.prep() + ec = EasyConfig(self.eb_file) + from easybuild.easyblocks.toy import EB_toy + eb = EB_toy(ec) + eb.cfg['stop'] = 'extensions' + with self.mocked_stdout_stderr(): + eb.run_all_steps(False) + logtxt = read_file(eb.logfile) + start_dir = os.path.join(eb.builddir, 'toy-0.0') + self.assertIn('start_dir in configure is %s/ &&' % start_dir, logtxt) + self.assertIn('start_dir in build is %s/ &&' % start_dir, logtxt) + ext_start_dir = os.path.join(eb.builddir, 'bar', 'bar-0.0') + self.assertIn('start_dir in extension configure is %s &&' % ext_start_dir, logtxt) + self.assertIn('start_dir in extension build is %s &&' % ext_start_dir, logtxt) + def test_constant_doc(self): """test constant documentation""" doc = avail_easyconfig_constants() @@ -1283,6 +1349,20 @@ def test_constant_doc(self): ] self.assertEqual(len(doc.split('\n')), sum([len(temps)] + [len(x) for x in temps])) + def test_constant_import(self): + """Test importing EC constants works""" + from easybuild.framework.easyconfig.constants import SYSTEM, OS_NAME, OS_VERSION + self.assertEqual(SYSTEM, {'name': 'system', 'version': 'system'}) + self.assertEqual(OS_NAME, get_os_name()) + self.assertEqual(OS_VERSION, get_os_version()) + + def test_constant_import_values(self): + """Test that importing an EC constant works as-if using EASYCONFIG_CONSTANTS""" + constants = __import__('easybuild.framework.easyconfig.constants', fromlist=[None]) + for name, (value, _doc) in easyconfig.constants.EASYCONFIG_CONSTANTS.items(): + self.assertTrue(hasattr(constants, name), 'Missing ' + name) + self.assertEqual(getattr(constants, name), value) + def test_build_options(self): """Test configure/build/install options, both strings and lists.""" orig_contents = '\n'.join([ @@ -1645,7 +1725,7 @@ def test_unknown_easyconfig_parameter(self): ]) self.prep() ec = EasyConfig(self.eb_file) - self.assertFalse('therenosucheasyconfigparameterlikethis' in ec) + self.assertNotIn('therenosucheasyconfigparameterlikethis', ec) error_regex = "unknown easyconfig parameter" self.assertErrorRegex(EasyBuildError, error_regex, lambda k: ec[k], 'therenosucheasyconfigparameterlikethis') @@ -2191,7 +2271,7 @@ def test_dump(self): "('GCC', '4.9.2', '', SYSTEM)", ] for pattern in patterns: - self.assertTrue(pattern in dumped_ec_txt, "Pattern '%s' should be found in: %s" % (pattern, dumped_ec_txt)) + self.assertIn(pattern, dumped_ec_txt) def test_toolchain_hierarchy_aware_dump(self): """Test that EasyConfig's dump() method is aware of the toolchain hierarchy.""" @@ -2237,14 +2317,14 @@ def test_toolchain_hierarchy_aware_dump(self): ectxt = read_file(test_ec) dumped_ec = EasyConfig(test_ec) self.assertEqual(ecdict, dumped_ec.asdict()) - self.assertTrue(r"'toy', '0.0')," in ectxt) + self.assertIn("'toy', '0.0'),", ectxt) # test case where we ask for explicit toolchains ec.dump(test_ec, explicit_toolchains=True) self.assertEqual(ecdict, ec.asdict()) ectxt = read_file(test_ec) dumped_ec = EasyConfig(test_ec) self.assertEqual(ecdict, dumped_ec.asdict()) - self.assertTrue(r"'toy', '0.0', '', ('gompi', '2018a'))," in ectxt) + self.assertIn("'toy', '0.0', '', ('gompi', '2018a')),", ectxt) def test_dump_order(self): """Test order of easyconfig parameters in dumped easyconfig.""" @@ -2985,7 +3065,7 @@ def test_software_license(self): # constant GPLv3 is resolved as string self.assertEqual(ec['software_license'], 'LicenseGPLv3') # software_license is defined as License subclass - self.assertTrue(isinstance(ec.software_license, LicenseGPLv3)) + self.assertIsInstance(ec.software_license, LicenseGPLv3) self.assertTrue(issubclass(ec.software_license.__class__, License)) ec['software_license'] = 'LicenseThatDoesNotExist' @@ -3029,7 +3109,7 @@ def test_eq_hash(self): ec2 = EasyConfig(os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb')) # different instances, same parsed easyconfig - self.assertFalse(ec1 is ec2) + self.assertIsNot(ec1, ec2) self.assertEqual(ec1, ec2) self.assertTrue(ec1 == ec2) self.assertFalse(ec1 != ec2) @@ -3083,7 +3163,7 @@ def test_copy_easyconfigs(self): for orig_ec, src_ec in test_ecs: orig_ec = os.path.basename(orig_ec) copied_ec = os.path.join(ecs_target_dir, orig_ec[0].lower(), orig_ec.split('-')[0], orig_ec) - self.assertTrue(os.path.exists(copied_ec), "File %s exists" % copied_ec) + self.assertExists(copied_ec) self.assertEqual(read_file(copied_ec), read_file(os.path.join(self.test_prefix, src_ec))) # create test easyconfig that includes comments & build stats, just like an archived easyconfig @@ -3157,7 +3237,7 @@ def test_template_constant_dict(self): res = template_constant_dict(ec) # 'arch' needs to be handled separately, since value depends on system architecture - self.assertTrue('arch' in res) + self.assertIn('arch', res) arch = res.pop('arch') self.assertTrue(arch_regex.match(arch), "'%s' matches with pattern '%s'" % (arch, arch_regex.pattern)) @@ -3244,7 +3324,7 @@ def test_template_constant_dict(self): dep_names = [x['name'] for x in ec['dependencies']] self.assertFalse('CMake' in dep_names, "CMake should not be included in list of dependencies: %s" % dep_names) - self.assertTrue('arch' in res) + self.assertIn('arch', res) arch = res.pop('arch') self.assertTrue(arch_regex.match(arch), "'%s' matches with pattern '%s'" % (arch, arch_regex.pattern)) @@ -3263,7 +3343,7 @@ def test_template_constant_dict(self): dep_names = [x[0] for x in ec['dependencies']] self.assertFalse('CMake' in dep_names, "CMake should not be included in list of dependencies: %s" % dep_names) - self.assertTrue('arch' in res) + self.assertIn('arch', res) arch = res.pop('arch') self.assertTrue(arch_regex.match(arch), "'%s' matches with pattern '%s'" % (arch, arch_regex.pattern)) @@ -3280,7 +3360,7 @@ def test_template_constant_dict(self): } res = template_constant_dict(ext_dict) - self.assertTrue('arch' in res) + self.assertIn('arch', res) arch = res.pop('arch') self.assertTrue(arch_regex.match(arch), "'%s' matches with pattern '%s'" % (arch, arch_regex.pattern)) @@ -3506,7 +3586,7 @@ def test_det_subtoolchain_version(self): self.assertEqual(versions, [None, '']) depr_msg = "WARNING: Deprecated functionality, will no longer work in v5.0: " depr_msg += "Use --add-system-to-minimal-toolchains instead of --add-dummy-to-minimal-toolchains" - self.assertTrue(depr_msg in stderr) + self.assertIn(depr_msg, stderr) # and GCCcore if existing too init_config(build_options={'add_system_to_minimal_toolchains': True}) @@ -3643,7 +3723,7 @@ def test_get_paths_for(self): self.assertEqual(res[-1], os.path.join(symlinked_prefix, 'easybuild', 'easyconfigs')) # wipe sys.path. then only path found via $EB_SCRIPT_PATH is found - sys.path = [] + sys.path[:] = [] res = get_paths_for(subdir='easyconfigs', robot_path=None) self.assertEqual(len(res), 1) self.assertEqual(res[0], os.path.join(symlinked_prefix, 'easybuild', 'easyconfigs')) @@ -3663,7 +3743,7 @@ def test_get_paths_for(self): os.environ['EB_SCRIPT_PATH'] = eb_symlink res = get_paths_for(subdir='easyconfigs', robot_path=None) - self.assertTrue(os.path.exists(res[0])) + self.assertExists(res[0]) self.assertTrue(os.path.samefile(res[0], os.path.join(someprefix, 'easybuild', 'easyconfigs'))) # Finally restore EB_SCRIPT_PATH value if set @@ -3782,7 +3862,7 @@ def test_check_sha256_checksums(self): ecs = [ec['ec'] for ec in ecs] res = check_sha256_checksums(ecs) - self.assertTrue(len(res) == 1) + self.assertEqual(len(res), 1) regex = re.compile(r"Non-SHA256 checksum\(s\) found for toy-0.0.tar.gz:.*not_really_a_sha256_checksum") self.assertTrue(regex.match(res[0]), "Pattern '%s' found in: %s" % (regex.pattern, res[0])) @@ -3795,6 +3875,26 @@ def test_deprecated(self): error_pattern = r"easyconfig file '.*/test.eb' is marked as deprecated:\nthis is just a test\n \(see also" self.assertErrorRegex(EasyBuildError, error_pattern, EasyConfig, test_ec) + with self.mocked_stdout_stderr(): + # But this can be silenced + init_config(build_options={'silence_deprecation_warnings': ['easyconfig']}) + EasyConfig(test_ec) + self.assertFalse(self.get_stderr()) + self.assertFalse(self.get_stdout()) + + def test_deprecated_toolchain(self): + """Test use of deprecated toolchain""" + topdir = os.path.dirname(os.path.abspath(__file__)) + deprecated_toolchain_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-gompi-2018a.eb') + init_config(build_options={'silence_deprecation_warnings': [], 'unit_testing_mode': False}) + error_pattern = r"toolchain 'gompi/2018a' is marked as deprecated \(see also" + self.assertErrorRegex(EasyBuildError, error_pattern, EasyConfig, deprecated_toolchain_ec) + with self.mocked_stdout_stderr(): + # But this can be silenced + init_config(build_options={'silence_deprecation_warnings': ['toolchain'], 'unit_testing_mode': False}) + EasyConfig(deprecated_toolchain_ec) + self.assertFalse(self.get_stderr()) + self.assertFalse(self.get_stdout()) def test_filename(self): """Test filename method of EasyConfig class.""" @@ -3866,7 +3966,7 @@ def test_multi_deps(self): # builddependencies should now be a non-empty list of lists, each with one entry corresponding to a GCC version builddeps = ec['builddependencies'] self.assertTrue(builddeps) - self.assertTrue(isinstance(builddeps, list)) + self.assertIsInstance(builddeps, list) self.assertEqual(len(builddeps), 3) self.assertTrue(all(isinstance(bd, list) for bd in builddeps)) self.assertTrue(all(len(bd) == 1 for bd in builddeps)) @@ -3875,7 +3975,7 @@ def test_multi_deps(self): # get_parsed_multi_deps() method basically returns same list multi_deps = ec.get_parsed_multi_deps() - self.assertTrue(isinstance(multi_deps, list)) + self.assertIsInstance(multi_deps, list) self.assertEqual(len(multi_deps), 3) self.assertTrue(all(isinstance(bd, list) for bd in multi_deps)) self.assertTrue(all(len(bd) == 1 for bd in multi_deps)) @@ -3887,7 +3987,7 @@ def test_multi_deps(self): ec = EasyConfig(test_ec) builddeps = ec['builddependencies'] self.assertTrue(builddeps) - self.assertTrue(isinstance(builddeps, list)) + self.assertIsInstance(builddeps, list) self.assertEqual(len(builddeps), 3) self.assertTrue(all(isinstance(bd, list) for bd in builddeps)) self.assertTrue(all(len(bd) == 3 for bd in builddeps)) @@ -3902,7 +4002,7 @@ def test_multi_deps(self): # get_parsed_multi_deps() method returns same list, but CMake & foo are not included multi_deps = ec.get_parsed_multi_deps() - self.assertTrue(isinstance(multi_deps, list)) + self.assertIsInstance(multi_deps, list) self.assertEqual(len(multi_deps), 3) self.assertTrue(all(isinstance(bd, list) for bd in multi_deps)) self.assertTrue(all(len(bd) == 1 for bd in multi_deps)) @@ -3939,7 +4039,7 @@ def test_multi_deps_templated_builddeps(self): builddeps = ec['builddependencies'] - self.assertTrue(isinstance(builddeps, list)) + self.assertIsInstance(builddeps, list) self.assertEqual(len(builddeps), 2) self.assertTrue(all(isinstance(bd, dict) for bd in builddeps)) @@ -3991,8 +4091,8 @@ def test_iter_builddeps_templates(self): ec = EasyConfig(test_ec) # %(pyver)s and %(pyshortver)s template are not defined when not in iterative mode - self.assertFalse('pyver' in ec.template_values) - self.assertFalse('pyshortver' in ec.template_values) + self.assertNotIn('pyver', ec.template_values) + self.assertNotIn('pyshortver', ec.template_values) # save reference to original list of lists of build dependencies builddeps = ec['builddependencies'] @@ -4003,18 +4103,18 @@ def test_iter_builddeps_templates(self): ec['builddependencies'] = builddeps[0] ec.generate_template_values() - self.assertTrue('pyver' in ec.template_values) + self.assertIn('pyver', ec.template_values) self.assertEqual(ec.template_values['pyver'], '2.7.15') - self.assertTrue('pyshortver' in ec.template_values) + self.assertIn('pyshortver', ec.template_values) self.assertEqual(ec.template_values['pyshortver'], '2.7') # put next list of build dependencies in place (i.e. Python 3.7.2) ec['builddependencies'] = builddeps[1] ec.generate_template_values() - self.assertTrue('pyver' in ec.template_values) + self.assertIn('pyver', ec.template_values) self.assertEqual(ec.template_values['pyver'], '3.6.6') - self.assertTrue('pyshortver' in ec.template_values) + self.assertIn('pyshortver', ec.template_values) self.assertEqual(ec.template_values['pyshortver'], '3.6') # check that extensions inherit these template values too @@ -4074,7 +4174,7 @@ def test_fix_deprecated_easyconfigs(self): self.mock_stderr(False) self.mock_stdout(False) self.assertFalse(stderr) - self.assertTrue("test.eb... FIXED!" in stdout) + self.assertIn("test.eb... FIXED!", stdout) # parsing now works ec = EasyConfig(test_ec) @@ -4106,7 +4206,7 @@ def test_fix_deprecated_easyconfigs(self): "Use of 'dummy' toolchain is deprecated, use 'system' toolchain instead", ] for warning in warnings: - self.assertTrue(warning in stderr, "Found warning '%s' in stderr output: %s" % (warning, stderr)) + self.assertIn(warning, stderr) init_config(build_options={'local_var_naming_check': 'error', 'silent': True}) @@ -4730,7 +4830,7 @@ def test_count_files(self): def test_ARCH(self): """Test ARCH easyconfig constant.""" arch = easyconfig.constants.EASYCONFIG_CONSTANTS['ARCH'][0] - self.assertTrue(arch in KNOWN_ARCH_CONSTANTS, "Unexpected value for ARCH constant: %s" % arch) + self.assertIn(arch, KNOWN_ARCH_CONSTANTS, "Unexpected value for ARCH constant: %s" % arch) def test_easyconfigs_caches(self): """ @@ -4747,7 +4847,7 @@ def test_easyconfigs_caches(self): ec1 = process_easyconfig(toy_ec)[0] self.assertEqual(ec1['ec'].name, 'toy') self.assertEqual(ec1['ec'].version, '0.0') - self.assertTrue(isinstance(ec1['ec'].toolchain, SystemToolchain)) + self.assertIsInstance(ec1['ec'].toolchain, SystemToolchain) self.assertTrue(os.path.samefile(ec1['ec'].path, toy_ec)) # wipe toy easyconfig (but path still needs to exist) @@ -4757,7 +4857,7 @@ def test_easyconfigs_caches(self): ec2 = process_easyconfig(toy_ec)[0] self.assertEqual(ec2['ec'].name, 'toy') self.assertEqual(ec2['ec'].version, '0.0') - self.assertTrue(isinstance(ec2['ec'].toolchain, SystemToolchain)) + self.assertIsInstance(ec2['ec'].toolchain, SystemToolchain) self.assertTrue(os.path.samefile(ec2['ec'].path, toy_ec)) # also check whether easyconfigs cache works with end-to-end test diff --git a/test/framework/easyconfigparser.py b/test/framework/easyconfigparser.py index bf303df38c..9a17d8f0a5 100644 --- a/test/framework/easyconfigparser.py +++ b/test/framework/easyconfigparser.py @@ -64,8 +64,8 @@ def test_v10(self): ec['sources'].append(fn) ec_bis = ecp.get_config_dict() - self.assertTrue(fn in ec['sources']) - self.assertFalse(fn in ec_bis['sources']) + self.assertIn(fn, ec['sources']) + self.assertNotIn(fn, ec_bis['sources']) def test_v20(self): """Test parsing of easyconfig in format v2.""" @@ -79,9 +79,9 @@ def test_v20(self): formatter = ecp._formatter self.assertEqual(formatter.VERSION, EasyVersion('2.0')) - self.assertTrue('name' in formatter.pyheader_localvars) - self.assertFalse('version' in formatter.pyheader_localvars) - self.assertFalse('toolchain' in formatter.pyheader_localvars) + self.assertIn('name', formatter.pyheader_localvars) + self.assertNotIn('version', formatter.pyheader_localvars) + self.assertNotIn('toolchain', formatter.pyheader_localvars) # this should be ok: ie the default values ec = ecp.get_config_dict() @@ -94,8 +94,8 @@ def test_v20(self): ec['sources'].append(fn) ec_bis = ecp.get_config_dict() - self.assertTrue(fn in ec['sources']) - self.assertFalse(fn in ec_bis['sources']) + self.assertIn(fn, ec['sources']) + self.assertNotIn(fn, ec_bis['sources']) # restore easybuild.tools.build_log.EXPERIMENTAL = orig_experimental @@ -112,9 +112,9 @@ def test_v20_extra(self): formatter = ecp._formatter self.assertEqual(formatter.VERSION, EasyVersion('2.0')) - self.assertTrue('name' in formatter.pyheader_localvars) - self.assertFalse('version' in formatter.pyheader_localvars) - self.assertFalse('toolchain' in formatter.pyheader_localvars) + self.assertIn('name', formatter.pyheader_localvars) + self.assertNotIn('version', formatter.pyheader_localvars) + self.assertNotIn('toolchain', formatter.pyheader_localvars) # restore easybuild.tools.build_log.EXPERIMENTAL = orig_experimental @@ -136,7 +136,7 @@ def test_v20_deps(self): # dependencies should be parsed correctly deps = ec['dependencies'] - self.assertTrue(isinstance(deps[0], Dependency)) + self.assertIsInstance(deps[0], Dependency) self.assertEqual(deps[0].name(), 'zlib') self.assertEqual(deps[0].version(), '1.2.5') @@ -198,11 +198,11 @@ def test_easyconfig_constants(self): # make sure both keys and values are of appropriate types for constant_name in constants: - self.assertTrue(isinstance(constant_name, string_type), "Constant name %s is a string" % constant_name) + self.assertIsInstance(constant_name, string_type, "Constant name %s is a string" % constant_name) val = constants[constant_name] fail_msg = "The constant %s should have an acceptable type, found %s (%s)" % (constant_name, type(val), str(val)) - self.assertTrue(isinstance(val, (string_type, dict, tuple)), fail_msg) + self.assertIsInstance(val, (string_type, dict, tuple), fail_msg) # check a couple of randomly picked constant values self.assertEqual(constants['SOURCE_TAR_GZ'], '%(name)s-%(version)s.tar.gz') diff --git a/test/framework/easystack.py b/test/framework/easystack.py index 4df3fcf976..ac36d7a26f 100644 --- a/test/framework/easystack.py +++ b/test/framework/easystack.py @@ -29,6 +29,7 @@ @author: Kenneth Hoste (Ghent University) """ import os +import re import sys from unittest import TextTestRunner @@ -127,6 +128,25 @@ def test_easystack_invalid_key2(self): error_pattern += r"instead found keys: .*, invalid_key" self.assertErrorRegex(EasyBuildError, error_pattern, parse_easystack, test_easystack) + def test_easystack_restore_env_after_each_build(self): + """Test that the build environment is reset for each easystack item""" + test_es_txt = '\n'.join([ + "easyconfigs:", + " - toy-0.0-gompi-2018a.eb:", + " - libtoy-0.0.eb:", + ]) + test_es_path = os.path.join(self.test_prefix, 'test.yml') + write_file(test_es_path, test_es_txt) + + args = [ + '--experimental', + '--easystack', + test_es_path + ] + stdout = self.eb_main(args, do_build=True, raise_error=True) + regex = re.compile(r"WARNING Loaded modules detected: \[.*gompi/2018.*\]\n") + self.assertFalse(regex.search(stdout), "Pattern '%s' should not be found in: %s" % (regex.pattern, stdout)) + def test_missing_easyconfigs_key(self): """Test that EasyStack file that doesn't contain an EasyConfigs key will fail with sane error message""" topdir = os.path.dirname(os.path.abspath(__file__)) diff --git a/test/framework/environment.py b/test/framework/environment.py index 61b9d3992e..9a81e17486 100644 --- a/test/framework/environment.py +++ b/test/framework/environment.py @@ -121,9 +121,9 @@ def test_unset_env_vars(self): res = env.unset_env_vars(['HOME', 'NO_SUCH_ENV_VAR', 'TEST_ENV_VAR']) - self.assertFalse('HOME' in os.environ) - self.assertFalse('NO_SUCH_ENV_VAR' in os.environ) - self.assertFalse('TEST_ENV_VAR' in os.environ) + self.assertNotIn('HOME', os.environ) + self.assertNotIn('NO_SUCH_ENV_VAR', os.environ) + self.assertNotIn('TEST_ENV_VAR', os.environ) expected = { 'HOME': home, diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 59fd292c40..87d3599c13 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -40,6 +40,7 @@ import sys import tempfile import time +from test.framework.github import requires_github_access from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config from unittest import TextTestRunner from easybuild.tools import run @@ -218,7 +219,7 @@ def test_patch_perl_script_autoflush(self): self.assertTrue(len(txt.split('\n')) == len(perl_lines) + 4) self.assertTrue(txt.startswith(perl_lines[0] + "\n\nuse IO::Handle qw();\nSTDOUT->autoflush(1);")) for line in perl_lines[1:]: - self.assertTrue(line in txt) + self.assertIn(line, txt) os.remove(fp) os.remove("%s.eb.orig" % fp) @@ -229,9 +230,9 @@ def test_which(self): invalid_cmd = 'i_really_do_not_expect_a_command_with_a_name_like_this_to_be_available' path = ft.which(invalid_cmd) - self.assertTrue(path is None) + self.assertIsNone(path) path = ft.which(invalid_cmd, on_error=IGNORE) - self.assertTrue(path is None) + self.assertIsNone(path) error_msg = "Could not find command '%s'" % invalid_cmd self.assertErrorRegex(EasyBuildError, error_msg, ft.which, invalid_cmd, on_error=ERROR) @@ -487,11 +488,11 @@ def test_download_file(self): self.mock_stdout(False) self.assertEqual(path, target_location) - self.assertFalse(os.path.exists(target_location)) + self.assertNotExists(target_location) self.assertTrue(re.match("^file written: .*/foo$", txt)) ft.download_file(fn, source_url, target_location, forced=True) - self.assertTrue(os.path.exists(target_location)) + self.assertExists(target_location) self.assertTrue(os.path.samefile(path, target_location)) def test_download_file_requests_fallback(self): @@ -512,7 +513,7 @@ def fake_urllib_open(*args, **kwargs): if ft.HAVE_REQUESTS: res = ft.download_file(fn, url, target) self.assertTrue(res and os.path.exists(res)) - self.assertTrue("https://easybuilders.github.io/easybuild" in ft.read_file(res)) + self.assertIn("https://easybuilders.github.io/easybuild", ft.read_file(res)) # without requests being available, error is raised ft.HAVE_REQUESTS = False @@ -528,7 +529,7 @@ def fake_urllib_open(*args, **kwargs): if ft.HAVE_REQUESTS: res = ft.download_file(fn, url, target) self.assertTrue(res and os.path.exists(res)) - self.assertTrue("https://easybuilders.github.io/easybuild" in ft.read_file(res)) + self.assertIn("https://easybuilders.github.io/easybuild", ft.read_file(res)) # without requests being available, error is raised ft.HAVE_REQUESTS = False @@ -569,8 +570,8 @@ def fake_urllib_open(url, *args, **kwargs): stderr = self.get_stderr() self.mock_stderr(False) - self.assertTrue("WARNING: Not checking server certificates while downloading toy-0.0.eb" in stderr) - self.assertTrue(os.path.exists(res)) + self.assertIn("WARNING: Not checking server certificates while downloading toy-0.0.eb", stderr) + self.assertExists(res) self.assertTrue(ft.read_file(res).startswith("name = 'toy'")) # also test insecure download via requests fallback @@ -606,9 +607,9 @@ def fake_requests_get(url, *args, **kwargs): stderr = self.get_stderr() self.mock_stderr(False) - self.assertTrue("WARNING: Not checking server certificates while downloading README.rst" in stderr) - self.assertTrue(os.path.exists(res)) - self.assertTrue("https://easybuilders.github.io/easybuild" in ft.read_file(res)) + self.assertIn("WARNING: Not checking server certificates while downloading README.rst", stderr) + self.assertExists(res) + self.assertIn("https://easybuilders.github.io/easybuild", ft.read_file(res)) def test_mkdir(self): """Test mkdir function.""" @@ -695,7 +696,7 @@ def test_symlink_resolve_path(self): link_dir = os.path.join(self.test_prefix, 'linkdir') ft.symlink(test_dir, link_dir) self.assertTrue(os.path.islink(link_dir)) - self.assertTrue(os.path.exists(link_dir)) + self.assertExists(link_dir) test_file = os.path.join(link_dir, 'test.txt') ft.write_file(test_file, "test123") @@ -706,7 +707,7 @@ def test_symlink_resolve_path(self): # checking if file is symlink self.assertTrue(os.path.islink(link)) - self.assertTrue(os.path.exists(link_dir)) + self.assertExists(link_dir) self.assertTrue(os.path.samefile(os.path.join(self.test_prefix, 'test', 'test.txt'), link)) @@ -742,7 +743,7 @@ def test_remove_symlinks(self): # Attempting to remove a valid symlink ft.remove_file(link) self.assertFalse(os.path.islink(link)) - self.assertFalse(os.path.exists(link)) + self.assertNotExists(link) # Testing the removal of invalid symlinks # Restoring the symlink and removing the file, this way the symlink is invalid @@ -751,7 +752,7 @@ def test_remove_symlinks(self): # attempting to remove the invalid symlink ft.remove_file(link) self.assertFalse(os.path.islink(link)) - self.assertFalse(os.path.exists(link)) + self.assertNotExists(link) def test_read_write_file(self): """Test reading/writing files.""" @@ -815,7 +816,7 @@ def test_read_write_file(self): self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) # by default, write_file will just blindly overwrite an already existing file - self.assertTrue(os.path.exists(fp)) + self.assertExists(fp) ft.write_file(fp, 'blah') self.assertEqual(ft.read_file(fp), 'blah') @@ -848,11 +849,11 @@ def test_read_write_file(self): txt = self.get_stdout() self.mock_stdout(False) - self.assertFalse(os.path.exists(foo)) + self.assertNotExists(foo) self.assertTrue(re.match("^file written: .*/foo.txt$", txt)) ft.write_file(foo, 'bar', forced=True) - self.assertTrue(os.path.exists(foo)) + self.assertExists(foo) self.assertEqual(ft.read_file(foo), 'bar') # test use of 'mode' in read_file @@ -1009,7 +1010,7 @@ def test_back_up_file(self): new_file = [x for x in test_files if x not in known_files][0] self.assertTrue(new_file.startswith('test.txt.bak_')) known_files = os.listdir(os.path.dirname(fp)) - self.assertTrue(ft.read_file(first_normal_backup), txt) + self.assertEqual(ft.read_file(first_normal_backup), txt) self.assertEqual(ft.read_file(os.path.join(os.path.dirname(fp), new_file)), new_txt) self.assertEqual(ft.read_file(fp), new_txt) @@ -1020,7 +1021,7 @@ def test_back_up_file(self): new_file = [x for x in test_files if x not in known_files][0] self.assertTrue(new_file.startswith('.test.txt_')) known_files = os.listdir(os.path.dirname(fp)) - self.assertTrue(ft.read_file(first_hidden_backup), txt) + self.assertEqual(ft.read_file(first_hidden_backup), txt) self.assertEqual(ft.read_file(os.path.join(os.path.dirname(fp), new_file)), new_txt) self.assertEqual(ft.read_file(fp), new_txt) @@ -1031,7 +1032,7 @@ def test_back_up_file(self): new_file = [x for x in test_files if x not in known_files][0] self.assertTrue(new_file.startswith('test.txt.bck_')) known_files = os.listdir(os.path.dirname(fp)) - self.assertTrue(ft.read_file(first_bck_backup), txt) + self.assertEqual(ft.read_file(first_bck_backup), txt) self.assertEqual(ft.read_file(os.path.join(os.path.dirname(fp), new_file)), new_txt) self.assertEqual(ft.read_file(fp), new_txt) @@ -1042,7 +1043,7 @@ def test_back_up_file(self): new_file = [x for x in test_files if x not in known_files][0] self.assertTrue(new_file.startswith('.test.txt.foobar_')) known_files = os.listdir(os.path.dirname(fp)) - self.assertTrue(ft.read_file(first_hidden_bck_backup), txt) + self.assertEqual(ft.read_file(first_hidden_bck_backup), txt) self.assertEqual(ft.read_file(os.path.join(os.path.dirname(fp), new_file)), new_txt) self.assertEqual(ft.read_file(fp), new_txt) @@ -1051,10 +1052,10 @@ def test_back_up_file(self): ft.copy_file(fp, fp2) res = ft.back_up_file(fp2) self.assertTrue(fp2.endswith('.lua')) - self.assertTrue('.lua' in os.path.basename(res)) + self.assertIn('.lua', os.path.basename(res)) res = ft.back_up_file(fp2, strip_fn='.lua') - self.assertFalse('.lua' in os.path.basename(res)) + self.assertNotIn('.lua', os.path.basename(res)) # strip_fn should not remove the first a in 'a.lua' expected = os.path.basename(fp) + 'a.bak_' res_fn = os.path.basename(res) @@ -1375,7 +1376,7 @@ def test_apply_regex_substitutions(self): # backup file is created by default backup = testfile + '.orig.eb' - self.assertTrue(os.path.exists(backup)) + self.assertExists(backup) self.assertEqual(ft.read_file(backup), testtxt) # cleanup @@ -1389,7 +1390,7 @@ def test_apply_regex_substitutions(self): self.assertEqual(new_testtxt, expected_testtxt) backup = testfile + '.backup' - self.assertTrue(os.path.exists(backup)) + self.assertExists(backup) self.assertEqual(ft.read_file(backup), testtxt) # cleanup @@ -1400,7 +1401,7 @@ def test_apply_regex_substitutions(self): ft.apply_regex_substitutions(testfile, regex_subs, backup=False) new_testtxt = ft.read_file(testfile) self.assertEqual(new_testtxt, expected_testtxt) - self.assertFalse(os.path.exists(backup)) + self.assertNotExists(backup) # passing empty list of substitions is a no-op ft.write_file(testfile, testtxt) @@ -1420,13 +1421,13 @@ def test_apply_regex_substitutions(self): with self.log_to_testlogfile(): ft.apply_regex_substitutions(testfile, regex_subs_no_match, on_missing_match=run.WARN) logtxt = ft.read_file(self.logfile) - self.assertTrue('WARNING ' + error_pat in logtxt) + self.assertIn('WARNING ' + error_pat, logtxt) # Ignore with self.log_to_testlogfile(): ft.apply_regex_substitutions(testfile, regex_subs_no_match, on_missing_match=run.IGNORE) logtxt = ft.read_file(self.logfile) - self.assertTrue('INFO ' + error_pat in logtxt) + self.assertIn('INFO ' + error_pat, logtxt) # clean error on non-existing file error_pat = "Failed to patch .*/nosuchfile.txt: .*No such file or directory" @@ -1584,8 +1585,8 @@ def test_pypi_source_urls(self): res = ft.pypi_source_urls('easybuild') eb340_url = 'https://pypi.python.org/packages/' eb340_url += '93/41/574d01f352671fbc8589a436167e15a7f3e27ac0aa635d208eb29ee8fd4e/' - eb340_url += 'easybuild-3.4.0.tar.gz#md5=267a056a77a8f77fccfbf56354364045' - self.assertTrue(eb340_url, res) + eb340_url += 'easybuild-3.4.0.tar.gz#sha256=d870b27211f2224aab89bfd3279834ffb89ff00ad849a0dc2bf5cc1691efa9d2' + self.assertIn(eb340_url, res) pattern = '^https://pypi.python.org/packages/[a-f0-9]{2}/[a-f0-9]{2}/[a-f0-9]{60}/' pattern_md5 = pattern + 'easybuild-[0-9a-z.]+.tar.gz#md5=[a-f0-9]{32}$' pattern_sha256 = pattern + 'easybuild-[0-9a-z.]+.tar.gz#sha256=[a-f0-9]{64}$' @@ -1605,7 +1606,7 @@ def test_pypi_source_urls(self): prefix = 'https://pypi.python.org/packages' for entry in res: self.assertTrue(entry.startswith(prefix), "'%s' should start with '%s'" % (entry, prefix)) - self.assertTrue('ipython' in entry, "Pattern 'ipython' should be found in '%s'" % entry) + self.assertIn('ipython', entry) def test_derive_alt_pypi_url(self): """Test derive_alt_pypi_url() function.""" @@ -1653,7 +1654,7 @@ def test_create_patch_info(self): expected_warning = "Use of patch file with filename that doesn't end with correct extension: foo.txt " expected_warning += "(should be any of: .patch, .patch.bz2, .patch.gz, .patch.xz)" fail_msg = "Warning '%s' should appear in stderr output: %s" % (expected_warning, stderr) - self.assertTrue(expected_warning in stderr, fail_msg) + self.assertIn(expected_warning, stderr, fail_msg) # deprecation warning is treated as an error in context of unit test suite expected_error = expected_warning.replace('(', '\\(').replace(')', '\\)') @@ -1687,21 +1688,21 @@ def test_apply_patch(self): backup_file = src_file + '.orig' patched = ft.read_file(src_file) pattern = "I'm a toy, and very proud of it" - self.assertTrue(pattern in patched) + self.assertIn(pattern, patched) if with_backup: - self.assertTrue(os.path.exists(backup_file)) - self.assertTrue(pattern not in ft.read_file(backup_file)) + self.assertExists(backup_file) + self.assertNotIn(pattern, ft.read_file(backup_file)) # Restore file to original after first(!) iteration ft.move_file(backup_file, src_file) else: - self.assertFalse(os.path.exists(backup_file)) + self.assertNotExists(backup_file) # This patch is dependent on the previous one toy_patch_gz = os.path.join(testdir, 'sandbox', 'sources', 'toy', 'toy-0.0_gzip.patch.gz') self.assertTrue(ft.apply_patch(toy_patch_gz, path)) patched_gz = ft.read_file(os.path.join(path, 'toy-0.0', 'toy.source')) pattern = "I'm a toy, and very very proud of it" - self.assertTrue(pattern in patched_gz) + self.assertIn(pattern, patched_gz) # trying the patch again should fail self.assertErrorRegex(EasyBuildError, "Couldn't apply patch file", ft.apply_patch, toy_patch, path) @@ -1737,10 +1738,10 @@ def test_apply_patch(self): # test applying of patch with git toy_source_path = os.path.join(self.test_prefix, 'toy-0.0', 'toy.source') - self.assertFalse("I'm a toy, and very proud of it" in ft.read_file(toy_source_path)) + self.assertNotIn("I'm a toy, and very proud of it", ft.read_file(toy_source_path)) ft.apply_patch(toy_patch, self.test_prefix, use_git=True) - self.assertTrue("I'm a toy, and very proud of it" in ft.read_file(toy_source_path)) + self.assertIn("I'm a toy, and very proud of it", ft.read_file(toy_source_path)) # construct patch that only adds a new file, # this shouldn't break applying a patch with git even when no level is specified @@ -1761,7 +1762,7 @@ def test_apply_patch(self): ft.remove_dir(path) path = ft.extract_file(toy_tar_gz, self.test_prefix, change_into_dir=False) - self.assertFalse("I'm a toy, and very proud of it" in ft.read_file(toy_source_path)) + self.assertNotIn("I'm a toy, and very proud of it", ft.read_file(toy_source_path)) # mock stderr to catch deprecation warning caused by setting 'use_git_am' self.allow_deprecated_behaviour() @@ -1769,8 +1770,8 @@ def test_apply_patch(self): ft.apply_patch(toy_patch, self.test_prefix, use_git_am=True) stderr = self.get_stderr() self.mock_stderr(False) - self.assertTrue("I'm a toy, and very proud of it" in ft.read_file(toy_source_path)) - self.assertTrue("'use_git_am' named argument in apply_patch function has been renamed to 'use_git'" in stderr) + self.assertIn("I'm a toy, and very proud of it", ft.read_file(toy_source_path)) + self.assertIn("'use_git_am' named argument in apply_patch function has been renamed to 'use_git'", stderr) def test_copy_file(self): """Test copy_file function.""" @@ -1778,7 +1779,7 @@ def test_copy_file(self): toy_ec = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') target_path = os.path.join(self.test_prefix, 'toy.eb') ft.copy_file(toy_ec, target_path) - self.assertTrue(os.path.exists(target_path)) + self.assertExists(target_path) self.assertTrue(ft.read_file(toy_ec) == ft.read_file(target_path)) # Make sure it doesn't fail if path is a symlink and target_path is a dir @@ -1790,7 +1791,7 @@ def test_copy_file(self): ft.copy_file(toy_link, dir_target_path) copied_file = os.path.join(dir_target_path, toy_link_fn) # symlinks that point to an existing file are resolved on copy (symlink itself is not copied) - self.assertTrue(os.path.exists(copied_file), "%s should exist" % copied_file) + self.assertExists(copied_file) self.assertTrue(os.path.isfile(copied_file), "%s should be a file" % copied_file) ft.remove_file(copied_file) @@ -1798,7 +1799,7 @@ def test_copy_file(self): ft.remove_file(target_path) ft.copy_file(toy_link, dir_target_path) self.assertTrue(os.path.islink(copied_file), "%s should be a broken symbolic link" % copied_file) - self.assertFalse(os.path.exists(copied_file), "%s should be a broken symbolic link" % copied_file) + self.assertNotExists(copied_file, "%s should be a broken symbolic link" % copied_file) self.assertEqual(os.readlink(os.path.join(dir_target_path, toy_link_fn)), os.readlink(toy_link)) ft.remove_file(copied_file) @@ -1837,14 +1838,14 @@ def test_copy_file(self): init_config(build_options=build_options) # make sure target file is not there, it shouldn't get copied under dry run - self.assertFalse(os.path.exists(target_path)) + self.assertNotExists(target_path) self.mock_stdout(True) ft.copy_file(toy_ec, target_path) txt = self.get_stdout() self.mock_stdout(False) - self.assertFalse(os.path.exists(target_path)) + self.assertNotExists(target_path) self.assertTrue(re.search("^copied file .*/toy-0.0.eb to .*/toy.eb", txt)) # forced copy, even in dry run mode @@ -1853,7 +1854,7 @@ def test_copy_file(self): txt = self.get_stdout() self.mock_stdout(False) - self.assertTrue(os.path.exists(target_path)) + self.assertExists(target_path) self.assertTrue(ft.read_file(toy_ec) == ft.read_file(target_path)) self.assertEqual(txt, '') @@ -1883,23 +1884,23 @@ def test_copy_files(self): target_dir = os.path.join(self.test_prefix, 'target_dir1') ft.copy_files([toy_ec], target_dir) copied_toy_ec = os.path.join(target_dir, 'toy-0.0.eb') - self.assertTrue(os.path.exists(copied_toy_ec)) + self.assertExists(copied_toy_ec) self.assertEqual(ft.read_file(copied_toy_ec), toy_ec_txt) # copying a single file to an existing directory ft.copy_files([bzip2_ec], target_dir) copied_bzip2_ec = os.path.join(target_dir, 'bzip2-1.0.6-GCC-4.9.2.eb') - self.assertTrue(os.path.exists(copied_bzip2_ec)) + self.assertExists(copied_bzip2_ec) self.assertEqual(ft.read_file(copied_bzip2_ec), bzip2_ec_txt) # copying multiple files to a non-existing directory target_dir = os.path.join(self.test_prefix, 'target_dir_multiple') ft.copy_files([toy_ec, bzip2_ec], target_dir) copied_toy_ec = os.path.join(target_dir, 'toy-0.0.eb') - self.assertTrue(os.path.exists(copied_toy_ec)) + self.assertExists(copied_toy_ec) self.assertEqual(ft.read_file(copied_toy_ec), toy_ec_txt) copied_bzip2_ec = os.path.join(target_dir, 'bzip2-1.0.6-GCC-4.9.2.eb') - self.assertTrue(os.path.exists(copied_bzip2_ec)) + self.assertExists(copied_bzip2_ec) self.assertEqual(ft.read_file(copied_bzip2_ec), bzip2_ec_txt) # copying files to an existing target that is not a directory results in an error @@ -1914,9 +1915,9 @@ def test_copy_files(self): # test special case: copying a single file to a file target via target_single_file=True target = os.path.join(self.test_prefix, 'target') - self.assertFalse(os.path.exists(target)) + self.assertNotExists(target) ft.copy_files([toy_ec], target, target_single_file=True) - self.assertTrue(os.path.exists(target)) + self.assertExists(target) self.assertTrue(os.path.isfile(target)) self.assertEqual(toy_ec_txt, ft.read_file(target)) @@ -1924,22 +1925,22 @@ def test_copy_files(self): # also test target_single_file=True with path including a missing subdirectory target = os.path.join(self.test_prefix, 'target_parent', 'target_subdir', 'target.txt') - self.assertFalse(os.path.exists(target)) - self.assertFalse(os.path.exists(os.path.dirname(target))) + self.assertNotExists(target) + self.assertNotExists(os.path.dirname(target)) ft.copy_files([toy_ec], target, target_single_file=True) - self.assertTrue(os.path.exists(target)) + self.assertExists(target) self.assertTrue(os.path.isfile(target)) self.assertEqual(toy_ec_txt, ft.read_file(target)) ft.remove_file(target) # default behaviour is to copy single file list to target *directory* - self.assertFalse(os.path.exists(target)) + self.assertNotExists(target) ft.copy_files([toy_ec], target) - self.assertTrue(os.path.exists(target)) + self.assertExists(target) self.assertTrue(os.path.isdir(target)) copied_toy_ec = os.path.join(target, 'toy-0.0.eb') - self.assertTrue(os.path.exists(copied_toy_ec)) + self.assertExists(copied_toy_ec) self.assertEqual(toy_ec_txt, ft.read_file(copied_toy_ec)) ft.remove_dir(target) @@ -1971,7 +1972,7 @@ def test_copy_files(self): # check behaviour under -x: only printing, no actual copying init_config(build_options={'extended_dry_run': True}) - self.assertFalse(os.path.exists(os.path.join(target, 'test.eb'))) + self.assertNotExists(os.path.join(target, 'test.eb')) self.mock_stderr(True) self.mock_stdout(True) @@ -1980,7 +1981,7 @@ def test_copy_files(self): self.mock_stderr(False) self.mock_stdout(False) - self.assertFalse(os.path.exists(os.path.join(target, 'test.eb'))) + self.assertNotExists(os.path.join(target, 'test.eb')) self.assertEqual(stderr, '') regex = re.compile("^copied test.eb to .*/target") @@ -1993,8 +1994,8 @@ def test_copy_files(self): self.mock_stderr(False) self.mock_stdout(False) - self.assertFalse(os.path.exists(os.path.join(target, 'bar.eb'))) - self.assertFalse(os.path.exists(os.path.join(target, 'foo.eb'))) + self.assertNotExists(os.path.join(target, 'bar.eb')) + self.assertNotExists(os.path.join(target, 'foo.eb')) self.assertEqual(stderr, '') regex = re.compile("^copied 2 files to .*/target") @@ -2046,17 +2047,17 @@ def test_copy_dir(self): to_copy = os.path.join(testdir, 'easyconfigs', 'test_ecs', 'g', 'GCC') target_dir = os.path.join(self.test_prefix, 'GCC') - self.assertFalse(os.path.exists(target_dir)) + self.assertNotExists(target_dir) - self.assertTrue(os.path.exists(os.path.join(to_copy, 'GCC-6.4.0-2.28.eb'))) + self.assertExists(os.path.join(to_copy, 'GCC-6.4.0-2.28.eb')) ft.copy_dir(to_copy, target_dir, ignore=lambda src, names: [x for x in names if '6.4.0-2.28' in x]) - self.assertTrue(os.path.exists(target_dir)) + self.assertExists(target_dir) expected = ['GCC-4.6.3.eb', 'GCC-4.6.4.eb', 'GCC-4.8.2.eb', 'GCC-4.8.3.eb', 'GCC-4.9.2.eb', 'GCC-4.9.3-2.25.eb', 'GCC-4.9.3-2.26.eb', 'GCC-7.3.0-2.30.eb'] self.assertEqual(sorted(os.listdir(target_dir)), expected) # GCC-6.4.0-2.28.eb should not get copied, since it's specified as file too ignore - self.assertFalse(os.path.exists(os.path.join(target_dir, 'GCC-6.4.0-2.28.eb'))) + self.assertNotExists(os.path.join(target_dir, 'GCC-6.4.0-2.28.eb')) # clean error when trying to copy a file with copy_dir src, target = os.path.join(to_copy, 'GCC-4.6.3.eb'), os.path.join(self.test_prefix, 'GCC-4.6.3.eb') @@ -2079,7 +2080,7 @@ def ignore_func(_, names): ft.mkdir(testdir) ft.copy_dir(to_copy, testdir, dirs_exist_ok=True, ignore=ignore_func) self.assertEqual(sorted(os.listdir(testdir)), expected) - self.assertFalse(os.path.exists(os.path.join(testdir, 'GCC-6.4.0-2.28.eb'))) + self.assertNotExists(os.path.join(testdir, 'GCC-6.4.0-2.28.eb')) # test copy_dir when broken symlinks are involved srcdir = os.path.join(self.test_prefix, 'topdir_to_copy') @@ -2115,10 +2116,10 @@ def ignore_func(_, names): ft.remove_dir(target_dir) os.symlink('.', os.path.join(subdir, 'recursive_link')) self.assertErrorRegex(EasyBuildError, 'Recursive symlinks detected', ft.copy_dir, srcdir, target_dir) - self.assertFalse(os.path.exists(target_dir)) + self.assertNotExists(target_dir) # Ok for symlinks=True ft.copy_dir(srcdir, target_dir, symlinks=True) - self.assertTrue(os.path.exists(target_dir)) + self.assertExists(target_dir) # also test behaviour of copy_file under --dry-run build_options = { @@ -2128,7 +2129,7 @@ def ignore_func(_, names): init_config(build_options=build_options) shutil.rmtree(target_dir) - self.assertFalse(os.path.exists(target_dir)) + self.assertNotExists(target_dir) # no actual copying in dry run mode, unless forced self.mock_stdout(True) @@ -2136,7 +2137,7 @@ def ignore_func(_, names): txt = self.get_stdout() self.mock_stdout(False) - self.assertFalse(os.path.exists(target_dir)) + self.assertNotExists(target_dir) self.assertTrue(re.search("^copied directory .*/GCC to .*/%s" % os.path.basename(target_dir), txt)) # forced copy, even in dry run mode @@ -2145,7 +2146,7 @@ def ignore_func(_, names): txt = self.get_stdout() self.mock_stdout(False) - self.assertTrue(os.path.exists(target_dir)) + self.assertExists(target_dir) self.assertTrue(sorted(os.listdir(to_copy)) == sorted(os.listdir(target_dir))) self.assertEqual(txt, '') @@ -2182,8 +2183,8 @@ def test_copy(self): txt = self.get_stdout() self.mock_stdout(False) - self.assertFalse(os.path.exists(os.path.join(self.test_prefix, 'toy'))) - self.assertFalse(os.path.exists(os.path.join(self.test_prefix, 'GCC-4.6.3.eb'))) + self.assertNotExists(os.path.join(self.test_prefix, 'toy')) + self.assertNotExists(os.path.join(self.test_prefix, 'GCC-4.6.3.eb')) self.assertTrue(re.search("^copied directory .*/toy to .*/toy", txt, re.M)) self.assertTrue(re.search("^copied file .*/GCC-4.6.3.eb to .*/GCC-4.6.3.eb", txt, re.M)) @@ -2230,9 +2231,9 @@ def test_extract_file(self): testdir = os.path.dirname(os.path.abspath(__file__)) toy_tarball = os.path.join(testdir, 'sandbox', 'sources', 'toy', 'toy-0.0.tar.gz') - self.assertFalse(os.path.exists(os.path.join(self.test_prefix, 'toy-0.0', 'toy.source'))) + self.assertNotExists(os.path.join(self.test_prefix, 'toy-0.0', 'toy.source')) path = ft.extract_file(toy_tarball, self.test_prefix, change_into_dir=False) - self.assertTrue(os.path.exists(os.path.join(self.test_prefix, 'toy-0.0', 'toy.source'))) + self.assertExists(os.path.join(self.test_prefix, 'toy-0.0', 'toy.source')) self.assertTrue(os.path.samefile(path, self.test_prefix)) # still in same directory as before if change_into_dir is set to False self.assertTrue(os.path.samefile(os.getcwd(), cwd)) @@ -2243,7 +2244,7 @@ def test_extract_file(self): path = ft.extract_file(toy_tarball_renamed, self.test_prefix, cmd="tar xfvz %s", change_into_dir=False) self.assertTrue(os.path.samefile(os.getcwd(), cwd)) - self.assertTrue(os.path.exists(os.path.join(self.test_prefix, 'toy-0.0', 'toy.source'))) + self.assertExists(os.path.join(self.test_prefix, 'toy-0.0', 'toy.source')) self.assertTrue(os.path.samefile(path, self.test_prefix)) shutil.rmtree(os.path.join(path, 'toy-0.0')) @@ -2261,11 +2262,11 @@ def test_extract_file(self): self.assertTrue(os.path.samefile(os.getcwd(), cwd)) self.assertTrue(os.path.samefile(path, self.test_prefix)) - self.assertFalse(os.path.exists(os.path.join(self.test_prefix, 'toy-0.0'))) + self.assertNotExists(os.path.join(self.test_prefix, 'toy-0.0')) self.assertTrue(re.search('running command "tar xzf .*/toy-0.0.tar.gz"', txt)) path = ft.extract_file(toy_tarball, self.test_prefix, forced=True, change_into_dir=False) - self.assertTrue(os.path.exists(os.path.join(self.test_prefix, 'toy-0.0', 'toy.source'))) + self.assertExists(os.path.join(self.test_prefix, 'toy-0.0', 'toy.source')) self.assertTrue(os.path.samefile(path, self.test_prefix)) self.assertTrue(os.path.samefile(os.getcwd(), cwd)) @@ -2313,24 +2314,26 @@ def test_remove(self): for remove_file_function in (ft.remove_file, ft.remove): ft.write_file(testfile, 'bar') - self.assertTrue(os.path.exists(testfile)) + self.assertExists(testfile) remove_file_function(testfile) - self.assertFalse(os.path.exists(testfile)) + self.assertNotExists(testfile) for remove_dir_function in (ft.remove_dir, ft.remove): ft.mkdir(test_dir) - self.assertTrue(os.path.exists(test_dir) and os.path.isdir(test_dir)) + self.assertExists(test_dir) + self.assertTrue(os.path.isdir(test_dir)) remove_dir_function(test_dir) - self.assertFalse(os.path.exists(test_dir) or os.path.isdir(test_dir)) + self.assertNotExists(test_dir) # remove also takes a list of paths ft.write_file(testfile, 'bar') ft.mkdir(test_dir) - self.assertTrue(os.path.exists(testfile)) - self.assertTrue(os.path.exists(test_dir) and os.path.isdir(test_dir)) + self.assertExists(testfile) + self.assertExists(test_dir) + self.assertTrue(os.path.isdir(test_dir)) ft.remove([testfile, test_dir]) - self.assertFalse(os.path.exists(testfile)) - self.assertFalse(os.path.exists(test_dir) or os.path.isdir(test_dir)) + self.assertNotExists(testfile) + self.assertNotExists(test_dir) # check error handling (after creating a permission problem with removing files/dirs) ft.write_file(testfile, 'bar') @@ -2395,7 +2398,7 @@ def test_index_functions(self): os.path.join('s', 'ScaLAPACK', 'ScaLAPACK-2.0.2-gompi-2018a-OpenBLAS-0.2.20.eb'), ] for fn in expected: - self.assertTrue(fn in index) + self.assertIn(fn, index) for fp in index: self.assertTrue(fp.endswith('.eb') or os.path.basename(fp) == 'checksums.json') @@ -2405,7 +2408,7 @@ def test_index_functions(self): # test dump_index function index_fp = ft.dump_index(self.test_prefix) - self.assertTrue(os.path.exists(index_fp)) + self.assertExists(index_fp) self.assertTrue(os.path.samefile(self.test_prefix, os.path.dirname(index_fp))) datestamp_pattern = r"[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]+" @@ -2438,7 +2441,7 @@ def test_index_functions(self): self.assertEqual(len(index), 26) for fn in expected: - self.assertTrue(fn in index, "%s should be found in %s" % (fn, sorted(index))) + self.assertIn(fn, index) # dump_index will not overwrite existing index without force error_pattern = "File exists, not overwriting it without --force" @@ -2468,7 +2471,7 @@ def test_index_functions(self): self.assertEqual(len(index), 26) for fn in expected: - self.assertTrue(fn in index, "%s should be found in %s" % (fn, sorted(index))) + self.assertIn(fn, index) ft.remove_file(index_fp) @@ -2482,7 +2485,7 @@ def test_index_functions(self): stdout = self.get_stdout() self.mock_stderr(False) self.mock_stdout(False) - self.assertTrue(index is None) + self.assertIsNone(index) self.assertFalse(stdout) regex = re.compile(r"WARNING: Index for %s is no longer valid \(too old\), so ignoring it" % self.test_prefix) self.assertTrue(regex.search(stderr), "Pattern '%s' found in: %s" % (regex.pattern, stderr)) @@ -2507,13 +2510,13 @@ def test_search_file(self): self.assertTrue(hits[4].endswith('/hwloc-1.11.8-GCC-7.3.0-2.30.eb')) # also test case-sensitive searching - var_defs, hits_bis = ft.search_file([test_ecs], 'HWLOC', silent=True, case_sensitive=True) + var_defs, hits_case_sensitive = ft.search_file([test_ecs], 'HWLOC', silent=True, case_sensitive=True) self.assertEqual(var_defs, []) - self.assertEqual(hits_bis, []) + self.assertEqual(hits_case_sensitive, []) - var_defs, hits_bis = ft.search_file([test_ecs], 'hwloc', silent=True, case_sensitive=True) + var_defs, hits_case_sensitive = ft.search_file([test_ecs], 'hwloc', silent=True, case_sensitive=True) self.assertEqual(var_defs, []) - self.assertEqual(hits_bis, hits) + self.assertEqual(hits_case_sensitive, hits) # check filename-only mode var_defs, hits = ft.search_file([test_ecs], 'HWLOC', silent=True, filename_only=True) @@ -2569,13 +2572,15 @@ def test_search_file(self): # no hits for any of these in test easyconfigs self.assertEqual(hits, []) - # create hit for netCDF-C++ search - test_ec = os.path.join(self.test_prefix, 'netCDF-C++-4.2-foss-2019a.eb') - ft.write_file(test_ec, '') + # create hit for netCDF-C++ search in empty directory, + # to avoid accidental matches in other files already present (log files, etc.) + ec_dir = tempfile.mkdtemp() + test_ec = os.path.join(ec_dir, 'netCDF-C++-4.2-foss-2019a.eb') + ft.write_file(test_ec, ''), for pattern in ['netCDF-C++', 'CDF', 'C++', '^netCDF']: - var_defs, hits = ft.search_file([self.test_prefix], pattern, terse=True, filename_only=True) - self.assertEqual(var_defs, []) - self.assertEqual(hits, ['netCDF-C++-4.2-foss-2019a.eb']) + var_defs, hits = ft.search_file([ec_dir], pattern, terse=True, filename_only=True) + self.assertEqual(var_defs, [], msg='For pattern ' + pattern) + self.assertEqual(hits, ['netCDF-C++-4.2-foss-2019a.eb'], msg='For pattern ' + pattern) # check how simply invalid queries are handled for pattern in ['*foo', '(foo', ')foo', 'foo)', 'foo(']: @@ -2617,8 +2622,8 @@ def test_find_eb_script(self): if 'EB_SCRIPT_PATH' in os.environ: del os.environ['EB_SCRIPT_PATH'] - self.assertTrue(os.path.exists(ft.find_eb_script('rpath_args.py'))) - self.assertTrue(os.path.exists(ft.find_eb_script('rpath_wrapper_template.sh.in'))) + self.assertExists(ft.find_eb_script('rpath_args.py')) + self.assertExists(ft.find_eb_script('rpath_wrapper_template.sh.in')) self.assertErrorRegex(EasyBuildError, "Script 'no_such_script' not found", ft.find_eb_script, 'no_such_script') # put test script in place relative to location of 'eb' @@ -2650,17 +2655,17 @@ def test_move_file(self): new_test_file = os.path.join(self.test_prefix, 'subdir', 'new_test.txt') ft.move_file(test_file, new_test_file) - self.assertFalse(os.path.exists(test_file)) - self.assertTrue(os.path.exists(new_test_file)) + self.assertNotExists(test_file) + self.assertExists(new_test_file) self.assertEqual(ft.read_file(new_test_file), 'test123') # test moving to an existing file ft.write_file(test_file, 'gibberish') ft.move_file(new_test_file, test_file) - self.assertTrue(os.path.exists(test_file)) + self.assertExists(test_file) self.assertEqual(ft.read_file(test_file), 'test123') - self.assertFalse(os.path.exists(new_test_file)) + self.assertNotExists(new_test_file) # also test behaviour of move_file under --dry-run build_options = { @@ -2682,9 +2687,9 @@ def test_move_file(self): self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) self.assertEqual(stderr, '') - self.assertTrue(os.path.exists(test_file)) + self.assertExists(test_file) self.assertEqual(ft.read_file(test_file), 'test123') - self.assertFalse(os.path.exists(new_test_file)) + self.assertNotExists(new_test_file) def test_find_backup_name_candidate(self): """Test find_backup_name_candidate""" @@ -2744,6 +2749,7 @@ def test_diff_files(self): regex = re.compile(r'^--- .*/foo\s*\n\+\+\+ .*/bar\s*$', re.M) self.assertTrue(regex.search(res), "Pattern '%s' found in: %s" % (regex.pattern, res)) + @requires_github_access() def test_get_source_tarball_from_git(self): """Test get_source_tarball_from_git function.""" @@ -2949,7 +2955,7 @@ def test_fake_vsc(self): self.mock_stdout(True) try: import vsc # noqa - self.assertTrue(False, "'import vsc' results in an error") + self.fail("'import vsc' results in an error") except SystemExit: pass @@ -2975,7 +2981,7 @@ def test_fake_vsc(self): self.mock_stdout(True) try: from test_fake_vsc import import_vsc # noqa - self.assertTrue(False, "'import vsc' results in an error") + self.fail("'import vsc' results in an error") except SystemExit: pass stderr = self.get_stderr() @@ -3146,7 +3152,7 @@ def test_copy_framework_files(self): orig_path = os.path.join(topdir, test_file) copied_path = os.path.join(target_dir, test_file) - self.assertTrue(os.path.exists(copied_path)) + self.assertExists(copied_path) self.assertEqual(ft.read_file(orig_path, mode='rb'), ft.read_file(copied_path, mode='rb')) self.assertTrue(os.path.samefile(copied_path, res['paths_in_repo'][idx])) @@ -3168,17 +3174,17 @@ def test_locks(self): # det_lock_path returns full path to lock with specified name # (used internally by create_lock, check_lock, remove_lock) lock_path = ft.det_lock_path(lock_name) - self.assertFalse(os.path.exists(lock_path)) + self.assertNotExists(lock_path) locks_dir = os.path.dirname(lock_path) - self.assertFalse(os.path.exists(locks_dir)) + self.assertNotExists(locks_dir) # if lock doesn't exist yet, check_lock just returns ft.check_lock(lock_name) # create lock, and check whether it actually was created ft.create_lock(lock_name) - self.assertTrue(os.path.exists(lock_path)) + self.assertExists(lock_path) # can't use os.path.samefile until locks_dir actually exists self.assertTrue(os.path.samefile(locks_dir, os.path.join(self.test_installpath, 'software', '.locks'))) @@ -3190,7 +3196,7 @@ def test_locks(self): # remove_lock should... remove the lock ft.remove_lock(lock_name) - self.assertFalse(os.path.exists(lock_path)) + self.assertNotExists(lock_path) self.assertEqual(os.listdir(locks_dir), []) # no harm done if remove_lock is called if lock is already gone @@ -3211,7 +3217,7 @@ def test_locks(self): ft.clean_up_locks() self.assertFalse(ft.global_lock_names) - self.assertFalse(os.path.exists(lock_path)) + self.assertNotExists(lock_path) self.assertEqual(os.listdir(locks_dir), []) # no problem with multiple locks @@ -3220,7 +3226,7 @@ def test_locks(self): for ln in lock_names: ft.create_lock(ln) for lp in lock_paths: - self.assertTrue(os.path.exists(lp), "Path %s should exist" % lp) + self.assertExists(lp) self.assertEqual(ft.global_lock_names, set(lock_names)) expected_locks = sorted(ln + '.lock' for ln in lock_names) @@ -3228,20 +3234,20 @@ def test_locks(self): ft.clean_up_locks() for lp in lock_paths: - self.assertFalse(os.path.exists(lp), "Path %s should not exist" % lp) + self.assertNotExists(lp) self.assertFalse(ft.global_lock_names) self.assertEqual(os.listdir(locks_dir), []) # also test signal handler that is supposed to clean up locks ft.create_lock(lock_name) self.assertTrue(ft.global_lock_names) - self.assertTrue(os.path.exists(lock_path)) + self.assertExists(lock_path) self.assertEqual(os.listdir(locks_dir), [lock_name + '.lock']) # clean_up_locks_signal_handler causes sys.exit with specified exit code self.assertErrorRegex(SystemExit, '15', ft.clean_up_locks_signal_handler, 15, None) self.assertFalse(ft.global_lock_names) - self.assertFalse(os.path.exists(lock_path)) + self.assertNotExists(lock_path) self.assertEqual(os.listdir(locks_dir), []) def test_locate_files(self): @@ -3266,7 +3272,7 @@ def test_locate_files(self): # files specified via absolute path don't have to be found res = ft.locate_files([one], []) - self.assertTrue(len(res) == 1) + self.assertEqual(len(res), 1) self.assertTrue(os.path.samefile(res[0], one)) # note: don't compare file paths directly but use os.path.samefile instead, @@ -3387,22 +3393,22 @@ def test_create_unused_dir(self): """Test create_unused_dir function.""" path = ft.create_unused_dir(self.test_prefix, 'folder') self.assertEqual(path, os.path.join(self.test_prefix, 'folder')) - self.assertTrue(os.path.exists(path)) + self.assertExists(path) # Repeat with existing folder(s) should create new ones for i in range(10): path = ft.create_unused_dir(self.test_prefix, 'folder') self.assertEqual(path, os.path.join(self.test_prefix, 'folder_%s' % i)) - self.assertTrue(os.path.exists(path)) + self.assertExists(path) # Not influenced by similar folder path = ft.create_unused_dir(self.test_prefix, 'folder2') self.assertEqual(path, os.path.join(self.test_prefix, 'folder2')) - self.assertTrue(os.path.exists(path)) + self.assertExists(path) for i in range(10): path = ft.create_unused_dir(self.test_prefix, 'folder2') self.assertEqual(path, os.path.join(self.test_prefix, 'folder2_%s' % i)) - self.assertTrue(os.path.exists(path)) + self.assertExists(path) # Fail cleanly if passed a readonly folder readonly_dir = os.path.join(self.test_prefix, 'ro_folder') @@ -3419,7 +3425,7 @@ def test_create_unused_dir(self): ft.write_file(os.path.join(self.test_prefix, 'file'), '') path = ft.create_unused_dir(self.test_prefix, 'file') self.assertEqual(path, os.path.join(self.test_prefix, 'file_0')) - self.assertTrue(os.path.exists(path)) + self.assertExists(path) def suite(): diff --git a/test/framework/general.py b/test/framework/general.py index 55bd9175f4..526fe07467 100644 --- a/test/framework/general.py +++ b/test/framework/general.py @@ -129,7 +129,7 @@ def test_import_available_modules(self): res = import_available_modules('easybuild.tools.repository') self.assertEqual(len(res), 5) # don't check all, since some required specific Python packages to be installed... - self.assertTrue(easybuild.tools.repository.filerepo in res) + self.assertIn(easybuild.tools.repository.filerepo, res) # replicate situation where import_available_modules failed when running in directory where modules are located # cfr. https://github.com/easybuilders/easybuild-framework/issues/2659 diff --git a/test/framework/github.py b/test/framework/github.py index 7e08ce7400..4a1e88298e 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -29,11 +29,13 @@ @author: Kenneth Hoste (Ghent University) """ import base64 +import functools import os import random import re import sys import textwrap +import unittest from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config from time import gmtime from unittest import TextTestRunner @@ -69,6 +71,23 @@ GITHUB_BRANCH = 'main' +def requires_github_access(): + """Silently skip for pull requests unless $FORCE_EB_GITHUB_TESTS is set + + Useful when the test uses e.g. `git` commands to download from Github and would run into rate limits + """ + if 'FORCE_EB_GITHUB_TESTS' in os.environ or os.getenv('GITHUB_EVENT_NAME') != 'pull_request': + return unittest.skipIf(False, None) + else: + # For pull requests silently skip to avoid rate limits + def decorator(test_item): + @functools.wraps(test_item) + def skip_wrapper(*args, **kwargs): + return + return skip_wrapper + return decorator + + class GithubTest(EnhancedTestCase): """ small test for The github package This should not be to much, since there is an hourly limit of request @@ -175,7 +194,7 @@ def test_github_add_pr_labels(self): stdout = self.get_stdout() self.mock_stdout(False) self.mock_stderr(False) - self.assertTrue("Could not determine any missing labels for PR #11262" in stdout) + self.assertIn("Could not determine any missing labels for PR #11262", stdout) self.mock_stdout(True) self.mock_stderr(True) @@ -183,7 +202,7 @@ def test_github_add_pr_labels(self): stdout = self.get_stdout() self.mock_stdout(False) self.mock_stderr(False) - self.assertTrue("PR #8006 should be labelled 'update'" in stdout) + self.assertIn("PR #8006 should be labelled 'update'", stdout) def test_github_fetch_pr_data(self): """Test fetch_pr_data function.""" @@ -258,7 +277,7 @@ def test_github_reasons_for_closing(self): self.mock_stdout(False) self.mock_stderr(False) - self.assertTrue(isinstance(res, list)) + self.assertIsInstance(res, list) self.assertEqual(stderr.strip(), "WARNING: Using easyconfigs from closed PR #1844") patterns = [ "Status of last commit is SUCCESS", @@ -267,7 +286,7 @@ def test_github_reasons_for_closing(self): "* QEMU-2.4.0", ] for pattern in patterns: - self.assertTrue(pattern in stdout, "Pattern '%s' found in: %s" % (pattern, stdout)) + self.assertIn(pattern, stdout) def test_github_close_pr(self): """Test close_pr function.""" @@ -295,7 +314,7 @@ def test_github_close_pr(self): "[DRY RUN] Closed easybuilders/testrepository PR #2", ] for pattern in patterns: - self.assertTrue(pattern in stdout, "Pattern '%s' found in: %s" % (pattern, stdout)) + self.assertIn(pattern, stdout) retest_msg = VALID_CLOSE_PR_REASONS['retest'] @@ -312,7 +331,7 @@ def test_github_close_pr(self): "[DRY RUN] Reopened easybuilders/testrepository PR #2", ] for pattern in patterns: - self.assertTrue(pattern in stdout, "Pattern '%s' found in: %s" % (pattern, stdout)) + self.assertIn(pattern, stdout) def test_github_fetch_easyblocks_from_pr(self): """Test fetch_easyblocks_from_pr function.""" @@ -427,7 +446,7 @@ def test_github_fetch_files_from_pr_cache(self): # github_account value is None (results in using default 'easybuilders') cache_key = (7159, None, 'easybuild-easyconfigs', self.test_prefix) - self.assertTrue(cache_key in gh.fetch_files_from_pr._cache.keys()) + self.assertIn(cache_key, gh.fetch_files_from_pr._cache.keys()) cache_entry = gh.fetch_files_from_pr._cache[cache_key] self.assertEqual(sorted([os.path.basename(f) for f in cache_entry]), sorted(pr7159_filenames)) @@ -477,10 +496,10 @@ def test_github_download_repo(self): path = gh.download_repo(path=self.test_prefix, github_user=GITHUB_TEST_ACCOUNT) repodir = os.path.join(self.test_prefix, 'easybuilders', 'easybuild-easyconfigs-main') self.assertTrue(os.path.samefile(path, repodir)) - self.assertTrue(os.path.exists(repodir)) + self.assertExists(repodir) shafile = os.path.join(repodir, 'latest-sha') self.assertTrue(re.match('^[0-9a-f]{40}$', read_file(shafile))) - self.assertTrue(os.path.exists(os.path.join(repodir, 'easybuild', 'easyconfigs', 'f', 'foss', 'foss-2019b.eb'))) + self.assertExists(os.path.join(repodir, 'easybuild', 'easyconfigs', 'f', 'foss', 'foss-2019b.eb')) # current directory should not have changed after calling download_repo self.assertTrue(os.path.samefile(cwd, os.getcwd())) @@ -503,9 +522,9 @@ def test_github_download_repo(self): path = gh.download_repo(repo=repo, branch=branch, account=account, path=self.test_prefix, github_user=GITHUB_TEST_ACCOUNT) self.assertTrue(os.path.samefile(path, repodir)) - self.assertTrue('easybuild' in os.listdir(repodir)) + self.assertIn('easybuild', os.listdir(repodir)) self.assertTrue(re.match('^[0-9a-f]{40}$', read_file(shafile))) - self.assertTrue(os.path.exists(os.path.join(repodir, 'easybuild', 'easyblocks', '__init__.py'))) + self.assertExists(os.path.join(repodir, 'easybuild', 'easyblocks', '__init__.py')) def test_install_github_token(self): """Test for install_github_token function.""" @@ -573,7 +592,7 @@ def test_github_find_easybuild_easyconfig(self): expected = os.path.join('e', 'EasyBuild', r'EasyBuild-[1-9]+\.[0-9]+\.[0-9]+\.eb') regex = re.compile(expected) self.assertTrue(regex.search(path), "Pattern '%s' found in '%s'" % (regex.pattern, path)) - self.assertTrue(os.path.exists(path), "Path %s exists" % path) + self.assertExists(path) def test_github_find_patches(self): """ Test for find_software_name_for_patch """ @@ -593,7 +612,7 @@ def test_github_find_patches(self): txt = self.get_stdout() self.mock_stdout(False) - self.assertTrue(ec == 'toy') + self.assertEqual(ec, 'toy') reg = re.compile(r'[1-9]+ of [1-9]+ easyconfigs checked') self.assertTrue(re.search(reg, txt)) @@ -667,7 +686,7 @@ def run_check(expected_result=False): self.mock_stderr(False) self.assertEqual(res, expected_result) self.assertEqual(stdout, expected_stdout) - self.assertTrue(expected_warning in stderr, "Found '%s' in: %s" % (expected_warning, stderr)) + self.assertIn(expected_warning, stderr) return stderr pr_data = { @@ -920,12 +939,12 @@ def test_github_restclient(self): status, headers = client.head() self.assertEqual(status, 200) self.assertTrue(headers) - self.assertTrue('X-GitHub-Media-Type' in headers) + self.assertIn('X-GitHub-Media-Type', headers) httperror_hit = False try: status, body = client.user.emails.post(body='test@example.com') - self.assertTrue(False, 'posting to unauthorized endpoint did not throw a http error') + self.fail('posting to unauthorized endpoint did not throw a http error') except HTTPError: httperror_hit = True self.assertTrue(httperror_hit, "expected HTTPError not encountered") @@ -933,7 +952,7 @@ def test_github_restclient(self): httperror_hit = False try: status, body = client.user.emails.delete(body='test@example.com') - self.assertTrue(False, 'deleting to unauthorized endpoint did not throw a http error') + self.fail('deleting to unauthorized endpoint did not throw a http error') except HTTPError: httperror_hit = True self.assertTrue(httperror_hit, "expected HTTPError not encountered") @@ -1011,9 +1030,9 @@ def test_github_det_pr_target_repo(self): github_py = os.path.join(test_dir, 'github.py') configuremake = os.path.join(test_dir, 'sandbox', 'easybuild', 'easyblocks', 'generic', 'configuremake.py') - self.assertTrue(os.path.exists(configuremake)) + self.assertExists(configuremake) toy_eb = os.path.join(test_dir, 'sandbox', 'easybuild', 'easyblocks', 't', 'toy.py') - self.assertTrue(os.path.exists(toy_eb)) + self.assertExists(toy_eb) self.assertEqual(build_option('pr_target_repo'), None) self.assertEqual(gh.det_pr_target_repo(categorize_files_by_type([github_py])), 'easybuild-framework') @@ -1038,6 +1057,7 @@ def test_github_det_pr_target_repo(self): self.assertEqual(gh.det_pr_target_repo(categorize_files_by_type([configuremake])), 'thisisjustatest') self.assertEqual(gh.det_pr_target_repo(categorize_files_by_type([toy_eb])), 'thisisjustatest') + @requires_github_access() def test_push_branch_to_github(self): """Test push_branch_to_github.""" @@ -1165,10 +1185,10 @@ def test_github_create_test_report(self): "EASYBUILD_DEBUG=1", ] for pattern in patterns: - self.assertTrue(pattern in res['full'], "Pattern '%s' found in: %s" % (pattern, res['full'])) + self.assertIn(pattern, res['full']) for pattern in patterns[:2]: - self.assertTrue(pattern in res['full'], "Pattern '%s' found in: %s" % (pattern, res['overview'])) + self.assertIn(pattern, res['full']) # mock create_gist function, we don't want to actually create a gist every time we run this test... def fake_create_gist(*args, **kwargs): @@ -1183,12 +1203,12 @@ def fake_create_gist(*args, **kwargs): "https://github.com/easybuilders/easybuild-easyconfigs/pull/123", ]) for pattern in patterns: - self.assertTrue(pattern in res['full'], "Pattern '%s' found in: %s" % (pattern, res['full'])) + self.assertIn(pattern, res['full']) for pattern in patterns[:3]: - self.assertTrue(pattern in res['full'], "Pattern '%s' found in: %s" % (pattern, res['overview'])) + self.assertIn(pattern, res['full']) - self.assertTrue("**SUCCESS** _test.eb_" in res['overview']) + self.assertIn("**SUCCESS** _test.eb_", res['overview']) def test_is_patch_for(self): """Test for is_patch_for function.""" diff --git a/test/framework/hooks.py b/test/framework/hooks.py index 5ee52a5d5f..fad251b040 100644 --- a/test/framework/hooks.py +++ b/test/framework/hooks.py @@ -61,6 +61,9 @@ def setUp(self): '', 'def pre_install_hook(self):', ' print("this is run before install step")', + '', + 'def pre_single_extension_hook(ext):', + ' print("this is run before installing an extension")', ]) write_file(self.test_hooks_pymod, test_hooks_pymod_txt) @@ -71,14 +74,21 @@ def test_load_hooks(self): hooks = load_hooks(self.test_hooks_pymod) - self.assertEqual(len(hooks), 4) - self.assertEqual(sorted(hooks.keys()), ['parse_hook', 'post_configure_hook', 'pre_install_hook', 'start_hook']) + self.assertEqual(len(hooks), 5) + expected = [ + 'parse_hook', + 'post_configure_hook', + 'pre_install_hook', + 'pre_single_extension_hook', + 'start_hook', + ] + self.assertEqual(sorted(hooks.keys()), expected) self.assertTrue(all(callable(h) for h in hooks.values())) # test caching of hooks remove_file(self.test_hooks_pymod) cached_hooks = load_hooks(self.test_hooks_pymod) - self.assertTrue(cached_hooks is hooks) + self.assertIs(cached_hooks, hooks) # hooks file can be empty empty_hooks_path = os.path.join(self.test_prefix, 'empty_hooks.py') @@ -88,7 +98,7 @@ def test_load_hooks(self): # loading another hooks file doesn't affect cached hooks prev_hooks = load_hooks(self.test_hooks_pymod) - self.assertTrue(prev_hooks is hooks) + self.assertIs(prev_hooks, hooks) # clearing cached hooks results in error because hooks file is not found easybuild.tools.hooks._cached_hooks = {} @@ -101,6 +111,7 @@ def test_find_hook(self): post_configure_hook = [hooks[k] for k in hooks if k == 'post_configure_hook'][0] pre_install_hook = [hooks[k] for k in hooks if k == 'pre_install_hook'][0] + pre_single_extension_hook = [hooks[k] for k in hooks if k == 'pre_single_extension_hook'][0] start_hook = [hooks[k] for k in hooks if k == 'start_hook'][0] self.assertEqual(find_hook('configure', hooks), None) @@ -111,6 +122,14 @@ def test_find_hook(self): self.assertEqual(find_hook('install', hooks, pre_step_hook=True), pre_install_hook) self.assertEqual(find_hook('install', hooks, post_step_hook=True), None) + self.assertEqual(find_hook('single_extension', hooks), None) + self.assertEqual(find_hook('single_extension', hooks, pre_step_hook=True), pre_single_extension_hook) + self.assertEqual(find_hook('single_extension', hooks, post_step_hook=True), None) + + self.assertEqual(find_hook('extensions', hooks), None) + self.assertEqual(find_hook('extensions', hooks, pre_step_hook=True), None) + self.assertEqual(find_hook('extensions', hooks, post_step_hook=True), None) + self.assertEqual(find_hook('build', hooks), None) self.assertEqual(find_hook('build', hooks, pre_step_hook=True), None) self.assertEqual(find_hook('build', hooks, post_step_hook=True), None) @@ -136,6 +155,11 @@ def test_run_hook(self): run_hook('build', hooks, post_step_hook=True, args=[None]) run_hook('install', hooks, pre_step_hook=True, args=[None]) run_hook('install', hooks, post_step_hook=True, args=[None]) + run_hook('extensions', hooks, pre_step_hook=True, args=[None]) + for _ in range(3): + run_hook('single_extension', hooks, pre_step_hook=True, args=[None]) + run_hook('single_extension', hooks, post_step_hook=True, args=[None]) + run_hook('extensions', hooks, post_step_hook=True, args=[None]) stdout = self.get_stdout() stderr = self.get_stderr() self.mock_stdout(False) @@ -151,6 +175,12 @@ def test_run_hook(self): "running foo helper method", "== Running pre-install hook...", "this is run before install step", + "== Running pre-single_extension hook...", + "this is run before installing an extension", + "== Running pre-single_extension hook...", + "this is run before installing an extension", + "== Running pre-single_extension hook...", + "this is run before installing an extension", ]) self.assertEqual(stdout.strip(), expected_stdout) diff --git a/test/framework/include.py b/test/framework/include.py index 6c9ab75864..b346f911ae 100644 --- a/test/framework/include.py +++ b/test/framework/include.py @@ -92,7 +92,7 @@ def test_include_easyblocks(self): 'easyblocks/generic/__init__.py', 'easyblocks/generic/mybar.py'] for filepath in expected_paths: fullpath = os.path.join(included_easyblocks_path, 'easybuild', filepath) - self.assertTrue(os.path.exists(fullpath), "%s exists" % fullpath) + self.assertExists(fullpath) # path to included easyblocks should be prepended to Python search path self.assertEqual(sys.path[0], included_easyblocks_path) @@ -185,7 +185,7 @@ def test_include_mns(self): 'tools/module_naming_scheme/my_mns.py'] for filepath in expected_paths: fullpath = os.path.join(included_mns_path, 'easybuild', filepath) - self.assertTrue(os.path.exists(fullpath), "%s exists" % fullpath) + self.assertExists(fullpath) # path to included MNSs should be prepended to Python search path self.assertEqual(sys.path[0], included_mns_path) @@ -232,7 +232,7 @@ def test_include_toolchains(self): 'toolchains/my_tc.py', 'toolchains/compiler/my_compiler.py'] for filepath in expected_paths: fullpath = os.path.join(included_tcs_path, 'easybuild', filepath) - self.assertTrue(os.path.exists(fullpath), "%s exists" % fullpath) + self.assertExists(fullpath) # path to included MNSs should be prepended to Python search path self.assertEqual(sys.path[0], included_tcs_path) diff --git a/test/framework/lib.py b/test/framework/lib.py index ed85866fcd..2f4149314c 100644 --- a/test/framework/lib.py +++ b/test/framework/lib.py @@ -72,9 +72,9 @@ def configure(self): if BuildOptions in BuildOptions._instances: del BuildOptions._instances[BuildOptions] - self.assertFalse(BuildOptions in BuildOptions._instances) + self.assertNotIn(BuildOptions, BuildOptions._instances) set_up_configuration(silent=True) - self.assertTrue(BuildOptions in BuildOptions._instances) + self.assertIn(BuildOptions, BuildOptions._instances) def test_run_cmd(self): """Test use of run_cmd function in the context of using EasyBuild framework as a library.""" @@ -102,9 +102,9 @@ def test_mkdir(self): self.configure() # mkdir works fine if set_up_configuration was called first - self.assertFalse(os.path.exists(test_dir)) + self.assertNotExists(test_dir) mkdir(test_dir) - self.assertTrue(os.path.exists(test_dir)) + self.assertExists(test_dir) def test_modules_tool(self): """Test use of modules_tool function in the context of using EasyBuild framework as a library.""" @@ -119,7 +119,7 @@ def test_modules_tool(self): modtool = modules_tool() modtool.use(test_mods_path) - self.assertTrue('GCC/6.4.0-2.28' in modtool.available()) + self.assertIn('GCC/6.4.0-2.28', modtool.available()) modtool.load(['GCC/6.4.0-2.28']) self.assertEqual(modtool.list(), [{'default': None, 'mod_name': 'GCC/6.4.0-2.28'}]) diff --git a/test/framework/license.py b/test/framework/license.py index 316b40c80b..80c741908f 100644 --- a/test/framework/license.py +++ b/test/framework/license.py @@ -44,7 +44,7 @@ def test_common_ones(self): lics = what_licenses() commonlicenses = ['LicenseVeryRestrictive', 'LicenseGPLv2', 'LicenseGPLv3'] for lic in commonlicenses: - self.assertTrue(lic in lics, "%s found in %s" % (lic, lics.keys())) + self.assertIn(lic, lics) def test_default_license(self): """Verify that the default License class is very restrictive""" @@ -62,7 +62,7 @@ def test_licenses(self): """Test format of available licenses.""" lics = what_licenses() for lic in lics: - self.assertTrue(isinstance(lic, string_type)) + self.assertIsInstance(lic, string_type) self.assertTrue(lic.startswith('License')) self.assertTrue(issubclass(lics[lic], License)) diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py index 12c2631c15..285f0a6cea 100644 --- a/test/framework/module_generator.py +++ b/test/framework/module_generator.py @@ -223,7 +223,7 @@ def test_set_default_module(self): self.modtool.load([module_name]) full_module_name = module_name + '/' + version_one - self.assertTrue(full_module_name in self.modtool.loaded_modules()) + self.assertIn(full_module_name, self.modtool.loaded_modules()) self.modtool.purge() # setting bar version as default @@ -231,7 +231,7 @@ def test_set_default_module(self): self.modtool.load([module_name]) full_module_name = module_name + '/' + version_two - self.assertTrue(full_module_name in self.modtool.loaded_modules()) + self.assertIn(full_module_name, self.modtool.loaded_modules()) self.modtool.purge() def test_is_loaded(self): @@ -970,7 +970,7 @@ def test_tcl_quoting(self): self.modtool.load([module_name]) full_module_name = module_name + '/' + version_one - self.assertTrue(full_module_name in self.modtool.loaded_modules()) + self.assertIn(full_module_name, self.modtool.loaded_modules()) self.assertEqual(os.getenv(test_envvar), test_flags) self.modtool.purge() @@ -1065,7 +1065,7 @@ def test_conditional_statement(self): self.assertEqual(if_else_cond, expected) else: - self.assertTrue(False, "Unknown module syntax") + self.fail("Unknown module syntax") def test_load_msg(self): """Test including a load message in the module file.""" @@ -1097,6 +1097,36 @@ def test_load_msg(self): ]) self.assertEqual(lua_load_msg, self.modgen.msg_on_load('test $test \\$test\ntest $foo \\$bar')) + def test_unload_msg(self): + """Test including an unload message in the module file.""" + if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl: + expected = "\nif { [ module-info mode unload ] } {\nputs stderr \"test\"\n}\n" + self.assertEqual(expected, self.modgen.msg_on_unload('test')) + + tcl_unload_msg = '\n'.join([ + '', + "if { [ module-info mode unload ] } {", + "puts stderr \"test \\$test \\$test", + "test \\$foo \\$bar\"", + "}", + '', + ]) + self.assertEqual(tcl_unload_msg, self.modgen.msg_on_unload('test $test \\$test\ntest $foo \\$bar')) + + else: + expected = '\nif mode() == "unload" then\nio.stderr:write([==[test]==])\nend\n' + self.assertEqual(expected, self.modgen.msg_on_unload('test')) + + lua_unload_msg = '\n'.join([ + '', + 'if mode() == "unload" then', + 'io.stderr:write([==[test $test \\$test', + 'test $foo \\$bar]==])', + 'end', + '', + ]) + self.assertEqual(lua_unload_msg, self.modgen.msg_on_unload('test $test \\$test\ntest $foo \\$bar')) + def test_module_naming_scheme(self): """Test using default module naming scheme.""" all_stops = [x[0] for x in EasyBlock.get_steps()] @@ -1239,21 +1269,32 @@ def test_mns(): ec2mod_map = default_ec2mod_map test_mns() + # check how an incorrect dependency specification is handled; + # accidentally using SYSTEM as 3rd tuple element should trigger a useful error, + # not a nasty crash; cfr. https://github.com/easybuilders/easybuild-framework/issues/4181 + faulty_dep_spec = { + 'name': 'test', + 'version': '1.2.3', + 'versionsuffix': {'name': 'system', 'version': 'system'}, + } + error_pattern = "versionsuffix value should be a string, found 'dict'" + self.assertErrorRegex(EasyBuildError, error_pattern, ActiveMNS().det_full_module_name, faulty_dep_spec) + def test_mod_name_validation(self): """Test module naming validation.""" # module name must be a string - self.assertTrue(not is_valid_module_name(('foo', 'bar'))) - self.assertTrue(not is_valid_module_name(['foo', 'bar'])) - self.assertTrue(not is_valid_module_name(123)) + self.assertFalse(is_valid_module_name(('foo', 'bar'))) + self.assertFalse(is_valid_module_name(['foo', 'bar'])) + self.assertFalse(is_valid_module_name(123)) # module name must be relative - self.assertTrue(not is_valid_module_name('/foo/bar')) + self.assertFalse(is_valid_module_name('/foo/bar')) # module name must only contain valid characters - self.assertTrue(not is_valid_module_name('foo\x0bbar')) - self.assertTrue(not is_valid_module_name('foo\x0cbar')) - self.assertTrue(not is_valid_module_name('foo\rbar')) - self.assertTrue(not is_valid_module_name('foo\0bar')) + self.assertFalse(is_valid_module_name('foo\x0bbar')) + self.assertFalse(is_valid_module_name('foo\x0cbar')) + self.assertFalse(is_valid_module_name('foo\rbar')) + self.assertFalse(is_valid_module_name('foo\0bar')) # valid module name must be accepted self.assertTrue(is_valid_module_name('gzip/foss-2018a-suffix')) diff --git a/test/framework/modules.py b/test/framework/modules.py index 0807f6a55a..4f2b3fc5d6 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -36,13 +36,13 @@ import shutil import stat import sys -from distutils.version import StrictVersion from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config from unittest import TextTestRunner import easybuild.tools.modules as mod from easybuild.framework.easyblock import EasyBlock from easybuild.framework.easyconfig.easyconfig import EasyConfig +from easybuild.tools import StrictVersion from easybuild.tools.build_log import EasyBuildError from easybuild.tools.environment import modify_env from easybuild.tools.filetools import adjust_permissions, copy_file, copy_dir, mkdir @@ -110,7 +110,7 @@ def test_run_module(self): modify_env(os.environ, self.orig_environ, verbose=False) self.reset_modulepath([os.path.join(testdir, 'modules')]) - self.assertFalse('EBROOTGCC' in os.environ) + self.assertNotIn('EBROOTGCC', os.environ) self.modtool.run_module(['load', 'GCC/6.4.0-2.28']) self.assertEqual(os.environ['EBROOTGCC'], '/prefix/software/GCC/6.4.0-2.28') @@ -149,7 +149,7 @@ def test_run_module(self): self.assertEqual(res, [{'mod_name': 'GCC/6.4.0-2.28', 'default': None}]) res = self.modtool.run_module('avail', 'GCC/4.6.3') - self.assertTrue(isinstance(res, list)) + self.assertIsInstance(res, list) self.assertEqual(sorted([x['mod_name'] for x in res]), ['GCC/4.6.3']) # loading a module produces no output, so we get an empty list @@ -170,7 +170,7 @@ def test_run_module(self): self.assertTrue(regex.search(out), "Pattern '%s' should be found in: %s" % (regex.pattern, out)) # OpenBLAS module did *not* get loaded - self.assertFalse('EBROOTOPENBLAS' in os.environ) + self.assertNotIn('EBROOTOPENBLAS', os.environ) res = self.modtool.list() expected = ['GCC/6.4.0-2.28', 'OpenMPI/2.1.2-GCC-6.4.0-2.28', 'hwloc/1.11.8-GCC-6.4.0-2.28'] self.assertEqual(sorted([x['mod_name'] for x in res]), expected) @@ -215,9 +215,9 @@ def test_avail(self): if isinstance(self.modtool, Lmod) and StrictVersion(self.modtool.version) >= StrictVersion('5.7.5'): # with recent versions of Lmod, also the hidden modules are included in the output of 'avail' self.assertEqual(len(ms), TEST_MODULES_COUNT + 3) - self.assertTrue('bzip2/.1.0.6' in ms) - self.assertTrue('toy/.0.0-deps' in ms) - self.assertTrue('OpenMPI/.2.1.2-GCC-6.4.0-2.28' in ms) + self.assertIn('bzip2/.1.0.6', ms) + self.assertIn('toy/.0.0-deps', ms) + self.assertIn('OpenMPI/.2.1.2-GCC-6.4.0-2.28', ms) else: self.assertEqual(len(ms), TEST_MODULES_COUNT) @@ -248,8 +248,8 @@ def test_exist(self): test_modules_path = os.path.abspath(os.path.join(os.path.dirname(__file__), 'modules')) if isinstance(self.modtool, Lmod): # make sure only the .lua module file is there, otherwise this test doesn't work as intended - self.assertTrue(os.path.exists(os.path.join(test_modules_path, 'bzip2', '.1.0.6.lua'))) - self.assertFalse(os.path.exists(os.path.join(test_modules_path, 'bzip2', '.1.0.6'))) + self.assertExists(os.path.join(test_modules_path, 'bzip2', '.1.0.6.lua')) + self.assertNotExists(os.path.join(test_modules_path, 'bzip2', '.1.0.6')) self.assertEqual(self.modtool.exist(['bzip2/.1.0.6']), [True]) # exist also works on lists of module names @@ -292,11 +292,11 @@ def test_exist(self): write_file(os.path.join(java_mod_dir, '.modulerc'), modulerc_tcl_txt) avail_mods = self.modtool.available() - self.assertTrue('Java/1.8.0_181' in avail_mods) + self.assertIn('Java/1.8.0_181', avail_mods) if isinstance(self.modtool, Lmod) and StrictVersion(self.modtool.version) >= StrictVersion('7.0'): - self.assertTrue('Java/1.8' in avail_mods) - self.assertTrue('Java/site_default' in avail_mods) - self.assertTrue('JavaAlias' in avail_mods) + self.assertIn('Java/1.8', avail_mods) + self.assertIn('Java/site_default', avail_mods) + self.assertIn('JavaAlias', avail_mods) self.assertEqual(self.modtool.exist(['JavaAlias']), [True]) self.assertEqual(self.modtool.exist(['Java/1.8', 'Java/1.8.0_181']), [True, True]) @@ -344,7 +344,7 @@ def test_exist(self): mkdir(os.path.join(self.test_prefix, 'Core')) shutil.move(java_mod_dir, os.path.join(self.test_prefix, 'Core', 'Java')) - self.assertTrue('Core/Java/1.8.0_181' in self.modtool.available()) + self.assertIn('Core/Java/1.8.0_181', self.modtool.available()) self.assertEqual(self.modtool.exist(['Core/Java/1.8.0_181']), [True]) # there's a workaround to ensure that module wrappers/aliases are recognized when they're # being checked with the full module name (see https://github.com/TACC/Lmod/issues/446); @@ -366,8 +366,8 @@ def test_exist(self): ])) avail_mods = self.modtool.available() - self.assertTrue('Java/1.8.0_181' in avail_mods) - self.assertTrue('Java/1.8' in avail_mods) + self.assertIn('Java/1.8.0_181', avail_mods) + self.assertIn('Java/1.8', avail_mods) self.assertEqual(self.modtool.exist(['Java/1.8', 'Java/1.8.0_181']), [True, True]) # check for an alias with a different version suffix than the base module @@ -379,7 +379,7 @@ def test_exist(self): # back to HMNS setup shutil.move(java_mod_dir, os.path.join(self.test_prefix, 'Core', 'Java')) - self.assertTrue('Core/Java/1.8.0_181' in self.modtool.available()) + self.assertIn('Core/Java/1.8.0_181', self.modtool.available()) self.assertEqual(self.modtool.exist(['Core/Java/1.8.0_181']), [True]) self.assertEqual(self.modtool.exist(['Core/Java/1.8']), [True]) self.assertEqual(self.modtool.exist(['Core/Java/site_default']), [True]) @@ -410,7 +410,7 @@ def test_load(self): for m in ms: self.modtool.load([m]) - self.assertTrue(m in self.modtool.loaded_modules()) + self.assertIn(m, self.modtool.loaded_modules()) self.modtool.purge() # trying to load a module not on the top level of a hierarchy should fail @@ -431,7 +431,7 @@ def test_load(self): # GCC should be loaded, but should not be listed last (OpenMPI was loaded last) loaded_modules = self.modtool.loaded_modules() - self.assertTrue('GCC/6.4.0-2.28' in loaded_modules) + self.assertIn('GCC/6.4.0-2.28', loaded_modules) self.assertFalse(loaded_modules[-1] == 'GCC/6.4.0-2.28') # if GCC is loaded again, $EBROOTGCC should be set again, and GCC should be listed last @@ -444,7 +444,7 @@ def test_load(self): if isinstance(self.modtool, Lmod): # order of loaded modules only changes with Lmod - self.assertTrue(self.modtool.loaded_modules()[-1] == 'GCC/6.4.0-2.28') + self.assertEqual(self.modtool.loaded_modules()[-1], 'GCC/6.4.0-2.28') # set things up for checking that GCC does *not* get reloaded when requested if 'EBROOTGCC' in os.environ: @@ -452,12 +452,12 @@ def test_load(self): self.modtool.load(['OpenMPI/2.1.2-GCC-6.4.0-2.28']) if isinstance(self.modtool, Lmod): # order of loaded modules only changes with Lmod - self.assertTrue(self.modtool.loaded_modules()[-1] == 'OpenMPI/2.1.2-GCC-6.4.0-2.28') + self.assertEqual(self.modtool.loaded_modules()[-1], 'OpenMPI/2.1.2-GCC-6.4.0-2.28') # reloading can be disabled using allow_reload=False self.modtool.load(['GCC/6.4.0-2.28'], allow_reload=False) self.assertEqual(os.environ.get('EBROOTGCC'), None) - self.assertFalse(loaded_modules[-1] == 'GCC/6.4.0-2.28') + self.assertNotEqual(loaded_modules[-1], 'GCC/6.4.0-2.28') def test_show(self): """Test for ModulesTool.show method.""" @@ -557,7 +557,7 @@ def test_check_module_path_hmns(self): mkdir(core_mod_dir, parents=True) doesnotexist = os.path.join(self.test_prefix, 'doesnotexist') - self.assertFalse(os.path.exists(doesnotexist)) + self.assertNotExists(doesnotexist) os.environ['MODULEPATH'] = '%s:%s' % (core_mod_dir, doesnotexist) modtool = modules_tool() @@ -638,13 +638,13 @@ def test_purge(self): ms = self.modtool.available() self.modtool.load([ms[0]]) - self.assertTrue(len(self.modtool.loaded_modules()) > 0) + self.assertGreater(len(self.modtool.loaded_modules()), 0) self.modtool.purge() - self.assertTrue(len(self.modtool.loaded_modules()) == 0) + self.assertEqual(len(self.modtool.loaded_modules()), 0) self.modtool.purge() - self.assertTrue(len(self.modtool.loaded_modules()) == 0) + self.assertEqual(len(self.modtool.loaded_modules()), 0) def test_get_software_root_version_libdir(self): """Test get_software_X functions.""" @@ -1038,6 +1038,7 @@ def test_modules_tool_stateless(self): # GCC/4.6.3 is *not* an available Core module os.environ['LC_ALL'] = 'C' + os.environ['LANG'] = 'C' self.assertErrorRegex(EasyBuildError, load_err_msg, self.modtool.load, ['GCC/4.6.3']) # GCC/6.4.0-2.28 is one of the available Core modules @@ -1080,7 +1081,7 @@ def test_mk_module_cache_key(self): """Test mk_module_cache_key method.""" os.environ['MODULEPATH'] = '%s:/tmp/test' % self.test_prefix res = self.modtool.mk_module_cache_key('thisisapartialkey') - self.assertTrue(isinstance(res, tuple)) + self.assertIsInstance(res, tuple) self.assertEqual(res, ('MODULEPATH=%s:/tmp/test' % self.test_prefix, self.modtool.COMMAND, 'thisisapartialkey')) del os.environ['MODULEPATH'] @@ -1119,20 +1120,20 @@ def test_module_caches(self): # fetch cache entry avail_cache_key = list(mod.MODULE_AVAIL_CACHE.keys())[0] cached_res = mod.MODULE_AVAIL_CACHE[avail_cache_key] - self.assertTrue(cached_res == res) + self.assertEqual(cached_res, res) # running avail again results in getting cached result, exactly the same result as before # depending on the modules tool being used, it may not be the same list instance, because of post-processing - self.assertTrue(self.modtool.available() == res) + self.assertEqual(self.modtool.available(), res) # run 'show', should be all cached show_res_gcc = self.modtool.show('GCC/6.4.0-2.28') show_res_fftw = self.modtool.show('FFTW') self.assertEqual(len(mod.MODULE_SHOW_CACHE), 2) - self.assertTrue(show_res_gcc in mod.MODULE_SHOW_CACHE.values()) - self.assertTrue(show_res_fftw in mod.MODULE_SHOW_CACHE.values()) - self.assertTrue(self.modtool.show('GCC/6.4.0-2.28') is show_res_gcc) - self.assertTrue(self.modtool.show('FFTW') is show_res_fftw) + self.assertIn(show_res_gcc, mod.MODULE_SHOW_CACHE.values()) + self.assertIn(show_res_fftw, mod.MODULE_SHOW_CACHE.values()) + self.assertIs(self.modtool.show('GCC/6.4.0-2.28'), show_res_gcc) + self.assertIs(self.modtool.show('FFTW'), show_res_fftw) # invalidate caches with correct path modulepaths = [p for p in os.environ.get('MODULEPATH', '').split(os.pathsep) if p] @@ -1158,7 +1159,7 @@ def test_module_use_unuse(self): ]) write_file(os.path.join(self.test_prefix, subdir, 'test'), modtxt) - self.assertFalse(test_dir1 in os.environ.get('MODULEPATH', '')) + self.assertNotIn(test_dir1, os.environ.get('MODULEPATH', '')) self.modtool.use(test_dir1) self.assertTrue(os.environ['MODULEPATH'].startswith('%s:' % test_dir1)) self.modtool.use(test_dir2) @@ -1177,21 +1178,21 @@ def test_module_use_unuse(self): self.modtool.unload(['test']) self.modtool.unuse(test_dir3) - self.assertFalse(test_dir3 in os.environ.get('MODULEPATH', '')) + self.assertNotIn(test_dir3, os.environ.get('MODULEPATH', '')) self.modtool.load(['test']) self.assertEqual(os.getenv('TEST123'), 'two') self.modtool.unload(['test']) self.modtool.unuse(test_dir2) - self.assertFalse(test_dir2 in os.environ.get('MODULEPATH', '')) + self.assertNotIn(test_dir2, os.environ.get('MODULEPATH', '')) self.modtool.load(['test']) self.assertEqual(os.getenv('TEST123'), 'one') self.modtool.unload(['test']) self.modtool.unuse(test_dir1) - self.assertFalse(test_dir1 in os.environ.get('MODULEPATH', '')) + self.assertNotIn(test_dir1, os.environ.get('MODULEPATH', '')) # also test use with high priority self.modtool.use(test_dir2, priority=10000) @@ -1228,7 +1229,7 @@ def test_module_use_unuse(self): self.modtool.use(test_dir1) self.assertEqual(os.environ['MODULEPATH'], test_dir1) self.modtool.unuse(test_dir1) - self.assertFalse('MODULEPATH' in os.environ) + self.assertNotIn('MODULEPATH', os.environ) os.environ['MODULEPATH'] = old_module_path # Restore def test_add_and_remove_module_path(self): @@ -1299,16 +1300,16 @@ def test_module_use_bash(self): # cfr. https://github.com/easybuilders/easybuild-framework/issues/1756, # https://bugzilla.redhat.com/show_bug.cgi?id=1326075 modules_dir = os.path.abspath(os.path.join(self.test_prefix, 'modules')) - self.assertFalse(modules_dir in os.environ['MODULEPATH']) + self.assertNotIn(modules_dir, os.environ['MODULEPATH']) mkdir(modules_dir, parents=True) self.modtool.use(modules_dir) modulepath = os.environ['MODULEPATH'] - self.assertTrue(modules_dir in modulepath) + self.assertIn(modules_dir, modulepath) out, _ = run_cmd("bash -c 'echo MODULEPATH: $MODULEPATH'", simple=False) self.assertEqual(out.strip(), "MODULEPATH: %s" % modulepath) - self.assertTrue(modules_dir in out) + self.assertIn(modules_dir, out) def test_load_in_hierarchy(self): """Test whether loading a module in a module hierarchy results in loading the correct module.""" @@ -1332,7 +1333,7 @@ def test_load_in_hierarchy(self): # make sure that compiler-dependent hwloc test module exists gcc_mod_dir = os.path.join(mod_dir, 'Compiler', 'GCC', '6.4.0-2.28') - self.assertTrue(os.path.exists(os.path.join(gcc_mod_dir, 'hwloc', '1.11.8'))) + self.assertExists(os.path.join(gcc_mod_dir, 'hwloc', '1.11.8')) # test loading of compiler-dependent hwloc test module self.modtool.purge() @@ -1455,7 +1456,7 @@ def check_loaded_modules(): stderr = check_loaded_modules() warning_msg = "WARNING: Found defined $EBROOT* environment variables without matching loaded module: " warning_msg = "$EBROOTSOFTWAREWITHOUTAMATCHINGMODULE\n" - self.assertTrue(warning_msg in stderr) + self.assertIn(warning_msg, stderr) build_options.update({'check_ebroot_env_vars': 'error'}) init_config(build_options=build_options) @@ -1474,7 +1475,7 @@ def check_loaded_modules(): warning_msg = "WARNING: Found defined $EBROOT* environment variables without matching loaded module: " warning_msg += "$EBROOTSOFTWAREWITHOUTAMATCHINGMODULE; unsetting them" self.assertEqual(stderr, warning_msg) - self.assertTrue(os.environ.get('EBROOTSOFTWAREWITHOUTAMATCHINGMODULE') is None) + self.assertIsNone(os.environ.get('EBROOTSOFTWAREWITHOUTAMATCHINGMODULE')) # specified action for detected loaded modules is verified early error_msg = "Unknown action specified to --detect-loaded-modules: sdvbfdgh" diff --git a/test/framework/modulestool.py b/test/framework/modulestool.py index 0330cb008c..59c6872b93 100644 --- a/test/framework/modulestool.py +++ b/test/framework/modulestool.py @@ -34,10 +34,9 @@ from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered from unittest import TextTestRunner -from distutils.version import StrictVersion from easybuild.base import fancylogger -from easybuild.tools import modules +from easybuild.tools import modules, StrictVersion from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import read_file, which, write_file from easybuild.tools.modules import Lmod @@ -91,10 +90,10 @@ def test_environment_command(self): try: bmmt = BrokenMockModulesTool(mod_paths=[], testing=True) # should never get here - self.assertTrue(False, 'BrokenMockModulesTool should fail') + self.fail('BrokenMockModulesTool should fail') except EasyBuildError as err: err_msg = "command is not available" - self.assertTrue(err_msg in str(err), "'%s' found in: %s" % (err_msg, err)) + self.assertIn(err_msg, str(err)) os.environ[BrokenMockModulesTool.COMMAND_ENVIRONMENT] = MockModulesTool.COMMAND os.environ['module'] = "() { /bin/echo $*\n}" @@ -130,7 +129,7 @@ def test_module_mismatch(self): # redefine 'module' function with correct module command os.environ['module'] = "() { eval `/bin/echo $*`\n}" mt = MockModulesTool(testing=True) - self.assertTrue(isinstance(mt.loaded_modules(), list)) # dummy usage + self.assertIsInstance(mt.loaded_modules(), list) # dummy usage # a warning should be logged if the 'module' function is undefined del os.environ['module'] diff --git a/test/framework/options.py b/test/framework/options.py index 7d0957e3d0..27928265ec 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -36,6 +36,7 @@ import sys import tempfile import textwrap +import warnings from easybuild.tools import LooseVersion from unittest import TextTestRunner @@ -176,7 +177,7 @@ def test_help_long(self): "Not all option groups included in short help (2)") # for boolean options, we mention in the help text how to disable them - regex = re.compile("default: True; disable with --disable-cleanup-builddir", re.M) + regex = re.compile(r"default: True; disable with\s*--disable-\s*cleanup-\s*builddir", re.M) self.assertTrue(regex.search(outtxt), "Pattern '%s' found in: %s" % (regex.pattern, outtxt)) def test_help_rst(self): @@ -390,12 +391,13 @@ def test_skip_test_step(self): # And now with the argument args.append('--skip-test-step') - self.mock_stdout(True) - outtxt = self.eb_main(args, do_build=True) - self.mock_stdout(False) + with self.mocked_stdout_stderr() as (_, stderr): + outtxt = self.eb_main(args, do_build=True) found_msg = "Skipping test step" found = re.search(found_msg, outtxt) self.assertTrue(found, "Message about test step being skipped is present, outtxt: %s" % outtxt) + # Warning should be printed to stderr + self.assertIn('Will not run the test step as requested via skip-test-step', stderr.getvalue()) found = re.search(test_run_msg, outtxt) self.assertFalse(found, "Test execution command is NOT present, outtxt: %s" % outtxt) @@ -492,8 +494,8 @@ def test_zzz_logtostdout(self): stdout = self.get_stdout() self.mock_stdout(False) - self.assertTrue("Auto-enabling streaming output" in stdout) - self.assertTrue("== (streaming) output for command 'gcc toy.c -o toy':" in stdout) + self.assertIn("Auto-enabling streaming output", stdout) + self.assertIn("== (streaming) output for command 'gcc toy.c -o toy':", stdout) if os.path.exists(dummylogfn): os.remove(dummylogfn) @@ -659,6 +661,66 @@ def run_test(custom=None, extra_params=[], fmt=None): run_test(custom='bar', extra_params=['bar_extra1', 'bar_extra2'], fmt=fmt) run_test(custom='EB_foofoo', extra_params=['foofoo_extra1', 'foofoo_extra2'], fmt=fmt) + def test_avail_hooks(self): + """ + Test listing available hooks via --avail-hooks + """ + + self.mock_stderr(True) + self.mock_stdout(True) + self.eb_main(['--avail-hooks'], verbose=True, raise_error=True) + stderr, stdout = self.get_stderr(), self.get_stdout() + self.mock_stderr(False) + self.mock_stdout(False) + + self.assertFalse(stderr) + + expected = '\n'.join([ + "List of supported hooks (in order of execution):", + " start_hook", + " parse_hook", + " pre_fetch_hook", + " post_fetch_hook", + " pre_ready_hook", + " post_ready_hook", + " pre_source_hook", + " post_source_hook", + " pre_patch_hook", + " post_patch_hook", + " pre_prepare_hook", + " post_prepare_hook", + " pre_configure_hook", + " post_configure_hook", + " pre_build_hook", + " post_build_hook", + " pre_test_hook", + " post_test_hook", + " pre_install_hook", + " post_install_hook", + " pre_extensions_hook", + " pre_single_extension_hook", + " post_single_extension_hook", + " post_extensions_hook", + " pre_postproc_hook", + " post_postproc_hook", + " pre_sanitycheck_hook", + " post_sanitycheck_hook", + " pre_cleanup_hook", + " post_cleanup_hook", + " pre_module_hook", + " module_write_hook", + " post_module_hook", + " pre_permissions_hook", + " post_permissions_hook", + " pre_package_hook", + " post_package_hook", + " pre_testcases_hook", + " post_testcases_hook", + " end_hook", + '', + ]) + self.assertEqual(stdout, expected) + # double underscore to make sure it runs first, which is required to detect certain types of bugs, # e.g. running with non-initialized EasyBuild config (truly mimicing 'eb --list-toolchains') def test__list_toolchains(self): @@ -1142,7 +1204,7 @@ def test_copy_ec(self): regex = re.compile(r'.*/toy-0.0.eb copied to %s' % test_ec) self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) - self.assertTrue(os.path.exists(test_ec)) + self.assertExists(test_ec) self.assertEqual(toy_ec_txt, read_file(test_ec)) remove_file(test_ec) @@ -1150,7 +1212,7 @@ def test_copy_ec(self): # basic test: copying one easyconfig file to a non-existing relative path cwd = change_dir(self.test_prefix) target_fn = 'test.eb' - self.assertFalse(os.path.exists(target_fn)) + self.assertNotExists(target_fn) args = ['--copy-ec', 'toy-0.0.eb', target_fn] stdout = self.mocked_main(args) @@ -1159,7 +1221,7 @@ def test_copy_ec(self): change_dir(cwd) - self.assertTrue(os.path.exists(test_ec)) + self.assertExists(test_ec) self.assertEqual(toy_ec_txt, read_file(test_ec)) # copying one easyconfig into an existing directory @@ -1171,20 +1233,20 @@ def test_copy_ec(self): self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) copied_toy_ec = os.path.join(test_target_dir, 'toy-0.0.eb') - self.assertTrue(os.path.exists(copied_toy_ec)) + self.assertExists(copied_toy_ec) self.assertEqual(toy_ec_txt, read_file(copied_toy_ec)) remove_dir(test_target_dir) def check_copied_files(): """Helper function to check result of copying multiple easyconfigs.""" - self.assertTrue(os.path.exists(test_target_dir)) + self.assertExists(test_target_dir) self.assertEqual(sorted(os.listdir(test_target_dir)), ['bzip2-1.0.6-GCC-4.9.2.eb', 'toy-0.0.eb']) copied_toy_ec = os.path.join(test_target_dir, 'toy-0.0.eb') - self.assertTrue(os.path.exists(copied_toy_ec)) + self.assertExists(copied_toy_ec) self.assertEqual(toy_ec_txt, read_file(copied_toy_ec)) copied_bzip2_ec = os.path.join(test_target_dir, 'bzip2-1.0.6-GCC-4.9.2.eb') - self.assertTrue(os.path.exists(copied_bzip2_ec)) + self.assertExists(copied_bzip2_ec) self.assertEqual(bzip2_ec_txt, read_file(copied_bzip2_ec)) # copying multiple easyconfig files to a non-existing target directory (which is created automatically) @@ -1199,7 +1261,7 @@ def check_copied_files(): # same but with relative path for target dir change_dir(self.test_prefix) args[-1] = os.path.basename(test_target_dir) - self.assertFalse(os.path.exists(args[-1])) + self.assertNotExists(args[-1]) stdout = self.mocked_main(args) self.assertEqual(stdout, '2 file(s) copied to test_target_dir') @@ -1223,7 +1285,7 @@ def check_copied_files(): regex = re.compile('.*/toy-0.0.eb copied to .*/%s' % os.path.basename(test_working_dir)) self.assertTrue(regex.match(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) copied_toy_cwd = os.path.join(test_working_dir, 'toy-0.0.eb') - self.assertTrue(os.path.exists(copied_toy_cwd)) + self.assertExists(copied_toy_cwd) self.assertEqual(read_file(copied_toy_cwd), toy_ec_txt) # --copy-ec without arguments results in a proper error @@ -1259,11 +1321,11 @@ def test_github_copy_ec_from_pr(self): # check that the files exist for pr_file in all_files_pr8007: - self.assertTrue(os.path.exists(os.path.join(test_working_dir, pr_file))) + self.assertExists(os.path.join(test_working_dir, pr_file)) remove_file(os.path.join(test_working_dir, pr_file)) # copying all files touched by PR to a non-existing target directory (which is created automatically) - self.assertFalse(os.path.exists(test_target_dir)) + self.assertNotExists(test_target_dir) args = ['--copy-ec', '--from-pr', '8007', test_target_dir] stdout = self.mocked_main(args) @@ -1271,7 +1333,7 @@ def test_github_copy_ec_from_pr(self): self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) for pr_file in all_files_pr8007: - self.assertTrue(os.path.exists(os.path.join(test_target_dir, pr_file))) + self.assertExists(os.path.join(test_target_dir, pr_file)) remove_dir(test_target_dir) # test where we select a single easyconfig file from a PR @@ -1284,7 +1346,7 @@ def test_github_copy_ec_from_pr(self): self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) self.assertEqual(os.listdir(test_target_dir), [ec_filename]) - self.assertTrue("name = 'bat'" in read_file(os.path.join(test_target_dir, ec_filename))) + self.assertIn("name = 'bat'", read_file(os.path.join(test_target_dir, ec_filename))) remove_dir(test_target_dir) # test copying of a single easyconfig file from a PR to a non-existing path @@ -1295,8 +1357,8 @@ def test_github_copy_ec_from_pr(self): regex = re.compile(r"%s copied to .*/bat.eb" % ec_filename) self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) - self.assertTrue(os.path.exists(bat_ec)) - self.assertTrue("name = 'bat'" in read_file(bat_ec)) + self.assertExists(bat_ec) + self.assertIn("name = 'bat'", read_file(bat_ec)) change_dir(cwd) remove_dir(test_working_dir) @@ -1310,7 +1372,7 @@ def test_github_copy_ec_from_pr(self): self.assertEqual(os.listdir(test_working_dir), [patch_fn]) patch_path = os.path.join(test_working_dir, patch_fn) - self.assertTrue(os.path.exists(patch_path)) + self.assertExists(patch_path) self.assertTrue(is_patch_file(patch_path)) remove_file(patch_path) @@ -1323,7 +1385,7 @@ def test_github_copy_ec_from_pr(self): self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) self.assertEqual(os.listdir(test_working_dir), [ec_filename]) - self.assertTrue("name = 'bat'" in read_file(os.path.join(test_working_dir, ec_filename))) + self.assertIn("name = 'bat'", read_file(os.path.join(test_working_dir, ec_filename))) # also test copying of patch file to current directory (without specifying target location) change_dir(test_working_dir) @@ -1346,8 +1408,8 @@ def test_github_copy_ec_from_pr(self): stdout = self.mocked_main(args) regex = re.compile(r'.*/%s copied to %s' % (ec_pr11521, test_ec)) self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) - self.assertTrue(os.path.exists(test_ec)) - self.assertTrue("name = 'ExifTool'" in read_file(test_ec)) + self.assertExists(test_ec) + self.assertIn("name = 'ExifTool'", read_file(test_ec)) remove_file(test_ec) def test_dry_run(self): @@ -1407,7 +1469,7 @@ def test_missing(self): self.mock_stderr(False) self.mock_stdout(False) self.assertFalse(stderr) - self.assertTrue(expected in stdout, "Pattern '%s' found in: %s" % (expected, stdout)) + self.assertIn(expected, stdout) def test_dry_run_short(self): """Test dry run (short format).""" @@ -1609,7 +1671,7 @@ def test_try_update_deps(self): warning_stub = "\nWARNING: There may be newer version(s) of dep 'OpenBLAS' available with a different " \ "versionsuffix to '-LAPACK-3.4.2'" self.mock_stderr(False) - self.assertTrue(warning_stub in errtxt) + self.assertIn(warning_stub, errtxt) patterns = [ # toolchain got updated r"^ \* \[x\] .*/test_ecs/g/GCC/GCC-6.4.0-2.28.eb \(module: GCC/6.4.0-2.28\)$", @@ -1832,9 +1894,9 @@ def test_github_from_pr_token_log(self): stderr = self.get_stderr() self.mock_stdout(False) self.mock_stderr(False) - self.assertFalse(self.github_token in outtxt) - self.assertFalse(self.github_token in stdout) - self.assertFalse(self.github_token in stderr) + self.assertNotIn(self.github_token, outtxt) + self.assertNotIn(self.github_token, stdout) + self.assertNotIn(self.github_token, stderr) except URLError as err: print("Ignoring URLError '%s' in test_from_pr" % err) @@ -1980,7 +2042,7 @@ def test_header_footer(self): 'setenv("SITE_SPECIFIC_FOOTER_ENV_VAR", "bar")', ]) else: - self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) + self.fail("Unknown module syntax: %s" % get_module_syntax()) # dump header/footer text to file handle, modules_footer = tempfile.mkstemp(prefix='modules-footer-') @@ -2158,10 +2220,10 @@ def test_experimental(self): try: log.experimental('x') # sanity check, should never be reached if it works. - self.assertTrue(False, "Experimental logging should be disabled by setting --disable-experimental option") + self.fail("Experimental logging should be disabled by setting --disable-experimental option") except easybuild.tools.build_log.EasyBuildError as err: # check error message - self.assertTrue('Experimental functionality.' in str(err)) + self.assertIn('Experimental functionality.', str(err)) # toggle experimental EasyBuildOptions( @@ -2170,7 +2232,7 @@ def test_experimental(self): try: log.experimental('x') except easybuild.tools.build_log.EasyBuildError as err: - self.assertTrue(False, "Experimental logging should be allowed by the --experimental option: %s" % err) + self.fail("Experimental logging should be allowed by the --experimental option: %s" % err) # set it back easybuild.tools.build_log.EXPERIMENTAL = orig_value @@ -2199,7 +2261,7 @@ def test_deprecated(self): stderr = self.get_stderr() self.mock_stderr(False) except easybuild.tools.build_log.EasyBuildError as err: - self.assertTrue(False, "Deprecated logging should work: %s" % err) + self.fail("Deprecated logging should work: %s" % err) stderr_regex = re.compile("^\nWARNING: Deprecated functionality, will no longer work in") self.assertTrue(stderr_regex.search(stderr), "Pattern '%s' found in: %s" % (stderr_regex.pattern, stderr)) @@ -2211,9 +2273,9 @@ def test_deprecated(self): try: log.deprecated('x', str(orig_value)) # not supposed to get here - self.assertTrue(False, 'Deprecated logging should throw EasyBuildError') + self.fail('Deprecated logging should throw EasyBuildError') except easybuild.tools.build_log.EasyBuildError as err2: - self.assertTrue('DEPRECATED' in str(err2)) + self.assertIn('DEPRECATED', str(err2)) # force higher version by prefixing it with 1, which should result in deprecation errors EasyBuildOptions( @@ -2222,9 +2284,9 @@ def test_deprecated(self): try: log.deprecated('x', str(orig_value)) # not supposed to get here - self.assertTrue(False, 'Deprecated logging should throw EasyBuildError') + self.fail('Deprecated logging should throw EasyBuildError') except easybuild.tools.build_log.EasyBuildError as err3: - self.assertTrue('DEPRECATED' in str(err3)) + self.assertIn('DEPRECATED', str(err3)) # set it back easybuild.tools.build_log.CURRENT_VERSION = orig_value @@ -2370,11 +2432,11 @@ def test_try_with_copy(self): self.eb_main(args + [copied_ec], verbose=True, raise_error=True) outtxt = self.get_stdout() errtxt = self.get_stderr() - self.assertTrue(r'toy-0.0-tweaked.eb copied to ' + copied_ec in outtxt) + self.assertIn(r'toy-0.0-tweaked.eb copied to ' + copied_ec, outtxt) self.assertFalse(errtxt) self.mock_stdout(False) self.mock_stderr(False) - self.assertTrue(os.path.exists(copied_ec)) + self.assertExists(copied_ec) self.mock_stdout(True) self.mock_stderr(True) @@ -2383,13 +2445,11 @@ def test_try_with_copy(self): verbose=True, raise_error=True) outtxt = self.get_stdout() errtxt = self.get_stderr() - self.assertTrue(r'1 file(s) copied to ' + tweaked_ecs_dir in outtxt) + self.assertIn(r'1 file(s) copied to ' + tweaked_ecs_dir, outtxt) self.assertFalse(errtxt) self.mock_stdout(False) self.mock_stderr(False) - self.assertTrue( - os.path.exists(os.path.join(self.test_buildpath, tweaked_ecs_dir, 'foo-1.2.3-GCC-6.4.0-2.28.eb')) - ) + self.assertExists(os.path.join(self.test_buildpath, tweaked_ecs_dir, 'foo-1.2.3-GCC-6.4.0-2.28.eb')) def test_software_version_ordering(self): """Test whether software versions are correctly ordered when using --software.""" @@ -2502,7 +2562,7 @@ def test_cleanup_builddir(self): # make sure --disable-cleanup-builddir works args.append('--disable-cleanup-builddir') self.eb_main(args, do_build=True, verbose=True) - self.assertTrue(os.path.exists(toy_buildpath), "Build dir %s is retained when requested" % toy_buildpath) + self.assertExists(toy_buildpath, "Build dir %s is retained when requested" % toy_buildpath) shutil.rmtree(toy_buildpath) # make sure build dir stays in case of failed build @@ -2512,7 +2572,7 @@ def test_cleanup_builddir(self): '--try-amend=prebuildopts=nosuchcommand &&', ] self.eb_main(args, do_build=True) - self.assertTrue(os.path.exists(toy_buildpath), "Build dir %s is retained after failed build" % toy_buildpath) + self.assertExists(toy_buildpath, "Build dir %s is retained after failed build" % toy_buildpath) def test_filter_deps(self): """Test use of --filter-deps.""" @@ -3034,7 +3094,7 @@ def test_robot_path_check(self): self.assertErrorRegex(SystemExit, '.*', self.eb_main, args, raise_error=True, raise_systemexit=True) stderr = self.get_stderr() self.mock_stderr(False) - self.assertTrue("error: no such option: -X" in stderr) + self.assertIn("error: no such option: -X", stderr) def test_missing_cfgfile(self): """Test behaviour when non-existing config file is specified.""" @@ -3103,7 +3163,7 @@ def test_show_default_configfiles(self): else: homecfgfile_str += " => not found" expected = expected_tmpl % ('(not set)', '(not set)', homecfgfile_str, '{/etc}') - self.assertTrue(expected in logtxt) + self.assertIn(expected, logtxt) # to predict the full output, we need to take control over $HOME and $XDG_CONFIG_DIRS os.environ['HOME'] = self.test_prefix @@ -3127,7 +3187,7 @@ def test_show_default_configfiles(self): logtxt = read_file(self.logfile) expected = expected_tmpl % ('(not set)', xdg_config_dirs, "%s => found" % homecfgfile, '{%s}' % xdg_config_dirs, '(no matches)', 1, homecfgfile) - self.assertTrue(expected in logtxt) + self.assertIn(expected, logtxt) xdg_config_home = os.path.join(self.test_prefix, 'home') os.environ['XDG_CONFIG_HOME'] = xdg_config_home @@ -3153,7 +3213,7 @@ def test_show_default_configfiles(self): "%s => found" % os.path.join(xdg_config_home, 'easybuild', 'config.cfg'), '{' + ', '.join(xdg_config_dirs) + '}', ', '.join(cfgfiles[:-1]), 4, ', '.join(cfgfiles)) - self.assertTrue(expected in logtxt) + self.assertIn(expected, logtxt) del os.environ['XDG_CONFIG_DIRS'] del os.environ['XDG_CONFIG_HOME'] @@ -3230,7 +3290,7 @@ def test_xxx_include_easyblocks(self): # 'undo' import of foo easyblock del sys.modules['easybuild.easyblocks.foo'] - sys.path = orig_local_sys_path + sys.path[:] = orig_local_sys_path import easybuild.easyblocks reload(easybuild.easyblocks) import easybuild.easyblocks.generic @@ -3350,7 +3410,7 @@ def test_xxx_include_generic_easyblocks(self): # 'undo' import of foobar easyblock del sys.modules['easybuild.easyblocks.generic.foobar'] os.remove(os.path.join(self.test_prefix, 'generic', 'foobar.py')) - sys.path = orig_local_sys_path + sys.path[:] = orig_local_sys_path import easybuild.easyblocks reload(easybuild.easyblocks) import easybuild.easyblocks.generic @@ -3453,7 +3513,7 @@ def test_github_xxx_include_easyblocks_from_pr(self): del sys.modules['easybuild.easyblocks.foo'] del sys.modules['easybuild.easyblocks.generic.cmakemake'] os.remove(os.path.join(self.test_prefix, 'foo.py')) - sys.path = orig_local_sys_path + sys.path[:] = orig_local_sys_path # include test cmakemake easyblock cmm_txt = '\n'.join([ @@ -3499,7 +3559,7 @@ def test_github_xxx_include_easyblocks_from_pr(self): del sys.modules['easybuild.easyblocks.foo'] del sys.modules['easybuild.easyblocks.generic.cmakemake'] os.remove(os.path.join(self.test_prefix, 'cmakemake.py')) - sys.path = orig_local_sys_path + sys.path[:] = orig_local_sys_path import easybuild.easyblocks reload(easybuild.easyblocks) import easybuild.easyblocks.generic @@ -3650,7 +3710,7 @@ def test_use_included_module_naming_scheme(self): toy_mod = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0') if get_module_syntax() == 'Lua': toy_mod += '.lua' - self.assertTrue(os.path.exists(toy_mod), "Found %s" % toy_mod) + self.assertExists(toy_mod) def test_include_toolchains(self): """Test --include-toolchains.""" @@ -3727,7 +3787,7 @@ def test_cleanup_tmpdir(self): # default: cleanup tmpdir & logfile self.eb_main(args, raise_error=True, testing=False) self.assertEqual(os.listdir(tmpdir), []) - self.assertFalse(os.path.exists(self.logfile)) + self.assertNotExists(self.logfile) # disable cleaning up tmpdir args.append('--disable-cleanup-tmpdir') @@ -3735,10 +3795,10 @@ def test_cleanup_tmpdir(self): tmpdir_files = os.listdir(tmpdir) # tmpdir and logfile are still there \o/ self.assertTrue(len(tmpdir_files) == 1) - self.assertTrue(os.path.exists(self.logfile)) + self.assertExists(self.logfile) # tweaked easyconfigs is still there \o/ tweaked_dir = os.path.join(tmpdir, tmpdir_files[0], 'tweaked_easyconfigs') - self.assertTrue(os.path.exists(os.path.join(tweaked_dir, 'toy-1.0.eb'))) + self.assertExists(os.path.join(tweaked_dir, 'toy-1.0.eb')) def test_github_preview_pr(self): """Test --preview-pr.""" @@ -3796,7 +3856,7 @@ def test_github_review_pr(self): txt = self.get_stdout() self.mock_stdout(False) self.mock_stderr(False) - self.assertTrue("This PR should be labelled with 'update'" in txt) + self.assertIn("This PR should be labelled with 'update'", txt) # test --review-pr-max self.mock_stdout(True) @@ -3811,7 +3871,7 @@ def test_github_review_pr(self): txt = self.get_stdout() self.mock_stdout(False) self.mock_stderr(False) - self.assertTrue("2016.04" not in txt) + self.assertNotIn("2016.04", txt) # test --review-pr-filter self.mock_stdout(True) @@ -3826,7 +3886,7 @@ def test_github_review_pr(self): txt = self.get_stdout() self.mock_stdout(False) self.mock_stderr(False) - self.assertTrue("2016.04" not in txt) + self.assertNotIn("2016.04", txt) def test_set_tmpdir(self): """Test set_tmpdir config function.""" @@ -3890,10 +3950,10 @@ def test_minimal_toolchains(self): # check requirements for test init_config([], build_options={'robot_path': os.environ['EASYBUILD_ROBOT_PATHS']}) - self.assertFalse(os.path.exists(robot_find_easyconfig('hwloc', '1.11.8-gompi-2018a') or 'nosuchfile')) - self.assertTrue(os.path.exists(robot_find_easyconfig('hwloc', '1.11.8-GCC-6.4.0-2.28'))) - self.assertTrue(os.path.exists(robot_find_easyconfig('SQLite', '3.8.10.2-gompi-2018a'))) - self.assertTrue(os.path.exists(robot_find_easyconfig('SQLite', '3.8.10.2-GCC-6.4.0-2.28'))) + self.assertNotExists(robot_find_easyconfig('hwloc', '1.11.8-gompi-2018a') or 'nosuchfile') + self.assertExists(robot_find_easyconfig('hwloc', '1.11.8-GCC-6.4.0-2.28')) + self.assertExists(robot_find_easyconfig('SQLite', '3.8.10.2-gompi-2018a')) + self.assertExists(robot_find_easyconfig('SQLite', '3.8.10.2-GCC-6.4.0-2.28')) args = [ ec_file, @@ -4225,7 +4285,7 @@ def test_github_new_update_pr(self): if len(dirs) == 1: git_working_dir = dirs[0] else: - self.assertTrue(False, "Failed to find temporary git working dir: %s" % dirs) + self.fail("Failed to find temporary git working dir: %s" % dirs) remote = 'git@github.com:%s/easybuild-easyconfigs.git' % GITHUB_TEST_ACCOUNT regexs = [ @@ -4262,9 +4322,9 @@ def test_github_new_update_pr(self): res = [d for d in res if os.path.basename(d) != os.path.basename(git_working_dir)] if len(res) == 1: unstaged_file_full = os.path.join(res[0], unstaged_file) - self.assertFalse(os.path.exists(unstaged_file_full), "%s not found in %s" % (unstaged_file, res[0])) + self.assertNotExists(unstaged_file_full), "%s not found in %s" % (unstaged_file, res[0]) else: - self.assertTrue(False, "Found copy of easybuild-easyconfigs working copy") + self.fail("Found copy of easybuild-easyconfigs working copy") # add required commit message, try again args.append('--pr-commit-msg=just a test') @@ -4321,7 +4381,7 @@ def test_github_new_update_pr(self): # modifying an existing easyconfig requires a custom PR title gcc_ec = os.path.join(test_ecs, 'g', 'GCC', 'GCC-4.9.2.eb') - self.assertTrue(os.path.exists(gcc_ec)) + self.assertExists(gcc_ec) args = [ '--new-pr', @@ -4629,7 +4689,7 @@ def test_new_pr_easyblock(self): topdir = os.path.dirname(os.path.abspath(__file__)) toy_eb = os.path.join(topdir, 'sandbox', 'easybuild', 'easyblocks', 't', 'toy.py') - self.assertTrue(os.path.exists(toy_eb)) + self.assertExists(toy_eb) args = [ '--github-user=%s' % GITHUB_TEST_ACCOUNT, @@ -4752,7 +4812,7 @@ def test_github_merge_pr(self): '', "Review OK, merging pull request!", ]) - self.assertTrue(expected_stdout in stdout) + self.assertIn(expected_stdout, stdout) def test_github_empty_pr(self): """Test use of --new-pr (dry run only) with no changes""" @@ -4953,7 +5013,7 @@ def test_dump_env_script(self): # check whether scripts were dumped env_script = os.path.join(self.test_prefix, '%s.env' % name) - self.assertTrue(os.path.exists(env_script)) + self.assertExists(env_script) # existing .env files are not overwritten, unless forced os.chdir(self.test_prefix) @@ -5033,7 +5093,7 @@ def test_parse_external_modules_metadata(self): # just a selection for mod in ['cray-libsci/13.2.0', 'cray-netcdf/4.3.2', 'fftw/3.3.4.3']: - self.assertTrue(mod in metadata) + self.assertIn(mod, metadata) netcdf = { 'name': ['netCDF', 'netCDF-Fortran'], @@ -5057,7 +5117,7 @@ def test_parse_external_modules_metadata(self): # default metadata is overruled, and not available anymore for mod in ['cray-libsci/13.2.0', 'cray-netcdf/4.3.2', 'fftw/3.3.4.3']: - self.assertFalse(mod in metadata) + self.assertNotIn(mod, metadata) foobar1 = { 'name': ['foo', 'bar'], @@ -5195,7 +5255,7 @@ def test_list_prs(self): args = ['--list-prs', 'closed,updated,asc'] txt, _ = self._run_mock_eb(args, testing=False) expected = "Listing PRs with parameters: direction=asc, per_page=100, sort=updated, state=closed" - self.assertTrue(expected in txt) + self.assertIn(expected, txt) def test_list_software(self): """Test --list-software and --list-installed-software.""" @@ -5518,7 +5578,7 @@ def test_verify_easyconfig_filenames(self): # filename of provided easyconfig doesn't matter by default self.eb_main(args, logfile=dummylogfn, raise_error=True) logtxt = read_file(self.logfile) - self.assertTrue('module: toy/0.0' in logtxt) + self.assertIn('module: toy/0.0', logtxt) write_file(self.logfile, '') @@ -5534,7 +5594,7 @@ def test_verify_easyconfig_filenames(self): args[0] = toy_ec self.eb_main(args, logfile=dummylogfn, raise_error=True) logtxt = read_file(self.logfile) - self.assertTrue('module: toy/0.0' in logtxt) + self.assertIn('module: toy/0.0', logtxt) def test_set_default_module(self): """Test use of --set-default-module""" @@ -5548,18 +5608,18 @@ def test_set_default_module(self): if get_module_syntax() == 'Lua': toy_mod += '.lua' - self.assertTrue(os.path.exists(toy_mod)) + self.assertExists(toy_mod) if get_module_syntax() == 'Lua': self.assertTrue(os.path.islink(os.path.join(toy_mod_dir, 'default'))) self.assertEqual(os.readlink(os.path.join(toy_mod_dir, 'default')), '0.0-deps.lua') elif get_module_syntax() == 'Tcl': toy_dot_version = os.path.join(toy_mod_dir, '.version') - self.assertTrue(os.path.exists(toy_dot_version)) + self.assertExists(toy_dot_version) toy_dot_version_txt = read_file(toy_dot_version) - self.assertTrue("set ModulesVersion 0.0-deps" in toy_dot_version_txt) + self.assertIn("set ModulesVersion 0.0-deps", toy_dot_version_txt) else: - self.assertTrue(False, "Uknown module syntax: %s" % get_module_syntax()) + self.fail("Uknown module syntax: %s" % get_module_syntax()) # make sure default is also set for moduleclass symlink toy_mod_symlink_dir = os.path.join(self.test_installpath, 'modules', 'tools', 'toy') @@ -5583,7 +5643,7 @@ def test_set_default_module(self): modfile_path = os.path.join(toy_mod_dir, '0.0-deps') self.assertTrue(os.path.samefile(os.readlink(mod_symlink), modfile_path)) else: - self.assertTrue(False, "Uknown module syntax: %s" % get_module_syntax()) + self.fail("Uknown module syntax: %s" % get_module_syntax()) def test_set_default_module_robot(self): """Test use of --set-default-module --robot.""" @@ -5635,9 +5695,9 @@ def test_set_default_module_robot(self): self.assertEqual(sorted(os.listdir(test_mod_dir)), ['.version', '1.0']) self.assertEqual(sorted(os.listdir(testdep_mod_dir)), ['3.14']) dot_version_file = os.path.join(test_mod_dir, '.version') - self.assertTrue("set ModulesVersion 1.0" in read_file(dot_version_file)) + self.assertIn("set ModulesVersion 1.0", read_file(dot_version_file)) else: - self.assertTrue(False, "Uknown module syntax: %s" % get_module_syntax()) + self.fail("Uknown module syntax: %s" % get_module_syntax()) def test_inject_checksums(self): """Test for --inject-checksums""" @@ -5659,10 +5719,10 @@ def test_inject_checksums(self): self.mock_stderr(False) # make sure software install directory is *not* created (see bug issue #3064) - self.assertFalse(os.path.exists(os.path.join(self.test_installpath, 'software', 'toy'))) + self.assertNotExists(os.path.join(self.test_installpath, 'software', 'toy')) # SHA256 is default type of checksums used - self.assertTrue("injecting sha256 checksums in" in stdout) + self.assertIn("injecting sha256 checksums in", stdout) self.assertEqual(stderr, '') args.append('--force') @@ -5699,7 +5759,7 @@ def test_inject_checksums(self): # some checks on 'raw' easyconfig contents # single-line checksum for barbar extension since there's only one - self.assertTrue("'checksums': ['d5bd9908cdefbe2d29c6f8d5b45b2aaed9fd904b5e6397418bb5094fbdb3d838']," in ec_txt) + self.assertIn("'checksums': ['d5bd9908cdefbe2d29c6f8d5b45b2aaed9fd904b5e6397418bb5094fbdb3d838'],", ec_txt) # single-line checksum entry for bar source tarball regex = re.compile("^[ ]*{'bar-0.0.tar.gz': '%s'},$" % bar_tar_gz_sha256, re.M) @@ -5724,7 +5784,7 @@ def test_inject_checksums(self): self.assertTrue(regex.search(ec_txt), "Pattern '%s' found in: %s" % (regex.pattern, ec_txt)) # name/version of toy should NOT be hardcoded in exts_list, 'name'/'version' parameters should be used - self.assertTrue(' (name, version, {' in ec_txt) + self.assertIn(' (name, version, {', ec_txt) # make sure checksums are only there once... # exactly one definition of 'checksums' easyconfig parameter @@ -5766,7 +5826,7 @@ def test_inject_checksums(self): self.assertEqual(len(ec_backups), 1) self.assertEqual(read_file(toy_ec), read_file(ec_backups[0])) - self.assertTrue("injecting sha256 checksums in" in stdout) + self.assertIn("injecting sha256 checksums in", stdout) self.assertEqual(stderr, warning_msg) remove_file(ec_backups[0]) @@ -5796,7 +5856,7 @@ def test_inject_checksums(self): # get rid of existing checksums regex = re.compile(r'^checksums(?:.|\n)*?\]\s*$', re.M) toy_ec_txt = regex.sub('', toy_ec_txt) - self.assertFalse('checksums = ' in toy_ec_txt) + self.assertNotIn('checksums = ', toy_ec_txt) write_file(test_ec, toy_ec_txt) args = [test_ec, '--inject-checksums=md5'] @@ -5869,7 +5929,7 @@ def test_inject_checksums(self): self.mock_stderr(False) self.assertEqual(stdout, '') - self.assertTrue("option --inject-checksums: invalid choice" in stderr) + self.assertIn("option --inject-checksums: invalid choice", stderr) def test_inject_checksums_to_json(self): """Test --inject-checksums-to-json.""" @@ -5954,7 +6014,7 @@ def test_enforce_checksums(self): # because of missing checksum for source of 'bar' extension regex = re.compile("^.*'checksums':.*$", re.M) test_ec_txt = regex.sub('', read_file(test_ec)) - self.assertFalse("'checksums':" in test_ec_txt) + self.assertNotIn("'checksums':", test_ec_txt) write_file(test_ec, test_ec_txt) error_pattern = r"Missing checksum for bar-0\.0\.tar\.gz" self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, do_build=True, raise_error=True) @@ -5964,7 +6024,7 @@ def test_enforce_checksums(self): for param in ['checksums', 'exts_list']: regex = re.compile(r'^%s(?:.|\n)*?\]\s*$' % param, re.M) test_ec_txt = regex.sub('', test_ec_txt) - self.assertFalse('%s = ' % param in test_ec_txt) + self.assertNotIn('%s = ' % param, test_ec_txt) write_file(test_ec, test_ec_txt) error_pattern = "Missing checksum for toy-0.0.tar.gz" @@ -6038,7 +6098,7 @@ def test_tmp_logdir(self): # purposely use a non-existing directory as log directory tmp_logdir = os.path.join(self.test_prefix, 'tmp-logs') - self.assertFalse(os.path.exists(tmp_logdir)) + self.assertNotExists(tmp_logdir) # force passing logfile=None to main in eb_main self.logfile = None @@ -6059,7 +6119,7 @@ def test_tmp_logdir(self): self.assertEqual(len(tmp_logs), 1) logtxt = read_file(os.path.join(tmp_logdir, tmp_logs[0])) - self.assertTrue("COMPLETED: Installation ended successfully" in logtxt) + self.assertIn("COMPLETED: Installation ended successfully", logtxt) def test_sanity_check_only(self): """Test use of --sanity-check-only.""" @@ -6082,7 +6142,7 @@ def test_sanity_check_only(self): # sanity check fails if software was not installed yet outtxt, error_thrown = self.eb_main([test_ec, '--sanity-check-only'], do_build=True, return_error=True) - self.assertTrue("Sanity check failed" in str(error_thrown)) + self.assertIn("Sanity check failed", str(error_thrown)) # actually install, then try --sanity-check-only again; # need to use --force to install toy because module already exists (but installation doesn't) @@ -6113,8 +6173,8 @@ def test_sanity_check_only(self): for skip in skipped: self.assertTrue("== %s [skipped]" % skip) - self.assertTrue("== sanity checking..." in stdout) - self.assertTrue("COMPLETED: Installation ended successfully" in stdout) + self.assertIn("== sanity checking...", stdout) + self.assertIn("COMPLETED: Installation ended successfully", stdout) msgs = [ " >> file 'bin/barbar' found: OK", " >> file 'bin/toy' found: OK", @@ -6124,7 +6184,7 @@ def test_sanity_check_only(self): "ls -l lib/libbarbar.a", # sanity check for extension barbar (via exts_filter) ] for msg in msgs: - self.assertTrue(msg in stdout, "'%s' found in: %s" % (msg, stdout)) + self.assertIn(msg, stdout) ebroottoy = os.path.join(self.test_installpath, 'software', 'toy', '0.0') @@ -6145,7 +6205,7 @@ def test_sanity_check_only(self): # failing sanity check for extension can be bypassed via --skip-extensions outtxt = self.eb_main(args + ['--skip-extensions'], do_build=True, raise_error=True) - self.assertTrue("Sanity check for toy successful" in outtxt) + self.assertIn("Sanity check for toy successful", outtxt) # restore fail, we want a passing sanity check for the next check move_file(libbarbar + '.moved', libbarbar) @@ -6176,11 +6236,11 @@ def test_sanity_check_only(self): toy_eb = os.path.join(test_ebs, 't', 'toy.py') toy_eb_txt = read_file(toy_eb) - self.assertFalse('self.build_in_installdir = True' in toy_eb_txt) + self.assertNotIn('self.build_in_installdir = True', toy_eb_txt) regex = re.compile(r'^(\s+)(super\(EB_toy, self\).__init__.*)\n', re.M) toy_eb_txt = regex.sub(r'\1\2\n\1self.build_in_installdir = True', toy_eb_txt) - self.assertTrue('self.build_in_installdir = True' in toy_eb_txt) + self.assertIn('self.build_in_installdir = True', toy_eb_txt) toy_eb = os.path.join(self.test_prefix, 'toy.py') write_file(toy_eb, toy_eb_txt) @@ -6228,12 +6288,12 @@ def test_skip_extensions(self): if get_module_syntax() == 'Lua': toy_mod += '.lua' - self.assertTrue(os.path.exists(toy_mod), "%s should exist" % toy_mod) + self.assertExists(toy_mod) toy_installdir = os.path.join(self.test_installpath, 'software', 'toy', '0.0') for path in (os.path.join('bin', 'barbar'), os.path.join('lib', 'libbarbar.a')): path = os.path.join(toy_installdir, path) - self.assertFalse(os.path.exists(path), "Path %s should not exist" % path) + self.assertNotExists(path) def test_fake_vsc_include(self): """Test whether fake 'vsc' namespace is triggered for modules included via --include-*.""" @@ -6296,12 +6356,15 @@ def test_sort_looseversions(self): else: version_class = LooseVersion - ver1 = version_class('1.2.3') - ver2 = version_class('4.5.6') - ver3 = version_class('1.2.3dev') - ver4 = version_class('system') - ver5 = version_class('rc3') - ver6 = version_class('v1802') + with warnings.catch_warnings(): + if use_distutils: + warnings.simplefilter("ignore", category=DeprecationWarning) + ver1 = version_class('1.2.3') + ver2 = version_class('4.5.6') + ver3 = version_class('1.2.3dev') + ver4 = version_class('system') + ver5 = version_class('rc3') + ver6 = version_class('v1802') # some versions are included multiple times on purpose, # to also test comparison between equal LooseVersion instances @@ -6380,7 +6443,7 @@ def test_create_index(self): def test_sysroot(self): """Test use of --sysroot option.""" - self.assertTrue(os.path.exists(self.test_prefix)) + self.assertExists(self.test_prefix) sysroot_arg = '--sysroot=' + self.test_prefix stdout, stderr = self._run_mock_eb([sysroot_arg, '--show-config'], raise_error=True) @@ -6433,18 +6496,18 @@ def test_accept_eula_for(self): for val in ('foo,toy,bar', '.*', 't.y'): self.eb_main(args + ['--accept-eula-for=' + val], do_build=True, raise_error=True) - self.assertTrue(os.path.exists(toy_modfile)) + self.assertExists(toy_modfile) remove_dir(self.test_installpath) - self.assertFalse(os.path.exists(toy_modfile)) + self.assertNotExists(toy_modfile) # also check use of $EASYBUILD_ACCEPT_EULA to accept EULA for specified software os.environ['EASYBUILD_ACCEPT_EULA_FOR'] = val self.eb_main(args, do_build=True, raise_error=True) - self.assertTrue(os.path.exists(toy_modfile)) + self.assertExists(toy_modfile) remove_dir(self.test_installpath) - self.assertFalse(os.path.exists(toy_modfile)) + self.assertNotExists(toy_modfile) del os.environ['EASYBUILD_ACCEPT_EULA_FOR'] @@ -6455,10 +6518,10 @@ def test_accept_eula_for(self): self.eb_main(args + ['--accept-eula=foo,toy,bar'], do_build=True, raise_error=True) stderr = self.get_stderr() self.mock_stderr(False) - self.assertTrue("Use accept-eula-for configuration setting rather than accept-eula" in stderr) + self.assertIn("Use accept-eula-for configuration setting rather than accept-eula", stderr) remove_dir(self.test_installpath) - self.assertFalse(os.path.exists(toy_modfile)) + self.assertNotExists(toy_modfile) # also via $EASYBUILD_ACCEPT_EULA self.mock_stderr(True) @@ -6467,18 +6530,18 @@ def test_accept_eula_for(self): stderr = self.get_stderr() self.mock_stderr(False) - self.assertTrue(os.path.exists(toy_modfile)) - self.assertTrue("Use accept-eula-for configuration setting rather than accept-eula" in stderr) + self.assertExists(toy_modfile) + self.assertIn("Use accept-eula-for configuration setting rather than accept-eula", stderr) remove_dir(self.test_installpath) - self.assertFalse(os.path.exists(toy_modfile)) + self.assertNotExists(toy_modfile) # also check accepting EULA via 'accept_eula = True' in easyconfig file self.disallow_deprecated_behaviour() del os.environ['EASYBUILD_ACCEPT_EULA'] write_file(test_ec, test_ec_txt + '\naccept_eula = True') self.eb_main(args, do_build=True, raise_error=True) - self.assertTrue(os.path.exists(toy_modfile)) + self.assertExists(toy_modfile) def test_config_abs_path(self): """Test ensuring of absolute path values for path configuration options.""" @@ -6640,14 +6703,14 @@ def test_easystack_opts(self): # libtoy module should be installed, module file should at least set EBROOTLIBTOY mod_dir = os.path.join(self.test_installpath, 'modules', 'all') mod_path = os.path.join(mod_dir, 'libtoy', '0.0') + mod_ext - self.assertTrue(os.path.exists(mod_path)) + self.assertExists(mod_path) self.modtool.use(mod_dir) self.modtool.load(['libtoy']) - self.assertTrue(os.path.exists(os.environ['EBROOTLIBTOY'])) + self.assertExists(os.environ['EBROOTLIBTOY']) # module should be hidden and in different install path mod_path = os.path.join(hidden_installpath, 'modules', 'all', 'toy', '.0.0') + mod_ext - self.assertTrue(os.path.exists(mod_path)) + self.assertExists(mod_path) # check build options that were put in place for last easyconfig self.assertFalse(build_option('hidden')) @@ -6704,7 +6767,7 @@ def test_set_up_configuration(self): self.assertFalse(build_option('debug')) self.assertFalse(build_option('hidden')) # tests may be configured to run with Tcl module syntax - self.assertTrue(get_module_syntax() in ('Lua', 'Tcl')) + self.assertIn(get_module_syntax(), ('Lua', 'Tcl')) # start with a clean slate, reset all configuration done by setUp method that prepares each test cleanup() @@ -6713,13 +6776,13 @@ def test_set_up_configuration(self): eb_go, settings = set_up_configuration(args=['--debug', '--module-syntax=Tcl'], silent=True) # 2nd part of return value is a tuple with various settings - self.assertTrue(isinstance(settings, tuple)) + self.assertIsInstance(settings, tuple) self.assertEqual(len(settings), 9) self.assertEqual(settings[0], {}) # build specs - self.assertTrue(isinstance(settings[1], EasyBuildLog)) # EasyBuildLog instance + self.assertIsInstance(settings[1], EasyBuildLog) # EasyBuildLog instance self.assertTrue(settings[2].endswith('.log')) # path to log file - self.assertTrue(os.path.exists(settings[2])) - self.assertTrue(isinstance(settings[3], list)) # list of robot paths + self.assertExists(settings[2]) + self.assertIsInstance(settings[3], list) # list of robot paths self.assertEqual(len(settings[3]), 1) self.assertTrue(os.path.samefile(settings[3][0], os.environ['EASYBUILD_ROBOT_PATHS'])) self.assertEqual(settings[4], None) # search query @@ -6749,7 +6812,7 @@ def test_set_up_configuration(self): stderr = self.get_stderr() self.mock_stderr(False) - self.assertTrue("WARNING: set_up_configuration is about to call init() and init_build_options()" in stderr) + self.assertIn("WARNING: set_up_configuration is about to call init() and init_build_options()", stderr) # 'hidden' option is enabled, but corresponding build option is still set to False! self.assertTrue(eb_go.options.hidden) @@ -6777,8 +6840,8 @@ def test_set_up_configuration(self): self.assertFalse(BuildOptions()['debug']) # tests may be configured to run with Tcl module syntax - self.assertTrue(ConfigurationVariables()['module_syntax'] in ('Lua', 'Tcl')) - self.assertTrue(get_module_syntax() in ('Lua', 'Tcl')) + self.assertIn(ConfigurationVariables()['module_syntax'], ('Lua', 'Tcl')) + self.assertIn(get_module_syntax(), ('Lua', 'Tcl')) def test_opts_dict_to_eb_opts(self): """Tests for opts_dict_to_eb_opts.""" diff --git a/test/framework/output.py b/test/framework/output.py index 101fb62f57..d1673a55d1 100644 --- a/test/framework/output.py +++ b/test/framework/output.py @@ -60,22 +60,20 @@ def test_status_bar(self): expected_progress_bar_class = DummyRich progress_bar = status_bar(ignore_cache=True) - error_msg = "%s should be instance of class %s" % (progress_bar, expected_progress_bar_class) - self.assertTrue(isinstance(progress_bar, expected_progress_bar_class), error_msg) + self.assertIsInstance(progress_bar, expected_progress_bar_class) update_build_option('output_style', 'basic') progress_bar = status_bar(ignore_cache=True) - self.assertTrue(isinstance(progress_bar, DummyRich)) + self.assertIsInstance(progress_bar, DummyRich) if HAVE_RICH: update_build_option('output_style', 'rich') progress_bar = status_bar(ignore_cache=True) - error_msg = "%s should be instance of class %s" % (progress_bar, expected_progress_bar_class) - self.assertTrue(isinstance(progress_bar, expected_progress_bar_class), error_msg) + self.assertIsInstance(progress_bar, expected_progress_bar_class) update_build_option('show_progress_bar', False) progress_bar = status_bar(ignore_cache=True) - self.assertTrue(isinstance(progress_bar, DummyRich)) + self.assertIsInstance(progress_bar, DummyRich) def test_get_output_style(self): """Test get_output_style function.""" @@ -152,9 +150,9 @@ def test_get_progress_bar(self): for pbar_type in PROGRESS_BAR_TYPES: pbar = get_progress_bar(pbar_type, ignore_cache=True) if HAVE_RICH: - self.assertTrue(isinstance(pbar, rich.progress.Progress)) + self.assertIsInstance(pbar, rich.progress.Progress) else: - self.assertTrue(isinstance(pbar, DummyRich)) + self.assertIsInstance(pbar, DummyRich) def test_get_start_update_stop_progress_bar(self): """ diff --git a/test/framework/package.py b/test/framework/package.py index 02c971d8f4..23e2e37e9c 100644 --- a/test/framework/package.py +++ b/test/framework/package.py @@ -36,7 +36,7 @@ from unittest import TextTestRunner from easybuild.framework.easyconfig.easyconfig import EasyConfig -from easybuild.tools.config import log_path +from easybuild.tools.config import get_package_naming_scheme, log_path from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import adjust_permissions, read_file, write_file from easybuild.tools.package.utilities import ActivePNS, avail_package_naming_schemes, check_pkg_support, package @@ -152,9 +152,11 @@ def mock_fpm(tmpdir): class PackageTest(EnhancedTestCase): """Tests for packaging support.""" + pnsNames = ['EasyBuildDebFriendlyPNS', 'EasyBuildPNS'] + def test_avail_package_naming_schemes(self): """Test avail_package_naming_schemes()""" - self.assertEqual(sorted(avail_package_naming_schemes().keys()), ['EasyBuildPNS']) + self.assertEqual(sorted(avail_package_naming_schemes().keys()), self.pnsNames) def test_check_pkg_support(self): """Test check_pkg_support().""" @@ -175,19 +177,25 @@ def test_check_pkg_support(self): def test_active_pns(self): """Test use of ActivePNS.""" - init_config(build_options={'silent': True}) - - topdir = os.path.dirname(os.path.abspath(__file__)) - test_easyconfigs = os.path.join(topdir, 'easyconfigs', 'test_ecs') - test_ec = os.path.join(test_easyconfigs, 'o', 'OpenMPI', 'OpenMPI-2.1.2-GCC-6.4.0-2.28.eb') - ec = EasyConfig(test_ec, validate=False) - - pns = ActivePNS() - - # default: EasyBuild package naming scheme, pkg release 1 - self.assertEqual(pns.name(ec), 'OpenMPI-2.1.2-GCC-6.4.0-2.28') - self.assertEqual(pns.version(ec), 'eb-%s' % EASYBUILD_VERSION) - self.assertEqual(pns.release(ec), '1') + for pns_type in self.pnsNames: + os.environ['EASYBUILD_PACKAGE_NAMING_SCHEME'] = pns_type + init_config(build_options={'silent': True}) + + topdir = os.path.dirname(os.path.abspath(__file__)) + test_easyconfigs = os.path.join(topdir, 'easyconfigs', 'test_ecs') + test_ec = os.path.join(test_easyconfigs, 'o', 'OpenMPI', 'OpenMPI-2.1.2-GCC-6.4.0-2.28.eb') + ec = EasyConfig(test_ec, validate=False) + + pns = ActivePNS() + + self.assertEqual(pns.name(ec), 'OpenMPI-2.1.2-GCC-6.4.0-2.28') + self.assertEqual(pns.release(ec), '1') + if get_package_naming_scheme() == "EasyBuildPNS": + # default: EasyBuild package naming scheme, pkg release 1 + self.assertEqual(pns.version(ec), 'eb-%s' % EASYBUILD_VERSION) + elif get_package_naming_scheme() == "EasyBuildDebFriendlyPNS": + # default: EasyBuild deb friendly package naming scheme, pkg release 1 + self.assertEqual(pns.version(ec), '%s-eb' % EASYBUILD_VERSION) def test_package(self): """Test package function.""" diff --git a/test/framework/parallelbuild.py b/test/framework/parallelbuild.py index 4b2a743683..3ad85ac3b1 100644 --- a/test/framework/parallelbuild.py +++ b/test/framework/parallelbuild.py @@ -167,18 +167,18 @@ def test_build_easyconfigs_in_parallel_pbs_python(self): self.assertEqual(len(jobs[1].deps), 0) # only dependency for toy/0.0-deps is intel/2018a (dep marked as external module is filtered out) - self.assertTrue('toy-0.0-deps.eb' in jobs[2].script) + self.assertIn('toy-0.0-deps.eb', jobs[2].script) self.assertEqual(len(jobs[2].deps), 1) - self.assertTrue('intel-2018a.eb' in jobs[2].deps[0].script) + self.assertIn('intel-2018a.eb', jobs[2].deps[0].script) # dependencies for gzip/1.4-GCC-4.6.3: GCC/4.6.3 (toolchain) + toy/.0.0-deps - self.assertTrue('gzip-1.4-GCC-4.6.3.eb' in jobs[3].script) + self.assertIn('gzip-1.4-GCC-4.6.3.eb', jobs[3].script) self.assertEqual(len(jobs[3].deps), 2) regex = re.compile(r'toy-0.0-deps\.eb.* --hidden') script_txt = jobs[3].deps[0].script fail_msg = "Pattern '%s' should be found in: %s" % (regex.pattern, script_txt) self.assertTrue(regex.search(script_txt), fail_msg) - self.assertTrue('GCC-4.6.3.eb' in jobs[3].deps[1].script) + self.assertIn('GCC-4.6.3.eb', jobs[3].deps[1].script) # also test use of --pre-create-installdir ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') @@ -186,16 +186,16 @@ def test_build_easyconfigs_in_parallel_pbs_python(self): # installation directory doesn't exist yet before submission toy_installdir = os.path.join(self.test_installpath, 'software', 'toy', '0.0') - self.assertFalse(os.path.exists(toy_installdir)) + self.assertNotExists(toy_installdir) jobs = submit_jobs(ordered_ecs, '', testing=False) self.assertEqual(len(jobs), 1) # software install dir is created (by default) as part of job submission process (fetch_step is run) - self.assertTrue(os.path.exists(toy_installdir)) + self.assertExists(toy_installdir) remove_dir(toy_installdir) remove_dir(os.path.dirname(toy_installdir)) - self.assertFalse(os.path.exists(toy_installdir)) + self.assertNotExists(toy_installdir) # installation directory does *not* get created when --pre-create-installdir is used build_options['pre_create_installdir'] = False @@ -203,7 +203,7 @@ def test_build_easyconfigs_in_parallel_pbs_python(self): jobs = submit_jobs(ordered_ecs, '', testing=False) self.assertEqual(len(jobs), 1) - self.assertFalse(os.path.exists(toy_installdir)) + self.assertNotExists(toy_installdir) # restore mocked stuff PbsPython.__init__ = PbsPython__init__ @@ -265,8 +265,8 @@ def test_build_easyconfigs_in_parallel_gc3pie(self): toy_modfile = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0') if get_module_syntax() == 'Lua': toy_modfile += '.lua' - self.assertTrue(os.path.exists(toy_modfile)) - self.assertTrue(os.path.exists(os.path.join(self.test_installpath, 'software', 'toy', '0.0', 'bin', 'toy'))) + self.assertExists(toy_modfile) + self.assertExists(os.path.join(self.test_installpath, 'software', 'toy', '0.0', 'bin', 'toy')) # also check what happens when a job fails (an error should be raised) test_ecfile = os.path.join(self.test_prefix, 'test.eb') diff --git a/test/framework/repository.py b/test/framework/repository.py index 8654d04cb1..aa1cf402b7 100644 --- a/test/framework/repository.py +++ b/test/framework/repository.py @@ -86,7 +86,7 @@ def test_gitrepo(self): try: repo.init() self.assertEqual(os.path.basename(repo.wc), 'testrepository') - self.assertTrue(os.path.exists(os.path.join(repo.wc, 'README.md'))) + self.assertExists(os.path.join(repo.wc, 'README.md')) shutil.rmtree(repo.wc) except EasyBuildError as err: print("ignoring failed subtest in test_gitrepo, testing offline?") @@ -126,7 +126,7 @@ def test_svnrepo(self): repo = SvnRepository(test_repo_url) repo.init() - self.assertTrue(os.path.exists(os.path.join(repo.wc, 'trunk', 'README.md'))) + self.assertExists(os.path.join(repo.wc, 'trunk', 'README.md')) shutil.rmtree(repo.wc) # this test is disabled because it fails in Travis as a result of bitbucket disabling TLS 1.0/1.1 @@ -147,7 +147,7 @@ def DISABLED_test_hgrepo(self): repo = HgRepository(test_repo_url) repo.init() - self.assertTrue(os.path.exists(os.path.join(repo.wc, 'README'))) + self.assertExists(os.path.join(repo.wc, 'README')) shutil.rmtree(repo.wc) def test_init_repository(self): @@ -170,10 +170,10 @@ def test_add_easyconfig(self): def check_ec(path, expected_buildstats): """Check easyconfig at specified path""" - self.assertTrue(os.path.exists(path)) + self.assertExists(path) ectxt = read_file(path) self.assertTrue(ectxt.startswith("# Built with EasyBuild version")) - self.assertTrue("# Build statistics" in ectxt) + self.assertIn("# Build statistics", ectxt) ecdict = EasyConfigParser(path).get_config_dict() self.assertEqual(ecdict['buildstats'], expected_buildstats) diff --git a/test/framework/robot.py b/test/framework/robot.py index b4421752d7..4f226bae35 100644 --- a/test/framework/robot.py +++ b/test/framework/robot.py @@ -202,15 +202,15 @@ def test_resolve_dependencies(self): res = resolve_dependencies([deepcopy(easyconfig_moredeps)], self.modtool) self.assertEqual(len(res), 2) full_mod_names = [ec['full_mod_name'] for ec in res] - self.assertFalse('toy/.0.0-deps' in full_mod_names) + self.assertNotIn('toy/.0.0-deps', full_mod_names) res = resolve_dependencies([deepcopy(easyconfig_moredeps)], self.modtool, retain_all_deps=True) self.assertEqual(len(res), 4) # hidden dep toy/.0.0-deps (+1) depends on (fake) intel/2018a (+1) self.assertEqual('gzip/1.4', res[0]['full_mod_name']) self.assertEqual('foo/1.2.3', res[-1]['full_mod_name']) full_mod_names = [ec['full_mod_name'] for ec in res] - self.assertTrue('toy/.0.0-deps' in full_mod_names) - self.assertTrue('intel/2018a' in full_mod_names) + self.assertIn('toy/.0.0-deps', full_mod_names) + self.assertIn('intel/2018a', full_mod_names) # here we have included a dependency in the easyconfig list easyconfig['full_mod_name'] = 'gzip/1.4' @@ -482,7 +482,7 @@ def test_resolve_dependencies_minimal(self): self.assertEqual(len(res), 10) mods = [x['full_mod_name'] for x in res] self.assertEqual(mods, all_mods_ordered) - self.assertTrue('SQLite/3.8.10.2-GCC-6.4.0-2.28' in mods) + self.assertIn('SQLite/3.8.10.2-GCC-6.4.0-2.28', mods) # test taking into account existing modules # with an SQLite module with foss/2018a in place, this toolchain should be used rather than GCC/6.4.0-2.28 @@ -497,8 +497,8 @@ def test_resolve_dependencies_minimal(self): res = resolve_dependencies([bar], self.modtool, retain_all_deps=True) self.assertEqual(len(res), 10) mods = [x['full_mod_name'] for x in res] - self.assertTrue('SQLite/3.8.10.2-foss-2018a' in mods) - self.assertFalse('SQLite/3.8.10.2-GCC-6.4.0-2.28' in mods) + self.assertIn('SQLite/3.8.10.2-foss-2018a', mods) + self.assertNotIn('SQLite/3.8.10.2-GCC-6.4.0-2.28', mods) # Check whether having 2 version of system toolchain is ok # Clear easyconfig and toolchain caches @@ -543,8 +543,8 @@ def test_resolve_dependencies_minimal(self): res = resolve_dependencies([bar], self.modtool, retain_all_deps=True) self.assertEqual(len(res), 11) mods = [x['full_mod_name'] for x in res] - self.assertTrue('impi/5.1.2.150' in mods) - self.assertTrue('gzip/1.4' in mods) + self.assertIn('impi/5.1.2.150', mods) + self.assertIn('gzip/1.4', mods) def test_resolve_dependencies_missing(self): """Test handling of missing dependencies in resolve_dependencies function.""" @@ -627,7 +627,7 @@ def test_det_easyconfig_paths(self): mkdir(subdir_hwloc, parents=True) shutil.copy2(os.path.join(test_ecs_path, 'h', 'hwloc', hwloc_ec), subdir_hwloc) shutil.copy2(os.path.join(test_ecs_path, 'i', 'intel', 'intel-2018a.eb'), self.test_prefix) - self.assertFalse(os.path.exists(test_ec)) + self.assertNotExists(test_ec) args = [ os.path.join(test_ecs_path, 't', 'toy', 'toy-0.0.eb'), @@ -696,7 +696,7 @@ def test_search_paths(self): test_ec = 'toy-0.0-deps.eb' shutil.copy2(os.path.join(test_ecs_path, 't', 'toy', test_ec), self.test_prefix) - self.assertFalse(os.path.exists(test_ec)) + self.assertNotExists(test_ec) args = [ '--search-paths=%s' % self.test_prefix, # add to search path @@ -727,7 +727,7 @@ def test_github_det_easyconfig_paths_from_pr(self): test_ec = 'toy-0.0-deps.eb' shutil.copy2(os.path.join(test_ecs_path, 't', 'toy', test_ec), self.test_prefix) shutil.copy2(os.path.join(test_ecs_path, 'i', 'intel', 'intel-2018a.eb'), self.test_prefix) - self.assertFalse(os.path.exists(test_ec)) + self.assertNotExists(test_ec) gompi_2018b_txt = '\n'.join([ "easyblock = 'Toolchain'", @@ -1109,10 +1109,10 @@ def test_tweak_robotpath(self): res = resolve_dependencies(easyconfigs, self.modtool, retain_all_deps=True) specs = [ec['spec'] for ec in res] # Check it picks up the tweaked OpenMPI - self.assertTrue(tweaked_openmpi in specs) + self.assertIn(tweaked_openmpi, specs) # Check it picks up the untweaked dependency of the tweaked OpenMPI untweaked_hwloc = os.path.join(test_easyconfigs, 'h', 'hwloc', 'hwloc-1.11.8-GCC-6.4.0-2.28.eb') - self.assertTrue(untweaked_hwloc in specs) + self.assertIn(untweaked_hwloc, specs) def test_robot_find_subtoolchain_for_dep(self): """Test robot_find_subtoolchain_for_dep.""" @@ -1392,7 +1392,7 @@ def test_check_conflicts(self): self.mock_stderr(False) self.assertTrue(conflicts) - self.assertTrue("Conflict found for dependencies of foss-2018a: GCC-4.6.4 vs GCC-6.4.0-2.28" in stderr) + self.assertIn("Conflict found for dependencies of foss-2018a: GCC-4.6.4 vs GCC-6.4.0-2.28", stderr) # conflicts between specified easyconfigs are also detected @@ -1407,7 +1407,7 @@ def test_check_conflicts(self): self.mock_stderr(False) self.assertTrue(conflicts) - self.assertTrue("Conflict between (dependencies of) easyconfigs: GCC-4.9.3-2.25 vs GCC-6.4.0-2.28" in stderr) + self.assertIn("Conflict between (dependencies of) easyconfigs: GCC-4.9.3-2.25 vs GCC-6.4.0-2.28", stderr) # indirect conflict on dependencies ecs, _ = parse_easyconfigs([ @@ -1420,7 +1420,7 @@ def test_check_conflicts(self): self.mock_stderr(False) self.assertTrue(conflicts) - self.assertTrue("Conflict between (dependencies of) easyconfigs: GCC-4.9.2 vs GCC-6.4.0-2.28" in stderr) + self.assertIn("Conflict between (dependencies of) easyconfigs: GCC-4.9.2 vs GCC-6.4.0-2.28", stderr) # test use of check_inter_ec_conflicts self.assertFalse(check_conflicts(ecs, self.modtool, check_inter_ec_conflicts=False), "No conflicts found") diff --git a/test/framework/run.py b/test/framework/run.py index 4389e73fec..48a0f75fd2 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -507,14 +507,14 @@ def test_dry_run(self): # check forced run outfile = os.path.join(self.test_prefix, 'cmd.out') - self.assertFalse(os.path.exists(outfile)) + self.assertNotExists(outfile) self.mock_stdout(True) run_cmd("echo 'This is always echoed' > %s" % outfile, force_in_dry_run=True) txt = self.get_stdout() self.mock_stdout(False) # nothing printed to stdout, but command was run self.assertEqual(txt, '') - self.assertTrue(os.path.exists(outfile)) + self.assertExists(outfile) self.assertEqual(read_file(outfile), "This is always echoed\n") # Q&A commands @@ -721,7 +721,7 @@ def test_check_log_for_errors(self): init_logging(logfile, silent=True) check_log_for_errors(input_text, [(r"\b(error|crashed)\b", WARN)]) stop_logging(logfile) - self.assertTrue(expected_msg in read_file(logfile)) + self.assertIn(expected_msg, read_file(logfile)) expected_msg = r"Found 2 error\(s\) in command output \(output: error found\n\ttest failed\)" write_file(logfile, '') @@ -734,7 +734,7 @@ def test_check_log_for_errors(self): ]) stop_logging(logfile) expected_msg = "Found 1 potential error(s) in command output (output: the process crashed with 0)" - self.assertTrue(expected_msg in read_file(logfile)) + self.assertIn(expected_msg, read_file(logfile)) def suite(): diff --git a/test/framework/sandbox/easybuild/easyblocks/__init__.py b/test/framework/sandbox/easybuild/easyblocks/__init__.py index 089b74322c..95ef9f40c3 100644 --- a/test/framework/sandbox/easybuild/easyblocks/__init__.py +++ b/test/framework/sandbox/easybuild/easyblocks/__init__.py @@ -1,5 +1,10 @@ import pkgutil +# Import fake version +import easybuild.tools.version + +VERSION = easybuild.tools.version.VERSION + subdirs = [chr(x) for x in range(ord('a'), ord('z') + 1)] + ['0'] for subdir in subdirs: __path__ = pkgutil.extend_path(__path__, '%s.%s' % (__name__, subdir)) diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py index 1d1d20e3a3..9c700cf779 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py @@ -64,7 +64,7 @@ def run(self, *args, **kwargs): Install toy extension. """ if self.src: - EB_toy.build_step(self.master, name=self.name, buildopts=self.cfg['buildopts']) + EB_toy.build_step(self.master, name=self.name, cfg=self.cfg) if self.cfg['toy_ext_param']: run_cmd(self.cfg['toy_ext_param']) @@ -79,7 +79,7 @@ def prerun(self): if self.src: super(Toy_Extension, self).run(unpack_src=True) - EB_toy.configure_step(self.master, name=self.name) + EB_toy.configure_step(self.master, name=self.name, cfg=self.cfg) def run_async(self): """ diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toy.py b/test/framework/sandbox/easybuild/easyblocks/t/toy.py index 92b818e63e..88ff864454 100644 --- a/test/framework/sandbox/easybuild/easyblocks/t/toy.py +++ b/test/framework/sandbox/easybuild/easyblocks/t/toy.py @@ -91,26 +91,37 @@ def run_all_steps(self, *args, **kwargs): return super(EB_toy, self).run_all_steps(*args, **kwargs) - def configure_step(self, name=None): + def configure_step(self, name=None, cfg=None): """Configure build of toy.""" if name is None: name = self.name + # Allow overwrite from Toy-Extension + if cfg is None: + cfg = self.cfg # make sure Python system dep is handled correctly when specified - if self.cfg['allow_system_deps']: + if cfg['allow_system_deps']: if get_software_root('Python') != 'Python' or get_software_version('Python') != platform.python_version(): raise EasyBuildError("Sanity check on allowed Python system dep failed.") + cmd = ' '.join([ + cfg['preconfigopts'], + 'echo "Configured"', + cfg['configopts'] + ]) + run_cmd(cmd) + if os.path.exists("%s.source" % name): os.rename('%s.source' % name, '%s.c' % name) - def build_step(self, name=None, buildopts=None): + def build_step(self, name=None, cfg=None): """Build toy.""" - if buildopts is None: - buildopts = self.cfg['buildopts'] + # Allow overwrite from Toy-Extension + if cfg is None: + cfg = self.cfg if name is None: name = self.name - cmd = compose_toy_build_cmd(self.cfg, name, self.cfg['prebuildopts'], buildopts) + cmd = compose_toy_build_cmd(self.cfg, name, cfg['prebuildopts'], cfg['buildopts']) run_cmd(cmd) def install_step(self, name=None): diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index d45935372e..0841e3b76f 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -51,7 +51,7 @@ from easybuild.tools.systemtools import get_cpu_family, get_cpu_features, get_cpu_model, get_cpu_speed, get_cpu_vendor from easybuild.tools.systemtools import get_gcc_version, get_glibc_version, get_os_type, get_os_name, get_os_version from easybuild.tools.systemtools import get_platform_name, get_shared_lib_ext, get_system_info, get_total_memory -from easybuild.tools.systemtools import find_library_path, locate_solib, pick_dep_version +from easybuild.tools.systemtools import find_library_path, locate_solib, pick_dep_version, pick_system_specific_value PROC_CPUINFO_TXT = None @@ -401,7 +401,7 @@ def tearDown(self): def test_avail_core_count_native(self): """Test getting core count.""" core_count = get_avail_core_count() - self.assertTrue(isinstance(core_count, int), "core_count has type int: %s, %s" % (core_count, type(core_count))) + self.assertIsInstance(core_count, int) self.assertTrue(core_count > 0, "core_count %d > 0" % core_count) def test_avail_core_count_linux(self): @@ -422,7 +422,7 @@ def test_avail_core_count_darwin(self): def test_cpu_model_native(self): """Test getting CPU model.""" cpu_model = get_cpu_model() - self.assertTrue(isinstance(cpu_model, string_type)) + self.assertIsInstance(cpu_model, string_type) def test_cpu_model_linux(self): """Test getting CPU model (mocked for Linux).""" @@ -460,8 +460,9 @@ def test_cpu_model_darwin(self): def test_cpu_speed_native(self): """Test getting CPU speed.""" cpu_speed = get_cpu_speed() - self.assertTrue(isinstance(cpu_speed, float) or cpu_speed is None) - self.assertTrue(cpu_speed is None or cpu_speed > 0.0) + if cpu_speed is not None: + self.assertIsInstance(cpu_speed, float) + self.assertTrue(cpu_speed > 0.0) def test_cpu_speed_linux(self): """Test getting CPU speed (mocked for Linux).""" @@ -494,7 +495,7 @@ def test_cpu_speed_darwin(self): def test_cpu_features_native(self): """Test getting CPU features.""" cpu_feat = get_cpu_features() - self.assertTrue(isinstance(cpu_feat, list)) + self.assertIsInstance(cpu_feat, list) self.assertTrue(len(cpu_feat) >= 0) self.assertTrue(all(isinstance(x, string_type) for x in cpu_feat)) @@ -556,7 +557,7 @@ def test_cpu_features_darwin(self): def test_cpu_architecture_native(self): """Test getting the CPU architecture.""" arch = get_cpu_architecture() - self.assertTrue(arch in CPU_ARCHITECTURES) + self.assertIn(arch, CPU_ARCHITECTURES) def test_cpu_architecture(self): """Test getting the CPU architecture (mocked).""" @@ -580,7 +581,7 @@ def test_cpu_architecture(self): def test_cpu_arch_name_native(self): """Test getting CPU arch name.""" arch_name = get_cpu_arch_name() - self.assertTrue(isinstance(arch_name, string_type)) + self.assertIsInstance(arch_name, string_type) def test_cpu_arch_name(self): """Test getting CPU arch name.""" @@ -601,7 +602,7 @@ def __init__(self, name): def test_cpu_vendor_native(self): """Test getting CPU vendor.""" cpu_vendor = get_cpu_vendor() - self.assertTrue(cpu_vendor in CPU_VENDORS) + self.assertIn(cpu_vendor, CPU_VENDORS) def test_cpu_vendor_linux(self): """Test getting CPU vendor (mocked for Linux).""" @@ -694,12 +695,12 @@ def test_cpu_family_darwin(self): def test_os_type(self): """Test getting OS type.""" os_type = get_os_type() - self.assertTrue(os_type in [DARWIN, LINUX]) + self.assertIn(os_type, [DARWIN, LINUX]) def test_shared_lib_ext_native(self): """Test getting extension for shared libraries.""" ext = get_shared_lib_ext() - self.assertTrue(ext in ['dylib', 'so']) + self.assertIn(ext, ['dylib', 'so']) def test_shared_lib_ext_linux(self): """Test getting extension for shared libraries (mocked for Linux).""" @@ -714,12 +715,12 @@ def test_shared_lib_ext_darwin(self): def test_platform_name_native(self): """Test getting platform name.""" platform_name_nover = get_platform_name() - self.assertTrue(isinstance(platform_name_nover, string_type)) + self.assertIsInstance(platform_name_nover, string_type) len_nover = len(platform_name_nover.split('-')) self.assertTrue(len_nover >= 3) platform_name_ver = get_platform_name(withversion=True) - self.assertTrue(isinstance(platform_name_ver, string_type)) + self.assertIsInstance(platform_name_ver, string_type) len_ver = len(platform_name_ver.split('-')) self.assertTrue(platform_name_ver.startswith(platform_name_ver)) self.assertTrue(len_ver >= len_nover) @@ -764,7 +765,8 @@ def test_os_version(self): def test_gcc_version_native(self): """Test getting gcc version.""" gcc_version = get_gcc_version() - self.assertTrue(isinstance(gcc_version, string_type) or gcc_version is None) + if gcc_version is not None: + self.assertIsInstance(gcc_version, string_type) def test_gcc_version_linux(self): """Test getting gcc version (mocked for Linux).""" @@ -816,12 +818,12 @@ def test_get_total_memory_darwin(self): def test_get_total_memory_native(self): """Test the function that gets the total memory.""" memtotal = get_total_memory() - self.assertTrue(isinstance(memtotal, int)) + self.assertIsInstance(memtotal, int) def test_system_info(self): """Test getting system info.""" system_info = get_system_info() - self.assertTrue(isinstance(system_info, dict)) + self.assertIsInstance(system_info, dict) def test_det_parallelism_native(self): """Test det_parallelism function (native calls).""" @@ -889,7 +891,7 @@ def mock_python_ver(py_maj_ver, py_min_ver): self.mock_stderr(False) self.assertFalse(stderr) - self.assertTrue(py_maj_ver in [2, 3]) + self.assertIn(py_maj_ver, [2, 3]) if py_maj_ver == 2: self.assertTrue(py_min_ver == 7) else: @@ -933,6 +935,38 @@ def test_pick_dep_version(self): error_pattern = r"Unknown value type for version: .* \(1.23\), should be string value" self.assertErrorRegex(EasyBuildError, error_pattern, pick_dep_version, 1.23) + def test_pick_system_specific_value(self): + """Test pick_system_specific_value function.""" + + self.assertEqual(pick_system_specific_value('test-desc', None), None) + self.assertEqual(pick_system_specific_value('test-desc', '1.2.3'), '1.2.3') + self.assertEqual(pick_system_specific_value('test-desc', (42, 'foobar')), (42, 'foobar')) + + option_dict = { + 'arch=x86_64': '1.2.3-amd64', + 'arch=POWER': '1.2.3-ppc64le', + 'arch=*': '1.2.3-other', + } + + st.get_cpu_architecture = lambda: X86_64 + self.assertEqual(pick_system_specific_value('test-desc', option_dict), '1.2.3-amd64') + + st.get_cpu_architecture = lambda: POWER + self.assertEqual(pick_system_specific_value('test-desc', option_dict), '1.2.3-ppc64le') + + st.get_cpu_architecture = lambda: "NON_EXISTING_ARCH" + self.assertEqual(pick_system_specific_value('test-desc', option_dict), '1.2.3-other') + + error_pattern = "Found empty dict as test-desc" + self.assertErrorRegex(EasyBuildError, error_pattern, pick_system_specific_value, 'test-desc', {}) + + error_pattern = r"Unexpected keys in test-desc: foo \(only 'arch=' keys are supported\)" + self.assertErrorRegex(EasyBuildError, error_pattern, pick_system_specific_value, 'test-desc', + {'foo': '1'}) + error_pattern = r"Unexpected keys in test-desc: foo \(only 'arch=' keys are supported\)" + self.assertErrorRegex(EasyBuildError, error_pattern, pick_system_specific_value, 'test-desc', + {'foo': '1', 'arch=POWER': '2'}) + def test_check_os_dependency(self): """Test check_os_dependency.""" @@ -1076,7 +1110,7 @@ def test_locate_solib(self): if libc_obj: libc_path = locate_solib(libc_obj) self.assertEqual(os.path.basename(libc_path), libname) - self.assertTrue(os.path.exists(libc_path), "%s should exist" % libname) + self.assertExists(libc_path) def test_find_library_path(self): """Test find_library_path function (Linux and Darwin only).""" @@ -1091,7 +1125,8 @@ def test_find_library_path(self): if libname: lib_path = find_library_path(libname) self.assertEqual(os.path.basename(lib_path), libname) - self.assertTrue(os.path.exists(lib_path) or os_type == DARWIN, "%s should exist" % libname) + if os_type != DARWIN: + self.assertExists(lib_path) def suite(): diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 389ad2c45a..1c36431c45 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -95,7 +95,7 @@ def test_toolchain(self): ec_file = find_full_path(os.path.join(test_ecs, 'g', 'gzip', 'gzip-1.4.eb')) ec = EasyConfig(ec_file, validate=False) tc = ec.toolchain - self.assertTrue('debug' in tc.options) + self.assertIn('debug', tc.options) def test_unknown_toolchain(self): """Test search_toolchain function for not available toolchains.""" @@ -107,7 +107,7 @@ def test_system_toolchain(self): """Test for system toolchain.""" for ver in ['system', '']: tc = self.get_toolchain('system', version=ver) - self.assertTrue(isinstance(tc, SystemToolchain)) + self.assertIsInstance(tc, SystemToolchain) def test_foss_toolchain(self): """Test for foss toolchain.""" @@ -168,7 +168,7 @@ def test_is_system_toolchain(self): stderr = self.get_stderr() self.mock_stderr(False) self.assertTrue(tc.is_system_toolchain()) - self.assertTrue(dummy_depr_warning in stderr, "Found '%s' in: %s" % (dummy_depr_warning, stderr)) + self.assertIn(dummy_depr_warning, stderr) def test_toolchain_prepare_sysroot(self): """Test build environment setup done by Toolchain.prepare in case --sysroot is specified.""" @@ -294,7 +294,7 @@ def test_toolchain_compiler_env_vars(self): for key, prev_val, new_val in [('CC', 'foo', 'gcc'), ('CXX', 'bar', 'g++')]: warning_msg = "WARNING: $%s was defined as '%s', " % (key, prev_val) warning_msg += "but is now set to '%s' in minimal build environment" % new_val - self.assertTrue(warning_msg in stderr) + self.assertIn(warning_msg, stderr) self.assertEqual(os.getenv('CC'), 'gcc') self.assertEqual(os.getenv('CXX'), 'g++') @@ -334,7 +334,7 @@ def test_toolchain_compiler_env_vars(self): warning_msg = "WARNING: 'nosuchcommand' command not found in $PATH, " warning_msg += "not setting $CC in minimal build environment" - self.assertTrue(warning_msg in stderr) + self.assertIn(warning_msg, stderr) self.assertEqual(stdout, '') self.assertEqual(os.getenv('CC'), None) @@ -519,9 +519,9 @@ def test_get_variable_libs_list(self): tc.prepare() ldflags = tc.get_variable('LDFLAGS', typ=list) - self.assertTrue(isinstance(ldflags, list)) + self.assertIsInstance(ldflags, list) if len(ldflags) > 0: - self.assertTrue(isinstance(ldflags[0], string_type)) + self.assertIsInstance(ldflags[0], string_type) def test_validate_pass_by_value(self): """ @@ -557,7 +557,7 @@ def test_optimization_flags(self): tc.prepare() for var in flag_vars: flags = tc.get_variable(var) - self.assertTrue(tc.COMPILER_SHARED_OPTION_MAP['defaultopt'] in flags) + self.assertIn(tc.COMPILER_SHARED_OPTION_MAP['defaultopt'], flags) # check other optimization flags for opt in ['noopt', 'lowopt', 'opt']: @@ -568,9 +568,9 @@ def test_optimization_flags(self): for var in flag_vars: flags = tc.get_variable(var) if enable: - self.assertTrue(tc.COMPILER_SHARED_OPTION_MAP[opt] in flags) + self.assertIn(tc.COMPILER_SHARED_OPTION_MAP[opt], flags) else: - self.assertTrue(tc.COMPILER_SHARED_OPTION_MAP[opt] in flags) + self.assertIn(tc.COMPILER_SHARED_OPTION_MAP[opt], flags) self.modtool.purge() def test_optimization_flags_combos(self): @@ -586,7 +586,7 @@ def test_optimization_flags_combos(self): for var in flag_vars: flags = tc.get_variable(var) flag = '-%s' % tc.COMPILER_SHARED_OPTION_MAP['lowopt'] - self.assertTrue(flag in flags) + self.assertIn(flag, flags) self.modtool.purge() tc = self.get_toolchain('foss', version='2018a') @@ -595,7 +595,7 @@ def test_optimization_flags_combos(self): for var in flag_vars: flags = tc.get_variable(var) flag = '-%s' % tc.COMPILER_SHARED_OPTION_MAP['noopt'] - self.assertTrue(flag in flags) + self.assertIn(flag, flags) self.modtool.purge() tc = self.get_toolchain('foss', version='2018a') @@ -604,7 +604,7 @@ def test_optimization_flags_combos(self): for var in flag_vars: flags = tc.get_variable(var) flag = '-%s' % tc.COMPILER_SHARED_OPTION_MAP['noopt'] - self.assertTrue(flag in flags) + self.assertIn(flag, flags) def test_misc_flags_shared(self): """Test whether shared compiler flags are set correctly.""" @@ -622,9 +622,9 @@ def test_misc_flags_shared(self): for var in flag_vars: flags = tc.get_variable(var).split() if enable: - self.assertTrue(flag in flags, "%s: True means %s in %s" % (opt, flag, flags)) + self.assertIn(flag, flags, "%s: True means %s in %s" % (opt, flag, flags)) else: - self.assertTrue(flag not in flags, "%s: False means no %s in %s" % (opt, flag, flags)) + self.assertNotIn(flag, flags, "%s: False means no %s in %s" % (opt, flag, flags)) self.modtool.purge() value = '--see-if-this-propagates' @@ -643,10 +643,10 @@ def test_misc_flags_shared(self): tc.prepare() self.assertTrue(tc.get_variable('CXXFLAGS').endswith(' ' + value)) for var in flag_vars: - self.assertTrue(value not in tc.get_variable(var)) + self.assertNotIn(value, tc.get_variable(var)) # https://github.com/easybuilders/easybuild-framework/pull/3571 # catch variable resued inside loop - self.assertTrue("-o -n -l -y" not in tc.get_variable(var)) + self.assertNotIn("-o -n -l -y", tc.get_variable(var)) self.modtool.purge() def test_misc_flags_unique(self): @@ -671,9 +671,9 @@ def test_misc_flags_unique(self): for key, value in option.items(): flag = "-%s" % value if enable == key: - self.assertTrue(flag in flags, "%s: %s means %s in %s" % (opt, enable, flag, flags)) + self.assertIn(flag, flags, "%s: %s means %s in %s" % (opt, enable, flag, flags)) else: - self.assertTrue(flag not in flags, "%s: %s means no %s in %s" % (opt, enable, flag, flags)) + self.assertNotIn(flag, flags, "%s: %s means no %s in %s" % (opt, enable, flag, flags)) self.modtool.purge() def test_override_optarch(self): @@ -695,9 +695,9 @@ def test_override_optarch(self): for var in flag_vars: flags = tc.get_variable(var) if enable: - self.assertTrue(flag in flags, "optarch: True means %s in %s" % (flag, flags)) + self.assertIn(flag, flags, "optarch: True means %s in %s" % (flag, flags)) else: - self.assertFalse(flag in flags, "optarch: False means no %s in %s" % (flag, flags)) + self.assertNotIn(flag, flags, "optarch: False means no %s in %s" % (flag, flags)) self.modtool.purge() def test_optarch_generic(self): @@ -739,10 +739,10 @@ def test_optarch_generic(self): tup = (key, tcversion, tcopts, generic_flags, val) if generic: error_msg = "(%s, %s, %s) '%s' flags should be found in: '%s'" - self.assertTrue(generic_flags in val, error_msg % tup) + self.assertIn(generic_flags, val, error_msg % tup) else: error_msg = "(%s, %s, %s) '%s' flags should not be found in: '%s'" - self.assertFalse(generic_flags in val, error_msg % tup) + self.assertNotIn(generic_flags, val, error_msg % tup) modules.modules_tool().purge() @@ -756,14 +756,14 @@ def test_optarch_aarch64_heuristic(self): tc.set_options({}) tc.prepare() self.assertEqual(tc.options.options_map['optarch'], 'mcpu=cortex-a53') - self.assertTrue('-mcpu=cortex-a53' in os.environ['CFLAGS']) + self.assertIn('-mcpu=cortex-a53', os.environ['CFLAGS']) self.modtool.purge() tc = self.get_toolchain("GCCcore", version="6.2.0") tc.set_options({}) tc.prepare() self.assertEqual(tc.options.options_map['optarch'], 'mcpu=native') - self.assertTrue('-mcpu=native' in os.environ['CFLAGS']) + self.assertIn('-mcpu=native', os.environ['CFLAGS']) self.modtool.purge() st.get_cpu_model = lambda: 'ARM Cortex-A53 + Cortex-A72' @@ -771,7 +771,7 @@ def test_optarch_aarch64_heuristic(self): tc.set_options({}) tc.prepare() self.assertEqual(tc.options.options_map['optarch'], 'mcpu=cortex-a72.cortex-a53') - self.assertTrue('-mcpu=cortex-a72.cortex-a53' in os.environ['CFLAGS']) + self.assertIn('-mcpu=cortex-a72.cortex-a53', os.environ['CFLAGS']) self.modtool.purge() def test_compiler_dependent_optarch(self): @@ -852,13 +852,13 @@ def test_compiler_dependent_optarch(self): # Check that the correct flags are there if enable and flags != '': error_msg = "optarch: True means '%s' in '%s'" % (flags, set_flags) - self.assertTrue(flags in set_flags, "optarch: True means '%s' in '%s'") + self.assertIn(flags, set_flags, "optarch: True means '%s' in '%s'") # Check that there aren't any unexpected flags else: for blacklisted_flag in blacklist: error_msg = "optarch: False means no '%s' in '%s'" % (blacklisted_flag, set_flags) - self.assertFalse(blacklisted_flag in set_flags, error_msg) + self.assertNotIn(blacklisted_flag, set_flags, error_msg) self.modtool.purge() @@ -900,9 +900,9 @@ def test_misc_flags_unique_fortran(self): for var in flag_vars: flags = tc.get_variable(var) if enable: - self.assertTrue(flag in flags, "%s: True means %s in %s" % (opt, flag, flags)) + self.assertIn(flag, flags, "%s: True means %s in %s" % (opt, flag, flags)) else: - self.assertTrue(flag not in flags, "%s: False means no %s in %s" % (opt, flag, flags)) + self.assertNotIn(flag, flags, "%s: False means no %s in %s" % (opt, flag, flags)) self.modtool.purge() def test_precision_flags(self): @@ -1261,7 +1261,7 @@ def test_fosscuda(self): self.assertEqual(tc.comp_family(prefix='CUDA'), "CUDA") # check CUDA runtime lib - self.assertTrue("-lrt -lcudart" in tc.get_variable('LIBS')) + self.assertIn("-lrt -lcudart", tc.get_variable('LIBS')) def setup_sandbox_for_foss_fftw(self, moddir, fftwver='3.3.7'): """Set up sandbox for foss FFTW and FFTW.MPI""" @@ -1377,12 +1377,12 @@ def test_intel_toolchain(self): for flag in ['-mt_mpi', '-fopenmp']: for var in ['CFLAGS', 'CXXFLAGS', 'FCFLAGS', 'FFLAGS', 'F90FLAGS']: - self.assertTrue(flag in tc.get_variable(var)) + self.assertIn(flag, tc.get_variable(var)) # -openmp is deprecated for new Intel compiler versions - self.assertFalse('-openmp' in tc.get_variable('CFLAGS')) - self.assertFalse('-openmp' in tc.get_variable('CXXFLAGS')) - self.assertFalse('-openmp' in tc.get_variable('FFLAGS')) + self.assertNotIn('-openmp', tc.get_variable('CFLAGS')) + self.assertNotIn('-openmp', tc.get_variable('CXXFLAGS')) + self.assertNotIn('-openmp', tc.get_variable('FFLAGS')) self.assertEqual(tc.get_variable('CC'), 'mpiicc') self.assertEqual(tc.get_variable('CXX'), 'mpiicpc') @@ -1405,7 +1405,7 @@ def test_intel_toolchain(self): tc.prepare() self.assertEqual(tc.get_variable('MPIFC'), 'mpiifort') for var in ['CFLAGS', 'CXXFLAGS', 'FCFLAGS', 'FFLAGS', 'F90FLAGS']: - self.assertTrue('-openmp' in tc.get_variable(var)) + self.assertIn('-openmp', tc.get_variable(var)) # with compiler-only toolchain the $MPI* variables are not defined mpi_vars = ('MPICC', 'MPICXX', 'MPIF77', 'MPIF90', 'MPIFC') @@ -1678,12 +1678,12 @@ def test_get_mpi_cmd_template(self): mpdbf = params['mpdbf'] regex = re.compile('^--file=.*/mpdboot$') self.assertTrue(regex.match(mpdbf), "'%s' should match pattern '%s'" % (mpdbf, regex.pattern)) - self.assertTrue(os.path.exists(mpdbf.split('=')[1])) + self.assertExists(mpdbf.split('=')[1]) nodesfile = params['nodesfile'] regex = re.compile('^-machinefile /.*/nodes$') self.assertTrue(regex.match(nodesfile), "'%s' should match pattern '%s'" % (nodesfile, regex.pattern)) - self.assertTrue(os.path.exists(nodesfile.split(' ')[1])) + self.assertExists(nodesfile.split(' ')[1]) def test_prepare_deps(self): """Test preparing for a toolchain when dependencies are involved.""" @@ -1700,7 +1700,7 @@ def test_prepare_deps(self): ] tc.prepare(deps=deps) mods = ['GCC/6.4.0-2.28', 'hwloc/1.11.8-GCC-6.4.0-2.28', 'OpenMPI/2.1.2-GCC-6.4.0-2.28'] - self.assertTrue([m['mod_name'] for m in self.modtool.list()], mods) + self.assertEqual(sorted(m['mod_name'] for m in self.modtool.list()), sorted(mods)) def test_prepare_deps_external(self): """Test preparing for a toolchain when dependencies and external modules are involved.""" @@ -1728,10 +1728,10 @@ def test_prepare_deps_external(self): tc = self.get_toolchain('GCC', version='6.4.0-2.28') tc.prepare(deps=deps) mods = ['GCC/6.4.0-2.28', 'hwloc/1.11.8-GCC-6.4.0-2.28', 'OpenMPI/2.1.2-GCC-6.4.0-2.28', 'toy/0.0'] - self.assertTrue([m['mod_name'] for m in self.modtool.list()], mods) + self.assertEqual(sorted(m['mod_name'] for m in self.modtool.list()), sorted(mods)) self.assertTrue(os.environ['EBROOTTOY'].endswith('software/toy/0.0')) self.assertEqual(os.environ['EBVERSIONTOY'], '0.0') - self.assertFalse('EBROOTFOOBAR' in os.environ) + self.assertNotIn('EBROOTFOOBAR', os.environ) # with metadata deps[1] = { @@ -1749,7 +1749,7 @@ def test_prepare_deps_external(self): os.environ['FOOBAR_PREFIX'] = '/foo/bar' tc.prepare(deps=deps) mods = ['GCC/6.4.0-2.28', 'hwloc/1.11.8-GCC-6.4.0-2.28', 'OpenMPI/2.1.2-GCC-6.4.0-2.28', 'toy/0.0'] - self.assertTrue([m['mod_name'] for m in self.modtool.list()], mods) + self.assertEqual(sorted(m['mod_name'] for m in self.modtool.list()), sorted(mods)) self.assertEqual(os.environ['EBROOTTOY'], '/foo/bar') self.assertEqual(os.environ['EBVERSIONTOY'], '1.2.3') self.assertEqual(os.environ['EBROOTFOOBAR'], '/foo/bar') @@ -1890,25 +1890,25 @@ def test_old_new_iccifort(self): self.assertEqual(os.environ.get('LIBBLAS_MT', "(not set)"), libblas_mt_intel4) self.assertEqual(os.environ.get('LIBFFT', "(not set)"), libfft_intel4) self.assertEqual(os.environ.get('LIBFFT_MT', "(not set)"), libfft_mt_intel4) - self.assertTrue(libscalack_intel4 in os.environ['LIBSCALAPACK']) + self.assertIn(libscalack_intel4, os.environ['LIBSCALAPACK']) self.modtool.purge() tc = self.get_toolchain('intel', version='2012a') tc.prepare() self.assertEqual(os.environ.get('LIBBLAS_MT', "(not set)"), libblas_mt_intel3) - self.assertTrue(libscalack_intel3 in os.environ['LIBSCALAPACK']) + self.assertIn(libscalack_intel3, os.environ['LIBSCALAPACK']) self.modtool.purge() tc = self.get_toolchain('intel', version='2018a') tc.prepare() self.assertEqual(os.environ.get('LIBBLAS_MT', "(not set)"), libblas_mt_intel4) - self.assertTrue(libscalack_intel4 in os.environ['LIBSCALAPACK']) + self.assertIn(libscalack_intel4, os.environ['LIBSCALAPACK']) self.modtool.purge() tc = self.get_toolchain('intel', version='2012a') tc.prepare() self.assertEqual(os.environ.get('LIBBLAS_MT', "(not set)"), libblas_mt_intel3) - self.assertTrue(libscalack_intel3 in os.environ['LIBSCALAPACK']) + self.assertIn(libscalack_intel3, os.environ['LIBSCALAPACK']) self.modtool.purge() libscalack_intel4 = libscalack_intel4.replace('_lp64', '_ilp64') @@ -1916,7 +1916,7 @@ def test_old_new_iccifort(self): opts = {'i8': True} tc.set_options(opts) tc.prepare() - self.assertTrue(libscalack_intel4 in os.environ['LIBSCALAPACK']) + self.assertIn(libscalack_intel4, os.environ['LIBSCALAPACK']) self.modtool.purge() tc = self.get_toolchain('fosscuda', version='2018a') @@ -1968,7 +1968,7 @@ def test_standalone_iccifort(self): self.modtool.purge() for key in ['EBROOTICC', 'EBROOTIFORT', 'EBVERSIONICC', 'EBVERSIONIFORT']: - self.assertTrue(os.getenv(key) is None) + self.assertIsNone(os.getenv(key)) # install fake iccifort module with no dependencies fake_iccifort = os.path.join(self.test_prefix, 'iccifort', '2018.1.163') @@ -2004,7 +2004,7 @@ def test_standalone_iccifortcuda(self): self.modtool.purge() for key in ['EBROOTICC', 'EBROOTIFORT', 'EBVERSIONICC', 'EBVERSIONIFORT', 'EBROOTCUDA', 'EBVERSIONCUDA']: - self.assertTrue(os.getenv(key) is None) + self.assertIsNone(os.getenv(key)) # install fake iccifortcuda module with no dependencies fake_iccifortcuda = os.path.join(self.test_prefix, 'iccifortcuda', '2018b') @@ -2892,12 +2892,12 @@ def prep(): # new $TMPDIR should be /tmp/xxxxxx tmpdir = os.environ.get('TMPDIR') self.assertTrue(tmpdir.startswith('/tmp')) - self.assertTrue(len(tmpdir) in (11, 13)) + self.assertIn(len(tmpdir), (11, 13)) # also test cleanup method to ensure short $TMPDIR is cleaned up properly - self.assertTrue(os.path.exists(tmpdir)) + self.assertExists(tmpdir) tc.cleanup() - self.assertFalse(os.path.exists(tmpdir)) + self.assertNotExists(tmpdir) os.environ['TMPDIR'] = orig_tmpdir diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index dcce9c228b..9d4fdf5de5 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -125,14 +125,14 @@ def check_toy(self, installpath, outtxt, name='toy', version='0.0', versionprefi msg = "module for toy build toy/%s found (path %s)" % (full_version, toy_module) if get_module_syntax() == 'Lua': toy_module += '.lua' - self.assertTrue(os.path.exists(toy_module), msg + error_msg) + self.assertExists(toy_module, msg + error_msg) # module file is symlinked according to moduleclass toy_module_symlink = os.path.join(installpath, 'modules', 'tools', name, full_version) if get_module_syntax() == 'Lua': toy_module_symlink += '.lua' self.assertTrue(os.path.islink(toy_module_symlink)) - self.assertTrue(os.path.exists(toy_module_symlink)) + self.assertExists(toy_module_symlink) # make sure installation log file and easyconfig file are copied to install dir software_path = os.path.join(installpath, 'software', name, full_version) @@ -147,10 +147,10 @@ def check_toy(self, installpath, outtxt, name='toy', version='0.0', versionprefi "Found at least 1 file at %s" % test_report_path_pattern) ec_file_path = os.path.join(software_path, 'easybuild', '%s-%s.eb' % (name, full_version)) - self.assertTrue(os.path.exists(ec_file_path)) + self.assertExists(ec_file_path) devel_module_path = os.path.join(software_path, 'easybuild', '%s-%s-easybuild-devel' % (name, full_version)) - self.assertTrue(os.path.exists(devel_module_path)) + self.assertExists(devel_module_path) def test_toy_build(self, extra_args=None, ec_file=None, tmpdir=None, verify=True, fails=False, verbose=True, raise_error=False, test_report=None, name='toy', versionsuffix='', testing=True, @@ -198,7 +198,7 @@ def test_toy_build(self, extra_args=None, ec_file=None, tmpdir=None, verify=True # make sure full test report was dumped, and contains sensible information if test_report is not None: - self.assertTrue(os.path.exists(test_report)) + self.assertExists(test_report) if test_report_regexs: regex_patterns = test_report_regexs else: @@ -327,7 +327,7 @@ def test_toy_tweaked(self): regex = re.compile(mod_load_msg, re.M) self.assertTrue(regex.search(toy_module_txt), "Pattern '%s' found in: %s" % (regex.pattern, toy_module_txt)) else: - self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) + self.fail("Unknown module syntax: %s" % get_module_syntax()) # newline between "I AM toy v0.0" (modloadmsg) and "oh hai!" (mod*footer) is added automatically expected = "\nTHANKS FOR LOADING ME\nI AM toy v0.0\n" @@ -421,7 +421,7 @@ def test_toy_build_with_blocks(self): # cleanup shutil.rmtree(tmpdir) - sys.path = orig_sys_path + sys.path[:] = orig_sys_path def test_toy_build_formatv2_sections(self): """Perform a toy build (format v2, using sections).""" @@ -481,7 +481,7 @@ def test_toy_download_sources(self): self.check_toy(tmpdir, outtxt) - self.assertTrue(os.path.exists(os.path.join(sourcepath, 't', 'toy', 'toy-0.0.tar.gz'))) + self.assertExists(os.path.join(sourcepath, 't', 'toy', 'toy-0.0.tar.gz')) shutil.rmtree(tmpdir) @@ -774,7 +774,7 @@ def test_toy_group_check(self): regex = re.compile(pattern % group_name, re.M) self.assertTrue(regex.search(outtxt), "Pattern '%s' found in: %s" % (regex.pattern, outtxt)) else: - self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) + self.fail("Unknown module syntax: %s" % get_module_syntax()) write_file(test_ec, read_file(toy_ec) + "\ngroup = ('%s', 'custom message', 'extra item')\n" % group_name) self.assertErrorRegex(SystemExit, '.*', self.eb_main, args, do_build=True, @@ -823,7 +823,7 @@ def test_toy_hierarchical(self): toy_module_path = os.path.join(mod_prefix, 'MPI', 'GCC', '6.4.0-2.28', 'OpenMPI', '2.1.2', 'toy', '0.0') if get_module_syntax() == 'Lua': toy_module_path += '.lua' - self.assertTrue(os.path.exists(toy_module_path)) + self.assertExists(toy_module_path) # check that toolchain load is expanded to loads for toolchain dependencies, # except for the ones that extend $MODULEPATH to make the toy module available @@ -832,7 +832,7 @@ def test_toy_hierarchical(self): elif get_module_syntax() == 'Lua': load_regex_template = r'load\("%s/.*"\)' else: - self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) + self.fail("Unknown module syntax: %s" % get_module_syntax()) modtxt = read_file(toy_module_path) for dep in ['foss', 'GCC', 'OpenMPI']: @@ -854,7 +854,7 @@ def test_toy_hierarchical(self): toy_module_path = os.path.join(mod_prefix, 'Compiler', 'GCC', '6.4.0-2.28', 'toy', '0.0') if get_module_syntax() == 'Lua': toy_module_path += '.lua' - self.assertTrue(os.path.exists(toy_module_path)) + self.assertExists(toy_module_path) # no dependencies or toolchain => no module load statements in module file modtxt = read_file(toy_module_path) @@ -871,7 +871,7 @@ def test_toy_hierarchical(self): toy_module_path = os.path.join(mod_prefix, 'Compiler', 'GCC', '6.4.0-2.28', 'toy', '0.0') if get_module_syntax() == 'Lua': toy_module_path += '.lua' - self.assertTrue(os.path.exists(toy_module_path)) + self.assertExists(toy_module_path) # 'module use' statements to extend $MODULEPATH are present modtxt = read_file(toy_module_path) @@ -883,7 +883,7 @@ def test_toy_hierarchical(self): regex = re.compile(r'^prepend_path\("MODULEPATH", "%s"\)' % fullmodpath_extension, re.M) self.assertTrue(regex.search(modtxt), "Pattern '%s' found in %s" % (regex.pattern, modtxt)) else: - self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) + self.fail("Unknown module syntax: %s" % get_module_syntax()) os.remove(toy_module_path) # ... unless they shouldn't be @@ -898,7 +898,7 @@ def test_toy_hierarchical(self): regex = re.compile(r'^prepend_path\("MODULEPATH", "%s"\)' % fullmodpath_extension, re.M) self.assertFalse(regex.search(modtxt), "Pattern '%s' found in %s" % (regex.pattern, modtxt)) else: - self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) + self.fail("Unknown module syntax: %s" % get_module_syntax()) os.remove(toy_module_path) # test module path with system/system build @@ -911,7 +911,7 @@ def test_toy_hierarchical(self): toy_module_path = os.path.join(mod_prefix, 'Core', 'toy', '0.0') if get_module_syntax() == 'Lua': toy_module_path += '.lua' - self.assertTrue(os.path.exists(toy_module_path)) + self.assertExists(toy_module_path) # no dependencies or toolchain => no module load statements in module file modtxt = read_file(toy_module_path) @@ -929,7 +929,7 @@ def test_toy_hierarchical(self): toy_module_path = os.path.join(mod_prefix, 'Core', 'toy', '0.0') if get_module_syntax() == 'Lua': toy_module_path += '.lua' - self.assertTrue(os.path.exists(toy_module_path)) + self.assertExists(toy_module_path) # no dependencies or toolchain => no module load statements in module file modtxt = read_file(toy_module_path) @@ -941,7 +941,7 @@ def test_toy_hierarchical(self): regex = re.compile(r'^prepend_path\("MODULEPATH", "%s"\)' % fullmodpath_extension, re.M) self.assertTrue(regex.search(modtxt), "Pattern '%s' found in %s" % (regex.pattern, modtxt)) else: - self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) + self.fail("Unknown module syntax: %s" % get_module_syntax()) os.remove(toy_module_path) # building a toolchain module should also work @@ -957,7 +957,7 @@ def test_toy_hierarchical(self): args[0] = os.path.join(test_easyconfigs, 'g', 'gompi', 'gompi-2018a.eb') self.modtool.purge() self.eb_main(args, logfile=self.dummylogfn, do_build=True, verbose=True, raise_error=True) - self.assertTrue(os.path.exists(gompi_module_path), "%s found" % gompi_module_path) + self.assertExists(gompi_module_path) def test_toy_hierarchical_subdir_user_modules(self): """ @@ -1116,7 +1116,7 @@ def test_toy_advanced_filter_deps(self): toy_module = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0-gompi-2018a-test') if get_module_syntax() == 'Lua': toy_module += '.lua' - self.assertTrue(os.path.exists(toy_module)) + self.assertExists(toy_module) def test_toy_hidden_cmdline(self): """Test installing a hidden module using the '--hidden' command line option.""" @@ -1127,10 +1127,10 @@ def test_toy_hidden_cmdline(self): toy_module = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '.0.0') if get_module_syntax() == 'Lua': toy_module += '.lua' - self.assertTrue(os.path.exists(toy_module), 'Found hidden module %s' % toy_module) + self.assertExists(toy_module, 'Found hidden module %s' % toy_module) # installed software is not hidden toybin = os.path.join(self.test_installpath, 'software', 'toy', '0.0', 'bin', 'toy') - self.assertTrue(os.path.exists(toybin)) + self.assertExists(toybin) def test_toy_hidden_easyconfig(self): """Test installing a hidden module using the 'hidden = True' easyconfig parameter.""" @@ -1145,10 +1145,10 @@ def test_toy_hidden_easyconfig(self): toy_module = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '.0.0') if get_module_syntax() == 'Lua': toy_module += '.lua' - self.assertTrue(os.path.exists(toy_module), 'Found hidden module %s' % toy_module) + self.assertExists(toy_module, 'Found hidden module %s' % toy_module) # installed software is not hidden toybin = os.path.join(self.test_installpath, 'software', 'toy', '0.0', 'bin', 'toy') - self.assertTrue(os.path.exists(toybin)) + self.assertExists(toybin) def test_module_filepath_tweaking(self): """Test using --suffix-modules-path.""" @@ -1173,10 +1173,10 @@ def test_module_filepath_tweaking(self): if get_module_syntax() == 'Lua': mod_file_suffix += '.lua' - self.assertTrue(os.path.exists(os.path.join(mod_file_prefix, 'foobarbaz', 'toy', '0.0' + mod_file_suffix))) - self.assertTrue(os.path.exists(os.path.join(mod_file_prefix, 'TOOLS', 'toy', '0.0' + mod_file_suffix))) + self.assertExists(os.path.join(mod_file_prefix, 'foobarbaz', 'toy', '0.0' + mod_file_suffix)) + self.assertExists(os.path.join(mod_file_prefix, 'TOOLS', 'toy', '0.0' + mod_file_suffix)) self.assertTrue(os.path.islink(os.path.join(mod_file_prefix, 'TOOLS', 'toy', '0.0' + mod_file_suffix))) - self.assertTrue(os.path.exists(os.path.join(mod_file_prefix, 't', 'toy', '0.0' + mod_file_suffix))) + self.assertExists(os.path.join(mod_file_prefix, 't', 'toy', '0.0' + mod_file_suffix)) self.assertTrue(os.path.islink(os.path.join(mod_file_prefix, 't', 'toy', '0.0' + mod_file_suffix))) def test_toy_archived_easyconfig(self): @@ -1189,7 +1189,7 @@ def test_toy_archived_easyconfig(self): self.test_toy_build(raise_error=True, extra_args=extra_args) archived_ec = os.path.join(repositorypath, 'toy', 'toy-0.0.eb') - self.assertTrue(os.path.exists(archived_ec)) + self.assertExists(archived_ec) ec = EasyConfig(archived_ec) self.assertEqual(ec.name, 'toy') self.assertEqual(ec.version, '0.0') @@ -1206,7 +1206,7 @@ def test_toy_patches(self): installdir = os.path.join(self.test_installpath, 'software', 'toy', '0.0') patch_file = os.path.join(installdir, 'easybuild', 'toy-0.0_fix-silly-typo-in-printf-statement.patch') - self.assertTrue(os.path.exists(patch_file)) + self.assertExists(patch_file) archived_patch_file = os.path.join(repositorypath, 'toy', 'toy-0.0_fix-silly-typo-in-printf-statement.patch') self.assertTrue(os.path.isfile(archived_patch_file)) @@ -1248,7 +1248,7 @@ def test_toy_extension_patches_postinstallcmds(self): # verify that post-install command for 'bar' extension was executed fn = 'created-via-postinstallcmds.txt' - self.assertTrue(os.path.exists(os.path.join(installdir, fn))) + self.assertExists(os.path.join(installdir, fn)) def test_toy_extension_sources(self): """Test install toy that includes extensions with 'sources' spec (as single-item list).""" @@ -1556,7 +1556,7 @@ def test_toy_module_fulltxt(self): r'puts stderr "oh hai\!"$', ]) else: - self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) + self.fail("Unknown module syntax: %s" % get_module_syntax()) mod_txt_regex = re.compile(mod_txt_regex_pattern) msg = "Pattern '%s' matches with: %s" % (mod_txt_regex.pattern, toy_mod_txt) @@ -1649,10 +1649,10 @@ def test_module_only(self): args = common_args + ['--module-only'] err_msg = "Sanity check failed" self.assertErrorRegex(EasyBuildError, err_msg, self.eb_main, args, do_build=True, raise_error=True) - self.assertFalse(os.path.exists(toy_mod)) + self.assertNotExists(toy_mod) self.eb_main(args + ['--force'], do_build=True, raise_error=True) - self.assertTrue(os.path.exists(toy_mod)) + self.assertExists(toy_mod) # make sure load statements for dependencies are included in additional module file generated with --module-only modtxt = read_file(toy_mod) @@ -1665,15 +1665,15 @@ def test_module_only(self): rebuild_args = args + ['--rebuild'] err_msg = "Sanity check failed" self.assertErrorRegex(EasyBuildError, err_msg, self.eb_main, rebuild_args, do_build=True, raise_error=True) - self.assertFalse(os.path.exists(toy_mod)) + self.assertNotExists(toy_mod) # installing another module under a different naming scheme and using Lua module syntax works fine # first actually build and install toy software + module prefix = os.path.join(self.test_installpath, 'software', 'toy', '0.0-deps') self.eb_main(common_args + ['--force'], do_build=True, raise_error=True) - self.assertTrue(os.path.exists(toy_mod)) - self.assertTrue(os.path.exists(os.path.join(self.test_installpath, 'software', 'toy', '0.0-deps', 'bin'))) + self.assertExists(toy_mod) + self.assertExists(os.path.join(self.test_installpath, 'software', 'toy', '0.0-deps', 'bin')) modtxt = read_file(toy_mod) self.assertTrue(re.search("set root %s" % prefix, modtxt)) self.assertEqual(len(os.listdir(os.path.join(self.test_installpath, 'software'))), 2) @@ -1685,9 +1685,9 @@ def test_module_only(self): '--module-naming-scheme=MigrateFromEBToHMNS', ] toy_core_mod = os.path.join(self.test_installpath, 'modules', 'all', 'Core', 'toy', '0.0-deps') - self.assertFalse(os.path.exists(toy_core_mod)) + self.assertNotExists(toy_core_mod) self.eb_main(args, do_build=True, raise_error=True) - self.assertTrue(os.path.exists(toy_core_mod)) + self.assertExists(toy_core_mod) # existing install is reused modtxt2 = read_file(toy_core_mod) self.assertTrue(re.search("set root %s" % prefix, modtxt2)) @@ -1702,9 +1702,9 @@ def test_module_only(self): os.remove(toy_core_mod) # remove the write permissions on the installation adjust_permissions(prefix, stat.S_IRUSR | stat.S_IXUSR, relative=False) - self.assertFalse(os.path.exists(toy_core_mod)) + self.assertNotExists(toy_core_mod) self.eb_main(args, do_build=True, raise_error=True) - self.assertTrue(os.path.exists(toy_core_mod)) + self.assertExists(toy_core_mod) # existing install is reused modtxt2 = read_file(toy_core_mod) self.assertTrue(re.search("set root %s" % prefix, modtxt2)) @@ -1727,9 +1727,9 @@ def test_module_only(self): '--module-syntax=Lua', '--modules-tool=Lmod', ] - self.assertFalse(os.path.exists(toy_mod + '.lua')) + self.assertNotExists(toy_mod + '.lua') self.eb_main(args, do_build=True, raise_error=True) - self.assertTrue(os.path.exists(toy_mod + '.lua')) + self.assertExists(toy_mod + '.lua') # existing install is reused modtxt3 = read_file(toy_mod + '.lua') self.assertTrue(re.search('local root = "%s"' % prefix, modtxt3)) @@ -1784,7 +1784,7 @@ def test_module_only_extensions(self): # first try normal --module-only, should work fine self.eb_main([test_ec, '--module-only'], do_build=True, raise_error=True) - self.assertTrue(os.path.exists(toy_mod)) + self.assertExists(toy_mod) remove_file(toy_mod) # rename file required for barbar extension, so we can check whether sanity check catches it @@ -1796,17 +1796,17 @@ def test_module_only_extensions(self): for extra_args in (['--module-only'], ['--module-only', '--rebuild']): self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, [test_ec] + extra_args, do_build=True, raise_error=True) - self.assertFalse(os.path.exists(toy_mod)) + self.assertNotExists(toy_mod) # failing sanity check for barbar extension is ignored when using --module-only --skip-extensions for extra_args in (['--module-only'], ['--module-only', '--rebuild']): self.eb_main([test_ec, '--skip-extensions'] + extra_args, do_build=True, raise_error=True) - self.assertTrue(os.path.exists(toy_mod)) + self.assertExists(toy_mod) remove_file(toy_mod) # we can force module generation via --force (which skips sanity check entirely) self.eb_main([test_ec, '--module-only', '--force'], do_build=True, raise_error=True) - self.assertTrue(os.path.exists(toy_mod)) + self.assertExists(toy_mod) def test_toy_exts_parallel(self): """ @@ -1921,11 +1921,11 @@ def test_backup_modules(self): # install module once (without --module-only), so it can be backed up self.eb_main(args, do_build=True, raise_error=True) - self.assertTrue(os.path.exists(toy_mod)) + self.assertExists(toy_mod) # forced reinstall, no backup of module file because --backup-modules (or --module-only) is not used self.eb_main(args, do_build=True, raise_error=True) - self.assertTrue(os.path.exists(toy_mod)) + self.assertExists(toy_mod) toy_mod_backups = glob.glob(os.path.join(toy_mod_dir, '.' + toy_mod_fn + '.bak_*')) self.assertEqual(len(toy_mod_backups), 0) @@ -1937,7 +1937,7 @@ def test_backup_modules(self): stdout = self.get_stdout() self.mock_stderr(False) self.mock_stdout(False) - self.assertTrue(os.path.exists(toy_mod)) + self.assertExists(toy_mod) toy_mod_backups = glob.glob(os.path.join(toy_mod_dir, '.' + toy_mod_fn + '.bak_*')) self.assertEqual(len(toy_mod_backups), 1) first_toy_mod_backup = toy_mod_backups[0] @@ -1989,7 +1989,7 @@ def test_backup_modules(self): # initial installation of Lua module file self.eb_main(args, do_build=True, raise_error=True) - self.assertTrue(os.path.exists(toy_mod)) + self.assertExists(toy_mod) lua_toy_mods = glob.glob(os.path.join(toy_mod_dir, '*.lua*')) self.assertEqual(len(lua_toy_mods), 1) self.assertEqual(os.path.basename(toy_mod), os.path.basename(lua_toy_mods[0])) @@ -2008,7 +2008,7 @@ def test_backup_modules(self): self.mock_stderr(False) self.mock_stdout(False) - self.assertTrue(os.path.exists(toy_mod)) + self.assertExists(toy_mod) lua_toy_mods = glob.glob(os.path.join(toy_mod_dir, '*.lua*')) self.assertEqual(len(lua_toy_mods), 1) self.assertEqual(os.path.basename(toy_mod), os.path.basename(lua_toy_mods[0])) @@ -2028,7 +2028,7 @@ def test_backup_modules(self): self.assertEqual(len(hidden_toy_mod_backups), backups_hidden) first_toy_lua_mod_backup = (toy_mod_backups or hidden_toy_mod_backups)[0] - self.assertTrue('.bak_' in os.path.basename(first_toy_lua_mod_backup)) + self.assertIn('.bak_', os.path.basename(first_toy_lua_mod_backup)) # check messages in stdout/stderr regex = re.compile("^== backup of existing module file stored at %s" % toy_mod_bak, re.M) @@ -2085,7 +2085,7 @@ def test_package(self): self.test_toy_build(extra_args=extra_args) toypkg = os.path.join(pkgpath, 'toy-0.0-eb-%s.321.foo' % EASYBUILD_VERSION) - self.assertTrue(os.path.exists(toypkg), "%s is there" % toypkg) + self.assertExists(toypkg) def test_package_skip(self): """Test use of --package with --skip.""" @@ -2093,14 +2093,14 @@ def test_package_skip(self): pkgpath = os.path.join(self.test_prefix, 'packages') # default path self.test_toy_build(['--packagepath=%s' % pkgpath]) - self.assertFalse(os.path.exists(pkgpath), "%s is not created without use of --package" % pkgpath) + self.assertNotExists(pkgpath, "%s is not created without use of --package" % pkgpath) self.mock_stdout(True) self.test_toy_build(extra_args=['--package', '--skip'], verify=False) self.mock_stdout(False) toypkg = os.path.join(pkgpath, 'toy-0.0-eb-%s.1.rpm' % EASYBUILD_VERSION) - self.assertTrue(os.path.exists(toypkg), "%s is there" % toypkg) + self.assertExists(toypkg) def test_regtest(self): """Test use of --regtest.""" @@ -2108,10 +2108,9 @@ def test_regtest(self): # just check whether module exists toy_module = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0') - msg = "module %s found" % toy_module if get_module_syntax() == 'Lua': toy_module += '.lua' - self.assertTrue(os.path.exists(toy_module), msg) + self.assertExists(toy_module) def test_minimal_toolchains(self): """Test toy build with --minimal-toolchains.""" @@ -2145,12 +2144,12 @@ def test_reproducibility(self): reprod_dir = os.path.join(self.test_installpath, 'software', 'toy', '0.0', 'easybuild', 'reprod') reprod_ec = os.path.join(reprod_dir, 'toy-0.0.eb') - self.assertTrue(os.path.exists(reprod_ec)) + self.assertExists(reprod_ec) # Also check that the dumpenv script is placed alongside it dumpenv_script = '%s.env' % os.path.splitext(reprod_ec)[0] reprod_dumpenv = os.path.join(reprod_dir, dumpenv_script) - self.assertTrue(os.path.exists(reprod_dumpenv)) + self.assertExists(reprod_dumpenv) # Check contents of the dumpenv script patterns = [ @@ -2162,7 +2161,7 @@ def test_reproducibility(self): ] env_file = read_file(reprod_dumpenv) for pattern in patterns: - self.assertTrue(pattern in env_file) + self.assertIn(pattern, env_file) # Check that the toytoy easyblock is recorded in the reprod easyconfig ec = EasyConfig(reprod_ec) @@ -2170,23 +2169,23 @@ def test_reproducibility(self): # make sure start_dir is not recorded in the dumped easyconfig, this does not appear in the original easyconfig # and is representative of values that are (typically) set by the easyblock steps (which are also dumped) - self.assertFalse('start_dir' in ec.parser.get_config_dict()) + self.assertNotIn('start_dir', ec.parser.get_config_dict()) # Check for child easyblock existence child_easyblock = os.path.join(reprod_dir, 'easyblocks', 'toytoy.py') - self.assertTrue(os.path.exists(child_easyblock)) + self.assertExists(child_easyblock) # Check for parent easyblock existence parent_easyblock = os.path.join(reprod_dir, 'easyblocks', 'toy.py') - self.assertTrue(os.path.exists(parent_easyblock)) + self.assertExists(parent_easyblock) # Make sure framework easyblock modules are not included for framework_easyblock in ['easyblock.py', 'extensioneasyblock.py']: path = os.path.join(reprod_dir, 'easyblocks', framework_easyblock) - self.assertFalse(os.path.exists(path)) + self.assertNotExists(path) # Make sure hooks are also copied reprod_hooks = os.path.join(reprod_dir, 'hooks', hooks_filename) - self.assertTrue(os.path.exists(reprod_hooks)) + self.assertExists(reprod_hooks) def test_reproducibility_ext_easyblocks(self): """Test toy build produces expected reproducibility files also when extensions are used""" @@ -2209,22 +2208,22 @@ def test_reproducibility_ext_easyblocks(self): reprod_dir = os.path.join(self.test_installpath, 'software', 'toy', '0.0', 'easybuild', 'reprod') reprod_ec = os.path.join(reprod_dir, 'toy-0.0.eb') - self.assertTrue(os.path.exists(reprod_ec)) + self.assertExists(reprod_ec) # Check for child easyblock existence child_easyblock = os.path.join(reprod_dir, 'easyblocks', 'toytoy.py') - self.assertTrue(os.path.exists(child_easyblock)) + self.assertExists(child_easyblock) # Check for parent easyblock existence parent_easyblock = os.path.join(reprod_dir, 'easyblocks', 'toy.py') - self.assertTrue(os.path.exists(parent_easyblock)) + self.assertExists(parent_easyblock) # Check for extension easyblock existence ext_easyblock = os.path.join(reprod_dir, 'easyblocks', 'toy_extension.py') - self.assertTrue(os.path.exists(ext_easyblock)) + self.assertExists(ext_easyblock) # Make sure framework easyblock modules are not included for framework_easyblock in ['easyblock.py', 'extensioneasyblock.py']: path = os.path.join(reprod_dir, 'easyblocks', framework_easyblock) - self.assertFalse(os.path.exists(path)) + self.assertNotExists(path) def test_toy_toy(self): """Test building two easyconfigs in a single go, with one depending on the other.""" @@ -2251,13 +2250,13 @@ def test_toy_toy(self): mod1 = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0-one') mod2 = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0-two') - self.assertTrue(os.path.exists(mod1) or os.path.exists('%s.lua' % mod1)) - self.assertTrue(os.path.exists(mod2) or os.path.exists('%s.lua' % mod2)) + if get_module_syntax() == 'Lua': + mod1 += '.lua' + mod2 += '.lua' + self.assertExists(mod1) + self.assertExists(mod2) - if os.path.exists(mod2): - mod2_txt = read_file(mod2) - else: - mod2_txt = read_file('%s.lua' % mod2) + mod2_txt = read_file(mod2) load1_regex = re.compile('load.*toy/0.0-one', re.M) self.assertTrue(load1_regex.search(mod2_txt), "Pattern '%s' found in: %s" % (load1_regex.pattern, mod2_txt)) @@ -2266,7 +2265,7 @@ def test_toy_toy(self): reprod_dir = os.path.join(self.test_installpath, 'software', 'toy', '0.0-two', 'easybuild', 'reprod') dumpenv_script = os.path.join(reprod_dir, 'toy-0.0-two.env') reprod_dumpenv = os.path.join(reprod_dir, dumpenv_script) - self.assertTrue(os.path.exists(reprod_dumpenv)) + self.assertExists(reprod_dumpenv) # Check contents of the dumpenv script patterns = [ @@ -2278,7 +2277,7 @@ def test_toy_toy(self): ] env_file = read_file(reprod_dumpenv) for pattern in patterns: - self.assertTrue(pattern in env_file) + self.assertIn(pattern, env_file) def test_toy_sanity_check_commands(self): """Test toy build with extra sanity check commands.""" @@ -2327,7 +2326,7 @@ def test_toy_sanity_check_commands(self): if get_module_syntax() == 'Lua': toy_modfile += '.lua' - self.assertTrue(os.path.exists(toy_modfile)) + self.assertExists(toy_modfile) def test_sanity_check_paths_lib64(self): """Test whether fallback in sanity check for lib64/ equivalents of library files works.""" @@ -2418,7 +2417,7 @@ def test_toy_build_enhanced_sanity_check(self): test_ec_txt = regex.sub('', toy_ec_txt) write_file(test_ec, test_ec_txt) - self.assertFalse('sanity_check_' in test_ec_txt) + self.assertNotIn('sanity_check_', test_ec_txt) # create custom easyblock for toy that has a custom sanity_check_step toy_easyblock = os.path.join(test_dir, 'sandbox', 'easybuild', 'easyblocks', 't', 'toy.py') @@ -2667,8 +2666,8 @@ def grab_gcc_rpath_wrapper_args(): # by default, /lib and /usr are included in RPATH filter, # together with temporary directory and build directory rpath_filter_paths = grab_gcc_rpath_wrapper_args()['filter_paths'].split(',') - self.assertTrue('/lib.*' in rpath_filter_paths) - self.assertTrue('/usr.*' in rpath_filter_paths) + self.assertIn('/lib.*', rpath_filter_paths) + self.assertIn('/usr.*', rpath_filter_paths) self.assertTrue(any(p.startswith(os.getenv('TMPDIR')) for p in rpath_filter_paths)) self.assertTrue(any(p.startswith(self.test_buildpath) for p in rpath_filter_paths)) @@ -2700,8 +2699,8 @@ def grab_gcc_rpath_wrapper_args(): # check whether rpath filter was set correctly rpath_filter_paths = grab_gcc_rpath_wrapper_args()['filter_paths'].split(',') - self.assertTrue('/test.*' in rpath_filter_paths) - self.assertTrue('/foo/bar.*' in rpath_filter_paths) + self.assertIn('/test.*', rpath_filter_paths) + self.assertIn('/foo/bar.*', rpath_filter_paths) self.assertTrue(any(p.startswith(os.getenv('TMPDIR')) for p in rpath_filter_paths)) self.assertTrue(any(p.startswith(self.test_buildpath) for p in rpath_filter_paths)) @@ -2808,8 +2807,8 @@ def test_toy_modaltsoftname(self): modules_path = os.path.join(self.test_installpath, 'modules', 'all', 'Core') # install dirs for both installations should be there (using original software name) - self.assertTrue(os.path.exists(os.path.join(software_path, 'toy', '0.0-one', 'bin', 'toy'))) - self.assertTrue(os.path.exists(os.path.join(software_path, 'toy', '0.0-two', 'bin', 'toy'))) + self.assertExists(os.path.join(software_path, 'toy', '0.0-one', 'bin', 'toy')) + self.assertExists(os.path.join(software_path, 'toy', '0.0-two', 'bin', 'toy')) toytwo_name = '0.0-two' yot_name = '0.0-one' @@ -2818,8 +2817,8 @@ def test_toy_modaltsoftname(self): yot_name += '.lua' # modules for both installations with alternative name should be there - self.assertTrue(os.path.exists(os.path.join(modules_path, 'toytwo', toytwo_name))) - self.assertTrue(os.path.exists(os.path.join(modules_path, 'yot', yot_name))) + self.assertExists(os.path.join(modules_path, 'toytwo', toytwo_name)) + self.assertExists(os.path.join(modules_path, 'yot', yot_name)) # only subdirectories for software should be created self.assertEqual(sorted(os.listdir(software_path)), sorted(['toy', '.locks'])) @@ -2878,6 +2877,11 @@ def test_toy_build_trace(self): def test_toy_build_hooks(self): """Test use of --hooks.""" + toy_ec = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') + test_ec = os.path.join(self.test_prefix, 'test.eb') + test_ec_txt = read_file(toy_ec) + "\nexts_list = [('bar', '0.0'), ('toy', '0.0')]" + write_file(test_ec, test_ec_txt) + hooks_file = os.path.join(self.test_prefix, 'my_hooks.py') hooks_file_txt = textwrap.dedent(""" import os @@ -2908,6 +2912,12 @@ def module_write_hook(self, module_path, module_txt): print('in module-write hook hook for %s' % os.path.basename(module_path)) return module_txt.replace('Toy C program, 100% toy.', 'Not a toy anymore') + def post_single_extension_hook(ext): + print('installing of extension %s is done!' % ext.name) + + def pre_sanitycheck_hook(self): + print('pre_sanity_check_hook') + def end_hook(): print('end hook triggered, all done!') """) @@ -2915,7 +2925,7 @@ def end_hook(): self.mock_stderr(True) self.mock_stdout(True) - self.test_toy_build(extra_args=['--hooks=%s' % hooks_file], raise_error=True) + self.test_toy_build(ec_file=test_ec, extra_args=['--hooks=%s' % hooks_file], raise_error=True) stderr = self.get_stderr() stdout = self.get_stdout() self.mock_stderr(False) @@ -2927,12 +2937,16 @@ def end_hook(): toy_mod_file += '.lua' self.assertEqual(stderr, '') - # There are 4 modules written: - # Sanitycheck for extensions and main easyblock (1 each), main and devel module + # parse hook is triggered 3 times: once for main install, and then again for each extension; + # module write hook is triggered 5 times: + # - before installing extensions + # - for fake module file being created during sanity check (triggered twice, for main + toy install) + # - for final module file + # - for devel module file expected_output = textwrap.dedent(""" == Running start hook... start hook triggered - == Running parse hook for toy-0.0.eb... + == Running parse hook for test.eb... toy 0.0 ['%(name)s-%(version)s.tar.gz'] echo toy @@ -2945,6 +2959,22 @@ def end_hook(): bin, lib == Running module_write hook... in module-write hook hook for {mod_name} + == Running parse hook... + toy 0.0 + ['%(name)s-%(version)s.tar.gz'] + echo toy + == Running parse hook... + toy 0.0 + ['%(name)s-%(version)s.tar.gz'] + echo toy + == Running post-single_extension hook... + installing of extension bar is done! + == Running post-single_extension hook... + installing of extension toy is done! + == Running pre-sanitycheck hook... + pre_sanity_check_hook + == Running module_write hook... + in module-write hook hook for {mod_name} == Running module_write hook... in module-write hook hook for {mod_name} == Running module_write hook... @@ -3009,7 +3039,7 @@ def test_toy_multi_deps(self): '}', ]) - self.assertTrue(expected in toy_mod_txt, "Pattern '%s' should be found in: %s" % (expected, toy_mod_txt)) + self.assertIn(expected, toy_mod_txt) # also check relevant parts of "module help" and whatis bits expected_descr = '\n'.join([ @@ -3019,7 +3049,7 @@ def test_toy_multi_deps(self): "* GCC/4.6.3 (default), GCC/7.3.0-2.30", ]) error_msg_descr = "Pattern '%s' should be found in: %s" % (expected_descr, toy_mod_txt) - self.assertTrue(expected_descr in toy_mod_txt, error_msg_descr) + self.assertIn(expected_descr, toy_mod_txt, error_msg_descr) if get_module_syntax() == 'Lua': expected_whatis = "whatis([==[Compatible modules: GCC/4.6.3 (default), GCC/7.3.0-2.30]==])" @@ -3027,22 +3057,22 @@ def test_toy_multi_deps(self): expected_whatis = "module-whatis {Compatible modules: GCC/4.6.3 (default), GCC/7.3.0-2.30}" error_msg_whatis = "Pattern '%s' should be found in: %s" % (expected_whatis, toy_mod_txt) - self.assertTrue(expected_whatis in toy_mod_txt, error_msg_whatis) + self.assertIn(expected_whatis, toy_mod_txt, error_msg_whatis) def check_toy_load(depends_on=False): # by default, toy/0.0 should load GCC/4.6.3 (first listed GCC version in multi_deps) self.modtool.load(['toy/0.0']) loaded_mod_names = [x['mod_name'] for x in self.modtool.list()] - self.assertTrue('toy/0.0' in loaded_mod_names) - self.assertTrue('GCC/4.6.3' in loaded_mod_names) - self.assertFalse('GCC/7.3.0-2.30' in loaded_mod_names) + self.assertIn('toy/0.0', loaded_mod_names) + self.assertIn('GCC/4.6.3', loaded_mod_names) + self.assertNotIn('GCC/7.3.0-2.30', loaded_mod_names) if depends_on: # check behaviour when unloading toy (should also unload GCC/4.6.3) self.modtool.unload(['toy/0.0']) loaded_mod_names = [x['mod_name'] for x in self.modtool.list()] - self.assertFalse('toy/0.0' in loaded_mod_names) - self.assertFalse('GCC/4.6.3' in loaded_mod_names) + self.assertNotIn('toy/0.0', loaded_mod_names) + self.assertNotIn('GCC/4.6.3', loaded_mod_names) else: # just undo (don't use 'purge', make cause problems in test environment), to prepare for next test self.modtool.unload(['toy/0.0', 'GCC/4.6.3']) @@ -3050,20 +3080,20 @@ def check_toy_load(depends_on=False): # if GCC/7.3.0-2.30 is loaded first, then GCC/4.6.3 is not loaded by loading toy/0.0 self.modtool.load(['GCC/7.3.0-2.30']) loaded_mod_names = [x['mod_name'] for x in self.modtool.list()] - self.assertTrue('GCC/7.3.0-2.30' in loaded_mod_names) + self.assertIn('GCC/7.3.0-2.30', loaded_mod_names) self.modtool.load(['toy/0.0']) loaded_mod_names = [x['mod_name'] for x in self.modtool.list()] - self.assertTrue('toy/0.0' in loaded_mod_names) - self.assertTrue('GCC/7.3.0-2.30' in loaded_mod_names) - self.assertFalse('GCC/4.6.3' in loaded_mod_names) + self.assertIn('toy/0.0', loaded_mod_names) + self.assertIn('GCC/7.3.0-2.30', loaded_mod_names) + self.assertNotIn('GCC/4.6.3', loaded_mod_names) if depends_on: # check behaviour when unloading toy (should *not* unload GCC/7.3.0-2.30) self.modtool.unload(['toy/0.0']) loaded_mod_names = [x['mod_name'] for x in self.modtool.list()] - self.assertFalse('toy/0.0' in loaded_mod_names) - self.assertTrue('GCC/7.3.0-2.30' in loaded_mod_names) + self.assertNotIn('toy/0.0', loaded_mod_names) + self.assertIn('GCC/7.3.0-2.30', loaded_mod_names) else: # just undo self.modtool.unload(['toy/0.0', 'GCC/7.3.0-2.30']) @@ -3071,20 +3101,20 @@ def check_toy_load(depends_on=False): # having GCC/4.6.3 loaded already is also fine self.modtool.load(['GCC/4.6.3']) loaded_mod_names = [x['mod_name'] for x in self.modtool.list()] - self.assertTrue('GCC/4.6.3' in loaded_mod_names) + self.assertIn('GCC/4.6.3', loaded_mod_names) self.modtool.load(['toy/0.0']) loaded_mod_names = [x['mod_name'] for x in self.modtool.list()] - self.assertTrue('toy/0.0' in loaded_mod_names) - self.assertTrue('GCC/4.6.3' in loaded_mod_names) - self.assertFalse('GCC/7.3.0-2.30' in loaded_mod_names) + self.assertIn('toy/0.0', loaded_mod_names) + self.assertIn('GCC/4.6.3', loaded_mod_names) + self.assertNotIn('GCC/7.3.0-2.30', loaded_mod_names) if depends_on: # check behaviour when unloading toy (should *not* unload GCC/4.6.3) self.modtool.unload(['toy/0.0']) loaded_mod_names = [x['mod_name'] for x in self.modtool.list()] - self.assertFalse('toy/0.0' in loaded_mod_names) - self.assertTrue('GCC/4.6.3' in loaded_mod_names) + self.assertNotIn('toy/0.0', loaded_mod_names) + self.assertIn('GCC/4.6.3', loaded_mod_names) else: # just undo self.modtool.unload(['toy/0.0', 'GCC/4.6.3']) @@ -3098,13 +3128,13 @@ def check_toy_load(depends_on=False): self.test_toy_build(ec_file=test_ec) toy_mod_txt = read_file(toy_mod_file) - self.assertFalse(expected in toy_mod_txt, "Pattern '%s' should not be found in: %s" % (expected, toy_mod_txt)) + self.assertNotIn(expected, toy_mod_txt) self.modtool.load(['toy/0.0']) loaded_mod_names = [x['mod_name'] for x in self.modtool.list()] - self.assertTrue('toy/0.0' in loaded_mod_names) - self.assertFalse('GCC/4.6.3' in loaded_mod_names) - self.assertFalse('GCC/7.3.0-2.30' in loaded_mod_names) + self.assertIn('toy/0.0', loaded_mod_names) + self.assertNotIn('GCC/4.6.3', loaded_mod_names) + self.assertNotIn('GCC/7.3.0-2.30', loaded_mod_names) # also check relevant parts of "module help" and whatis bits (no '(default)' here!) expected_descr_no_default = '\n'.join([ @@ -3114,7 +3144,7 @@ def check_toy_load(depends_on=False): "* GCC/4.6.3, GCC/7.3.0-2.30", ]) error_msg_descr = "Pattern '%s' should be found in: %s" % (expected_descr_no_default, toy_mod_txt) - self.assertTrue(expected_descr_no_default in toy_mod_txt, error_msg_descr) + self.assertIn(expected_descr_no_default, toy_mod_txt, error_msg_descr) if get_module_syntax() == 'Lua': expected_whatis_no_default = "whatis([==[Compatible modules: GCC/4.6.3, GCC/7.3.0-2.30]==])" @@ -3122,7 +3152,7 @@ def check_toy_load(depends_on=False): expected_whatis_no_default = "module-whatis {Compatible modules: GCC/4.6.3, GCC/7.3.0-2.30}" error_msg_whatis = "Pattern '%s' should be found in: %s" % (expected_whatis_no_default, toy_mod_txt) - self.assertTrue(expected_whatis_no_default in toy_mod_txt, error_msg_whatis) + self.assertIn(expected_whatis_no_default, toy_mod_txt, error_msg_whatis) # restore original environment to continue testing with a clean slate modify_env(os.environ, self.orig_environ, verbose=False) @@ -3159,11 +3189,11 @@ def check_toy_load(depends_on=False): '}', ]) - self.assertTrue(expected in toy_mod_txt, "Pattern '%s' should be found in: %s" % (expected, toy_mod_txt)) + self.assertIn(expected, toy_mod_txt) error_msg_descr = "Pattern '%s' should be found in: %s" % (expected_descr, toy_mod_txt) - self.assertTrue(expected_descr in toy_mod_txt, error_msg_descr) + self.assertIn(expected_descr, toy_mod_txt, error_msg_descr) error_msg_whatis = "Pattern '%s' should be found in: %s" % (expected_whatis, toy_mod_txt) - self.assertTrue(expected_whatis in toy_mod_txt, error_msg_whatis) + self.assertIn(expected_whatis, toy_mod_txt, error_msg_whatis) check_toy_load(depends_on=True) @@ -3183,58 +3213,60 @@ def test_fix_shebang(self): " 'cp -a %(installdir)s/bin/toy %(installdir)s/bin/toy.sh',", # hardcoded path to bin/python - " 'echo \"#!/usr/bin/python\\n# test\" > %(installdir)s/bin/t1.py',", + " 'echo \"#!/usr/bin/python\\n# test\" > %(installdir)s/bin/t1.py',", # hardcoded path to bin/python3.6 - " 'echo \"#!/software/Python/3.6.6-foss-2018b/bin/python3.6\\n# test\" > %(installdir)s/bin/t2.py',", + " 'echo \"#!/software/Python/3.6.6-foss-2018b/bin/python3.6\\n# test\" > %(installdir)s/bin/t2.py',", # already OK, should remain the same - " 'echo \"#!/usr/bin/env python\\n# test\" > %(installdir)s/bin/t3.py',", + " 'echo \"#!/usr/bin/env python\\n# test\" > %(installdir)s/bin/t3.py',", # space after #! + 'env python3' - " 'echo \"#! /usr/bin/env python3\\n# test\" > %(installdir)s/bin/t4.py',", + " 'echo \"#! /usr/bin/env python3\\n# test\" > %(installdir)s/bin/t4.py',", # 'env python3.6' - " 'echo \"#!/usr/bin/env python3.6\\n# test\" > %(installdir)s/bin/t5.py',", + " 'echo \"#!/usr/bin/env python3.6\\n# test\" > %(installdir)s/bin/t5.py',", # shebang with space, should strip the space - " 'echo \"#! /usr/bin/env python\\n# test\" > %(installdir)s/bin/t6.py',", + " 'echo \"#! /usr/bin/env python\\n# test\" > %(installdir)s/bin/t6.py',", # no shebang python - " 'echo \"# test\" > %(installdir)s/bin/t7.py',", + " 'echo \"# test\" > %(installdir)s/bin/t7.py',", # shebang bash - " 'echo \"#!/usr/bin/env bash\\n# test\" > %(installdir)s/bin/b1.sh',", + " 'echo \"#!/usr/bin/env bash\\n# test\" > %(installdir)s/bin/b1.sh',", + # directory, should be skipped (not crash) + " 'mkdir %(installdir)s/bin/testdir',", # tests for perl shebang # hardcoded path to bin/perl - " 'echo \"#!/usr/bin/perl\\n# test\" > %(installdir)s/bin/t1.pl',", + " 'echo \"#!/usr/bin/perl\\n# test\" > %(installdir)s/bin/t1.pl',", # hardcoded path to bin/perl5 - " 'echo \"#!/software/Perl/5.28.1-GCCcore-7.3.0/bin/perl5\\n# test\" > %(installdir)s/bin/t2.pl',", + " 'echo \"#!/software/Perl/5.28.1-GCCcore-7.3.0/bin/perl5\\n# test\" > %(installdir)s/bin/t2.pl',", # already OK, should remain the same - " 'echo \"#!/usr/bin/env perl\\n# test\" > %(installdir)s/bin/t3.pl',", + " 'echo \"#!/usr/bin/env perl\\n# test\" > %(installdir)s/bin/t3.pl',", # hardcoded perl with extra arguments - " 'echo \"#!/usr/bin/perl -w\\n# test\" > %(installdir)s/bin/t4.pl',", + " 'echo \"#!/usr/bin/perl -w\\n# test\" > %(installdir)s/bin/t4.pl',", # space after #! + 'env perl5' - " 'echo \"#!/usr/bin/env perl5\\n# test\" > %(installdir)s/bin/t5.pl',", + " 'echo \"#!/usr/bin/env perl5\\n# test\" > %(installdir)s/bin/t5.pl',", # shebang with space, should strip the space - " 'echo \"#! /usr/bin/env perl\\n# test\" > %(installdir)s/bin/t6.pl',", + " 'echo \"#! /usr/bin/env perl\\n# test\" > %(installdir)s/bin/t6.pl',", # no shebang perl - " 'echo \"# test\" > %(installdir)s/bin/t7.pl',", + " 'echo \"# test\" > %(installdir)s/bin/t7.pl',", # shebang bash - " 'echo \"#!/usr/bin/env bash\\n# test\" > %(installdir)s/bin/b2.sh',", + " 'echo \"#!/usr/bin/env bash\\n# test\" > %(installdir)s/bin/b2.sh',", # tests for bash shebang # hardcoded path to bin/bash - " 'echo \"#!/bin/bash\\n# test\" > %(installdir)s/bin/t1.sh',", + " 'echo \"#!/bin/bash\\n# test\" > %(installdir)s/bin/t1.sh',", # hardcoded path to usr/bin/bash - " 'echo \"#!/usr/bin/bash\\n# test\" > %(installdir)s/bin/t2.sh',", + " 'echo \"#!/usr/bin/bash\\n# test\" > %(installdir)s/bin/t2.sh',", # already OK, should remain the same - " 'echo \"#!/usr/bin/env bash\\n# test\" > %(installdir)s/bin/t3.sh',", + " 'echo \"#!/usr/bin/env bash\\n# test\" > %(installdir)s/bin/t3.sh',", # shebang with space, should strip the space - " 'echo \"#! /usr/bin/env bash\\n# test\" > %(installdir)s/bin/t4.sh',", + " 'echo \"#! /usr/bin/env bash\\n# test\" > %(installdir)s/bin/t4.sh',", # no shebang sh - " 'echo \"# test\" > %(installdir)s/bin/t5.sh',", + " 'echo \"# test\" > %(installdir)s/bin/t5.sh',", # shebang python - " 'echo \"#!/usr/bin/env python\\n# test\" > %(installdir)s/bin/b1.py',", + " 'echo \"#!/usr/bin/env python\\n# test\" > %(installdir)s/bin/b1.py',", # shebang perl - " 'echo \"#!/usr/bin/env perl\\n# test\" > %(installdir)s/bin/b1.pl',", - + " 'echo \"#!/usr/bin/env perl\\n# test\" > %(installdir)s/bin/b1.pl',", "]", "fix_python_shebang_for = ['bin/t1.py', 'bin/t*.py', 'nosuchdir/*.py', 'bin/toy.python', 'bin/b1.sh']", + "fix_python_shebang_for += ['bin/testdir']", "fix_perl_shebang_for = ['bin/t*.pl', 'bin/b2.sh', 'bin/toy.perl']", "fix_bash_shebang_for = ['bin/t*.sh', 'bin/b1.py', 'bin/b1.pl', 'bin/toy.sh']", ]) @@ -3380,7 +3412,7 @@ def test_toy_ghost_installdir(self): self.assertFalse(stdout) regex = re.compile("WARNING: Likely ghost installation directory detected: %s" % toy_installdir) self.assertTrue(regex.search(stderr), "Pattern '%s' found in: %s" % (regex.pattern, stderr)) - self.assertTrue(os.path.exists(toy_installdir)) + self.assertExists(toy_installdir) # cleanup of ghost installation directories can be enable via --remove-ghost-install-dirs write_file(toy_modfile, dummy_toy_mod_txt) @@ -3391,7 +3423,7 @@ def test_toy_ghost_installdir(self): regex = re.compile("^== Ghost installation directory %s removed" % toy_installdir) self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) - self.assertFalse(os.path.exists(toy_installdir)) + self.assertNotExists(toy_installdir) def test_toy_build_lock(self): """Test toy installation when a lock is already in place.""" @@ -3407,11 +3439,11 @@ def test_toy_build_lock(self): self.assertErrorRegex(EasyBuildError, error_pattern, self.test_toy_build, raise_error=True, verbose=False) # lock should still be there after it was hit - self.assertTrue(os.path.exists(toy_lock_path)) + self.assertExists(toy_lock_path) # trying again should give same result self.assertErrorRegex(EasyBuildError, error_pattern, self.test_toy_build, raise_error=True, verbose=False) - self.assertTrue(os.path.exists(toy_lock_path)) + self.assertExists(toy_lock_path) locks_dir = os.path.join(self.test_prefix, 'locks') @@ -3474,7 +3506,7 @@ def __exit__(self, type, value, traceback): if not os.path.exists(toy_lock_path): mkdir(toy_lock_path) - self.assertTrue(os.path.exists(toy_lock_path)) + self.assertExists(toy_lock_path) all_args = extra_args + opts @@ -3488,13 +3520,13 @@ def __exit__(self, type, value, traceback): self.mock_stdout(False) if any('--wait-on-lock=' in x for x in all_args): - self.assertTrue("Use of --wait-on-lock is deprecated" in stderr) + self.assertIn("Use of --wait-on-lock is deprecated", stderr) else: self.assertEqual(stderr, '') wait_matches = wait_regex.findall(stdout) # we can't rely on an exact number of 'waiting' messages, so let's go with a range... - self.assertTrue(len(wait_matches) in range(2, 5)) + self.assertIn(len(wait_matches), range(2, 5)) self.assertTrue(ok_regex.search(stdout), "Pattern '%s' found in: %s" % (ok_regex.pattern, stdout)) @@ -3511,13 +3543,13 @@ def __exit__(self, type, value, traceback): self.mock_stdout(False) wait_matches = wait_regex.findall(stdout) - self.assertTrue(len(wait_matches) in range(2, 5)) + self.assertIn(len(wait_matches), range(2, 5)) # when there is no lock in place, --wait-on-lock* has no impact remove_dir(toy_lock_path) for opt in ['--wait-on-lock=1', '--wait-on-lock-limit=3', '--wait-on-lock-interval=1']: all_args = extra_args + [opt] - self.assertFalse(os.path.exists(toy_lock_path)) + self.assertNotExists(toy_lock_path) self.mock_stderr(True) self.mock_stdout(True) self.test_toy_build(extra_args=all_args, verify=False, raise_error=True, testing=False) @@ -3542,7 +3574,7 @@ def test_toy_lock_cleanup_signals(self): orig_wd = os.getcwd() locks_dir = os.path.join(self.test_installpath, 'software', '.locks') - self.assertFalse(os.path.exists(locks_dir)) + self.assertNotExists(locks_dir) orig_sigalrm_handler = signal.getsignal(signal.SIGALRM) @@ -3611,7 +3643,7 @@ def test_toy_build_unicode_description(self): # the tilde character included here is a Unicode tilde character, not a regular ASCII tilde (~) descr = "This description includes a unicode tilde character: ∼, for your entertainment." - self.assertFalse('~' in descr) + self.assertNotIn('~', descr) regex = re.compile(r'^description\s*=.*', re.M) test_ec_txt = regex.sub(r'description = "%s"' % descr, toy_ec_txt) @@ -3634,12 +3666,12 @@ def test_toy_build_lib64_lib_symlink(self): lib64_path = os.path.join(toy_installdir, 'lib64') # lib64 subdir exists, is not a symlink - self.assertTrue(os.path.exists(lib_path)) + self.assertExists(lib_path) self.assertTrue(os.path.isdir(lib_path)) self.assertFalse(os.path.islink(lib_path)) # lib64 subdir is a symlink to lib subdir - self.assertTrue(os.path.exists(lib64_path)) + self.assertExists(lib64_path) self.assertTrue(os.path.islink(lib64_path)) self.assertTrue(os.path.samefile(lib_path, lib64_path)) @@ -3650,9 +3682,9 @@ def test_toy_build_lib64_lib_symlink(self): remove_dir(self.test_installpath) self.test_toy_build(extra_args=['--disable-lib64-lib-symlink']) - self.assertTrue(os.path.exists(lib_path)) - self.assertFalse(os.path.exists(lib64_path)) - self.assertFalse('lib64' in os.listdir(toy_installdir)) + self.assertExists(lib_path) + self.assertNotExists(lib64_path) + self.assertNotIn('lib64', os.listdir(toy_installdir)) self.assertTrue(os.path.isdir(lib_path)) self.assertFalse(os.path.islink(lib_path)) @@ -3676,12 +3708,12 @@ def test_toy_build_lib_lib64_symlink(self): lib64_path = os.path.join(toy_installdir, 'lib64') # lib64 subdir exists, is not a symlink - self.assertTrue(os.path.exists(lib64_path)) + self.assertExists(lib64_path) self.assertTrue(os.path.isdir(lib64_path)) self.assertFalse(os.path.islink(lib64_path)) # lib subdir is a symlink to lib64 subdir - self.assertTrue(os.path.exists(lib_path)) + self.assertExists(lib_path) self.assertTrue(os.path.isdir(lib_path)) self.assertTrue(os.path.islink(lib_path)) self.assertTrue(os.path.samefile(lib_path, lib64_path)) @@ -3693,9 +3725,9 @@ def test_toy_build_lib_lib64_symlink(self): remove_dir(self.test_installpath) self.test_toy_build(ec_file=test_ec, extra_args=['--disable-lib-lib64-symlink']) - self.assertTrue(os.path.exists(lib64_path)) - self.assertFalse(os.path.exists(lib_path)) - self.assertFalse('lib' in os.listdir(toy_installdir)) + self.assertExists(lib64_path) + self.assertNotExists(lib_path) + self.assertNotIn('lib', os.listdir(toy_installdir)) self.assertTrue(os.path.isdir(lib64_path)) self.assertFalse(os.path.islink(lib64_path)) @@ -3784,7 +3816,7 @@ def test_toy_ignore_test_failure(self): args = ['--ignore-test-failure'] stdout, stderr = self.run_test_toy_build_with_output(extra_args=args, verify=False, testing=False) - self.assertTrue("Build succeeded (with --ignore-test-failure) for 1 out of 1" in stdout) + self.assertIn("Build succeeded (with --ignore-test-failure) for 1 out of 1", stdout) self.assertFalse(stderr) def test_toy_post_install_patches(self): diff --git a/test/framework/tweak.py b/test/framework/tweak.py index e31226570e..eda8046d52 100644 --- a/test/framework/tweak.py +++ b/test/framework/tweak.py @@ -85,7 +85,7 @@ def test_find_matching_easyconfigs(self): ecs_basename = [os.path.basename(ec) for ec in ecs] for gccver in gccvers: gcc_ec = 'GCC-%s.eb' % gccver - self.assertTrue(gcc_ec in ecs_basename, "%s is included in %s" % (gcc_ec, ecs_basename)) + self.assertIn(gcc_ec, ecs_basename) def test_obtain_ec_for(self): """Test obtain_ec_for function.""" @@ -156,7 +156,7 @@ def test_tweak_one_version(self): self.assertEqual(tweaked_toy_ec_parsed['version'], '1.2.3') for key in [k for k in toy_ec_parsed.keys() if k not in ['checksums', 'version']]: val = toy_ec_parsed[key] - self.assertTrue(key in tweaked_toy_ec_parsed, "Parameter '%s' not defined in tweaked easyconfig file" % key) + self.assertIn(key, tweaked_toy_ec_parsed, "Parameter '%s' not defined in tweaked easyconfig file" % key) tweaked_val = tweaked_toy_ec_parsed.get(key) self.assertEqual(val, tweaked_val, "Different value for %s parameter: %s vs %s" % (key, val, tweaked_val)) @@ -436,11 +436,12 @@ def test_map_easyconfig_to_target_tc_hierarchy(self): tweaked_dict = tweaked_ec['ec'].asdict() # First check the mapped toolchain key, value = 'toolchain', iccifort_binutils_tc - self.assertTrue(key in tweaked_dict and value == tweaked_dict[key]) + self.assertIn(key, tweaked_dict) + self.assertEqual(value, tweaked_dict[key]) # Also check that binutils has been mapped for key, value in {'name': 'binutils', 'version': '2.25', 'versionsuffix': ''}.items(): - self.assertTrue(key in tweaked_dict['builddependencies'][0] and - value == tweaked_dict['builddependencies'][0][key]) + self.assertIn(key, tweaked_dict['builddependencies'][0]) + self.assertEqual(tweaked_dict['builddependencies'][0][key], value) # Now test the case where we try to update the dependencies init_config(build_options=build_options) @@ -450,15 +451,16 @@ def test_map_easyconfig_to_target_tc_hierarchy(self): tweaked_dict = tweaked_ec['ec'].asdict() # First check the mapped toolchain key, value = 'toolchain', iccifort_binutils_tc - self.assertTrue(key in tweaked_dict and value == tweaked_dict[key]) + self.assertIn(key, tweaked_dict) + self.assertEqual(value, tweaked_dict[key]) # Also check that binutils has been mapped for key, value in {'name': 'binutils', 'version': '2.25', 'versionsuffix': ''}.items(): - self.assertTrue( - key in tweaked_dict['builddependencies'][0] and value == tweaked_dict['builddependencies'][0][key] - ) + self.assertIn(key, tweaked_dict['builddependencies'][0]) + self.assertEqual(tweaked_dict['builddependencies'][0][key], value) # Also check that the gzip dependency was upgraded for key, value in {'name': 'gzip', 'version': '1.6', 'versionsuffix': ''}.items(): - self.assertTrue(key in tweaked_dict['dependencies'][0] and value == tweaked_dict['dependencies'][0][key]) + self.assertIn(key, tweaked_dict['dependencies'][0]) + self.assertEqual(tweaked_dict['dependencies'][0][key], value) # Make sure there are checksums for our next test self.assertTrue(tweaked_dict['checksums']) @@ -475,19 +477,21 @@ def test_map_easyconfig_to_target_tc_hierarchy(self): tweaked_dict = tweaked_ec['ec'].asdict() # First check the mapped toolchain key, value = 'toolchain', iccifort_binutils_tc - self.assertTrue(key in tweaked_dict and value == tweaked_dict[key]) + self.assertIn(key, tweaked_dict) + self.assertEqual(tweaked_dict[key], value) # Also check that binutils has been mapped for key, value in {'name': 'binutils', 'version': '2.25', 'versionsuffix': ''}.items(): - self.assertTrue( - key in tweaked_dict['builddependencies'][0] and value == tweaked_dict['builddependencies'][0][key] - ) + self.assertIn(key, tweaked_dict['builddependencies'][0]) + self.assertEqual(tweaked_dict['builddependencies'][0][key], value) # Also check that the gzip dependency was upgraded for key, value in {'name': 'gzip', 'version': '1.6', 'versionsuffix': ''}.items(): - self.assertTrue(key in tweaked_dict['dependencies'][0] and value == tweaked_dict['dependencies'][0][key]) + self.assertIn(key, tweaked_dict['dependencies'][0]) + self.assertEqual(tweaked_dict['dependencies'][0][key], value) # Finally check that the version was upgraded key, value = 'version', new_version - self.assertTrue(key in tweaked_dict and value == tweaked_dict[key]) + self.assertIn(key, tweaked_dict) + self.assertEqual(tweaked_dict[key], value) # and that the checksum was removed self.assertFalse(tweaked_dict['checksums']) @@ -513,7 +517,7 @@ def test_map_easyconfig_to_target_tc_hierarchy(self): if isinstance(extension, tuple) and extension[0] == 'toy': self.assertEqual(extension[1], new_version) # Make sure checksum has been purged - self.assertFalse('checksums' in extension[2]) + self.assertNotIn('checksums', extension[2]) hit_extension += 1 self.assertEqual(hit_extension, 1, "Should only have updated one extension") diff --git a/test/framework/type_checking.py b/test/framework/type_checking.py index c6659e527b..83f1ef41ac 100644 --- a/test/framework/type_checking.py +++ b/test/framework/type_checking.py @@ -701,7 +701,7 @@ def test_to_sanity_check_paths_dict(self): self.assertErrorRegex(EasyBuildError, "Expected value to be a dict", to_sanity_check_paths_dict, []) error_msg = "Expected value to be a list" self.assertErrorRegex(EasyBuildError, error_msg, to_sanity_check_paths_dict, {'files': 'foo', 'dirs': []}) - error_msg = "Expected elements to be of type string, tuple or list" + error_msg = "Expected elements to be of type string, tuple/list or dict" self.assertErrorRegex(EasyBuildError, error_msg, to_sanity_check_paths_dict, {'files': [], 'dirs': [1]}) def test_to_checksums(self): diff --git a/test/framework/utilities.py b/test/framework/utilities.py index a268f2a593..37a9b29318 100644 --- a/test/framework/utilities.py +++ b/test/framework/utilities.py @@ -236,7 +236,7 @@ def tearDown(self): self.allow_deprecated_behaviour() # restore original Python search path - sys.path = self.orig_sys_path + sys.path[:] = self.orig_sys_path import easybuild.easyblocks reload(easybuild.easyblocks) import easybuild.easyblocks.generic diff --git a/test/framework/yeb.py b/test/framework/yeb.py index 584e6ab0ae..74a2a47875 100644 --- a/test/framework/yeb.py +++ b/test/framework/yeb.py @@ -122,8 +122,8 @@ def test_yeb_get_config_obj(self): ecdict['sources'].append(fn) ecdict_bis = ec.parser.get_config_dict() - self.assertTrue(fn in ecdict['sources']) - self.assertFalse(fn in ecdict_bis['sources']) + self.assertIn(fn, ecdict['sources']) + self.assertNotIn(fn, ecdict_bis['sources']) def test_is_yeb_format(self): """ Test is_yeb_format function """