Merge pull request #3278 from easybuilders/4.2.x

release EasyBuild v4.2.0
easybuilders · Apr 14, 2020 · e9e8532 · e9e8532
2 parents 210743d + ef67c31
commit e9e8532
Show file tree

Hide file tree

Showing 64 changed files with 4,128 additions and 663 deletions.
diff --git a/.coveragerc b/.coveragerc
@@ -0,0 +1,26 @@
+# .coveragerc to control coverage.py
+[run]
+branch = True
+
+[report]
+# Regexes for lines to exclude from consideration
+exclude_lines =
+    # Have to re-enable the standard pragma
+    pragma: no cover
+
+    # Don't complain about missing debug-only code:
+    def __repr__
+    if self\.debug
+
+    # Don't complain if tests don't hit defensive assertion code:
+    raise AssertionError
+    raise NotImplementedError
+
+    # Don't complain if non-runnable code isn't run:
+    if 0:
+    if __name__ == .__main__.:
+
+ignore_errors = True
+
+[html]
+directory = coverage_html_report
diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml
@@ -39,7 +39,7 @@ jobs:
             python: 3.8
       fail-fast: false
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v2
 
     - name: set up Python
       uses: actions/setup-python@v1
@@ -49,6 +49,9 @@ jobs:
 
     - name: install OS & Python packages
       run: |
+        # disable apt-get update, we don't really need it,
+        # and it does more harm than good (it's fairly expensive, and it results in flaky test runs)
+        # sudo apt-get update
         # for modules tool
         sudo apt-get install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev
         # fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082
@@ -153,19 +156,15 @@ jobs:
           EB_BOOTSTRAP_VERSION=$(grep '^EB_BOOTSTRAP_VERSION' easybuild/scripts/bootstrap_eb.py | sed 's/[^0-9.]//g')
           EB_BOOTSTRAP_SHA256SUM=$(sha256sum easybuild/scripts/bootstrap_eb.py | cut -f1 -d' ')
           EB_BOOTSTRAP_FOUND="$EB_BOOTSTRAP_VERSION $EB_BOOTSTRAP_SHA256SUM"
-          EB_BOOTSTRAP_EXPECTED="20190922.01 7927513e7448d886decfb1bb5daf840e85dc7367f57cc75e51b68f21fe109d53"
+          EB_BOOTSTRAP_EXPECTED="20200203.01 616bf3ce812c0844bf9ea3e690f9d88b394ed48f834ddb8424a73cf45fc64ea5"
           test "$EB_BOOTSTRAP_FOUND" = "$EB_BOOTSTRAP_EXPECTED" || (echo "Version check on bootstrap script failed $EB_BOOTSTRAP_FOUND" && exit 1)
 
-          # test bootstrap script (only compatible with Python 2 for now)
-          if [[ ${{matrix.python}} =~ '2.' ]]; then
-              export PREFIX=/tmp/$USER/$GITHUB_SHA/eb_bootstrap
-              python easybuild/scripts/bootstrap_eb.py $PREFIX
-              # unset $PYTHONPATH to avoid mixing two EasyBuild 'installations' when testing bootstrapped EasyBuild module
-              unset PYTHONPATH
-              # simple sanity check on bootstrapped EasyBuild module (skip when testing with Python 3, for now)
-              module use $PREFIX/modules/all
-              module load EasyBuild
-              eb --version
-          else
-              echo "Testing of bootstrap script skipped when testing with Python ${{matrix.python}}"
-          fi
+          # test bootstrap script
+          export PREFIX=/tmp/$USER/$GITHUB_SHA/eb_bootstrap
+          python easybuild/scripts/bootstrap_eb.py $PREFIX
+          # unset $PYTHONPATH to avoid mixing two EasyBuild 'installations' when testing bootstrapped EasyBuild module
+          unset PYTHONPATH
+          # simple sanity check on bootstrapped EasyBuild module
+          module use $PREFIX/modules/all
+          module load EasyBuild
+          eb --version
diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,4 @@
+.idea
 .pydevproject
 .project
 LICENSE_HEADER

diff --git a/.travis.yml b/.travis.yml
@@ -38,6 +38,8 @@ before_install:
     - pip --version
     - pip install --upgrade pip
     - pip --version
+    # coveralls doesn't support Python 2.6 anymore, so don't try to install it when testing with Python 2.6
+    - if [ "x$TRAVIS_PYTHON_VERSION" != 'x2.6' ]; then pip install coveralls; fi    
     - pip install -r requirements.txt
     # git config is required to make actual git commits (cfr. tests for GitRepository)
     - git config --global user.name "Travis CI"
@@ -55,14 +57,18 @@ script:
     - if [ ! -z $MOD_INIT ] && [ ! -z $LMOD_VERSION ]; then alias ml=foobar; fi
     # set up environment for modules tool (if $MOD_INIT is defined)
     - if [ ! -z $MOD_INIT ]; then source $MOD_INIT; type module; fi
-    # install GitHub token
+    # install GitHub token;
+    # unset $GITHUB_TOKEN environment variable after installing token,
+    # to avoid that it is included in environment dump that is included in EasyBuild debug logs,
+    # which causes test_from_pr_token_log to fail...
     - if [ ! -z $GITHUB_TOKEN ]; then
         if [ "x$TRAVIS_PYTHON_VERSION" == 'x2.6' ];
             then SET_KEYRING="keyring.set_keyring(keyring.backends.file.PlaintextKeyring())";
             else SET_KEYRING="import keyrings; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())";
         fi;
         python -c "import keyring; $SET_KEYRING; keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')";
-      fi
+      fi;
+      unset GITHUB_TOKEN;
     - if [ ! -z $TEST_EASYBUILD_MODULES_TOOL ]; then export EASYBUILD_MODULES_TOOL=$TEST_EASYBUILD_MODULES_TOOL; fi
     - if [ ! -z $TEST_EASYBUILD_MODULE_SYNTAX ]; then export EASYBUILD_MODULE_SYNTAX=$TEST_EASYBUILD_MODULE_SYNTAX; fi
     # create 'source distribution' tarball, like we do when publishing a release to PyPI
@@ -88,8 +94,10 @@ script:
     # create file owned by root but writable by anyone (used by test_copy_file)
     - sudo touch /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt
     - sudo chmod o+w /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt
-    # run test suite
-    - python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log
+    # run coverage on all travis builds except for Python 2.6
+    - if [ "x$TRAVIS_PYTHON_VERSION" != 'x2.6' ]; then coverage run -m test.framework.suite 2>&1 | tee test_framework_suite.log; coverage report -m --ignore-errors; fi  
+    # invoke the regression test for Python 2.6 the original way without coverage
+    - if [ "x$TRAVIS_PYTHON_VERSION" == 'x2.6' ]; then python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log; fi
     # try and make sure output of running tests is clean (no printed messages/warnings)
     - IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|lib/python2.6/site-packages|requires Lmod as modules tool"
     # '|| true' is needed to avoid that Travis stops the job on non-zero exit of grep (i.e. when there are no matches)
@@ -100,12 +108,15 @@ script:
     - EB_BOOTSTRAP_VERSION=$(grep '^EB_BOOTSTRAP_VERSION' $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py | sed 's/[^0-9.]//g')
     - EB_BOOTSTRAP_SHA256SUM=$(sha256sum $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py | cut -f1 -d' ')
     - EB_BOOTSTRAP_FOUND="$EB_BOOTSTRAP_VERSION $EB_BOOTSTRAP_SHA256SUM"
-    - EB_BOOTSTRAP_EXPECTED="20190922.01 7927513e7448d886decfb1bb5daf840e85dc7367f57cc75e51b68f21fe109d53"
+    - EB_BOOTSTRAP_EXPECTED="20200203.01 616bf3ce812c0844bf9ea3e690f9d88b394ed48f834ddb8424a73cf45fc64ea5"
     - test "$EB_BOOTSTRAP_FOUND" = "$EB_BOOTSTRAP_EXPECTED" || (echo "Version check on bootstrap script failed $EB_BOOTSTRAP_FOUND" && exit 1)
-    # test bootstrap script (skip when testing with Python 3 for now, since latest EasyBuild release is not compatible with Python 3 yet)
-    - if [ ! "x$TRAVIS_PYTHON_VERSION" =~ x3.[0-9] ]; then python $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py /tmp/$TRAVIS_JOB_ID/eb_bootstrap; fi
+    # test bootstrap script
+    - python $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py /tmp/$TRAVIS_JOB_ID/eb_bootstrap
     # unset $PYTHONPATH to avoid mixing two EasyBuild 'installations' when testing bootstrapped EasyBuild module
     - unset PYTHONPATH
-    # simply sanity check on bootstrapped EasyBuild module (skip when testing with Python 3, for now)
-    - if [ ! "x$TRAVIS_PYTHON_VERSION" =~ x3.[0-9] ]; then module use /tmp/$TRAVIS_JOB_ID/eb_bootstrap/modules/all; fi
-    - if [ ! "x$TRAVIS_PYTHON_VERSION" =~ x3.[0-9] ]; then module load EasyBuild; eb --version; fi
+    # simply sanity check on bootstrapped EasyBuild module
+    - module use /tmp/$TRAVIS_JOB_ID/eb_bootstrap/modules/all
+    - module load EasyBuild; eb --version
+after_success:
+  - if [ "x$TRAVIS_PYTHON_VERSION" != 'x2.6' ]; then coveralls; fi  
+
diff --git a/RELEASE_NOTES b/RELEASE_NOTES
@@ -3,6 +3,69 @@ For more detailed information, please see the git log.
 
 These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
 
+v4.2.0 (April 14th 2020)
+------------------------
+
+feature release
+
+- add support for --try-update-deps (experimental feature), to upgrade dependencies based on available easyconfigs (#2599)
+- adding locking to prevent two parallel builds of the same installation directory (#3009)
+  - for more information, see https://easybuild.readthedocs.io/en/latest/Locks.html
+- significantly speed up -D/--dry-run by avoiding useless 'module show' (#3203)
+- add support for creating an index & using it when searching for easyconfigs (#3210)
+  - for more information, see https://easybuild.readthedocs.io/en/latest/Easyconfigs_index.html
+- additional GitHub integration features:
+    - add support for targeting easyblocks and framework repositories in --new-pr (#1876, #3189)
+    - add support for --include-easyblocks-from-pr (#3206)
+    - for more information, https://easybuild.readthedocs.io/en/latest/Integration_with_GitHub.html
+- various other enhancements, including:
+  - add a contrib/hooks dir with some examples of hooks used (#2777)
+  - also mention working directory + input passed via stdin (if any) in trace output of run_cmd (#3168)
+  - probe external modules for missing metadata that is not provided via extermal module metadata file (#3174)
+  - also update $CMAKE_PREFIX_PATH and $CMAKE_LIBRARY_PATH in generated module file (#3176)
+  - optionally call log.warning in print_warning (#3195)
+  - add an option to git_config to retain the .git directory (#3197)
+  - allow use of SYSTEM as --try-toolchain option (#3213)
+  - mention CPU arch name in comment for uploaded test report, if it's known by archspec (#3227)
+  - make --merge-pr take into account --pr-target-branch (#3236)
+  - make --check-contrib print a warning when None is used for checksums (#3244)
+  - update install-EasyBuild-develop.sh script and create script for 2020a merge sprint (#3245)
+  - add template for mpi_cmd_prefix (#3264)
+  - update copy_dir to include option to merge directories (#3270)
+  - support template name for CUDA version (#3274)
+- various bug fixes, including:
+  - use correct module for errors_found_in_log (#3119)
+  - fix EasyConfig.update code to handle both strings and lists as input (#3170)
+  - fix removing temporary branch on --check-github (#3182)
+  - fix shebang even if first line doesn't start with '#!' (#3183)
+  - make boostrap script work with Python 3 (#3186)
+  - read patch files as bytestring to avoid UnicodeDecodeError for patches that include funky characters (#3191)
+  - fix online check in --check-github: try repeatedly and with different URLs to cater for HTTP issues (#3194)
+  - don't crash when GitPython is not installed when using Python 3 (#3198)
+  - fix os_name_map for RHEL8 (#3201)
+  - don't add shebang to binary files (#3208)
+  - use checkout@v2 in GitHub Actions to fix broken re-triggered tests (#3219)
+  - don't filter out None values in to_checksums, leave them in place (#3225)
+  - fix defining of $MPI_INC_* and $MPI_LIB_* environment variables for impi version 2019 and later (#3229)
+  - use correct target account/repo when creating test report & posting comment in PR (#3234)
+  - reorganize EasyBlock.extensions_step to ensure correct filtering of extensions (#3235)
+  - also support %(installdir)s and %(builddir)s templates for extensions (#3237)
+  - unset $GITHUB_TOKEN in Travis after installing token, to avoid failing test_from_pr_token_log (#3252)
+  - fix reporting when skipping extensions (#3254)
+  - avoid API rate limit errors on online check in --check-github (#3257)
+  - show easyconfig filenames for parallel build (#3258)
+  - make EasyConfig.dump aware of toolchain hierarchy, to avoid hardcoded subtoolchains in dependencies easyconfig parameters (#3261)
+  - fix use of --copy-ec with a single argument, assume copy to current working directory (#3262)
+  - fix posting of comment in PR with --upload-test-report (#3272)
+  - take into account dependencies marked as external modules when composing template values like %(pyver)s (#3273)
+- other changes:
+  - increase timeout & use api.github.com for connectivity check in check_github (#3192)
+  - restore flake8 default ignores (#3193)
+  - enable tracking of test suite coverage using coveralls (#3204)
+  - make tests use easybuilders/testrepository rather than hpcugent/testrepository after it was moved (#3238)
+  - improve raised error in remove_dir and deprecate rmtree2 (#3228)
+
+
 v4.1.2 (March 16th 2020)
 ------------------------
 
@@ -11,6 +74,8 @@ bugfix release
 - fix gitdb dependency on Python 2.6 in test configuration (#3212)
 - fix broken test for --review-pr by using different PR to test with (#3226)
 - censor authorization part of headers before logging ReST API request (#3248)
+  - see also https://github.com/easybuilders/easybuild-framework/security/advisories/GHSA-2wx6-wc87-rmjm
+
 
 v4.1.1 (January 16th 2020)
 --------------------------

diff --git a/contrib/hooks/README.rst b/contrib/hooks/README.rst
@@ -0,0 +1,15 @@
+Example implementations of EasyBuild hooks
+=================================
+
+.. image:: https://easybuilders.github.io/easybuild/images/easybuild_logo_small.png
+   :align: center
+
+EasyBuild website: https://easybuilders.github.io/easybuild/
+docs: https://easybuild.readthedocs.io
+
+This directory contain examples of implementations of EasyBuild hooks
+used at various sites, along with a couple of small examples with
+explanations.
+
+See https://easybuild.readthedocs.io/en/latest/Hooks.html for
+documentation on hooks in EasyBuild.
diff --git a/contrib/hooks/add_delete_configopt.py b/contrib/hooks/add_delete_configopt.py
@@ -0,0 +1,32 @@
+# Small example of how to add/delete a configure option.
+#
+# Author: Åke Sandgren, HPC2N
+
+# We need to be able to distinguish between versions of OpenMPI
+from distutils.version import LooseVersion
+
+
+def pre_configure_hook(self, *args, **kwargs):
+    # Check that we're dealing with the correct easyconfig file
+    if self.name == 'OpenMPI':
+        extra_opts = ""
+        # Enable using pmi from slurm
+        extra_opts += "--with-pmi=/lap/slurm "
+
+        # And enable munge for OpenMPI versions that knows about it
+        if LooseVersion(self.version) >= LooseVersion('2'):
+            extra_opts += "--with-munge "
+
+        # Now add the options
+        self.log.info("[pre-configure hook] Adding %s" % extra_opts)
+        self.cfg.update('configopts', extra_opts)
+
+        # Now we delete some options
+        # For newer versions of OpenMPI we can re-enable ucx, i.e. delete the --without-ucx flag
+        if LooseVersion(self.version) >= LooseVersion('2.1'):
+            self.log.info("[pre-configure hook] Re-enabling ucx")
+            self.cfg['configopts'] = self.cfg['configopts'].replace('--without-ucx', ' ')
+
+        # And we can remove the --disable-dlopen option from the easyconfig file
+        self.log.info("[pre-configure hook] Re-enabling dlopen")
+        self.cfg['configopts'] = self.cfg['configopts'].replace('--disable-dlopen', ' ')