Skip to content

Commit

Permalink
Merge pull request #3720 from easybuilders/4.4.x
Browse files Browse the repository at this point in the history
release EasyBuild v4.4.0
  • Loading branch information
boegel committed Jun 2, 2021
2 parents 74a32f8 + 5f2813d commit 05ab036
Show file tree
Hide file tree
Showing 75 changed files with 3,600 additions and 683 deletions.
121 changes: 121 additions & 0 deletions .github/workflows/bootstrap_script.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
name: test EasyBuild bootstrap script
on: [push, pull_request]
jobs:
setup:
runs-on: ubuntu-latest
outputs:
lmod7: Lmod-7.8.22
lmod8: Lmod-8.4.27
modulesTcl: modules-tcl-1.147
modules3: modules-3.2.10
modules4: modules-4.1.4
steps:
- run: "true"
build:
needs: setup
runs-on: ubuntu-18.04
strategy:
matrix:
# Don't run for Python 3.8, 3.9 , people should just use `pip install easybuild`
python: [2.7, 3.6, 3.7]
modules_tool:
# use variables defined by 'setup' job above, see also
# https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#needs-context
- ${{needs.setup.outputs.lmod7}}
- ${{needs.setup.outputs.lmod8}}
module_syntax: [Lua]
lc_all: [""]
include:
# also test with module tools other than Lmod (only Tcl syntax)
- modules_tool: ${{needs.setup.outputs.modulesTcl}}
module_syntax: Tcl
python: 2.7
- modules_tool: ${{needs.setup.outputs.modulesTcl}}
module_syntax: Tcl
python: 3.6
- modules_tool: ${{needs.setup.outputs.modules3}}
module_syntax: Tcl
python: 2.7
- modules_tool: ${{needs.setup.outputs.modules3}}
module_syntax: Tcl
python: 3.6
- modules_tool: ${{needs.setup.outputs.modules4}}
module_syntax: Tcl
python: 2.7
- modules_tool: ${{needs.setup.outputs.modules4}}
module_syntax: Tcl
python: 3.6
# There may be encoding errors in Python 3 which are hidden when an UTF-8 encoding is set
# Hence run the tests (again) with LC_ALL=C and Python 3.6 (or any < 3.7)
- python: 3.6
modules_tool: ${{needs.setup.outputs.lmod8}}
module_syntax: Lua
lc_all: C
fail-fast: false
steps:
- uses: actions/checkout@v2

- name: set up Python
uses: actions/setup-python@v2
with:
python-version: ${{matrix.python}}
architecture: x64

- name: install OS & Python packages
run: |
# disable apt-get update, we don't really need it,
# and it does more harm than good (it's fairly expensive, and it results in flaky test runs)
# sudo apt-get update
# for modules tool
sudo apt-get install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev
# fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082
# needed for Ubuntu 18.04, but not for Ubuntu 20.04, so skipping symlinking if posix.so already exists
if [ ! -e /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so ] ; then
sudo ln -s /usr/lib/x86_64-linux-gnu/lua/5.2/posix_c.so /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so
fi
- name: install modules tool
run: |
# avoid downloading modules tool sources into easybuild-framework dir
cd $HOME
export INSTALL_DEP=$GITHUB_WORKSPACE/easybuild/scripts/install_eb_dep.sh
# install Lmod
source $INSTALL_DEP ${{matrix.modules_tool}} $HOME
# changes in environment are not passed to other steps, so need to create files...
echo $MOD_INIT > mod_init
echo $PATH > path
if [ ! -z $MODULESHOME ]; then echo $MODULESHOME > moduleshome; fi
- name: test bootstrap script
run: |
# (re)initialize environment for modules tool
if [ -f $HOME/moduleshome ]; then export MODULESHOME=$(cat $HOME/moduleshome); fi
source $(cat $HOME/mod_init); type module
# also pick up changes to $PATH set by sourcing $HOME/mod_init
export PATH=$(cat $HOME/path)
# define $EASYBUILD_MODULES_TOOL only for oldest module tools
# (for Lmod and EnvironmentModules 4.x the bootstrap script should correctly auto-detect the modules tool)
if [[ ${{matrix.modules_tool}} =~ ^modules-tcl- ]]; then
export EASYBUILD_MODULES_TOOL=EnvironmentModulesTcl
elif [[ ${{matrix.modules_tool}} =~ ^modules-3 ]]; then
export EASYBUILD_MODULES_TOOL=EnvironmentModulesC
fi
# version and SHA256 checksum are hardcoded below to avoid forgetting to update the version in the script along with contents
EB_BOOTSTRAP_VERSION=$(grep '^EB_BOOTSTRAP_VERSION' easybuild/scripts/bootstrap_eb.py | sed 's/[^0-9.]//g')
EB_BOOTSTRAP_SHA256SUM=$(sha256sum easybuild/scripts/bootstrap_eb.py | cut -f1 -d' ')
EB_BOOTSTRAP_FOUND="$EB_BOOTSTRAP_VERSION $EB_BOOTSTRAP_SHA256SUM"
EB_BOOTSTRAP_EXPECTED="20210106.01 c2d93de0dd91123eb4f51cfc16d1f5efb80f1d238b3d6cd100994086887a1ae0"
test "$EB_BOOTSTRAP_FOUND" = "$EB_BOOTSTRAP_EXPECTED" || (echo "Version check on bootstrap script failed $EB_BOOTSTRAP_FOUND" && exit 1)
# test bootstrap script
export PREFIX=/tmp/$USER/$GITHUB_SHA/eb_bootstrap
python easybuild/scripts/bootstrap_eb.py $PREFIX
# unset $PYTHONPATH to avoid mixing two EasyBuild 'installations' when testing bootstrapped EasyBuild module
unset PYTHONPATH
# simple sanity check on bootstrapped EasyBuild module
module use $PREFIX/modules/all
module load EasyBuild
eb --version
33 changes: 0 additions & 33 deletions .github/workflows/unit_tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -197,36 +197,3 @@ jobs:
# '|| true' is needed to avoid that Travis stops the job on non-zero exit of grep (i.e. when there are no matches)
PRINTED_MSG=$(egrep -v "${IGNORE_PATTERNS}" test_framework_suite.log | grep '\.\n*[A-Za-z]' || true)
test "x$PRINTED_MSG" = "x" || (echo "ERROR: Found printed messages in output of test suite\n${PRINTED_MSG}" && exit 1)
- name: test bootstrap script
run: |
# (re)initialize environment for modules tool
if [ -f $HOME/moduleshome ]; then export MODULESHOME=$(cat $HOME/moduleshome); fi
source $(cat $HOME/mod_init); type module
# also pick up changes to $PATH set by sourcing $HOME/mod_init
export PATH=$(cat $HOME/path)
# define $EASYBUILD_MODULES_TOOL only for oldest module tools
# (for Lmod and EnvironmentModules 4.x the bootstrap script should correctly auto-detect the modules tool)
if [[ ${{matrix.modules_tool}} =~ ^modules-tcl- ]]; then
export EASYBUILD_MODULES_TOOL=EnvironmentModulesTcl
elif [[ ${{matrix.modules_tool}} =~ ^modules-3 ]]; then
export EASYBUILD_MODULES_TOOL=EnvironmentModulesC
fi
# version and SHA256 checksum are hardcoded below to avoid forgetting to update the version in the script along with contents
EB_BOOTSTRAP_VERSION=$(grep '^EB_BOOTSTRAP_VERSION' easybuild/scripts/bootstrap_eb.py | sed 's/[^0-9.]//g')
EB_BOOTSTRAP_SHA256SUM=$(sha256sum easybuild/scripts/bootstrap_eb.py | cut -f1 -d' ')
EB_BOOTSTRAP_FOUND="$EB_BOOTSTRAP_VERSION $EB_BOOTSTRAP_SHA256SUM"
EB_BOOTSTRAP_EXPECTED="20210106.01 c2d93de0dd91123eb4f51cfc16d1f5efb80f1d238b3d6cd100994086887a1ae0"
test "$EB_BOOTSTRAP_FOUND" = "$EB_BOOTSTRAP_EXPECTED" || (echo "Version check on bootstrap script failed $EB_BOOTSTRAP_FOUND" && exit 1)
# test bootstrap script
export PREFIX=/tmp/$USER/$GITHUB_SHA/eb_bootstrap
python easybuild/scripts/bootstrap_eb.py $PREFIX
# unset $PYTHONPATH to avoid mixing two EasyBuild 'installations' when testing bootstrapped EasyBuild module
unset PYTHONPATH
# simple sanity check on bootstrapped EasyBuild module
module use $PREFIX/modules/all
module load EasyBuild
eb --version
62 changes: 62 additions & 0 deletions RELEASE_NOTES
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,68 @@ For more detailed information, please see the git log.
These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.


v4.4.0 (June 2nd 2021)
----------------------

feature release

- various enhancements, including:
- enhance apply_regex_substitutions to allow specifying action to take in case there are no matches (#3440)
- performance improvements for easyconfig parsing and eb startup (#3555)
- add support for downloading easyconfigs from multiple PRs with --from-pr (#3605, #3707, #3708)
- add support for prepending custom library paths in RPATH section via --rpath-override-dirs (#3650)
- allow amending easyconfig parameters which are not the default (#3651)
- update HierarchicalMNS for Intel OneAPI compilers (#3653)
- add support for --sanity-check-only (#3655)
- add support for running commands asynchronously via run_cmd + complete_cmd functions (#3656)
- add support for using oneAPI versions of 'intel' toolchain components (#3665)
- add toolchain definition for gofbf (foss with FlexiBLAS rather than OpenBLAS) (#3666)
- add support for using ARCH constant in easyconfig files (#3670)
- honor keyboard interrupt in 'eb' command (#3674)
- add toolchain definition for Fujitsu toolchain (#3677, #3704, #3712, #3713, #3714, #3717)
- extend sanity check step to check whether specific libraries are not linked into installed binaries/libraries (#3678)
- via banned-linked-shared-libs and required-linked-shared-libs EasyBuild configuration options
- via banned_linked_shared_libs and required_linked_shared_libs methods in toolchain support
- via banned_linked_shared_libs and required_linked_shared_libs methods in easyblock
- via banned_linked_shared_libs and required_linked_shared_libs easyconfig parameters
- add locate_solib function to locate Linux shared libraries (#3682)
- add system agnostic function to locate shared libraries (#3683)
- add update_build_option function to update specific build options after initializing the EasyBuild configuration (#3684)
- allow opting out of recursively unloaded of modules via recursive_module_unload easyconfig parameter (#3689)
- check for correct version values when parsing easystack file (#3693)
- run post-install commands specified for a specific extension (#3696)
- add support for --skip-extensions (#3702)
- include PR title in output produced by --merge-pr (#3706)
- various bug fixes, including:
- avoid metadata greedy behaviour when probing for external module metadata (mostly relevant for integration with Cray Programming Environment) (#3559)
- catch problems early on if --github-user is not specified for --new-pr & co (#3644)
- re-enable write permissions when installing with read-only-installdir (#3649)
- also run sanity check for extensions when using --module-only (#3655)
- improve logging when failing to load class from exts_classmap (#3657)
- fix use of --module-only on existing installations without write permissions (#3659)
- correct help text for subdir-user-modules (#3660)
- avoid picking up easyblocks outside of sandbox in framework tests (#3680)
- use unload/load in ModuleGeneratorLua.swap_module, since 'swap' is not supported by Lmod (#3685)
- update HierarchicalMNS to also return 'Toolchain/<name>/<version>' as $MODULEPATH extension for cpe* Cray toolchains (#3686)
- make EasyConfigParser.get_config_dict return a copy rather than a reference (#3692)
- make sure that $TAPE is unset when using piped tar (#3698)
- fix extending message for changed files in new_pr_from_branch (#3699)
- enhance sched_getaffinity function to avoid early crash when counting available cores on systems with more than 1024 cores (#3701)
- correctly strip extension from filename in extract_cmd and back_up_file functions (#3705)
- other changes:
- deprecate adding a non-existing path to $MODULEPATH (#3637)
- bump cryptography requirement from 3.2.1 to 3.3.2 (#3643, #3648)
- test bootstrap script in separate workflow, and limit test configurations a bit (#3646)
- update setup.py to indicate compatibility with Python 3.8 and 3.9 (#3647)
- replace log_error parameter of which() by on_error (#3661, #3664)
- don't skip sanity check for --module-only --rebuild (#3645)
- move disable_templating function into the EasyConfig class (#3668)
- pin GitPython version for Python<3.6, don't test bootstrap script against Python 3.8/3.9 (#3675)
- tweak foss toolchain definition to switch from OpenBLAS to FlexiBLAS in foss/2021a (#3679)
- suggest missing SSH key when not able to read from remote repository in --check-github (#3681)
- drop support for Python 2.6 (#3715)


v4.3.4 (April 9th 2021)
-----------------------

Expand Down
44 changes: 19 additions & 25 deletions contrib/hooks/hpc2n_hooks.py
Original file line number Diff line number Diff line change
Expand Up @@ -162,21 +162,19 @@ def pre_module_hook(self, *args, **kwargs):
self.log.info("[pre-module hook] Set I_MPI_PMI_LIBRARY in impi module")
# Must be done this way, updating self.cfg['modextravars']
# directly doesn't work due to templating.
en_templ = self.cfg.enable_templating
self.cfg.enable_templating = False
shlib_ext = get_shared_lib_ext()
pmix_root = get_software_root('PMIx')
if pmix_root:
mpi_type = 'pmix_v3'
self.cfg['modextravars'].update({
'I_MPI_PMI_LIBRARY': os.path.join(pmix_root, "lib", "libpmi." + shlib_ext)
})
self.cfg['modextravars'].update({'SLURM_MPI_TYPE': mpi_type})
# Unfortunately UCX doesn't yet work for unknown reasons. Make sure it is off.
self.cfg['modextravars'].update({'SLURM_PMIX_DIRECT_CONN_UCX': 'false'})
else:
self.cfg['modextravars'].update({'I_MPI_PMI_LIBRARY': "/lap/slurm/lib/libpmi.so"})
self.cfg.enable_templating = en_templ
with self.cfg.disable_templating():
shlib_ext = get_shared_lib_ext()
pmix_root = get_software_root('PMIx')
if pmix_root:
mpi_type = 'pmix_v3'
self.cfg['modextravars'].update({
'I_MPI_PMI_LIBRARY': os.path.join(pmix_root, "lib", "libpmi." + shlib_ext)
})
self.cfg['modextravars'].update({'SLURM_MPI_TYPE': mpi_type})
# Unfortunately UCX doesn't yet work for unknown reasons. Make sure it is off.
self.cfg['modextravars'].update({'SLURM_PMIX_DIRECT_CONN_UCX': 'false'})
else:
self.cfg['modextravars'].update({'I_MPI_PMI_LIBRARY': "/lap/slurm/lib/libpmi.so"})

if self.name == 'OpenBLAS':
self.log.info("[pre-module hook] Set OMP_NUM_THREADS=1 in OpenBLAS module")
Expand All @@ -197,18 +195,14 @@ def pre_module_hook(self, *args, **kwargs):
self.log.info("[pre-module hook] Set SLURM_MPI_TYPE=%s in OpenMPI module" % mpi_type)
# Must be done this way, updating self.cfg['modextravars']
# directly doesn't work due to templating.
en_templ = self.cfg.enable_templating
self.cfg.enable_templating = False
self.cfg['modextravars'].update({'SLURM_MPI_TYPE': mpi_type})
# Unfortunately UCX doesn't yet work for unknown reasons. Make sure it is off.
self.cfg['modextravars'].update({'SLURM_PMIX_DIRECT_CONN_UCX': 'false'})
self.cfg.enable_templating = en_templ
with self.cfg.disable_templating():
self.cfg['modextravars'].update({'SLURM_MPI_TYPE': mpi_type})
# Unfortunately UCX doesn't yet work for unknown reasons. Make sure it is off.
self.cfg['modextravars'].update({'SLURM_PMIX_DIRECT_CONN_UCX': 'false'})

if self.name == 'PMIx':
# This is a, hopefully, temporary workaround for https://github.com/pmix/pmix/issues/1114
if LooseVersion(self.version) > LooseVersion('2') and LooseVersion(self.version) < LooseVersion('3'):
self.log.info("[pre-module hook] Set PMIX_MCA_gds=^ds21 in PMIx module")
en_templ = self.cfg.enable_templating
self.cfg.enable_templating = False
self.cfg['modextravars'].update({'PMIX_MCA_gds': '^ds21'})
self.cfg.enable_templating = en_templ
with self.cfg.disable_templating():
self.cfg['modextravars'].update({'PMIX_MCA_gds': '^ds21'})
19 changes: 18 additions & 1 deletion easybuild/base/generaloption.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,13 +40,30 @@
from functools import reduce
from optparse import Option, OptionGroup, OptionParser, OptionValueError, Values
from optparse import SUPPRESS_HELP as nohelp # supported in optparse of python v2.4
from optparse import gettext as _gettext # this is gettext.gettext normally

from easybuild.base.fancylogger import getLogger, setroot, setLogLevel, getDetailsLogLevels
from easybuild.base.optcomplete import autocomplete, CompleterOption
from easybuild.tools.py2vs3 import StringIO, configparser, string_type
from easybuild.tools.utilities import mk_rst_table, nub, shell_quote

try:
import gettext
eb_translation = None

def get_translation():
global eb_translation
if not eb_translation:
# Finding a translation is expensive, so do only once
domain = gettext.textdomain()
eb_translation = gettext.translation(domain, gettext.bindtextdomain(domain), fallback=True)
return eb_translation

def _gettext(message):
return get_translation().gettext(message)
except ImportError:
def _gettext(message):
return message


HELP_OUTPUT_FORMATS = ['', 'rst', 'short', 'config']

Expand Down
Loading

0 comments on commit 05ab036

Please sign in to comment.