release EasyBuild v4.3.4

.github/workflows/eb_command.yml

@@ -0,0 +1,88 @@
+# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
+name: Tests for the 'eb' command
+on: [push, pull_request]
+jobs:
+  test-eb:
+    runs-on: ubuntu-18.04
+    strategy:
+      matrix:
+        python: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9]
+      fail-fast: false
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{matrix.python}}
+        architecture: x64
+
+    - name: install OS & Python packages
+      run: |
+        # check Python version
+        python -V
+        # update to latest pip, check version
+        pip install --upgrade pip
+        pip --version
+        # install packages required for modules tool
+        sudo apt-get install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev
+        # fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082
+        # needed for Ubuntu 18.04, but not for Ubuntu 20.04, so skipping symlinking if posix.so already exists
+        if [ ! -e /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so ] ; then
+            sudo ln -s /usr/lib/x86_64-linux-gnu/lua/5.2/posix_c.so /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so
+        fi
+
+    - name: install modules tool
+      run: |
+          # avoid downloading modules tool sources into easybuild-framework dir
+          cd $HOME
+          export INSTALL_DEP=$GITHUB_WORKSPACE/easybuild/scripts/install_eb_dep.sh
+          # install Lmod
+          source $INSTALL_DEP Lmod-8.4.26 $HOME
+          # changes in environment are not passed to other steps, so need to create files...
+          echo $MOD_INIT > mod_init
+          echo $PATH > path
+          if [ ! -z $MODULESHOME ]; then echo $MODULESHOME > moduleshome; fi
+
+    - name: install EasyBuild framework
+      run: |
+          # install from source distribution tarball, to test release as published on PyPI
+          python setup.py sdist
+          ls dist
+          export PREFIX=/tmp/$USER/$GITHUB_SHA
+          pip install --prefix $PREFIX dist/easybuild-framework*tar.gz
+
+    - name: run tests for 'eb' command
+      env:
+        EB_VERBOSE: 1
+      run: |
+          # run tests *outside* of checked out easybuild-framework directory,
+          # to ensure we're testing installed version (see previous step)
+          cd $HOME
+          # initialize environment for modules tool
+          if [ -f $HOME/moduleshome ]; then export MODULESHOME=$(cat $HOME/moduleshome); fi
+          source $(cat $HOME/mod_init); type module
+          # make sure 'eb' is available via $PATH, and that $PYTHONPATH is set (some tests expect that);
+          # also pick up changes to $PATH set by sourcing $MOD_INIT
+          export PREFIX=/tmp/$USER/$GITHUB_SHA
+          export PATH=$PREFIX/bin:$(cat $HOME/path)
+          export PYTHONPATH=$PREFIX/lib/python${{matrix.python}}/site-packages:$PYTHONPATH
+          # run --version, capture (verbose) output
+          eb --version | tee eb_version.out 2>&1
+          # determine active Python version
+          pymajver=$(python -c 'import sys; print(sys.version_info[0])')
+          pymajminver=$(python -c 'import sys; print(".".join(str(x) for x in sys.version_info[:2]))')
+          # check patterns in verbose output
+          for pattern in "^>> Considering .python.\.\.\." "^>> .python. version: ${pymajminver}\.[0-9]\+, which matches Python ${pymajver} version requirement" "^>> 'python' is able to import 'easybuild.main', so retaining it" "^>> Selected Python command: python \(.*/bin/python\)" "^This is EasyBuild 4\.[0-9.]\+"; do
+              echo "Looking for pattern \"${pattern}\" in eb_version.out..."
+              grep "$pattern" eb_version.out
+          done
+          # also check when specifying Python command via $EB_PYTHON
+          for eb_python in "python${pymajver}" "python${pymajminver}"; do
+              export EB_PYTHON="${eb_python}"
+              eb --version | tee eb_version.out 2>&1
+              for pattern in "^>> Considering .${eb_python}.\.\.\." "^>> .${eb_python}. version: ${pymajminver}\.[0-9]\+, which matches Python ${pymajver} version requirement" "^>> '${eb_python}' is able to import 'easybuild.main', so retaining it" "^>> Selected Python command: ${eb_python} \(.*/bin/${eb_python}\)" "^This is EasyBuild 4\.[0-9.]\+"; do
+                  echo "Looking for pattern \"${pattern}\" in eb_version.out..."
+                  grep "$pattern" eb_version.out
+              done
+          done

.github/workflows/unit_tests.yml

@@ -2,61 +2,70 @@
 name: EasyBuild framework unit tests
 on: [push, pull_request]
 jobs:
+  setup:
+    runs-on: ubuntu-latest
+    outputs:
+        lmod7: Lmod-7.8.22
+        lmod8: Lmod-8.4.27
+        modulesTcl: modules-tcl-1.147
+        modules3: modules-3.2.10
+        modules4: modules-4.1.4
+    steps:
+      - run: "true"
   build:
+    needs: setup
     runs-on: ubuntu-18.04
     strategy:
       matrix:
-        python: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9]
-        modules_tool: [Lmod-7.8.22, Lmod-8.2.9, modules-tcl-1.147, modules-3.2.10, modules-4.1.4]
+        python: [2.7, 3.6]
+        modules_tool:
+          # use variables defined by 'setup' job above, see also
+          # https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#needs-context
+          - ${{needs.setup.outputs.lmod7}}
+          - ${{needs.setup.outputs.lmod8}}
+          - ${{needs.setup.outputs.modulesTcl}}
+          - ${{needs.setup.outputs.modules3}}
+          - ${{needs.setup.outputs.modules4}}
         module_syntax: [Lua, Tcl]
         lc_all: [""]
-        # exclude some configuration for non-Lmod modules tool:
-        # - don't test with Lua module syntax (only supported in Lmod)
-        # - exclude Python 3.x versions other than 3.6, to limit test configurations
+        # don't test with Lua module syntax (only supported in Lmod)
         exclude:
-          - modules_tool: modules-tcl-1.147
+          - modules_tool: ${{needs.setup.outputs.modulesTcl}}
             module_syntax: Lua
-          - modules_tool: modules-3.2.10
+          - modules_tool: ${{needs.setup.outputs.modules3}}
             module_syntax: Lua
-          - modules_tool: modules-4.1.4
+          - modules_tool: ${{needs.setup.outputs.modules4}}
             module_syntax: Lua
-          - modules_tool: modules-tcl-1.147
-            python: 3.5
-          - modules_tool: modules-tcl-1.147
-            python: 3.7
-          - modules_tool: modules-tcl-1.147
-            python: 3.8
-          - modules_tool: modules-tcl-1.147
-            python: 3.9
-          - modules_tool: modules-3.2.10
-            python: 3.5
-          - modules_tool: modules-3.2.10
-            python: 3.7
-          - modules_tool: modules-3.2.10
-            python: 3.8
-          - modules_tool: modules-3.2.10
-            python: 3.9
-          - modules_tool: modules-4.1.4
-            python: 3.5
-          - modules_tool: modules-4.1.4
-            python: 3.7
-          - modules_tool: modules-4.1.4
-            python: 3.8
-          - modules_tool: modules-4.1.4
-            python: 3.9
-          - modules_tool: Lmod-7.8.22
-            python: 3.5
-          - modules_tool: Lmod-7.8.22
-            python: 3.7
-          - modules_tool: Lmod-7.8.22
-            python: 3.8
-          - modules_tool: Lmod-7.8.22
-            python: 3.9
-        # There may be encoding errors in Python 3 which are hidden when an UTF-8 encoding is set
-        # Hence run the tests (again) with LC_ALL=C and Python 3.6 (or any < 3.7)
         include:
+          # Test different Python 3 versions with Lmod 8.x (with both Lua and Tcl module syntax)
+          - python: 3.5
+            modules_tool: ${{needs.setup.outputs.lmod8}}
+            module_syntax: Lua
+          - python: 3.5
+            modules_tool: ${{needs.setup.outputs.lmod8}}
+            module_syntax: Tcl
+          - python: 3.7
+            modules_tool: ${{needs.setup.outputs.lmod8}}
+            module_syntax: Lua
+          - python: 3.7
+            modules_tool: ${{needs.setup.outputs.lmod8}}
+            module_syntax: Tcl
+          - python: 3.8
+            modules_tool: ${{needs.setup.outputs.lmod8}}
+            module_syntax: Lua
+          - python: 3.8
+            modules_tool: ${{needs.setup.outputs.lmod8}}
+            module_syntax: Tcl
+          - python: 3.9
+            modules_tool: ${{needs.setup.outputs.lmod8}}
+            module_syntax: Lua
+          - python: 3.9
+            modules_tool: ${{needs.setup.outputs.lmod8}}
+            module_syntax: Tcl
+          # There may be encoding errors in Python 3 which are hidden when an UTF-8 encoding is set
+          # Hence run the tests (again) with LC_ALL=C and Python 3.6 (or any < 3.7)
           - python: 3.6
-            modules_tool: Lmod-8.2.9
+            modules_tool: ${{needs.setup.outputs.lmod8}}
             module_syntax: Lua
             lc_all: C
       fail-fast: false

RELEASE_NOTES

@@ -4,6 +4,40 @@ For more detailed information, please see the git log.
 These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
 
 
+v4.3.4 (April 9th 2021)
+-----------------------
+
+update/bugfix release
+
+- various enhancements, including:
+  - add support for filtering dependencies by using False as version (#3506)
+  - add create_unused_dir function to create a directory which does not yet exist (#3551)
+  - avoid running expensive 'module use' and 'module unuse' commands when using Lmod as modules tool, update $MODULEPATH directly instead (#3557, #3633)
+  - create CUDA cache (for JIT compiled PTX code) in build dir instead of $HOME (#3569)
+  - add "Citing" section to module files (#3596)
+  - add support for using fallback 'arch=*' key in dependency version specified as arch->version mapping (#3600)
+  - also check for pending change requests and mergeable_state in check_pr_eligible_to_merge (#3604)
+  - ignore undismissed 'changes requested' review if there is an 'approved' review by the same user (#3607, #3608)
+  - sort output of 'eb --search' in natural order (respecting numbers) (#3609)
+  - enhance 'eb' command to ensure that easybuild.main can be imported before settling on python* command to use (#3610)
+  - add --env-for-shebang configuration option to define the env command to use for shebangs (#3613)
+  - add templates for architecture independent Python wheels (#3618)
+  - mention easyblocks PR in gist when uploading test report for it + fix clean_gists.py script (#3622)
+  - also accept regular expression value for --accept-eula-for (#3630)
+  - update validate_github_token function to accept GitHub token in new format (#3632)
+- various bug fixes, including:
+  - fix $BLAS_LIB_MT for OpenBLAS, ensure -lpthread is included (#3584)
+  - use '--opt=val' for passing settings from config file to option parser to avoid error for values starting with '-' or '--' (#3594)
+  - avoid raised exception when getting output from interactive command in run_cmd_qa (#3599)
+  - add option to write file from file-like object and use in download_file (#3614)
+  - make sure that path to eb is always found by tests (#3617)
+- other changes:
+  - add pick_default_branch function to clean up duplicate code in tools/github.py (#3592)
+  - refactor the CI configuration to use inclusion instead of exclusion (#3616)
+  - use develop branch when testing push access in --check-github (#3629)
+  - deprecate --accept-eula, rename to --accept-eula-for (#3630)
+
+
 v4.3.3 (February 23rd 2021)
 ---------------------------
 
@@ -17,7 +51,7 @@ update/bugfix release
   - detect 'SYSTEM' toolchain as special case in easystack files (#3543)
   - enhance extract_cmd function to use 'cp -a' for shell scripts (.sh) (#3545)
   - allow use of alternate envvar(s) to $HOME for user modules (#3558)
-  - use https://sources/easybuild.io as fallback source URL (#3572, #3576)
+  - use https://sources.easybuild.io as fallback source URL (#3572, #3576)
   - add toolchain definition for iibff toolchain (#3574)
   - add %(cuda_cc_space_sep)s and %(cuda_cc_semicolon_sep)s templates (#3578)
   - add support for intel-compiler toolchain (>= 2021.x versions, oneAPI) (#3581, #3582)
@@ -34,6 +68,7 @@ update/bugfix release
 - other changes:
   - rename EasyBlock._skip_step to EasyBlock.skip_step, to make it part of the public API (#3561)
   - make symlinking of posix_c.so to posix.so in test suite configuration conditional (#3570)
+  - use 'main' rather than 'master' branch in GitHub integration functionality (#3589)
 
 
 v4.3.2 (December 10th 2020)

easybuild/base/generaloption.py

@@ -1376,8 +1376,7 @@ def parseconfigfiles(self):
                             configfile_values[opt_dest] = newval
                     else:
                         configfile_cmdline_dest.append(opt_dest)
-                        configfile_cmdline.append("--%s" % opt_name)
-                        configfile_cmdline.append(val)
+                        configfile_cmdline.append("--%s=%s" % (opt_name, val))
 
         # reparse
         self.log.debug('parseconfigfiles: going to parse options through cmdline %s' % configfile_cmdline)

easybuild/framework/easyblock.py

@@ -1034,6 +1034,27 @@ def make_dir(self, dir_name, clean, dontcreateinstalldir=False):
 
         mkdir(dir_name, parents=True)
 
+    def set_up_cuda_cache(self):
+        """Set up CUDA PTX cache."""
+
+        cuda_cache_maxsize = build_option('cuda_cache_maxsize')
+        if cuda_cache_maxsize is None:
+            cuda_cache_maxsize = 1 * 1024  # 1 GiB default value
+        else:
+            cuda_cache_maxsize = int(cuda_cache_maxsize)
+
+        if cuda_cache_maxsize == 0:
+            self.log.info("Disabling CUDA PTX cache since cache size was set to zero")
+            env.setvar('CUDA_CACHE_DISABLE', '1')
+        else:
+            cuda_cache_dir = build_option('cuda_cache_dir')
+            if not cuda_cache_dir:
+                cuda_cache_dir = os.path.join(self.builddir, 'eb-cuda-cache')
+            self.log.info("Enabling CUDA PTX cache of size %s MiB at %s", cuda_cache_maxsize, cuda_cache_dir)
+            env.setvar('CUDA_CACHE_DISABLE', '0')
+            env.setvar('CUDA_CACHE_PATH', cuda_cache_dir)
+            env.setvar('CUDA_CACHE_MAXSIZE', str(cuda_cache_maxsize * 1024 * 1024))
+
     #
     # MODULE UTILITY FUNCTIONS
     #
@@ -1655,8 +1676,8 @@ def check_accepted_eula(self, name=None, more_info=None):
         if name is None:
             name = self.name
 
-        accepted_eulas = build_option('accept_eula') or []
-        if self.cfg['accept_eula'] or name in accepted_eulas:
+        accepted_eulas = build_option('accept_eula_for') or []
+        if self.cfg['accept_eula'] or name in accepted_eulas or any(re.match(x, name) for x in accepted_eulas):
             self.log.info("EULA for %s is accepted", name)
         else:
             error_lines = [
@@ -1667,7 +1688,7 @@ def check_accepted_eula(self, name=None, more_info=None):
 
             error_lines.extend([
                 "You should either:",
-                "- add --accept-eula=%(name)s to the 'eb' command;",
+                "- add --accept-eula-for=%(name)s to the 'eb' command;",
                 "- update your EasyBuild configuration to always accept the EULA for %(name)s;",
                 "- add 'accept_eula = True' to the easyconfig file you are using;",
                 '',
@@ -2163,6 +2184,10 @@ def prepare_step(self, start_dir=True, load_tc_deps_modules=True):
             self.log.info("Loading extra modules: %s", extra_modules)
             self.modules_tool.load(extra_modules)
 
+        # Setup CUDA cache if required. If we don't do this, CUDA will use the $HOME for its cache files
+        if get_software_root('CUDA') or get_software_root('CUDAcore'):
+            self.set_up_cuda_cache()
+
         # guess directory to start configure/build/install process in, and move there
         if start_dir:
             self.guess_start_dir()
@@ -2367,7 +2392,7 @@ def fix_shebang(self):
                 if isinstance(fix_shebang_for, string_type):
                     fix_shebang_for = [fix_shebang_for]
 
-                shebang = '#!/usr/bin/env %s' % lang
+                shebang = '#!%s %s' % (build_option('env_for_shebang'), lang)
                 for glob_pattern in fix_shebang_for:
                     paths = glob.glob(os.path.join(self.installdir, glob_pattern))
                     self.log.info("Fixing '%s' shebang to '%s' for files that match '%s': %s",

easybuild/framework/easyconfig/default.py

@@ -194,6 +194,7 @@
 
     # MODULES documentation easyconfig parameters
     #    (docurls is part of MANDATORY)
+    'citing': [None, "Free-form text that describes how the software should be cited in publications", MODULES],
     'docpaths': [None, "List of paths for documentation relative to installation directory", MODULES],
     'examples': [None, "Free-form text with examples on using the software", MODULES],
     'site_contacts': [None, "String/list of strings with site contacts for the software", MODULES],

easybuild/framework/easyconfig/easyconfig.py

@@ -702,10 +702,13 @@ def parse(self):
             # parse dependency specifications
             # it's important that templating is still disabled at this stage!
             self.log.info("Parsing dependency specifications...")
-            self['dependencies'] = [self._parse_dependency(dep) for dep in self['dependencies']]
-            self['hiddendependencies'] = [
-                self._parse_dependency(dep, hidden=True) for dep in self['hiddendependencies']
-            ]
+
+            def remove_false_versions(deps):
+                return [dep for dep in deps if not (isinstance(dep, dict) and dep['version'] is False)]
+
+            self['dependencies'] = remove_false_versions(self._parse_dependency(dep) for dep in self['dependencies'])
+            self['hiddendependencies'] = remove_false_versions(self._parse_dependency(dep, hidden=True) for dep in
+                                                               self['hiddendependencies'])
 
             # need to take into account that builddependencies may need to be iterated over,
             # i.e. when the value is a list of lists of tuples
@@ -715,7 +718,7 @@ def parse(self):
                 builddeps = [[self._parse_dependency(dep, build_only=True) for dep in x] for x in builddeps]
             else:
                 builddeps = [self._parse_dependency(dep, build_only=True) for dep in builddeps]
-            self['builddependencies'] = builddeps
+            self['builddependencies'] = remove_false_versions(builddeps)
 
             # keep track of parsed multi deps, they'll come in handy during sanity check & module steps...
             self.multi_deps = self.get_parsed_multi_deps()

easybuild/framework/easyconfig/templates.py

@@ -155,6 +155,19 @@
         ('SOURCE_%s' % suffix, '%(name)s-%(version)s.' + ext, "Source .%s bundle" % ext),
         ('SOURCELOWER_%s' % suffix, '%(namelower)s-%(version)s.' + ext, "Source .%s bundle with lowercase name" % ext),
     ]
+for pyver in ('py2.py3', 'py2', 'py3'):
+    if pyver == 'py2.py3':
+        desc = 'Python 2 & Python 3'
+        name_infix = ''
+    else:
+        desc = 'Python ' + pyver[-1]
+        name_infix = pyver.upper() + '_'
+    TEMPLATE_CONSTANTS += [
+        ('SOURCE_%sWHL' % name_infix, '%%(name)s-%%(version)s-%s-none-any.whl' % pyver,
+         'Generic (non-compiled) %s wheel package' % desc),
+        ('SOURCELOWER_%sWHL' % name_infix, '%%(namelower)s-%%(version)s-%s-none-any.whl' % pyver,
+         'Generic (non-compiled) %s wheel package with lowercase name' % desc),
+    ]
 
 # TODO derived config templates
 # versionmajor, versionminor, versionmajorminor (eg '.'.join(version.split('.')[:2])) )