diff --git a/easybuild/toolchains/mpi/craympich.py b/easybuild/toolchains/mpi/craympich.py index cf32237451..bd01662002 100644 --- a/easybuild/toolchains/mpi/craympich.py +++ b/easybuild/toolchains/mpi/craympich.py @@ -40,7 +40,7 @@ class CrayMPICH(Mpi): """Generic support for using Cray compiler wrappers""" # MPI support # no separate module, Cray compiler drivers always provide MPI support - MPI_MODULE_NAME = [] + MPI_MODULE_NAME = None MPI_FAMILY = TC_CONSTANT_MPICH MPI_TYPE = TC_CONSTANT_MPI_TYPE_MPICH diff --git a/easybuild/tools/module_naming_scheme/hierarchical_mns.py b/easybuild/tools/module_naming_scheme/hierarchical_mns.py index f6d7cf1423..1647f4c96e 100644 --- a/easybuild/tools/module_naming_scheme/hierarchical_mns.py +++ b/easybuild/tools/module_naming_scheme/hierarchical_mns.py @@ -41,9 +41,11 @@ CORE = 'Core' COMPILER = 'Compiler' MPI = 'MPI' +TOOLCHAIN = 'Toolchain' MODULECLASS_COMPILER = 'compiler' MODULECLASS_MPI = 'mpi' +MODULECLASS_TOOLCHAIN = 'toolchain' GCCCORE = GCCcore.NAME @@ -107,7 +109,11 @@ def det_toolchain_compilers_name_version(self, tc_comps): # no compiler in toolchain, system toolchain res = None elif len(tc_comps) == 1: - res = (tc_comps[0]['name'], self.det_full_version(tc_comps[0])) + tc_comp = tc_comps[0] + if tc_comp is None: + res = None + else: + res = (tc_comp['name'], self.det_full_version(tc_comp)) else: comp_versions = dict([(comp['name'], self.det_full_version(comp)) for comp in tc_comps]) comp_names = comp_versions.keys() @@ -135,6 +141,10 @@ def det_module_subdir(self, ec): if tc_comps is None: # no compiler in toolchain, system toolchain => Core module subdir = CORE + elif tc_comps == [None]: + # no info on toolchain compiler (cfr. Cray toolchains), + # then use toolchain name/version + subdir = os.path.join(TOOLCHAIN, ec.toolchain.name, ec.toolchain.version) else: tc_comp_name, tc_comp_ver = self.det_toolchain_compilers_name_version(tc_comps) tc_mpi = det_toolchain_mpi(ec) @@ -223,6 +233,10 @@ def det_modpath_extensions(self, ec): fullver = self.det_full_version(ec) paths.append(os.path.join(MPI, tc_comp_name, tc_comp_ver, ec['name'], fullver)) + # special case for Cray toolchains + elif modclass == MODULECLASS_TOOLCHAIN and tc_comp_info is None and ec.name.startswith('Cray'): + paths.append(os.path.join(TOOLCHAIN, ec.name, ec.version)) + return paths def expand_toolchain_load(self, ec=None): diff --git a/easybuild/tools/toolchain/mpi.py b/easybuild/tools/toolchain/mpi.py index 93f833b07a..abd24e4809 100644 --- a/easybuild/tools/toolchain/mpi.py +++ b/easybuild/tools/toolchain/mpi.py @@ -242,7 +242,8 @@ def _set_mpi_variables(self): if not self.options.get('32bit', None): suffix = '64' - for root in self.get_software_root(self.MPI_MODULE_NAME): + # take into account that MPI_MODULE_NAME could be None (see Cray toolchains) + for root in self.get_software_root(self.MPI_MODULE_NAME or []): self.variables.append_exists('MPI_LIB_STATIC', root, lib_dir, filename="lib%s.a" % self.MPI_LIBRARY_NAME, suffix=suffix) self.variables.append_exists('MPI_LIB_SHARED', root, lib_dir, filename="lib%s.so" % self.MPI_LIBRARY_NAME, diff --git a/test/framework/easyconfigs/test_ecs/h/HPL/HPL-2.1-CrayCCE-5.1.29.eb b/test/framework/easyconfigs/test_ecs/h/HPL/HPL-2.1-CrayCCE-5.1.29.eb new file mode 100644 index 0000000000..607821faf2 --- /dev/null +++ b/test/framework/easyconfigs/test_ecs/h/HPL/HPL-2.1-CrayCCE-5.1.29.eb @@ -0,0 +1,14 @@ +easyblock = 'ConfigureMake' + +name = 'HPL' +version = '2.1' + +homepage = 'http://www.netlib.org/benchmark/hpl/' +description = "HPL, you know, LINPACK" + +toolchain = {'name': 'CrayCCE', 'version': '5.1.29'} + +source_urls = ['http://www.netlib.org/benchmark/%(namelower)s'] +sources = [SOURCELOWER_TAR_GZ] + +moduleclass = 'tools' diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 2fefc2979b..fd2e4e5c68 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1816,7 +1816,7 @@ def test_index_functions(self): # test with specified path with and without trailing '/'s for path in [test_ecs, test_ecs + '/', test_ecs + '//']: index = ft.create_index(path) - self.assertEqual(len(index), 81) + self.assertEqual(len(index), 82) expected = [ os.path.join('b', 'bzip2', 'bzip2-1.0.6-GCC-4.9.2.eb'), diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py index 81ce794218..439091a3a3 100644 --- a/test/framework/module_generator.py +++ b/test/framework/module_generator.py @@ -1273,6 +1273,11 @@ def test_ec(ecfile, short_modname, mod_subdir, modpath_exts, user_modpath_exts, ['MPI/intel-CUDA/%s-5.5.22/impi/5.1.2.150' % iccver], ['MPI/intel-CUDA/%s-5.5.22/impi/5.1.2.150' % iccver], ['Core']), + 'CrayCCE-5.1.29.eb': ('CrayCCE/5.1.29', 'Core', + ['Toolchain/CrayCCE/5.1.29'], + ['Toolchain/CrayCCE/5.1.29'], + ['Core']), + 'HPL-2.1-CrayCCE-5.1.29.eb': ('HPL/2.1', 'Toolchain/CrayCCE/5.1.29', [], [], ['Core']), } for ecfile, mns_vals in test_ecs.items(): test_ec(ecfile, *mns_vals) diff --git a/test/framework/options.py b/test/framework/options.py index a0a4e39d62..938c2d8182 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -4265,54 +4265,128 @@ def test_list_prs(self): def test_list_software(self): """Test --list-software and --list-installed-software.""" - test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'v1.0') - args = [ - '--list-software', - '--robot-paths=%s' % test_ecs, - ] - txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False) - expected = '\n'.join([ - "== Processed 5/5 easyconfigs...", - "== Found 2 different software packages", - '', - "* GCC", - "* gzip", - '', + + # copy selected test easyconfigs for testing --list-*software options with; + # full test is a nuisance, because all dependencies must be available and toolchains like intel must have + # all expected components when testing with HierarchicalMNS (which the test easyconfigs don't always have) + topdir = os.path.dirname(os.path.abspath(__file__)) + + cray_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'c', 'CrayCCE', 'CrayCCE-5.1.29.eb') + gcc_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'g', 'GCC', 'GCC-4.6.3.eb') + gzip_ec = os.path.join(topdir, 'easyconfigs', 'v1.0', 'g', 'gzip', 'gzip-1.4-GCC-4.6.3.eb') + gzip_system_ec = os.path.join(topdir, 'easyconfigs', 'v1.0', 'g', 'gzip', 'gzip-1.4.eb') + + test_ecs = os.path.join(self.test_prefix, 'test_ecs') + for ec in [cray_ec, gcc_ec, gzip_ec, gzip_system_ec]: + subdirs = os.path.dirname(ec).split(os.path.sep)[-2:] + target_dir = os.path.join(test_ecs, *subdirs) + mkdir(target_dir, parents=True) + copy_file(ec, target_dir) + + # add (fake) HPL easyconfig using CrayCCE toolchain + # (required to trigger bug reported in https://github.com/easybuilders/easybuild-framework/issues/3265) + hpl_cray_ec_txt = '\n'.join([ + 'easyblock = "ConfigureMake"', + 'name = "HPL"', + 'version = "2.3"', + "homepage = 'http://www.netlib.org/benchmark/hpl/'", + 'description = "HPL"', + 'toolchain = {"name": "CrayCCE", "version": "5.1.29"}', ]) - self.assertTrue(txt.endswith(expected)) + hpl_cray_ec = os.path.join(self.test_prefix, 'test_ecs', 'h', 'HPL', 'HPL-2.3-CrayCCE-5.1.29.eb') + write_file(hpl_cray_ec, hpl_cray_ec_txt) - args = [ - '--list-software=detailed', - '--output-format=rst', - '--robot-paths=%s' % test_ecs, - ] - txt, _ = self._run_mock_eb(args, testing=False) - self.assertTrue(re.search(r'^\*GCC\*', txt, re.M)) - self.assertTrue(re.search(r'^``4.6.3``\s+``system``', txt, re.M)) - self.assertTrue(re.search(r'^\*gzip\*', txt, re.M)) - self.assertTrue(re.search(r'^``1.5``\s+``foss/2018a``,\s+``intel/2018a``', txt, re.M)) + # put dummy Core/GCC/4.6.3 in place + modpath = os.path.join(self.test_prefix, 'modules') + write_file(os.path.join(modpath, 'Core', 'GCC', '4.6.3'), '#%Module') + self.modtool.use(modpath) - args = [ - '--list-installed-software', - '--output-format=rst', - '--robot-paths=%s' % test_ecs, - ] - txt, _ = self._run_mock_eb(args, testing=False, raise_error=True) - self.assertTrue(re.search(r'== Processed 5/5 easyconfigs...', txt, re.M)) - self.assertTrue(re.search(r'== Found 2 different software packages', txt, re.M)) - self.assertTrue(re.search(r'== Retained 1 installed software packages', txt, re.M)) - self.assertTrue(re.search(r'^\* GCC', txt, re.M)) - self.assertFalse(re.search(r'gzip', txt, re.M)) + # test with different module naming scheme active + # (see https://github.com/easybuilders/easybuild-framework/issues/3265) + for mns in ['EasyBuildMNS', 'HierarchicalMNS']: - args = [ - '--list-installed-software=detailed', - '--robot-paths=%s' % test_ecs, - ] - txt, _ = self._run_mock_eb(args, testing=False) - self.assertTrue(re.search(r'^== Retained 1 installed software packages', txt, re.M)) - self.assertTrue(re.search(r'^\* GCC', txt, re.M)) - self.assertTrue(re.search(r'^\s+\* GCC v4.6.3: system', txt, re.M)) - self.assertFalse(re.search(r'gzip', txt, re.M)) + args = [ + '--list-software', + '--robot-paths=%s' % test_ecs, + '--module-naming-scheme=%s' % mns, + ] + txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False, verbose=True) + + patterns = [ + r"^.*\s*== Processed 5/5 easyconfigs...", + r"^== Found 4 different software packages", + r"^\* CrayCCE", + r"^\* GCC", + r"^\* gzip", + r"^\* HPL", + ] + for pattern in patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) + + args = [ + '--list-software=detailed', + '--output-format=rst', + '--robot-paths=%s' % test_ecs, + '--module-naming-scheme=%s' % mns, + ] + txt, _ = self._run_mock_eb(args, testing=False, raise_error=True, verbose=True) + + patterns = [ + r"^.*\s*== Processed 5/5 easyconfigs...", + r"^== Found 4 different software packages", + r'^\*CrayCCE\*', + r'^``5.1.29``\s+``system``', + r'^\*GCC\*', + r'^``4.6.3``\s+``system``', + r'^\*gzip\*', + r'^``1.4`` ``GCC/4.6.3``, ``system``', + ] + for pattern in patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) + + args = [ + '--list-installed-software', + '--output-format=rst', + '--robot-paths=%s' % test_ecs, + '--module-naming-scheme=%s' % mns, + ] + txt, _ = self._run_mock_eb(args, testing=False, raise_error=True, verbose=True) + + patterns = [ + r"^.*\s*== Processed 5/5 easyconfigs...", + r"^== Found 4 different software packages", + r"^== Retained 1 installed software packages", + r'^\* GCC', + ] + for pattern in patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) + + self.assertFalse(re.search(r'gzip', txt, re.M)) + self.assertFalse(re.search(r'CrayCCE', txt, re.M)) + + args = [ + '--list-installed-software=detailed', + '--robot-paths=%s' % test_ecs, + '--module-naming-scheme=%s' % mns, + ] + txt, _ = self._run_mock_eb(args, testing=False, raise_error=True, verbose=True) + + patterns = [ + r"^.*\s*== Processed 5/5 easyconfigs...", + r"^== Found 4 different software packages", + r"^== Retained 1 installed software packages", + r'^\* GCC', + r'^\s+\* GCC v4.6.3: system', + ] + for pattern in patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) + + self.assertFalse(re.search(r'gzip', txt, re.M)) + self.assertFalse(re.search(r'CrayCCE', txt, re.M)) def test_parse_optarch(self): """Test correct parsing of optarch option."""