Merge pull request #3319 from akesandgren/fix-crayce-problems

Fix problems with CrayCCE processing when there are no actual external modules

easybuild/toolchains/mpi/craympich.py

@@ -40,7 +40,7 @@ class CrayMPICH(Mpi):
     """Generic support for using Cray compiler wrappers"""
     # MPI support
     # no separate module, Cray compiler drivers always provide MPI support
-    MPI_MODULE_NAME = []
+    MPI_MODULE_NAME = None
     MPI_FAMILY = TC_CONSTANT_MPICH
     MPI_TYPE = TC_CONSTANT_MPI_TYPE_MPICH
 

easybuild/tools/module_naming_scheme/hierarchical_mns.py

@@ -41,9 +41,11 @@
 CORE = 'Core'
 COMPILER = 'Compiler'
 MPI = 'MPI'
+TOOLCHAIN = 'Toolchain'
 
 MODULECLASS_COMPILER = 'compiler'
 MODULECLASS_MPI = 'mpi'
+MODULECLASS_TOOLCHAIN = 'toolchain'
 
 GCCCORE = GCCcore.NAME
 
@@ -107,7 +109,11 @@ def det_toolchain_compilers_name_version(self, tc_comps):
             # no compiler in toolchain, system toolchain
             res = None
         elif len(tc_comps) == 1:
-            res = (tc_comps[0]['name'], self.det_full_version(tc_comps[0]))
+            tc_comp = tc_comps[0]
+            if tc_comp is None:
+                res = None
+            else:
+                res = (tc_comp['name'], self.det_full_version(tc_comp))
         else:
             comp_versions = dict([(comp['name'], self.det_full_version(comp)) for comp in tc_comps])
             comp_names = comp_versions.keys()
@@ -135,6 +141,10 @@ def det_module_subdir(self, ec):
         if tc_comps is None:
             # no compiler in toolchain, system toolchain => Core module
             subdir = CORE
+        elif tc_comps == [None]:
+            # no info on toolchain compiler (cfr. Cray toolchains),
+            # then use toolchain name/version
+            subdir = os.path.join(TOOLCHAIN, ec.toolchain.name, ec.toolchain.version)
         else:
             tc_comp_name, tc_comp_ver = self.det_toolchain_compilers_name_version(tc_comps)
             tc_mpi = det_toolchain_mpi(ec)
@@ -223,6 +233,10 @@ def det_modpath_extensions(self, ec):
                 fullver = self.det_full_version(ec)
                 paths.append(os.path.join(MPI, tc_comp_name, tc_comp_ver, ec['name'], fullver))
 
+        # special case for Cray toolchains
+        elif modclass == MODULECLASS_TOOLCHAIN and tc_comp_info is None and ec.name.startswith('Cray'):
+            paths.append(os.path.join(TOOLCHAIN, ec.name, ec.version))
+
         return paths
 
     def expand_toolchain_load(self, ec=None):

easybuild/tools/toolchain/mpi.py

@@ -242,7 +242,8 @@ def _set_mpi_variables(self):
         if not self.options.get('32bit', None):
             suffix = '64'
 
-        for root in self.get_software_root(self.MPI_MODULE_NAME):
+        # take into account that MPI_MODULE_NAME could be None (see Cray toolchains)
+        for root in self.get_software_root(self.MPI_MODULE_NAME or []):
             self.variables.append_exists('MPI_LIB_STATIC', root, lib_dir, filename="lib%s.a" % self.MPI_LIBRARY_NAME,
                                          suffix=suffix)
             self.variables.append_exists('MPI_LIB_SHARED', root, lib_dir, filename="lib%s.so" % self.MPI_LIBRARY_NAME,

test/framework/easyconfigs/test_ecs/h/HPL/HPL-2.1-CrayCCE-5.1.29.eb

@@ -0,0 +1,14 @@
+easyblock = 'ConfigureMake'
+
+name = 'HPL'
+version = '2.1'
+
+homepage = 'http://www.netlib.org/benchmark/hpl/'
+description = "HPL, you know, LINPACK"
+
+toolchain = {'name': 'CrayCCE', 'version': '5.1.29'}
+
+source_urls = ['http://www.netlib.org/benchmark/%(namelower)s']
+sources = [SOURCELOWER_TAR_GZ]
+
+moduleclass = 'tools'

test/framework/filetools.py

@@ -1816,7 +1816,7 @@ def test_index_functions(self):
         # test with specified path with and without trailing '/'s
         for path in [test_ecs, test_ecs + '/', test_ecs + '//']:
             index = ft.create_index(path)
-            self.assertEqual(len(index), 81)
+            self.assertEqual(len(index), 82)
 
             expected = [
                 os.path.join('b', 'bzip2', 'bzip2-1.0.6-GCC-4.9.2.eb'),

test/framework/module_generator.py

@@ -1273,6 +1273,11 @@ def test_ec(ecfile, short_modname, mod_subdir, modpath_exts, user_modpath_exts,
                                                           ['MPI/intel-CUDA/%s-5.5.22/impi/5.1.2.150' % iccver],
                                                           ['MPI/intel-CUDA/%s-5.5.22/impi/5.1.2.150' % iccver],
                                                           ['Core']),
+            'CrayCCE-5.1.29.eb': ('CrayCCE/5.1.29', 'Core',
+                                  ['Toolchain/CrayCCE/5.1.29'],
+                                  ['Toolchain/CrayCCE/5.1.29'],
+                                  ['Core']),
+            'HPL-2.1-CrayCCE-5.1.29.eb': ('HPL/2.1', 'Toolchain/CrayCCE/5.1.29', [], [], ['Core']),
         }
         for ecfile, mns_vals in test_ecs.items():
             test_ec(ecfile, *mns_vals)

test/framework/options.py

@@ -4265,54 +4265,128 @@ def test_list_prs(self):
 
     def test_list_software(self):
         """Test --list-software and --list-installed-software."""
-        test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'v1.0')
-        args = [
-            '--list-software',
-            '--robot-paths=%s' % test_ecs,
-        ]
-        txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False)
-        expected = '\n'.join([
-            "== Processed 5/5 easyconfigs...",
-            "== Found 2 different software packages",
-            '',
-            "* GCC",
-            "* gzip",
-            '',
+
+        # copy selected test easyconfigs for testing --list-*software options with;
+        # full test is a nuisance, because all dependencies must be available and toolchains like intel must have
+        # all expected components when testing with HierarchicalMNS (which the test easyconfigs don't always have)
+        topdir = os.path.dirname(os.path.abspath(__file__))
+
+        cray_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'c', 'CrayCCE', 'CrayCCE-5.1.29.eb')
+        gcc_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'g', 'GCC', 'GCC-4.6.3.eb')
+        gzip_ec = os.path.join(topdir, 'easyconfigs', 'v1.0', 'g', 'gzip', 'gzip-1.4-GCC-4.6.3.eb')
+        gzip_system_ec = os.path.join(topdir, 'easyconfigs', 'v1.0', 'g', 'gzip', 'gzip-1.4.eb')
+
+        test_ecs = os.path.join(self.test_prefix, 'test_ecs')
+        for ec in [cray_ec, gcc_ec, gzip_ec, gzip_system_ec]:
+            subdirs = os.path.dirname(ec).split(os.path.sep)[-2:]
+            target_dir = os.path.join(test_ecs, *subdirs)
+            mkdir(target_dir, parents=True)
+            copy_file(ec, target_dir)
+
+        # add (fake) HPL easyconfig using CrayCCE toolchain
+        # (required to trigger bug reported in https://github.com/easybuilders/easybuild-framework/issues/3265)
+        hpl_cray_ec_txt = '\n'.join([
+            'easyblock = "ConfigureMake"',
+            'name = "HPL"',
+            'version = "2.3"',
+            "homepage = 'http://www.netlib.org/benchmark/hpl/'",
+            'description = "HPL"',
+            'toolchain = {"name": "CrayCCE", "version": "5.1.29"}',
         ])
-        self.assertTrue(txt.endswith(expected))
+        hpl_cray_ec = os.path.join(self.test_prefix, 'test_ecs', 'h', 'HPL', 'HPL-2.3-CrayCCE-5.1.29.eb')
+        write_file(hpl_cray_ec, hpl_cray_ec_txt)
 
-        args = [
-            '--list-software=detailed',
-            '--output-format=rst',
-            '--robot-paths=%s' % test_ecs,
-        ]
-        txt, _ = self._run_mock_eb(args, testing=False)
-        self.assertTrue(re.search(r'^\*GCC\*', txt, re.M))
-        self.assertTrue(re.search(r'^``4.6.3``\s+``system``', txt, re.M))
-        self.assertTrue(re.search(r'^\*gzip\*', txt, re.M))
-        self.assertTrue(re.search(r'^``1.5``\s+``foss/2018a``,\s+``intel/2018a``', txt, re.M))
+        # put dummy Core/GCC/4.6.3 in place
+        modpath = os.path.join(self.test_prefix, 'modules')
+        write_file(os.path.join(modpath, 'Core', 'GCC', '4.6.3'), '#%Module')
+        self.modtool.use(modpath)
 
-        args = [
-            '--list-installed-software',
-            '--output-format=rst',
-            '--robot-paths=%s' % test_ecs,
-        ]
-        txt, _ = self._run_mock_eb(args, testing=False, raise_error=True)
-        self.assertTrue(re.search(r'== Processed 5/5 easyconfigs...', txt, re.M))
-        self.assertTrue(re.search(r'== Found 2 different software packages', txt, re.M))
-        self.assertTrue(re.search(r'== Retained 1 installed software packages', txt, re.M))
-        self.assertTrue(re.search(r'^\* GCC', txt, re.M))
-        self.assertFalse(re.search(r'gzip', txt, re.M))
+        # test with different module naming scheme active
+        # (see https://github.com/easybuilders/easybuild-framework/issues/3265)
+        for mns in ['EasyBuildMNS', 'HierarchicalMNS']:
 
-        args = [
-            '--list-installed-software=detailed',
-            '--robot-paths=%s' % test_ecs,
-        ]
-        txt, _ = self._run_mock_eb(args, testing=False)
-        self.assertTrue(re.search(r'^== Retained 1 installed software packages', txt, re.M))
-        self.assertTrue(re.search(r'^\* GCC', txt, re.M))
-        self.assertTrue(re.search(r'^\s+\* GCC v4.6.3: system', txt, re.M))
-        self.assertFalse(re.search(r'gzip', txt, re.M))
+            args = [
+                '--list-software',
+                '--robot-paths=%s' % test_ecs,
+                '--module-naming-scheme=%s' % mns,
+            ]
+            txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False, verbose=True)
+
+            patterns = [
+                r"^.*\s*== Processed 5/5 easyconfigs...",
+                r"^== Found 4 different software packages",
+                r"^\* CrayCCE",
+                r"^\* GCC",
+                r"^\* gzip",
+                r"^\* HPL",
+            ]
+            for pattern in patterns:
+                regex = re.compile(pattern, re.M)
+                self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt))
+
+            args = [
+                '--list-software=detailed',
+                '--output-format=rst',
+                '--robot-paths=%s' % test_ecs,
+                '--module-naming-scheme=%s' % mns,
+            ]
+            txt, _ = self._run_mock_eb(args, testing=False, raise_error=True, verbose=True)
+
+            patterns = [
+                r"^.*\s*== Processed 5/5 easyconfigs...",
+                r"^== Found 4 different software packages",
+                r'^\*CrayCCE\*',
+                r'^``5.1.29``\s+``system``',
+                r'^\*GCC\*',
+                r'^``4.6.3``\s+``system``',
+                r'^\*gzip\*',
+                r'^``1.4``    ``GCC/4.6.3``, ``system``',
+            ]
+            for pattern in patterns:
+                regex = re.compile(pattern, re.M)
+                self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt))
+
+            args = [
+                '--list-installed-software',
+                '--output-format=rst',
+                '--robot-paths=%s' % test_ecs,
+                '--module-naming-scheme=%s' % mns,
+            ]
+            txt, _ = self._run_mock_eb(args, testing=False, raise_error=True, verbose=True)
+
+            patterns = [
+                r"^.*\s*== Processed 5/5 easyconfigs...",
+                r"^== Found 4 different software packages",
+                r"^== Retained 1 installed software packages",
+                r'^\* GCC',
+            ]
+            for pattern in patterns:
+                regex = re.compile(pattern, re.M)
+                self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt))
+
+            self.assertFalse(re.search(r'gzip', txt, re.M))
+            self.assertFalse(re.search(r'CrayCCE', txt, re.M))
+
+            args = [
+                '--list-installed-software=detailed',
+                '--robot-paths=%s' % test_ecs,
+                '--module-naming-scheme=%s' % mns,
+            ]
+            txt, _ = self._run_mock_eb(args, testing=False, raise_error=True, verbose=True)
+
+            patterns = [
+                r"^.*\s*== Processed 5/5 easyconfigs...",
+                r"^== Found 4 different software packages",
+                r"^== Retained 1 installed software packages",
+                r'^\* GCC',
+                r'^\s+\* GCC v4.6.3: system',
+            ]
+            for pattern in patterns:
+                regex = re.compile(pattern, re.M)
+                self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt))
+
+            self.assertFalse(re.search(r'gzip', txt, re.M))
+            self.assertFalse(re.search(r'CrayCCE', txt, re.M))
 
     def test_parse_optarch(self):
         """Test correct parsing of optarch option."""