From 75a553234f14e4e8647bc29e304a00b236ae8d9a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 7 Jun 2022 10:53:10 +0200 Subject: [PATCH 01/44] add support for easystack file that contains easyconfig filenames --- easybuild/framework/easystack.py | 62 ++++++++++++++++--- test/framework/easystack.py | 48 ++++++++++++-- .../test_easystack_easyconfigs.yaml | 5 ++ 3 files changed, 99 insertions(+), 16 deletions(-) create mode 100644 test/framework/easystacks/test_easystack_easyconfigs.yaml diff --git a/easybuild/framework/easystack.py b/easybuild/framework/easystack.py index fe56253b3f..58a9789370 100644 --- a/easybuild/framework/easystack.py +++ b/easybuild/framework/easystack.py @@ -64,14 +64,19 @@ def check_value(value, context): class EasyStack(object): """One class instance per easystack. General options + list of all SoftwareSpecs instances""" - def __init__(self): + def __init__(self, easyconfigs=None): self.easybuild_version = None self.robot = False self.software_list = [] + self.easyconfigs = [] + if easyconfigs is not None: + self.easyconfigs.extend(easyconfigs) def compose_ec_filenames(self): """Returns a list of all easyconfig names""" ec_filenames = [] + + # entries specified via 'software' top-level key for sw in self.software_list: full_ec_version = det_full_ec_version({ 'toolchain': {'name': sw.toolchain_name, 'version': sw.toolchain_version}, @@ -80,6 +85,11 @@ def compose_ec_filenames(self): }) ec_filename = '%s-%s.eb' % (sw.name, full_ec_version) ec_filenames.append(ec_filename) + + # entries specified via 'easyconfigs' top-level key + for ec in self.easyconfigs: + ec_filenames.append(ec + '.eb') + return ec_filenames # flags applicable to all sw (i.e. robot) @@ -108,7 +118,8 @@ class EasyStackParser(object): @staticmethod def parse(filepath): - """Parses YAML file and assigns obtained values to SW config instances as well as general config instance""" + """ + Parses YAML file and assigns obtained values to SW config instances as well as general config instance""" yaml_txt = read_file(filepath) try: @@ -116,13 +127,44 @@ def parse(filepath): except yaml.YAMLError as err: raise EasyBuildError("Failed to parse %s: %s" % (filepath, err)) - easystack = EasyStack() + easystack_data = None + top_keys = ('easyconfigs', 'software') + for key in top_keys: + if key in easystack_raw: + easystack_data = easystack_raw[key] + break - try: - software = easystack_raw["software"] - except KeyError: - wrong_structure_file = "Not a valid EasyStack YAML file: no 'software' key found" - raise EasyBuildError(wrong_structure_file) + if easystack_data is None: + msg = "Not a valid EasyStack YAML file: no 'easyconfigs' or 'software' top-level key found" + raise EasyBuildError(msg) + else: + parse_method_name = 'parse_by_' + key + parse_method = getattr(EasyStackParser, 'parse_by_%s' % key, None) + if parse_method is None: + raise EasyBuildError("Easystack parse method '%s' not found!", parse_method_name) + + # assign general easystack attributes + easybuild_version = easystack_raw.get('easybuild_version', None) + robot = easystack_raw.get('robot', False) + + return parse_method(filepath, easystack_data, easybuild_version=easybuild_version, robot=robot) + + @staticmethod + def parse_by_easyconfigs(filepath, easyconfigs, easybuild_version=None, robot=False): + """ + Parse easystack file with 'easyconfigs' as top-level key. + """ + easystack = EasyStack(easyconfigs=easyconfigs) + + return easystack + + @staticmethod + def parse_by_software(filepath, software, easybuild_version=None, robot=False): + """ + Parse easystack file with 'software' as top-level key. + """ + + easystack = EasyStack() # assign software-specific easystack attributes for name in software: @@ -224,8 +266,8 @@ def parse(filepath): easystack.software_list.append(sw) # assign general easystack attributes - easystack.easybuild_version = easystack_raw.get('easybuild_version', None) - easystack.robot = easystack_raw.get('robot', False) + easystack.easybuild_version = easybuild_version + easystack.robot = robot return easystack diff --git a/test/framework/easystack.py b/test/framework/easystack.py index 78be785fb5..fa79693474 100644 --- a/test/framework/easystack.py +++ b/test/framework/easystack.py @@ -56,6 +56,36 @@ def tearDown(self): easybuild.tools.build_log.EXPERIMENTAL = self.orig_experimental super(EasyStackTest, self).tearDown() + def test_easystack_basic(self): + """Test for basic easystack file.""" + topdir = os.path.dirname(os.path.abspath(__file__)) + test_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_basic.yaml') + + ec_fns, opts = parse_easystack(test_easystack) + expected = [ + 'binutils-2.25-GCCcore-4.9.3.eb', + 'binutils-2.26-GCCcore-4.9.3.eb', + 'foss-2018a.eb', + 'toy-0.0-gompi-2018a-test.eb', + ] + self.assertEqual(sorted(ec_fns), sorted(expected)) + self.assertEqual(opts, {}) + + def test_easystack_easyconfigs(self): + """Test for easystack file using 'easyconfigs' key.""" + topdir = os.path.dirname(os.path.abspath(__file__)) + test_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_easyconfigs.yaml') + + ec_fns, opts = parse_easystack(test_easystack) + expected = [ + 'binutils-2.25-GCCcore-4.9.3.eb', + 'binutils-2.26-GCCcore-4.9.3.eb', + 'foss-2018a.eb', + 'toy-0.0-gompi-2018a-test.eb', + ] + self.assertEqual(sorted(ec_fns), sorted(expected)) + self.assertEqual(opts, {}) + def test_parse_fail(self): """Test for clean error when easystack file fails to parse.""" test_yml = os.path.join(self.test_prefix, 'test.yml') @@ -120,15 +150,17 @@ def test_easystack_versions(self): versions = ('1.2.3', '1.2.30', '2021a', '1.2.3') for version in versions: write_file(test_easystack, tmpl_easystack_txt + ' ' + version) - ec_fns, _ = parse_easystack(test_easystack) + ec_fns, opts = parse_easystack(test_easystack) self.assertEqual(ec_fns, ['foo-%s.eb' % version]) + self.assertEqual(opts, {}) # multiple versions as a list test_easystack_txt = tmpl_easystack_txt + " [1.2.3, 3.2.1]" write_file(test_easystack, test_easystack_txt) - ec_fns, _ = parse_easystack(test_easystack) + ec_fns, opts = parse_easystack(test_easystack) expected = ['foo-1.2.3.eb', 'foo-3.2.1.eb'] self.assertEqual(sorted(ec_fns), sorted(expected)) + self.assertEqual(opts, {}) # multiple versions listed with more info test_easystack_txt = '\n'.join([ @@ -139,9 +171,10 @@ def test_easystack_versions(self): " versionsuffix: -foo", ]) write_file(test_easystack, test_easystack_txt) - ec_fns, _ = parse_easystack(test_easystack) + ec_fns, opts = parse_easystack(test_easystack) expected = ['foo-1.2.3.eb', 'foo-2021a.eb', 'foo-3.2.1-foo.eb'] self.assertEqual(sorted(ec_fns), sorted(expected)) + self.assertEqual(opts, {}) # versions that get interpreted by YAML as float or int, single quotes required for version in ('1.2', '123', '3.50', '100', '2.44_01'): @@ -152,8 +185,9 @@ def test_easystack_versions(self): # all is fine when wrapping the value in single quotes write_file(test_easystack, tmpl_easystack_txt + " '" + version + "'") - ec_fns, _ = parse_easystack(test_easystack) + ec_fns, opts = parse_easystack(test_easystack) self.assertEqual(ec_fns, ['foo-%s.eb' % version]) + self.assertEqual(opts, {}) # one rotten apple in the basket is enough test_easystack_txt = tmpl_easystack_txt + " [1.2.3, %s, 3.2.1]" % version @@ -179,9 +213,10 @@ def test_easystack_versions(self): " versionsuffix: -foo", ]) write_file(test_easystack, test_easystack_txt) - ec_fns, _ = parse_easystack(test_easystack) + ec_fns, opts = parse_easystack(test_easystack) expected = ['foo-1.2.3.eb', 'foo-%s.eb' % version, 'foo-3.2.1-foo.eb'] self.assertEqual(sorted(ec_fns), sorted(expected)) + self.assertEqual(opts, {}) # also check toolchain version that could be interpreted as a non-string value... test_easystack_txt = '\n'.join([ @@ -192,9 +227,10 @@ def test_easystack_versions(self): " versions: [1.2.3, '2.3']", ]) write_file(test_easystack, test_easystack_txt) - ec_fns, _ = parse_easystack(test_easystack) + ec_fns, opts = parse_easystack(test_easystack) expected = ['test-1.2.3-intel-2021.03.eb', 'test-2.3-intel-2021.03.eb'] self.assertEqual(sorted(ec_fns), sorted(expected)) + self.assertEqual(opts, {}) def suite(): diff --git a/test/framework/easystacks/test_easystack_easyconfigs.yaml b/test/framework/easystacks/test_easystack_easyconfigs.yaml new file mode 100644 index 0000000000..1d7bfe073d --- /dev/null +++ b/test/framework/easystacks/test_easystack_easyconfigs.yaml @@ -0,0 +1,5 @@ +easyconfigs: + - binutils-2.25-GCCcore-4.9.3 + - binutils-2.26-GCCcore-4.9.3 + - foss-2018a + - toy-0.0-gompi-2018a-test From db4310e017761ef096ce670841bc57f21bc48afc Mon Sep 17 00:00:00 2001 From: casparl Date: Fri, 29 Jul 2022 13:53:45 +0200 Subject: [PATCH 02/44] Allow for specifying EasyConfigs in the yaml file both with and without .eb extesion --- easybuild/framework/easystack.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easystack.py b/easybuild/framework/easystack.py index 58a9789370..f8bbde9c20 100644 --- a/easybuild/framework/easystack.py +++ b/easybuild/framework/easystack.py @@ -88,8 +88,10 @@ def compose_ec_filenames(self): # entries specified via 'easyconfigs' top-level key for ec in self.easyconfigs: - ec_filenames.append(ec + '.eb') - + if not ec.endswith('.eb'): + ec_filenames.append(ec + '.eb') + else: + ec_filenames.append(ec) return ec_filenames # flags applicable to all sw (i.e. robot) From a28e15ad821eab7c65cd37d427fdce26e8292097 Mon Sep 17 00:00:00 2001 From: casparl Date: Fri, 29 Jul 2022 15:47:25 +0200 Subject: [PATCH 03/44] Added support for specifying EasyConfigs in EasyStack files with and without explicit '.eb' extension --- easybuild/framework/easystack.py | 14 ++++++++++++-- test/framework/easystack.py | 15 +++++++++++++++ .../test_easystack_easyconfigs_with_eb_ext.yaml | 5 +++++ 3 files changed, 32 insertions(+), 2 deletions(-) create mode 100644 test/framework/easystacks/test_easystack_easyconfigs_with_eb_ext.yaml diff --git a/easybuild/framework/easystack.py b/easybuild/framework/easystack.py index f8bbde9c20..c67c09e7eb 100644 --- a/easybuild/framework/easystack.py +++ b/easybuild/framework/easystack.py @@ -131,10 +131,20 @@ def parse(filepath): easystack_data = None top_keys = ('easyconfigs', 'software') + keys_found = [] for key in top_keys: if key in easystack_raw: - easystack_data = easystack_raw[key] - break + keys_found.append(key) + # For now, we don't support mixing multiple top_keys, so check that only one was defined + if len(keys_found) > 1: + keys_string = ', '.join(keys_found) + msg = "Specifying multiple top level keys (%s) in one EasyStack file is not supported." % keys_string + raise EasyBuildError(msg) + elif len(keys_found) == 0: + raise EasyBuildError("An EasyStack file needs to contain at least one of the keys: %s" % (top_keys,)) + else: + key = keys_found[0] + easystack_data = easystack_raw[key] if easystack_data is None: msg = "Not a valid EasyStack YAML file: no 'easyconfigs' or 'software' top-level key found" diff --git a/test/framework/easystack.py b/test/framework/easystack.py index fa79693474..d6c5093a2d 100644 --- a/test/framework/easystack.py +++ b/test/framework/easystack.py @@ -86,6 +86,21 @@ def test_easystack_easyconfigs(self): self.assertEqual(sorted(ec_fns), sorted(expected)) self.assertEqual(opts, {}) + def test_easystack_easyconfigs_with_eb_ext(self): + """Test for easystack file using 'easyconfigs' key, where eb extension is included in the easystack file""" + topdir = os.path.dirname(os.path.abspath(__file__)) + test_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_easyconfigs_with_eb_ext.yaml') + + ec_fns, opts = parse_easystack(test_easystack) + expected = [ + 'binutils-2.25-GCCcore-4.9.3.eb', + 'binutils-2.26-GCCcore-4.9.3.eb', + 'foss-2018a.eb', + 'toy-0.0-gompi-2018a-test.eb', + ] + self.assertEqual(sorted(ec_fns), sorted(expected)) + self.assertEqual(opts, {}) + def test_parse_fail(self): """Test for clean error when easystack file fails to parse.""" test_yml = os.path.join(self.test_prefix, 'test.yml') diff --git a/test/framework/easystacks/test_easystack_easyconfigs_with_eb_ext.yaml b/test/framework/easystacks/test_easystack_easyconfigs_with_eb_ext.yaml new file mode 100644 index 0000000000..bafab2396f --- /dev/null +++ b/test/framework/easystacks/test_easystack_easyconfigs_with_eb_ext.yaml @@ -0,0 +1,5 @@ +easyconfigs: + - binutils-2.25-GCCcore-4.9.3.eb + - binutils-2.26-GCCcore-4.9.3.eb + - foss-2018a.eb + - toy-0.0-gompi-2018a-test.eb From 5a41961b1defcc05323bf522c1fef468ac3e712a Mon Sep 17 00:00:00 2001 From: casparl Date: Fri, 29 Jul 2022 16:02:17 +0200 Subject: [PATCH 04/44] Cleaned up the logic, there were two checks that caught cases where no top level item was specified in the YAML --- easybuild/framework/easystack.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/easybuild/framework/easystack.py b/easybuild/framework/easystack.py index c67c09e7eb..ad5e70eba3 100644 --- a/easybuild/framework/easystack.py +++ b/easybuild/framework/easystack.py @@ -141,19 +141,16 @@ def parse(filepath): msg = "Specifying multiple top level keys (%s) in one EasyStack file is not supported." % keys_string raise EasyBuildError(msg) elif len(keys_found) == 0: - raise EasyBuildError("An EasyStack file needs to contain at least one of the keys: %s" % (top_keys,)) + msg = "Not a valid EasyStack YAML file: no 'easyconfigs' or 'software' top-level key found" + raise EasyBuildError(msg) else: key = keys_found[0] easystack_data = easystack_raw[key] - if easystack_data is None: - msg = "Not a valid EasyStack YAML file: no 'easyconfigs' or 'software' top-level key found" - raise EasyBuildError(msg) - else: - parse_method_name = 'parse_by_' + key - parse_method = getattr(EasyStackParser, 'parse_by_%s' % key, None) - if parse_method is None: - raise EasyBuildError("Easystack parse method '%s' not found!", parse_method_name) + parse_method_name = 'parse_by_' + key + parse_method = getattr(EasyStackParser, 'parse_by_%s' % key, None) + if parse_method is None: + raise EasyBuildError("Easystack parse method '%s' not found!", parse_method_name) # assign general easystack attributes easybuild_version = easystack_raw.get('easybuild_version', None) From 19c317b4a8892f3c1c5d0a01bf7b89cf15d26ccf Mon Sep 17 00:00:00 2001 From: casparl Date: Fri, 29 Jul 2022 17:06:26 +0200 Subject: [PATCH 05/44] Added potential for future support of passing per-easyconfig options in easystack files. As discussed here https://github.com/easybuilders/easybuild-framework/pull/4021#issuecomment-1199484019 --- easybuild/framework/easystack.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/easybuild/framework/easystack.py b/easybuild/framework/easystack.py index ad5e70eba3..6ce93bd658 100644 --- a/easybuild/framework/easystack.py +++ b/easybuild/framework/easystack.py @@ -64,13 +64,11 @@ def check_value(value, context): class EasyStack(object): """One class instance per easystack. General options + list of all SoftwareSpecs instances""" - def __init__(self, easyconfigs=None): + def __init__(self): self.easybuild_version = None self.robot = False self.software_list = [] - self.easyconfigs = [] - if easyconfigs is not None: - self.easyconfigs.extend(easyconfigs) + self.easyconfigs = [] # A list of easyconfig names. May or may not include .eb extension def compose_ec_filenames(self): """Returns a list of all easyconfig names""" @@ -163,8 +161,23 @@ def parse_by_easyconfigs(filepath, easyconfigs, easybuild_version=None, robot=Fa """ Parse easystack file with 'easyconfigs' as top-level key. """ - easystack = EasyStack(easyconfigs=easyconfigs) + easystack = EasyStack() + + for easyconfig in easyconfigs: + if isinstance(easyconfig, str): + easystack.easyconfigs.append(easyconfig) + elif isinstance(easyconfig, dict): + if len(easyconfig) == 1: + # Get single key from dictionary 'easyconfig' + easyconf_name = list(easyconfig.keys())[0] + easystack.easyconfigs.append(easyconf_name) + else: + dict_keys = ', '.join(easyconfig.keys()) + msg = "Failed to parse easystack file: expected a dictionary with one key (the EasyConfig name). " + msg += "Instead found keys: %s" % dict_keys + raise EasyBuildError(msg) + return easystack @staticmethod From a73b3c98ef63a90fab46dcdf24cfda2d6026395f Mon Sep 17 00:00:00 2001 From: casparl Date: Fri, 29 Jul 2022 17:07:16 +0200 Subject: [PATCH 06/44] Make the hound happy: remove whitespace --- easybuild/framework/easystack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easystack.py b/easybuild/framework/easystack.py index 6ce93bd658..d54dcd5f17 100644 --- a/easybuild/framework/easystack.py +++ b/easybuild/framework/easystack.py @@ -177,7 +177,7 @@ def parse_by_easyconfigs(filepath, easyconfigs, easybuild_version=None, robot=Fa msg = "Failed to parse easystack file: expected a dictionary with one key (the EasyConfig name). " msg += "Instead found keys: %s" % dict_keys raise EasyBuildError(msg) - + return easystack @staticmethod From 372ef1351626557beb48c8c7cc17e53667f33428 Mon Sep 17 00:00:00 2001 From: casparl Date: Fri, 29 Jul 2022 17:09:13 +0200 Subject: [PATCH 07/44] Make hound happy about this comment... --- easybuild/framework/easystack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easystack.py b/easybuild/framework/easystack.py index d54dcd5f17..de6203fcb8 100644 --- a/easybuild/framework/easystack.py +++ b/easybuild/framework/easystack.py @@ -68,7 +68,7 @@ def __init__(self): self.easybuild_version = None self.robot = False self.software_list = [] - self.easyconfigs = [] # A list of easyconfig names. May or may not include .eb extension + self.easyconfigs = [] # A list of easyconfig names. May or may not include .eb extension def compose_ec_filenames(self): """Returns a list of all easyconfig names""" From f7fa73dde9c87ec6d2a5430246fdfff2b41594d2 Mon Sep 17 00:00:00 2001 From: casparl Date: Fri, 29 Jul 2022 17:50:06 +0200 Subject: [PATCH 08/44] Added support for specifying EasyConfig specific options, for easystack files based on the 'easyconfig' top level keyword --- easybuild/framework/easystack.py | 24 +++++++++++++------ easybuild/main.py | 4 ++-- test/framework/easystack.py | 19 +++++++++++++++ .../test_easystack_easyconfig_opts.yaml | 11 +++++++++ 4 files changed, 49 insertions(+), 9 deletions(-) create mode 100644 test/framework/easystacks/test_easystack_easyconfig_opts.yaml diff --git a/easybuild/framework/easystack.py b/easybuild/framework/easystack.py index de6203fcb8..924bde33ea 100644 --- a/easybuild/framework/easystack.py +++ b/easybuild/framework/easystack.py @@ -69,6 +69,8 @@ def __init__(self): self.robot = False self.software_list = [] self.easyconfigs = [] # A list of easyconfig names. May or may not include .eb extension + # A dict where keys are easyconfig names, values are dictionary of options that should be applied for that easyconfig + self.ec_opts = {} def compose_ec_filenames(self): """Returns a list of all easyconfig names""" @@ -171,7 +173,11 @@ def parse_by_easyconfigs(filepath, easyconfigs, easybuild_version=None, robot=Fa if len(easyconfig) == 1: # Get single key from dictionary 'easyconfig' easyconf_name = list(easyconfig.keys())[0] + # Add easyconfig name to the list easystack.easyconfigs.append(easyconf_name) + # Add options to the ec_opts dict + if 'options' in easyconfig[easyconf_name].keys(): + easystack.ec_opts[easyconf_name] = easyconfig[easyconf_name]['options'] else: dict_keys = ', '.join(easyconfig.keys()) msg = "Failed to parse easystack file: expected a dictionary with one key (the EasyConfig name). " @@ -307,12 +313,16 @@ def parse_easystack(filepath): easyconfig_names = easystack.compose_ec_filenames() - general_options = easystack.get_general_options() + # Disabled general options for now. We weren't using them, and first want support for EasyConfig-specific options. + # Then, we need a method to resolve conflicts (specific options should win) + # general_options = easystack.get_general_options() _log.debug("EasyStack parsed. Proceeding to install these Easyconfigs: %s" % ', '.join(sorted(easyconfig_names))) - if len(general_options) != 0: - _log.debug("General options for installation are: \n%s" % str(general_options)) - else: - _log.debug("No general options were specified in easystack") - - return easyconfig_names, general_options + _log.debug("Using EasyConfig specific options based on the following dict:") + _log.debug(easystack.ec_opts) + # if len(general_options) != 0: + # _log.debug("General options for installation are: \n%s" % str(general_options)) + # else: + # _log.debug("No general options were specified in easystack") + + return easyconfig_names, easystack.ec_opts diff --git a/easybuild/main.py b/easybuild/main.py index c43c0583df..4d3810ed35 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -43,7 +43,7 @@ # IMPORTANT this has to be the first easybuild import as it customises the logging # expect missing log output when this not the case! -from easybuild.tools.build_log import EasyBuildError, print_error, print_msg, stop_logging +from easybuild.tools.build_log import EasyBuildError, print_error, print_msg, stop_logging, print_warning from easybuild.framework.easyblock import build_and_install_one, inject_checksums from easybuild.framework.easyconfig import EASYCONFIGS_PKG_SUBDIR @@ -261,7 +261,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): # TODO add general_options (i.e. robot) to build options orig_paths, general_options = parse_easystack(options.easystack) if general_options: - raise EasyBuildError("Specifying general configuration options in easystack file is not supported yet.") + print_warning("Specifying options in easystack files is not supported yet. They are parsed, but ignored.") # check whether packaging is supported when it's being used if options.package: diff --git a/test/framework/easystack.py b/test/framework/easystack.py index d6c5093a2d..5ac222b330 100644 --- a/test/framework/easystack.py +++ b/test/framework/easystack.py @@ -101,6 +101,25 @@ def test_easystack_easyconfigs_with_eb_ext(self): self.assertEqual(sorted(ec_fns), sorted(expected)) self.assertEqual(opts, {}) + def test_easystack_easyconfig_opts(self): + """Teast an easystack file using the 'easyconfigs' key, where additonal options are defined for some easyconfigs""" + topdir = os.path.dirname(os.path.abspath(__file__)) + test_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_easyconfigs_opts.yaml') + + ec_fns, opts = parse_easystack(test_easystack) + expected = [ + 'binutils-2.25-GCCcore-4.9.3.eb', + 'binutils-2.26-GCCcore-4.9.3.eb', + 'foss-2018a.eb', + 'toy-0.0-gompi-2018a-test.eb', + ] + expected_opts = { + 'binutils-2.25-GCCcore-4.9.3.eb': {'debug': True}, + 'foss-2018a.eb': {'robot': True}, + } + self.assertEqual(sorted(ec_fns), sorted(expected)) + self.assertEqual(opts, expected_opts) + def test_parse_fail(self): """Test for clean error when easystack file fails to parse.""" test_yml = os.path.join(self.test_prefix, 'test.yml') diff --git a/test/framework/easystacks/test_easystack_easyconfig_opts.yaml b/test/framework/easystacks/test_easystack_easyconfig_opts.yaml new file mode 100644 index 0000000000..96477259ff --- /dev/null +++ b/test/framework/easystacks/test_easystack_easyconfig_opts.yaml @@ -0,0 +1,11 @@ +easyconfigs: + - binutils-2.25-GCCcore-4.9.3: + options: { + 'debug': True, + } + - binutils-2.26-GCCcore-4.9.3 + - foss-2018a: + options: { + 'robot': True, + } + - toy-0.0-gompi-2018a-test From a7e731d1883dc5fc8475ffe5863d454a4622983d Mon Sep 17 00:00:00 2001 From: casparl Date: Fri, 29 Jul 2022 17:50:54 +0200 Subject: [PATCH 09/44] Removed whitspace on blank line --- easybuild/framework/easystack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easystack.py b/easybuild/framework/easystack.py index 924bde33ea..da5e7eaf3c 100644 --- a/easybuild/framework/easystack.py +++ b/easybuild/framework/easystack.py @@ -183,7 +183,7 @@ def parse_by_easyconfigs(filepath, easyconfigs, easybuild_version=None, robot=Fa msg = "Failed to parse easystack file: expected a dictionary with one key (the EasyConfig name). " msg += "Instead found keys: %s" % dict_keys raise EasyBuildError(msg) - + return easystack @staticmethod From 4e9a51c78173552222c1804aa3caf6279bd473ec Mon Sep 17 00:00:00 2001 From: casparl Date: Fri, 29 Jul 2022 17:52:34 +0200 Subject: [PATCH 10/44] Shortened lines --- easybuild/framework/easystack.py | 3 ++- test/framework/easystack.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easystack.py b/easybuild/framework/easystack.py index da5e7eaf3c..69d6b03847 100644 --- a/easybuild/framework/easystack.py +++ b/easybuild/framework/easystack.py @@ -69,7 +69,8 @@ def __init__(self): self.robot = False self.software_list = [] self.easyconfigs = [] # A list of easyconfig names. May or may not include .eb extension - # A dict where keys are easyconfig names, values are dictionary of options that should be applied for that easyconfig + # A dict where keys are easyconfig names, values are dictionary of options that should be + # applied for that easyconfig self.ec_opts = {} def compose_ec_filenames(self): diff --git a/test/framework/easystack.py b/test/framework/easystack.py index 5ac222b330..0c5ad55dcb 100644 --- a/test/framework/easystack.py +++ b/test/framework/easystack.py @@ -102,7 +102,7 @@ def test_easystack_easyconfigs_with_eb_ext(self): self.assertEqual(opts, {}) def test_easystack_easyconfig_opts(self): - """Teast an easystack file using the 'easyconfigs' key, where additonal options are defined for some easyconfigs""" + """Test an easystack file using the 'easyconfigs' key, with additonal options for some easyconfigs""" topdir = os.path.dirname(os.path.abspath(__file__)) test_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_easyconfigs_opts.yaml') From c0334e0c7ecd786817b73c27d7815c77cb8bc220 Mon Sep 17 00:00:00 2001 From: casparl Date: Fri, 29 Jul 2022 17:53:38 +0200 Subject: [PATCH 11/44] Removed trailing whitespace. It must be friday afternoon... --- easybuild/framework/easystack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easystack.py b/easybuild/framework/easystack.py index 69d6b03847..8eee68ca11 100644 --- a/easybuild/framework/easystack.py +++ b/easybuild/framework/easystack.py @@ -69,7 +69,7 @@ def __init__(self): self.robot = False self.software_list = [] self.easyconfigs = [] # A list of easyconfig names. May or may not include .eb extension - # A dict where keys are easyconfig names, values are dictionary of options that should be + # A dict where keys are easyconfig names, values are dictionary of options that should be # applied for that easyconfig self.ec_opts = {} From eb0f6dbad355a097fa7edd523262a850ae11fac4 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 3 Aug 2022 14:50:21 +0200 Subject: [PATCH 12/44] remove duplicate import for print_warning in main.py --- easybuild/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/easybuild/main.py b/easybuild/main.py index 4d3810ed35..5f052cf701 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -55,7 +55,6 @@ from easybuild.framework.easyconfig.tools import det_easyconfig_paths, dump_env_script, get_paths_for from easybuild.framework.easyconfig.tools import parse_easyconfigs, review_pr, run_contrib_checks, skip_available from easybuild.framework.easyconfig.tweak import obtain_ec_for, tweak -from easybuild.tools.build_log import print_warning from easybuild.tools.config import find_last_log, get_repository, get_repositorypath, build_option from easybuild.tools.containers.common import containerize from easybuild.tools.docs import list_software From 5a8eb3c641069462dd2dee94f0c30daa735eb675 Mon Sep 17 00:00:00 2001 From: casparl Date: Wed, 3 Aug 2022 15:20:54 +0200 Subject: [PATCH 13/44] Renamed, using plural for easyconfigs, consistent with other test files and the code --- ..._easyconfig_opts.yaml => test_easystack_easyconfigs_opts.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename test/framework/easystacks/{test_easystack_easyconfig_opts.yaml => test_easystack_easyconfigs_opts.yaml} (100%) diff --git a/test/framework/easystacks/test_easystack_easyconfig_opts.yaml b/test/framework/easystacks/test_easystack_easyconfigs_opts.yaml similarity index 100% rename from test/framework/easystacks/test_easystack_easyconfig_opts.yaml rename to test/framework/easystacks/test_easystack_easyconfigs_opts.yaml From 8f3e439da66b6f3dd38b9917c56c796fb294d416 Mon Sep 17 00:00:00 2001 From: casparl Date: Thu, 4 Aug 2022 15:00:40 +0200 Subject: [PATCH 14/44] Make the parser already add the .eb suffix if it is not there. This is neededso that the keys in the option dictionary can be expected to be the full EasyConfig names --- easybuild/framework/easystack.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/easybuild/framework/easystack.py b/easybuild/framework/easystack.py index 8eee68ca11..06897915cf 100644 --- a/easybuild/framework/easystack.py +++ b/easybuild/framework/easystack.py @@ -89,10 +89,7 @@ def compose_ec_filenames(self): # entries specified via 'easyconfigs' top-level key for ec in self.easyconfigs: - if not ec.endswith('.eb'): - ec_filenames.append(ec + '.eb') - else: - ec_filenames.append(ec) + ec_filenames.append(ec) return ec_filenames # flags applicable to all sw (i.e. robot) @@ -169,12 +166,16 @@ def parse_by_easyconfigs(filepath, easyconfigs, easybuild_version=None, robot=Fa for easyconfig in easyconfigs: if isinstance(easyconfig, str): + if not easyconfig.endswith('.eb'): + easyconfig = easyconfig + '.eb' easystack.easyconfigs.append(easyconfig) elif isinstance(easyconfig, dict): if len(easyconfig) == 1: # Get single key from dictionary 'easyconfig' easyconf_name = list(easyconfig.keys())[0] # Add easyconfig name to the list + if not easyconf_name.endswith('.eb'): + easyconf_name = easyconf_name + '.eb' easystack.easyconfigs.append(easyconf_name) # Add options to the ec_opts dict if 'options' in easyconfig[easyconf_name].keys(): From 56e3bfc21507b5b509e919cfc139345d186ed43b Mon Sep 17 00:00:00 2001 From: casparl Date: Thu, 4 Aug 2022 15:53:55 +0200 Subject: [PATCH 15/44] test_easystack_easyconfig_opts doesn't pass because we're trying to index the yaml dict _with_ the eb suffix, even though that dict contains the names _without_ suffix --- easybuild/framework/easystack.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easystack.py b/easybuild/framework/easystack.py index 06897915cf..e9daaf6762 100644 --- a/easybuild/framework/easystack.py +++ b/easybuild/framework/easystack.py @@ -175,11 +175,11 @@ def parse_by_easyconfigs(filepath, easyconfigs, easybuild_version=None, robot=Fa easyconf_name = list(easyconfig.keys())[0] # Add easyconfig name to the list if not easyconf_name.endswith('.eb'): - easyconf_name = easyconf_name + '.eb' - easystack.easyconfigs.append(easyconf_name) + easyconf_name_with_eb = easyconf_name + '.eb' + easystack.easyconfigs.append(easyconf_name_with_eb) # Add options to the ec_opts dict if 'options' in easyconfig[easyconf_name].keys(): - easystack.ec_opts[easyconf_name] = easyconfig[easyconf_name]['options'] + easystack.ec_opts[easyconf_name_with_eb] = easyconfig[easyconf_name]['options'] else: dict_keys = ', '.join(easyconfig.keys()) msg = "Failed to parse easystack file: expected a dictionary with one key (the EasyConfig name). " From 64486961a0896c69c60e9d9785036c198b3ff4da Mon Sep 17 00:00:00 2001 From: casparl Date: Wed, 10 Aug 2022 16:54:30 +0200 Subject: [PATCH 16/44] Fixed 'local variable 'easyconf_name_with_eb' referenced before assignment' that one would get if the easystack file already included .eb extensions --- easybuild/framework/easystack.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/easybuild/framework/easystack.py b/easybuild/framework/easystack.py index e9daaf6762..a7bcbfb5d4 100644 --- a/easybuild/framework/easystack.py +++ b/easybuild/framework/easystack.py @@ -176,6 +176,8 @@ def parse_by_easyconfigs(filepath, easyconfigs, easybuild_version=None, robot=Fa # Add easyconfig name to the list if not easyconf_name.endswith('.eb'): easyconf_name_with_eb = easyconf_name + '.eb' + else: + easyconf_name_with_eb = easyconf_name easystack.easyconfigs.append(easyconf_name_with_eb) # Add options to the ec_opts dict if 'options' in easyconfig[easyconf_name].keys(): From 4ca4a3fe54e7557cf07427a040f94e18ec3515a7 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 11 Sep 2022 13:51:00 +0200 Subject: [PATCH 17/44] make check_sha256_checksums verify all checksums if they're specified as a dict value --- easybuild/framework/easyblock.py | 49 ++++++++++++++++---------------- test/framework/easyconfig.py | 18 ++++++++++++ 2 files changed, 43 insertions(+), 24 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 971342f1b8..860eac3a21 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2375,33 +2375,34 @@ def check_checksums_for(self, ent, sub='', source_cnt=None): checksum_issues.append(msg) for fn, checksum in zip(sources + patches, checksums): + + # a checksum may be specified as a dictionary which maps filename to actual checksum + # for example when different source files are used for different CPU architectures if isinstance(checksum, dict): - # sources entry may be a dictionary rather than just a string value with filename - if isinstance(fn, dict): - filename = fn['filename'] - else: - filename = fn - checksum = checksum.get(filename) - - # take into account that we may encounter a tuple of valid SHA256 checksums - # (see https://github.com/easybuilders/easybuild-framework/pull/2958) - if isinstance(checksum, tuple): - # 1st tuple item may indicate checksum type, must be SHA256 or else it's blatently ignored here - if len(checksum) == 2 and checksum[0] == CHECKSUM_TYPE_SHA256: - valid_checksums = (checksum[1],) - else: - valid_checksums = checksum + checksums_to_check = checksum.values() else: - valid_checksums = (checksum,) - - non_sha256_checksums = [c for c in valid_checksums if not is_sha256_checksum(c)] - if non_sha256_checksums: - if all(c is None for c in non_sha256_checksums): - print_warning("Found %d None checksum value(s), please make sure this is intended!" % - len(non_sha256_checksums)) + checksums_to_check = [checksum] + + for checksum in checksums_to_check: + # take into account that we may encounter a tuple of valid SHA256 checksums + # (see https://github.com/easybuilders/easybuild-framework/pull/2958) + if isinstance(checksum, tuple): + # 1st tuple item may indicate checksum type, must be SHA256 or else it's blatently ignored here + if len(checksum) == 2 and checksum[0] == CHECKSUM_TYPE_SHA256: + valid_checksums = (checksum[1],) + else: + valid_checksums = checksum else: - msg = "Non-SHA256 checksum(s) found for %s: %s" % (fn, valid_checksums) - checksum_issues.append(msg) + valid_checksums = (checksum,) + + non_sha256_checksums = [c for c in valid_checksums if not is_sha256_checksum(c)] + if non_sha256_checksums: + if all(c is None for c in non_sha256_checksums): + print_warning("Found %d None checksum value(s), please make sure this is intended!" % + len(non_sha256_checksums)) + else: + msg = "Non-SHA256 checksum(s) found for %s: %s" % (fn, valid_checksums) + checksum_issues.append(msg) return checksum_issues diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 1e8db1c812..190a3c584b 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -3750,6 +3750,24 @@ def test_check_sha256_checksums(self): # multiple checksums listed for source tarball, while exactly one (SHA256) checksum is expected self.assertTrue(res[1].startswith("Non-SHA256 checksum(s) found for toy-0.0.tar.gz: ")) + checksums_dict = textwrap.dedent("""checksums = [{ + 'toy-0.0-aarch64.tar.gz': 'not_really_a_sha256_checksum', + 'toy-0.0-x86_64.tar.gz': '%s', + }]""" % toy_sha256) + + test_ec_txt = checksums_regex.sub(checksums_dict, toy_ec_txt) + test_ec_txt = re.sub(r'patches = \[(.|\n)*\]', '', test_ec_txt) + + test_ec = os.path.join(self.test_prefix, 'toy-checksums-dict.eb') + write_file(test_ec, test_ec_txt) + ecs, _ = parse_easyconfigs([(test_ec, False)]) + ecs = [ec['ec'] for ec in ecs] + + res = check_sha256_checksums(ecs) + self.assertTrue(len(res) == 1) + regex = re.compile(r"Non-SHA256 checksum\(s\) found for toy-0.0.tar.gz:.*not_really_a_sha256_checksum") + self.assertTrue(regex.match(res[0]), "Pattern '%s' found in: %s" % (regex.pattern, res[0])) + def test_deprecated(self): """Test use of 'deprecated' easyconfig parameter.""" topdir = os.path.dirname(os.path.abspath(__file__)) From 2c9a8311e8a034674d14b4c09023f570d58fb7ea Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 12 Sep 2022 09:03:48 +0200 Subject: [PATCH 18/44] bump version to 4.6.2dev --- easybuild/tools/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index 155733e5d1..4fec6dcddb 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.6.1') +VERSION = LooseVersion('4.6.2.dev0') UNKNOWN = 'UNKNOWN' From 46b613b3fbb89060fd57b55dbb5c21280d6e23ae Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 20 Sep 2022 09:00:55 +0200 Subject: [PATCH 19/44] skip over unset $EB_PYTHON/$EB_INSTALLPYTHON Check if the current $python_cmd is empty and skip all checks in that case. This avoids useless checks and log output in VERBOSE mode. Also extend the CI test to catch this issue. --- .github/workflows/eb_command.yml | 4 ++++ eb | 3 +++ 2 files changed, 7 insertions(+) diff --git a/.github/workflows/eb_command.yml b/.github/workflows/eb_command.yml index 8fa5cfb1fc..6994ededee 100644 --- a/.github/workflows/eb_command.yml +++ b/.github/workflows/eb_command.yml @@ -77,6 +77,10 @@ jobs: echo "Looking for pattern \"${pattern}\" in eb_version.out..." grep "$pattern" eb_version.out done + if grep -q "Considering ''" eb_version.out; then + echo '`eb` did wrongly consider an empty command' + false + fi # also check when specifying Python command via $EB_PYTHON for eb_python in "python${pymajver}" "python${pymajminver}"; do export EB_PYTHON="${eb_python}" diff --git a/eb b/eb index 2387c24035..bd916ad115 100755 --- a/eb +++ b/eb @@ -62,6 +62,9 @@ PYTHON= # time, this variable preserves that choice). for python_cmd in "${EB_PYTHON}" "${EB_INSTALLPYTHON}" 'python' 'python3' 'python2'; do + # Only consider non-empty values, i.e. continue if e.g. $EB_PYTHON is not set + [ -n "${python_cmd}" ] || continue + verbose "Considering '${python_cmd}'..." # check whether python* command being considered is available From 1228b7e3dd2a3192477fdc7c46334d5d331d64d6 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 20 Sep 2022 13:26:48 +0200 Subject: [PATCH 20/44] Make scripts executable --- easybuild/scripts/bootstrap_eb.py | 0 easybuild/scripts/clean_gists.py | 0 easybuild/scripts/fix_docs.py | 3 ++- easybuild/scripts/mk_tmpl_easyblock_for.py | 0 4 files changed, 2 insertions(+), 1 deletion(-) mode change 100644 => 100755 easybuild/scripts/bootstrap_eb.py mode change 100644 => 100755 easybuild/scripts/clean_gists.py mode change 100644 => 100755 easybuild/scripts/fix_docs.py mode change 100644 => 100755 easybuild/scripts/mk_tmpl_easyblock_for.py diff --git a/easybuild/scripts/bootstrap_eb.py b/easybuild/scripts/bootstrap_eb.py old mode 100644 new mode 100755 diff --git a/easybuild/scripts/clean_gists.py b/easybuild/scripts/clean_gists.py old mode 100644 new mode 100755 diff --git a/easybuild/scripts/fix_docs.py b/easybuild/scripts/fix_docs.py old mode 100644 new mode 100755 index 5a02fde0b0..d25f071a22 --- a/easybuild/scripts/fix_docs.py +++ b/easybuild/scripts/fix_docs.py @@ -1,4 +1,5 @@ -# # +#!/usr/bin/env python +## # Copyright 2016-2022 Ghent University # # This file is part of EasyBuild, diff --git a/easybuild/scripts/mk_tmpl_easyblock_for.py b/easybuild/scripts/mk_tmpl_easyblock_for.py old mode 100644 new mode 100755 From c6740056c21ed005f3a9c0d5ce06b400293546b5 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 24 Sep 2022 11:04:57 +0200 Subject: [PATCH 21/44] improve error reporting for parsing of easystack file, mention docs URL where it makes sense --- easybuild/framework/easystack.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easystack.py b/easybuild/framework/easystack.py index a7bcbfb5d4..6a9be46043 100644 --- a/easybuild/framework/easystack.py +++ b/easybuild/framework/easystack.py @@ -136,10 +136,12 @@ def parse(filepath): # For now, we don't support mixing multiple top_keys, so check that only one was defined if len(keys_found) > 1: keys_string = ', '.join(keys_found) - msg = "Specifying multiple top level keys (%s) in one EasyStack file is not supported." % keys_string + msg = "Specifying multiple top level keys (%s) " % keys_string + msg += "in one EasyStack file is currently not supported." raise EasyBuildError(msg) elif len(keys_found) == 0: msg = "Not a valid EasyStack YAML file: no 'easyconfigs' or 'software' top-level key found" + msg += ", see https://docs.easybuild.io/en/latest/Easystack-files.html for documentation." raise EasyBuildError(msg) else: key = keys_found[0] @@ -184,8 +186,9 @@ def parse_by_easyconfigs(filepath, easyconfigs, easybuild_version=None, robot=Fa easystack.ec_opts[easyconf_name_with_eb] = easyconfig[easyconf_name]['options'] else: dict_keys = ', '.join(easyconfig.keys()) - msg = "Failed to parse easystack file: expected a dictionary with one key (the EasyConfig name). " - msg += "Instead found keys: %s" % dict_keys + msg = "Failed to parse easystack file: expected a dictionary with one key (the EasyConfig name), " + msg += "instead found keys: %s" % dict_keys + msg += "; see https://docs.easybuild.io/en/latest/Easystack-files.html for documentation." raise EasyBuildError(msg) return easystack From 2f63290a9c6444b8a331ca07c867f4a82d85e02f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 24 Sep 2022 11:05:10 +0200 Subject: [PATCH 22/44] fix alphabetical ordering in import statement in main.py --- easybuild/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/main.py b/easybuild/main.py index 5f052cf701..2a2aa61541 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -43,7 +43,7 @@ # IMPORTANT this has to be the first easybuild import as it customises the logging # expect missing log output when this not the case! -from easybuild.tools.build_log import EasyBuildError, print_error, print_msg, stop_logging, print_warning +from easybuild.tools.build_log import EasyBuildError, print_error, print_msg, print_warning, stop_logging from easybuild.framework.easyblock import build_and_install_one, inject_checksums from easybuild.framework.easyconfig import EASYCONFIGS_PKG_SUBDIR From 6a12a44ca3d036d086fd42c1a8b411da312c33df Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 24 Sep 2022 11:05:18 +0200 Subject: [PATCH 23/44] use multiple options in test easystack file + fix test_easystack_easyconfig_opts --- test/framework/easystack.py | 4 ++-- .../framework/easystacks/test_easystack_easyconfigs_opts.yaml | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/test/framework/easystack.py b/test/framework/easystack.py index 0c5ad55dcb..a0e2595531 100644 --- a/test/framework/easystack.py +++ b/test/framework/easystack.py @@ -114,8 +114,8 @@ def test_easystack_easyconfig_opts(self): 'toy-0.0-gompi-2018a-test.eb', ] expected_opts = { - 'binutils-2.25-GCCcore-4.9.3.eb': {'debug': True}, - 'foss-2018a.eb': {'robot': True}, + 'binutils-2.25-GCCcore-4.9.3.eb': {'debug': True, 'from-pr': 12345}, + 'foss-2018a.eb': {'enforce-checksums': True, 'robot': True}, } self.assertEqual(sorted(ec_fns), sorted(expected)) self.assertEqual(opts, expected_opts) diff --git a/test/framework/easystacks/test_easystack_easyconfigs_opts.yaml b/test/framework/easystacks/test_easystack_easyconfigs_opts.yaml index 96477259ff..d2bc703986 100644 --- a/test/framework/easystacks/test_easystack_easyconfigs_opts.yaml +++ b/test/framework/easystacks/test_easystack_easyconfigs_opts.yaml @@ -2,10 +2,12 @@ easyconfigs: - binutils-2.25-GCCcore-4.9.3: options: { 'debug': True, + 'from-pr': 12345, } - binutils-2.26-GCCcore-4.9.3 - foss-2018a: options: { + 'enforce-checksums': True, 'robot': True, } - toy-0.0-gompi-2018a-test From 34453721ef144517c386672f5e3f6aff77a11233 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 26 Sep 2022 16:02:36 +0200 Subject: [PATCH 24/44] Replace use of symlink with copied files --- test/framework/sandbox/sources/alt_toy | 1 - .../sandbox/sources/alt_toy/toy-0.0.tar.gz | Bin 0 -> 273 bytes ...y-0.0_fix-silly-typo-in-printf-statement.patch | 10 ++++++++++ 3 files changed, 10 insertions(+), 1 deletion(-) delete mode 120000 test/framework/sandbox/sources/alt_toy create mode 100644 test/framework/sandbox/sources/alt_toy/toy-0.0.tar.gz create mode 100644 test/framework/sandbox/sources/alt_toy/toy-0.0_fix-silly-typo-in-printf-statement.patch diff --git a/test/framework/sandbox/sources/alt_toy b/test/framework/sandbox/sources/alt_toy deleted file mode 120000 index ee6141256d..0000000000 --- a/test/framework/sandbox/sources/alt_toy +++ /dev/null @@ -1 +0,0 @@ -toy \ No newline at end of file diff --git a/test/framework/sandbox/sources/alt_toy/toy-0.0.tar.gz b/test/framework/sandbox/sources/alt_toy/toy-0.0.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..6d21e801284995ecd7407a141a0a395a7f249991 GIT binary patch literal 273 zcmV+s0q*`EiwFRIwHQ+X1MSkwPQx$|1<TzISvv+ z{5#V&bc4bIQB}Y_o3WCSHTJc)FP9wf)#QknIZspC)ofomO7?X?VH9%~#!;Ll#9@@@ z;e^s-4)vk$ymm6ms%mWKtIzKG0RR9100000 X00000000000RD=1w&~v204M+eJA#KZ literal 0 HcmV?d00001 diff --git a/test/framework/sandbox/sources/alt_toy/toy-0.0_fix-silly-typo-in-printf-statement.patch b/test/framework/sandbox/sources/alt_toy/toy-0.0_fix-silly-typo-in-printf-statement.patch new file mode 100644 index 0000000000..e2e4afec56 --- /dev/null +++ b/test/framework/sandbox/sources/alt_toy/toy-0.0_fix-silly-typo-in-printf-statement.patch @@ -0,0 +1,10 @@ +--- a/toy-0.0.orig/toy.source 2014-03-06 18:48:16.000000000 +0100 ++++ b/toy-0.0/toy.source 2020-08-18 12:19:35.000000000 +0200 +@@ -2,6 +2,6 @@ + + int main(int argc, char* argv[]){ + +- printf("I'm a toy, and proud of it.\n"); ++ printf("I'm a toy, and very proud of it.\n"); + return 0; + } From b7526eef7d87806fd97ae0f906140720de5b7d74 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 28 Sep 2022 17:01:10 +0200 Subject: [PATCH 25/44] Add GITHUB_RELEASE and GITHUB_LOWER_RELEASE templates --- easybuild/framework/easyconfig/templates.py | 4 ++ test/framework/easyconfig.py | 61 +++++++++++++-------- 2 files changed, 41 insertions(+), 24 deletions(-) diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index d268536c63..0c49085f37 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -113,6 +113,10 @@ 'GitHub source URL (namelower is used if github_account easyconfig parameter is not specified)'), ('GITHUB_LOWER_SOURCE', 'https://github.com/%(github_account)s/%(namelower)s/archive', 'GitHub source URL (lowercase name, namelower is used if github_account easyconfig parameter is not specified)'), + ('GITHUB_RELEASE', 'https://github.com/%(github_account)s/%(name)s/releases/download/v%(version)s', + 'GitHub release URL (namelower is used if github_account easyconfig parameter is not specified)'), + ('GITHUB_LOWER_RELEASE', 'https://github.com/%(github_account)s/%(namelower)s/releases/download/v%(version)s', + 'GitHub release URL (namelower is used if github_account easyconfig parameter is not specified)'), ('GNU_SAVANNAH_SOURCE', 'https://download-mirror.savannah.gnu.org/releases/%(namelower)s', 'download.savannah.gnu.org source url'), ('GNU_SOURCE', 'https://ftpmirror.gnu.org/gnu/%(namelower)s', diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 190a3c584b..629cc2c076 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1063,7 +1063,7 @@ def test_templating_constants(self): 'homepage = "http://example.com/%%(nameletter)s/%%(nameletterlower)s/v%%(version_major)s/"', 'description = "test easyconfig %%(name)s"', 'toolchain = SYSTEM', - 'source_urls = [GOOGLECODE_SOURCE, GITHUB_SOURCE]', + 'source_urls = [GOOGLECODE_SOURCE, GITHUB_SOURCE, GITHUB_RELEASE, GITHUB_LOWER_RELEASE]', 'sources = [SOURCE_TAR_GZ, (SOURCELOWER_TAR_BZ2, "%(cmd)s")]', 'sanity_check_paths = {', ' "files": ["bin/pi_%%(version_major)s_%%(version_minor)s", "lib/python%%(pyshortver)s/site-packages"],', @@ -1090,43 +1090,56 @@ def test_templating_constants(self): "github_account = 'easybuilders'", ]) % inp self.prep() - eb = EasyConfig(self.eb_file, validate=False) - eb.validate() + ec = EasyConfig(self.eb_file, validate=False) + ec.validate() # temporarily disable templating, just so we can check later whether it's *still* disabled - with eb.disable_templating(): - eb.generate_template_values() - self.assertFalse(eb.enable_templating) - - self.assertEqual(eb['description'], "test easyconfig PI") - self.assertEqual(eb['sources'][0], 'PI-3.04.tar.gz') - self.assertEqual(eb['sources'][1], ('pi-3.04.tar.bz2', "tar xfvz %s")) - self.assertEqual(eb['source_urls'][0], 'http://pi.googlecode.com/files') - self.assertEqual(eb['source_urls'][1], 'https://github.com/easybuilders/PI/archive') - self.assertEqual(eb['versionsuffix'], '-Python-2.7.10') - self.assertEqual(eb['sanity_check_paths']['files'][0], 'bin/pi_3_04') - self.assertEqual(eb['sanity_check_paths']['files'][1], 'lib/python2.7/site-packages') - self.assertEqual(eb['sanity_check_paths']['dirs'][0], 'libfoo.%s' % get_shared_lib_ext()) + with ec.disable_templating(): + ec.generate_template_values() + self.assertFalse(ec.enable_templating) + + self.assertEqual(ec['description'], "test easyconfig PI") + self.assertEqual(ec['sources'][0], 'PI-3.04.tar.gz') + self.assertEqual(ec['sources'][1], ('pi-3.04.tar.bz2', "tar xfvz %s")) + self.assertEqual(ec['source_urls'][0], 'http://pi.googlecode.com/files') + self.assertEqual(ec['source_urls'][1], 'https://github.com/easybuilders/PI/archive') + self.assertEqual(ec['source_urls'][2], 'https://github.com/easybuilders/PI/releases/download/v3.04') + self.assertEqual(ec['source_urls'][3], 'https://github.com/easybuilders/pi/releases/download/v3.04') + + self.assertEqual(ec['versionsuffix'], '-Python-2.7.10') + self.assertEqual(ec['sanity_check_paths']['files'][0], 'bin/pi_3_04') + self.assertEqual(ec['sanity_check_paths']['files'][1], 'lib/python2.7/site-packages') + self.assertEqual(ec['sanity_check_paths']['dirs'][0], 'libfoo.%s' % get_shared_lib_ext()) # should match lib/x86_64/2.7.18, lib/aarch64/3.8.6, lib/ppc64le/3.9.2, etc. lib_arch_regex = re.compile(r'^lib/[a-z0-9_]+/[23]\.[0-9]+\.[0-9]+$') - dirs1 = eb['sanity_check_paths']['dirs'][1] + dirs1 = ec['sanity_check_paths']['dirs'][1] self.assertTrue(lib_arch_regex.match(dirs1), "Pattern '%s' should match '%s'" % (lib_arch_regex.pattern, dirs1)) inc_regex = re.compile('^include/(aarch64|ppc64le|x86_64)$') - dirs2 = eb['sanity_check_paths']['dirs'][2] + dirs2 = ec['sanity_check_paths']['dirs'][2] self.assertTrue(inc_regex.match(dirs2), "Pattern '%s' should match '%s'" % (inc_regex, dirs2)) - self.assertEqual(eb['homepage'], "http://example.com/P/p/v3/") + self.assertEqual(ec['homepage'], "http://example.com/P/p/v3/") expected = ("CUDA: 10.1.105, 10, 1, 10.1; " "Java: 1.7.80, 1, 7, 1.7; " "Python: 2.7.10, 2, 7, 2.7; " "Perl: 5.22.0, 5, 22, 5.22; " "R: 3.2.3, 3, 2, 3.2") - self.assertEqual(eb['modloadmsg'], expected) - self.assertEqual(eb['modextrapaths'], {'PI_MOD_NAME': 'PI/3.04-Python-2.7.10'}) - self.assertEqual(eb['license_file'], os.path.join(os.environ['HOME'], 'licenses', 'PI', 'license.txt')) + self.assertEqual(ec['modloadmsg'], expected) + self.assertEqual(ec['modextrapaths'], {'PI_MOD_NAME': 'PI/3.04-Python-2.7.10'}) + self.assertEqual(ec['license_file'], os.path.join(os.environ['HOME'], 'licenses', 'PI', 'license.txt')) # test the escaping insanity here (ie all the crap we allow in easyconfigs) - eb['description'] = "test easyconfig % %% %s% %%% %(name)s %%(name)s %%%(name)s %%%%(name)s" - self.assertEqual(eb['description'], "test easyconfig % %% %s% %%% PI %(name)s %PI %%(name)s") + ec['description'] = "test easyconfig % %% %s% %%% %(name)s %%(name)s %%%(name)s %%%%(name)s" + self.assertEqual(ec['description'], "test easyconfig % %% %s% %%% PI %(name)s %PI %%(name)s") + + # Remove github_account + self.contents = re.sub(r'github_account =.*$', '', self.contents, flags=re.MULTILINE) + self.prep() + ec = EasyConfig(self.eb_file, validate=False) + ec.generate_template_values() + # and retest GITHUB_* templates that namelower is used instead + self.assertEqual(ec['source_urls'][1], 'https://github.com/pi/PI/archive') + self.assertEqual(ec['source_urls'][2], 'https://github.com/pi/PI/releases/download/v3.04') + self.assertEqual(ec['source_urls'][3], 'https://github.com/pi/pi/releases/download/v3.04') # test use of %(mpi_cmd_prefix)s template test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') From 45ce12dd5eb37ec6c3c426e41d0d10010e4a5618 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 30 Sep 2022 12:54:38 +0200 Subject: [PATCH 26/44] Make inject_checksums inject dictionaries --- easybuild/framework/easyblock.py | 6 +++--- test/framework/options.py | 33 ++++++++++++++++++-------------- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 860eac3a21..20c63b413f 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -4416,11 +4416,11 @@ def make_checksum_lines(checksums, indent_level): line_indent = INDENT_4SPACES * indent_level checksum_lines = [] for fn, checksum in checksums: - checksum_line = "%s'%s', # %s" % (line_indent, checksum, fn) + checksum_line = "%s{'%s': '%s'}," % (line_indent, fn, checksum) if len(checksum_line) > MAX_LINE_LENGTH: checksum_lines.extend([ - "%s# %s" % (line_indent, fn), - "%s'%s'," % (line_indent, checksum), + "%s{'%s':" % (line_indent, fn), + "%s%s'%s'}," % (line_indent, INDENT_4SPACES, checksum), ]) else: checksum_lines.append(checksum_line) diff --git a/test/framework/options.py b/test/framework/options.py index 308a1670cc..7e9b328fa6 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -5699,13 +5699,13 @@ def test_inject_checksums(self): self.assertTrue("'checksums': ['d5bd9908cdefbe2d29c6f8d5b45b2aaed9fd904b5e6397418bb5094fbdb3d838']," in ec_txt) # single-line checksum entry for bar source tarball - regex = re.compile("^[ ]*'%s', # bar-0.0.tar.gz$" % bar_tar_gz_sha256, re.M) + regex = re.compile("^[ ]*{'bar-0.0.tar.gz': '%s'},$" % bar_tar_gz_sha256, re.M) self.assertTrue(regex.search(ec_txt), "Pattern '%s' found in: %s" % (regex.pattern, ec_txt)) # no single-line checksum entry for bar patches, since line would be > 120 chars bar_patch_patterns = [ - r"^[ ]*# %s\n[ ]*'%s',$" % (bar_patch, bar_patch_sha256), - r"^[ ]*# %s\n[ ]*'%s',$" % (bar_patch_bis, bar_patch_bis_sha256), + r"^[ ]*{'%s':\n[ ]*'%s'},$" % (bar_patch, bar_patch_sha256), + r"^[ ]*{'%s':\n[ ]*'%s'},$" % (bar_patch_bis, bar_patch_bis_sha256), ] for pattern in bar_patch_patterns: regex = re.compile(pattern, re.M) @@ -5736,15 +5736,16 @@ def test_inject_checksums(self): ec = EasyConfigParser(test_ec).get_config_dict() self.assertEqual(ec['sources'], ['%(name)s-%(version)s.tar.gz']) self.assertEqual(ec['patches'], ['toy-0.0_fix-silly-typo-in-printf-statement.patch']) - self.assertEqual(ec['checksums'], [toy_source_sha256, toy_patch_sha256]) + self.assertEqual(ec['checksums'], [{'toy-0.0.tar.gz': toy_source_sha256}, + {'toy-0.0_fix-silly-typo-in-printf-statement.patch': toy_patch_sha256}]) self.assertEqual(ec['exts_default_options'], {'source_urls': ['http://example.com/%(name)s']}) self.assertEqual(ec['exts_list'][0], 'ls') self.assertEqual(ec['exts_list'][1], ('bar', '0.0', { 'buildopts': " && gcc bar.c -o anotherbar", 'checksums': [ - bar_tar_gz_sha256, - bar_patch_sha256, - bar_patch_bis_sha256, + {'bar-0.0.tar.gz': bar_tar_gz_sha256}, + {'bar-0.0_fix-silly-typo-in-printf-statement.patch': bar_patch_sha256}, + {'bar-0.0_fix-very-silly-typo-in-printf-statement.patch': bar_patch_bis_sha256}, ], 'exts_filter': ("cat | grep '^bar$'", '%(name)s'), 'patches': [bar_patch, bar_patch_bis], @@ -5770,13 +5771,16 @@ def test_inject_checksums(self): # if any checksums are present already, it doesn't matter if they're wrong (since they will be replaced) ectxt = read_file(test_ec) for chksum in ec['checksums'] + [c for e in ec['exts_list'][1:] for c in e[2]['checksums']]: + if isinstance(chksum, dict): + chksum = list(chksum.values())[0] ectxt = ectxt.replace(chksum, chksum[::-1]) write_file(test_ec, ectxt) stdout, stderr = self._run_mock_eb(args, raise_error=True, strip=True) ec = EasyConfigParser(test_ec).get_config_dict() - self.assertEqual(ec['checksums'], [toy_source_sha256, toy_patch_sha256]) + self.assertEqual(ec['checksums'], [{'toy-0.0.tar.gz': toy_source_sha256}, + {'toy-0.0_fix-silly-typo-in-printf-statement.patch': toy_patch_sha256}]) ec_backups = glob.glob(test_ec + '.bak_*') self.assertEqual(len(ec_backups), 1) @@ -5819,9 +5823,9 @@ def test_inject_checksums(self): # no parse errors for updated easyconfig file... ec = EasyConfigParser(test_ec).get_config_dict() checksums = [ - 'be662daa971a640e40be5c804d9d7d10', - 'a99f2a72cee1689a2f7e3ace0356efb1', - '3b0787b3bf36603ae1398c4a49097893', + {'toy-0.0.tar.gz': 'be662daa971a640e40be5c804d9d7d10'}, + {'toy-0.0_fix-silly-typo-in-printf-statement.patch': 'a99f2a72cee1689a2f7e3ace0356efb1'}, + {'toy-extra.txt': '3b0787b3bf36603ae1398c4a49097893'}, ] self.assertEqual(ec['checksums'], checksums) @@ -5843,9 +5847,10 @@ def test_inject_checksums(self): ec = EasyConfigParser(test_ec).get_config_dict() expected_checksums = [ - '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc', - '81a3accc894592152f81814fbf133d39afad52885ab52c25018722c7bda92487', - '4196b56771140d8e2468fb77f0240bc48ddbf5dabafe0713d612df7fafb1e458' + {'toy-0.0.tar.gz': '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc'}, + {'toy-0.0_fix-silly-typo-in-printf-statement.patch': + '81a3accc894592152f81814fbf133d39afad52885ab52c25018722c7bda92487'}, + {'toy-extra.txt': '4196b56771140d8e2468fb77f0240bc48ddbf5dabafe0713d612df7fafb1e458'} ] self.assertEqual(ec['checksums'], expected_checksums) From 58d680b1d0574eb308545ddd823c173632e62a60 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 30 Sep 2022 13:17:35 +0200 Subject: [PATCH 27/44] Stick to flake8 indent rules --- easybuild/framework/easyblock.py | 2 +- test/framework/options.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 20c63b413f..666cdec540 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -4420,7 +4420,7 @@ def make_checksum_lines(checksums, indent_level): if len(checksum_line) > MAX_LINE_LENGTH: checksum_lines.extend([ "%s{'%s':" % (line_indent, fn), - "%s%s'%s'}," % (line_indent, INDENT_4SPACES, checksum), + "%s '%s'}," % (line_indent, checksum), ]) else: checksum_lines.append(checksum_line) diff --git a/test/framework/options.py b/test/framework/options.py index 7e9b328fa6..b541c024ae 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -5849,7 +5849,7 @@ def test_inject_checksums(self): expected_checksums = [ {'toy-0.0.tar.gz': '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc'}, {'toy-0.0_fix-silly-typo-in-printf-statement.patch': - '81a3accc894592152f81814fbf133d39afad52885ab52c25018722c7bda92487'}, + '81a3accc894592152f81814fbf133d39afad52885ab52c25018722c7bda92487'}, {'toy-extra.txt': '4196b56771140d8e2468fb77f0240bc48ddbf5dabafe0713d612df7fafb1e458'} ] self.assertEqual(ec['checksums'], expected_checksums) From 2b0e40f31858a9ec088168bd25b84f66650e7709 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 5 Oct 2022 10:24:10 +0200 Subject: [PATCH 28/44] Add `%(cuda_cc_cmake)s` template This can be used with CMake 3.18+ projects via the env variable `$CUDAARCHS` or the CMake variable `CMAKE_CUDA_ARCHITECTURES` (via `-DCMAKE_CUDA_ARCHITECTURES=...`) See https://cmake.org/cmake/help/v3.18/variable/CMAKE_CUDA_ARCHITECTURES.html --- easybuild/framework/easyconfig/templates.py | 2 + test/framework/easyconfig.py | 45 +++++++++++---------- 2 files changed, 25 insertions(+), 22 deletions(-) diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index d268536c63..e0bfb90541 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -90,6 +90,7 @@ ('mpi_cmd_prefix', "Prefix command for running MPI programs (with default number of ranks)"), ('cuda_compute_capabilities', "Comma-separated list of CUDA compute capabilities, as specified via " "--cuda-compute-capabilities configuration option or via cuda_compute_capabilities easyconfig parameter"), + ('cuda_cc_cmake', "List of CUDA compute capabilities suitable for use with $CUDAARCHS in CMake 3.18+"), ('cuda_cc_space_sep', "Space-separated list of CUDA compute capabilities"), ('cuda_cc_semicolon_sep', "Semicolon-separated list of CUDA compute capabilities"), ('cuda_sm_comma_sep', "Comma-separated list of sm_* values that correspond with CUDA compute capabilities"), @@ -353,6 +354,7 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None) template_values['cuda_compute_capabilities'] = ','.join(cuda_compute_capabilities) template_values['cuda_cc_space_sep'] = ' '.join(cuda_compute_capabilities) template_values['cuda_cc_semicolon_sep'] = ';'.join(cuda_compute_capabilities) + template_values['cuda_cc_cmake'] = ';'.join(cc.replace('.', '') for cc in cuda_compute_capabilities) sm_values = ['sm_' + cc.replace('.', '') for cc in cuda_compute_capabilities] template_values['cuda_sm_comma_sep'] = ','.join(sm_values) template_values['cuda_sm_space_sep'] = ' '.join(sm_values) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 190a3c584b..a255e21063 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -4340,38 +4340,39 @@ def __init__(self, values): def test_cuda_compute_capabilities(self): """Tests that the cuda_compute_capabilities templates are correct""" - test_ec = os.path.join(self.test_prefix, 'test.eb') - test_ectxt = '\n'.join([ - "easyblock = 'ConfigureMake'", - "name = 'test'", - "version = '0.2'", - "homepage = 'https://example.com'", - "description = 'test'", - "toolchain = SYSTEM", - "cuda_compute_capabilities = ['5.1', '7.0', '7.1']", - "installopts = '%(cuda_compute_capabilities)s'", - "preinstallopts = '%(cuda_cc_space_sep)s'", - "prebuildopts = '%(cuda_cc_semicolon_sep)s'", - "configopts = '%(cuda_sm_comma_sep)s'", - "preconfigopts = '%(cuda_sm_space_sep)s'", - ]) - write_file(test_ec, test_ectxt) + self.contents = textwrap.dedent(""" + easyblock = 'ConfigureMake' + name = 'test' + version = '0.2' + homepage = 'https://example.com' + description = 'test' + toolchain = SYSTEM + cuda_compute_capabilities = ['5.1', '7.0', '7.1'] + installopts = '%(cuda_compute_capabilities)s' + preinstallopts = '%(cuda_cc_space_sep)s' + prebuildopts = '%(cuda_cc_semicolon_sep)s' + configopts = 'comma="%(cuda_sm_comma_sep)s" space="%(cuda_sm_space_sep)s"' + preconfigopts = 'CUDAARCHS="%(cuda_cc_cmake)s"' + """) + self.prep() - ec = EasyConfig(test_ec) + ec = EasyConfig(self.eb_file) self.assertEqual(ec['installopts'], '5.1,7.0,7.1') self.assertEqual(ec['preinstallopts'], '5.1 7.0 7.1') self.assertEqual(ec['prebuildopts'], '5.1;7.0;7.1') - self.assertEqual(ec['configopts'], 'sm_51,sm_70,sm_71') - self.assertEqual(ec['preconfigopts'], 'sm_51 sm_70 sm_71') + self.assertEqual(ec['configopts'], 'comma="sm_51,sm_70,sm_71" ' + 'space="sm_51 sm_70 sm_71"') + self.assertEqual(ec['preconfigopts'], 'CUDAARCHS="51;70;71"') # build options overwrite it init_config(build_options={'cuda_compute_capabilities': ['4.2', '6.3']}) - ec = EasyConfig(test_ec) + ec = EasyConfig(self.eb_file) self.assertEqual(ec['installopts'], '4.2,6.3') self.assertEqual(ec['preinstallopts'], '4.2 6.3') self.assertEqual(ec['prebuildopts'], '4.2;6.3') - self.assertEqual(ec['configopts'], 'sm_42,sm_63') - self.assertEqual(ec['preconfigopts'], 'sm_42 sm_63') + self.assertEqual(ec['configopts'], 'comma="sm_42,sm_63" ' + 'space="sm_42 sm_63"') + self.assertEqual(ec['preconfigopts'], 'CUDAARCHS="42;63"') def test_det_copy_ec_specs(self): """Test det_copy_ec_specs function.""" From aae2bf4e1a57fe04b792f2ceeadf5c27d0037ca9 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 5 Oct 2022 18:24:27 +0200 Subject: [PATCH 29/44] self.compilers() returns (['clang', 'clang++'],[None, None, None]). The 'which' call in 'prepare_rpath_wrappers' fails when looping over those None objects. Thus, skip those and continue with the next loop iteration --- easybuild/tools/toolchain/toolchain.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index e21bd66869..3792464f78 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -988,6 +988,9 @@ def prepare_rpath_wrappers(self, rpath_filter_dirs=None, rpath_include_dirs=None # create wrappers for cmd in nub(c_comps + fortran_comps + ['ld', 'ld.gold', 'ld.bfd']): + # Not all toolchains have fortran compilers (e.g. Clang), in which case they are 'None' + if cmd is None: + continue orig_cmd = which(cmd) if orig_cmd: From 3b036eb3df60282f0f42ef0ea803aa2f3166e2b1 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 10 Oct 2022 19:47:33 +0200 Subject: [PATCH 30/44] skip GitPython 3.1.28, since it prints 'git' as version which messes up the version check in check_github() --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 4a7c734e58..ad471b002c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,8 @@ keyrings.alt # GitPython 3.1.15 deprecates Python 3.5 GitPython==3.1.14; python_version >= '3.0' and python_version < '3.6' -GitPython; python_version >= '3.6' or python_version <= '3.0' +# skip GitPython 3.1.28, since it prints 'git' as version which messes up the version check in check_github() +GitPython!=3.1.28; python_version >= '3.6' or python_version <= '3.0' # autopep8 # stick to older autopep8 with Python 2.7, since autopep8 1.7.0 requires pycodestyle>=2.9.1 (which is Python 3 only) From f230d884e81f3302e22b9d56053387c409b0f5ac Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 10 Oct 2022 20:07:49 +0200 Subject: [PATCH 31/44] update to v3 of actions/checkout and actions/setup-python in CI workflows --- .github/workflows/bootstrap_script.yml | 4 ++-- .github/workflows/container_tests.yml | 4 ++-- .github/workflows/eb_command.yml | 4 ++-- .github/workflows/linting.yml | 4 ++-- .github/workflows/unit_tests.yml | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/bootstrap_script.yml b/.github/workflows/bootstrap_script.yml index e8d4bce985..4613be8464 100644 --- a/.github/workflows/bootstrap_script.yml +++ b/.github/workflows/bootstrap_script.yml @@ -54,10 +54,10 @@ jobs: lc_all: C fail-fast: false steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: ${{matrix.python}} architecture: x64 diff --git a/.github/workflows/container_tests.yml b/.github/workflows/container_tests.yml index 6e4e080cee..b3305314fb 100644 --- a/.github/workflows/container_tests.yml +++ b/.github/workflows/container_tests.yml @@ -10,10 +10,10 @@ jobs: python: [2.7, 3.6] fail-fast: false steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: ${{matrix.python}} architecture: x64 diff --git a/.github/workflows/eb_command.yml b/.github/workflows/eb_command.yml index 6994ededee..c50e3079fe 100644 --- a/.github/workflows/eb_command.yml +++ b/.github/workflows/eb_command.yml @@ -9,10 +9,10 @@ jobs: python: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10'] fail-fast: false steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: ${{matrix.python}} architecture: x64 diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index f22011ba88..ec9b075960 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -8,10 +8,10 @@ jobs: python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10'] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 1795f076c1..494bb55efa 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -76,10 +76,10 @@ jobs: lc_all: C fail-fast: false steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: ${{matrix.python}} architecture: x64 From 7d1333683163ca0092d6b93c9e8f1c934274e54d Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 10 Oct 2022 20:23:48 +0200 Subject: [PATCH 32/44] make sure that GitPython version is a proper version before checking minimal required version --- easybuild/tools/github.py | 3 ++- requirements.txt | 3 +-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 234c3fb3fc..8f880beb41 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -2086,7 +2086,8 @@ def check_github(): elif github_user: if 'git' in sys.modules: ver, req_ver = git.__version__, '1.0' - if LooseVersion(ver) < LooseVersion(req_ver): + version_regex = re.compile('^[0-9.]+$') + if version_regex.match(ver) and LooseVersion(ver) < LooseVersion(req_ver): check_res = "FAIL (GitPython version %s is too old, should be version %s or newer)" % (ver, req_ver) elif "Could not read from remote repository" in str(push_err): check_res = "FAIL (GitHub SSH key missing? %s)" % push_err diff --git a/requirements.txt b/requirements.txt index ad471b002c..4a7c734e58 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,8 +5,7 @@ keyrings.alt # GitPython 3.1.15 deprecates Python 3.5 GitPython==3.1.14; python_version >= '3.0' and python_version < '3.6' -# skip GitPython 3.1.28, since it prints 'git' as version which messes up the version check in check_github() -GitPython!=3.1.28; python_version >= '3.6' or python_version <= '3.0' +GitPython; python_version >= '3.6' or python_version <= '3.0' # autopep8 # stick to older autopep8 with Python 2.7, since autopep8 1.7.0 requires pycodestyle>=2.9.1 (which is Python 3 only) From 2ba2124839e343b59a01395bb4435b3f298462d4 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 12 Oct 2022 13:39:37 +0200 Subject: [PATCH 33/44] consistently include pointer to docs in easystack parser errors --- easybuild/framework/easystack.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easystack.py b/easybuild/framework/easystack.py index 6a9be46043..cccf0bc202 100644 --- a/easybuild/framework/easystack.py +++ b/easybuild/framework/easystack.py @@ -41,6 +41,8 @@ pass _log = fancylogger.getLogger('easystack', fname=False) +EASYSTACK_DOC_URL = 'https://docs.easybuild.io/en/latest/Easystack-files.html' + def check_value(value, context): """ @@ -137,11 +139,12 @@ def parse(filepath): if len(keys_found) > 1: keys_string = ', '.join(keys_found) msg = "Specifying multiple top level keys (%s) " % keys_string - msg += "in one EasyStack file is currently not supported." + msg += "in one EasyStack file is currently not supported" + msg += ", see %s for documentation." % EASYSTACK_DOC_URL raise EasyBuildError(msg) elif len(keys_found) == 0: msg = "Not a valid EasyStack YAML file: no 'easyconfigs' or 'software' top-level key found" - msg += ", see https://docs.easybuild.io/en/latest/Easystack-files.html for documentation." + msg += ", see %s for documentation." % EASYSTACK_DOC_URL raise EasyBuildError(msg) else: key = keys_found[0] @@ -188,7 +191,7 @@ def parse_by_easyconfigs(filepath, easyconfigs, easybuild_version=None, robot=Fa dict_keys = ', '.join(easyconfig.keys()) msg = "Failed to parse easystack file: expected a dictionary with one key (the EasyConfig name), " msg += "instead found keys: %s" % dict_keys - msg += "; see https://docs.easybuild.io/en/latest/Easystack-files.html for documentation." + msg += ", see %s for documentation." % EASYSTACK_DOC_URL raise EasyBuildError(msg) return easystack From 0685188448bfc8e6f2e5f9d8a2ce2ea76f0c66bb Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Wed, 12 Oct 2022 15:45:11 +0200 Subject: [PATCH 34/44] first look in alt_location for patch when specified When alt_location is specified we should not look in the directory of the easyconfig for patches before alt_location. --- easybuild/framework/easyblock.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 666cdec540..f4de7276ce 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -753,8 +753,11 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No foundfile = None failedpaths = [] - # always look first in the dir of the current eb file - ebpath = [os.path.dirname(self.cfg.path)] + # always look first in the dir of the current eb file unless alt_location is set + if alt_location is None: + ebpath = [os.path.dirname(self.cfg.path)] + else: + ebpath = [] # always consider robot + easyconfigs install paths as a fall back (e.g. for patch files, test cases, ...) common_filepaths = [] From 48db126f1fcd8113193bed02de16af86e657376e Mon Sep 17 00:00:00 2001 From: jfgrimm Date: Wed, 12 Oct 2022 15:54:59 +0100 Subject: [PATCH 35/44] use SYSTEM template constant in dependencies instead of True --- test/framework/easyblock.py | 2 +- test/framework/easyconfig.py | 2 +- .../test_ecs/b/binutils/binutils-2.25-GCCcore-4.9.3.eb | 2 +- .../test_ecs/b/binutils/binutils-2.26-GCCcore-4.9.3.eb | 2 +- .../framework/easyconfigs/test_ecs/g/gzip/gzip-1.4-GCC-4.6.3.eb | 2 +- test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-deps.eb | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 4006fae68f..979b9ed696 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -870,7 +870,7 @@ def test_make_module_dep_of_dep_hmns(self): 'description = "test easyconfig"', "toolchain = {'name': 'foss', 'version': '2018a'}", 'dependencies = [', - " ('GCC', '6.4.0-2.28', '', True)," + " ('GCC', '6.4.0-2.28', '', SYSTEM)," " ('hwloc', '1.11.8', '', ('GCC', '6.4.0-2.28')),", " ('OpenMPI', '2.1.2', '', ('GCC', '6.4.0-2.28'))," ']', diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index a40c8c1df5..a27c52731d 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1188,7 +1188,7 @@ def test_java_wrapper_templating(self): 'homepage = "https://example.com"', 'description = "test easyconfig"', 'toolchain = {"name":"GCC", "version": "4.6.3"}', - 'dependencies = [("Java", "11", "", True)]', + 'dependencies = [("Java", "11", "", SYSTEM)]', 'modloadmsg = "Java: %(javaver)s, %(javamajver)s, %(javashortver)s"', ]) self.prep() diff --git a/test/framework/easyconfigs/test_ecs/b/binutils/binutils-2.25-GCCcore-4.9.3.eb b/test/framework/easyconfigs/test_ecs/b/binutils/binutils-2.25-GCCcore-4.9.3.eb index 47ffbc2e76..6969ac4b08 100644 --- a/test/framework/easyconfigs/test_ecs/b/binutils/binutils-2.25-GCCcore-4.9.3.eb +++ b/test/framework/easyconfigs/test_ecs/b/binutils/binutils-2.25-GCCcore-4.9.3.eb @@ -20,7 +20,7 @@ source_urls = [GNU_SOURCE] # # zlib required, but being linked in statically, so not a runtime dep # ('zlib', '1.2.8'), # # use same binutils version that was used when building GCC toolchain, to 'bootstrap' this binutils -# ('binutils', version, '', True) +# ('binutils', version, '', SYSTEM) #] moduleclass = 'tools' diff --git a/test/framework/easyconfigs/test_ecs/b/binutils/binutils-2.26-GCCcore-4.9.3.eb b/test/framework/easyconfigs/test_ecs/b/binutils/binutils-2.26-GCCcore-4.9.3.eb index e6e318b505..c9cb4abfff 100644 --- a/test/framework/easyconfigs/test_ecs/b/binutils/binutils-2.26-GCCcore-4.9.3.eb +++ b/test/framework/easyconfigs/test_ecs/b/binutils/binutils-2.26-GCCcore-4.9.3.eb @@ -20,7 +20,7 @@ source_urls = [GNU_SOURCE] # # zlib required, but being linked in statically, so not a runtime dep # ('zlib', '1.2.8'), # # use same binutils version that was used when building GCC toolchain, to 'bootstrap' this binutils -# ('binutils', version, '', True) +# ('binutils', version, '', SYSTEM) #] moduleclass = 'tools' diff --git a/test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.4-GCC-4.6.3.eb b/test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.4-GCC-4.6.3.eb index 3ef134f492..54c18d79e0 100644 --- a/test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.4-GCC-4.6.3.eb +++ b/test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.4-GCC-4.6.3.eb @@ -26,7 +26,7 @@ sources = ['%(name)s-%(version)s.tar.gz'] # download location for source files source_urls = [GNU_SOURCE] -hiddendependencies = [('toy', '0.0', '-deps', True)] +hiddendependencies = [('toy', '0.0', '-deps', SYSTEM)] dependencies = hiddendependencies # hidden deps must be included in list of deps # make sure the gzip and gunzip binaries are available after installation diff --git a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-deps.eb b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-deps.eb index d2a222a21c..4ae349d0a0 100644 --- a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-deps.eb +++ b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-deps.eb @@ -20,7 +20,7 @@ checksums = [[ patches = ['toy-0.0_fix-silly-typo-in-printf-statement.patch'] dependencies = [ - ('intel', '2018a', '', True), + ('intel', '2018a', '', SYSTEM), ('GCC/6.4.0-2.28', EXTERNAL_MODULE), ] From 522393acebba7d9dbcbb3aafdecd58a6e103223d Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Wed, 12 Oct 2022 17:57:19 +0200 Subject: [PATCH 36/44] Log when directory of the easyconfig file is being ignored due to alt_location --- easybuild/framework/easyblock.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index f4de7276ce..63530f15c2 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -757,6 +757,8 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No if alt_location is None: ebpath = [os.path.dirname(self.cfg.path)] else: + self.log.info("Directory of the easyconfig file ignored when searching for %s " + "because alt_location is set to %s", filename, alt_location) ebpath = [] # always consider robot + easyconfigs install paths as a fall back (e.g. for patch files, test cases, ...) From d57a64afd6721917130eff45978a4b76be994209 Mon Sep 17 00:00:00 2001 From: casparl Date: Fri, 14 Oct 2022 13:00:12 +0200 Subject: [PATCH 37/44] Expand test to test if wrappers can be put in place for a toolchain for which the fortran compilers are 'None', such as Clang --- test/framework/toolchain.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 225a47510e..5189581154 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -2617,6 +2617,26 @@ def test_toolchain_prepare_rpath(self): # any other available 'g++' commands should not be a wrapper or our fake g++ self.assertFalse(any(os.path.samefile(x, fake_gxx) for x in res[2:])) + # Check that we can create a wrapper for a toolchain for which self.compilers() returns 'None' for the Fortran + # compilers (i.e. Clang) + fake_clang = os.path.join(self.test_prefix, 'fake', 'clang') + write_file(fake_clang, '#!/bin/bash\necho "$@"') + adjust_permissions(fake_clang, stat.S_IXUSR) + tc_clang = self.get_toolchain('clang', version='13.0.1') + tc_clang.prepare() + + # Check that the clang wrapper is indeed in place + res = which('clang', retain_all=True) + # there should be at least 2 hits: the RPATH wrapper, and our fake 'clang' command (there may be real ones too) + self.assertTrue(len(res) >= 2) + self.assertTrue(tc_clang.is_rpath_wrapper(res[0])) + self.assertEqual(os.path.basename(res[0]), 'clang') + self.assertEqual(os.path.basename(os.path.dirname(res[0])), 'clang_wrapper') + self.assertFalse(any(tc_clang.is_rpath_wrapper(x) for x in res[1:])) + self.assertTrue(os.path.samefile(res[1], fake_clang)) + # any other available 'clang' commands should not be a wrapper or our fake clang + self.assertFalse(any(os.path.samefile(x, fake_clang) for x in res[2:])) + # RPATH wrapper should be robust against Python environment variables & site-packages magic, # so we set up a weird environment here to verify that # (see https://github.com/easybuilders/easybuild-framework/issues/3421) From cf7d3b24f059537b9ec5a4f9090ff39398dbc19c Mon Sep 17 00:00:00 2001 From: casparl Date: Fri, 14 Oct 2022 15:46:26 +0200 Subject: [PATCH 38/44] Clang toolchain name should probably be capitalized... --- test/framework/toolchain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 5189581154..ab84a7e091 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -2622,7 +2622,7 @@ def test_toolchain_prepare_rpath(self): fake_clang = os.path.join(self.test_prefix, 'fake', 'clang') write_file(fake_clang, '#!/bin/bash\necho "$@"') adjust_permissions(fake_clang, stat.S_IXUSR) - tc_clang = self.get_toolchain('clang', version='13.0.1') + tc_clang = self.get_toolchain('Clang', version='13.0.1') tc_clang.prepare() # Check that the clang wrapper is indeed in place From a10363dcf5e8a0b33e597765b1c3488169f94799 Mon Sep 17 00:00:00 2001 From: casparl Date: Fri, 14 Oct 2022 16:37:51 +0200 Subject: [PATCH 39/44] TOolchain name should probably be ClangGCC, not Clang --- test/framework/toolchain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index ab84a7e091..2ca3b000de 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -2622,7 +2622,7 @@ def test_toolchain_prepare_rpath(self): fake_clang = os.path.join(self.test_prefix, 'fake', 'clang') write_file(fake_clang, '#!/bin/bash\necho "$@"') adjust_permissions(fake_clang, stat.S_IXUSR) - tc_clang = self.get_toolchain('Clang', version='13.0.1') + tc_clang = self.get_toolchain('ClangGCC', version='0.0.0') tc_clang.prepare() # Check that the clang wrapper is indeed in place From 3d3bfb04e53a93e3d6535fa9e4ac04d151a84d6a Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Fri, 14 Oct 2022 17:54:26 +0200 Subject: [PATCH 40/44] Ok, there is no full Clang toolchain, as we have zero EasyConfigs for those. I Can however instatiate a Clang toolchain similar to how it's done here https://github.com/easybuilders/easybuild-easyblocks/blob/b436f55669e5e09fe49555c7ce6a3c456dbacc4a/easybuild/easyblocks/c/clang.py#L420 . That 'prepare_rpath_wrappers()' call would cause an error without the fix in this current PR --- test/framework/toolchain.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 2ca3b000de..fbce1305d6 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -56,6 +56,7 @@ from easybuild.tools.toolchain.mpi import get_mpi_cmd_template from easybuild.tools.toolchain.toolchain import env_vars_external_module from easybuild.tools.toolchain.utilities import get_toolchain, search_toolchain +from easybuild.toolchains.compiler.clang import Clang easybuild.tools.toolchain.compiler.systemtools.get_compiler_family = lambda: st.POWER @@ -2622,8 +2623,8 @@ def test_toolchain_prepare_rpath(self): fake_clang = os.path.join(self.test_prefix, 'fake', 'clang') write_file(fake_clang, '#!/bin/bash\necho "$@"') adjust_permissions(fake_clang, stat.S_IXUSR) - tc_clang = self.get_toolchain('ClangGCC', version='0.0.0') - tc_clang.prepare() + tc_clang = Clang(name='Clang', version='1') + tc_clang.prepare_rpath_wrappers() # Check that the clang wrapper is indeed in place res = which('clang', retain_all=True) From bb046138c2904423af9ab770434f88af8b52a3ab Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 17 Oct 2022 11:10:23 +0200 Subject: [PATCH 41/44] always log a message in obtain_file to indicate whether or directory in which easyconfig file is located is being consider to locate file --- easybuild/framework/easyblock.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 63530f15c2..1fd59b932d 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -756,10 +756,12 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No # always look first in the dir of the current eb file unless alt_location is set if alt_location is None: ebpath = [os.path.dirname(self.cfg.path)] + self.log.info("Considering directory in which easyconfig file is located when searching for %s: %s", + filename, ebpath[0]) else: - self.log.info("Directory of the easyconfig file ignored when searching for %s " - "because alt_location is set to %s", filename, alt_location) ebpath = [] + self.log.info("Not considering directory in which easyconfig file is located when searching for %s " + "because alt_location is set to %s", filename, alt_location) # always consider robot + easyconfigs install paths as a fall back (e.g. for patch files, test cases, ...) common_filepaths = [] From dc7a61ff5e026a182dd208c4b6f3c86b87110ad7 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 17 Oct 2022 11:10:54 +0200 Subject: [PATCH 42/44] enhance test_obtain_file to check that directory in which easyconfig file is located is ignored when alt_location is used --- test/framework/easyblock.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 5653f43d8d..98416ea638 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -1684,7 +1684,11 @@ def test_obtain_file(self): mkdir(tmpdir_subdir, parents=True) del os.environ['EASYBUILD_SOURCEPATH'] # defined by setUp - ec = process_easyconfig(os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb'))[0] + toy_ec = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') + test_ec = os.path.join(tmpdir, 'ecs', 'test.eb') + copy_file(toy_ec, test_ec) + + ec = process_easyconfig(test_ec)[0] eb = EasyBlock(ec['ec']) # 'downloading' a file to (first) sourcepath works @@ -1697,6 +1701,15 @@ def test_obtain_file(self): res = eb.obtain_file(toy_tarball, urls=['file://%s' % tmpdir_subdir], alt_location='alt_toy') self.assertEqual(res, os.path.join(tmpdir, 'a', 'alt_toy', toy_tarball)) + # make sure that directory in which easyconfig file is located is *ignored* when alt_location is used + dummy_toy_tar_gz = os.path.join(os.path.dirname(test_ec), 'toy-0.0.tar.gz') + write_file(dummy_toy_tar_gz, '') + res = eb.obtain_file(toy_tarball, urls=['file://%s' % tmpdir_subdir]) + self.assertEqual(res, dummy_toy_tar_gz) + res = eb.obtain_file(toy_tarball, urls=['file://%s' % tmpdir_subdir], alt_location='alt_toy') + self.assertEqual(res, os.path.join(tmpdir, 'a', 'alt_toy', toy_tarball)) + remove_file(dummy_toy_tar_gz) + # finding a file in sourcepath works init_config(args=["--sourcepath=%s:/no/such/dir:%s" % (sandbox_sources, tmpdir)]) res = eb.obtain_file(toy_tarball) From bfc14f667a8d3bafe408add40c1a224cf75740cf Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Fri, 21 Oct 2022 12:04:24 +0800 Subject: [PATCH 43/44] prepare release notes for EasyBuild v4.6.2 + bump version to 4.6.2 --- RELEASE_NOTES | 24 ++++++++++++++++++++++++ easybuild/tools/version.py | 2 +- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 44ae86e67f..ea19b6a2e4 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -4,6 +4,30 @@ For more detailed information, please see the git log. These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html. +v4.6.2 (October 21st 2022) +-------------------------- + +update/bugfix release + +- various enhancements, including: + - add support for easystack file that contains easyconfig filenames + implement parsing of configuration options (#4021) + - skip over unset $EB_PYTHON/$EB_INSTALLPYTHON (#4080) + - add GITHUB_RELEASE and GITHUB_LOWER_RELEASE templates (#4084) + - add `%(cuda_cc_cmake)s` template (#4087) +- various bug fixes, including: + - make check_sha256_checksums verify all checksums if they're specified as a dict value (#4076) + - replace use of symlink with copied files in `alt_location` tests (#4083) + - fix trying to generate RPATH wrappers for Clang (#4088) + - skip GitPython 3.1.28, since it prints 'git' as version which messes up the version check in check_github() (#4090) + - make sure that GitPython version is a proper version before checking minimal required version (#4091) + - first look for patch in alt_location when it is specified (#4093) +- other changes: + - make scripts executable (#4081) + - make --inject-checksums inject dictionary value for checksums which maps filename to SHA256 checksum (#4085) + - update to v3 of actions/checkout and actions/setup-python in CI workflows (#4089) + - use SYSTEM template constant in dependencies instead of True in framework tests (#4094) + + v4.6.1 (September 12th 2022) ---------------------------- diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index 4fec6dcddb..1bb3c2b05a 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.6.2.dev0') +VERSION = LooseVersion('4.6.2') UNKNOWN = 'UNKNOWN' From dc80788f2891b3bb3027daf66479274c967caf93 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 21 Oct 2022 06:26:37 +0200 Subject: [PATCH 44/44] minor tweak release notes for EasyBuild v4.6.2 --- RELEASE_NOTES | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index ea19b6a2e4..cd5479e7a1 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -11,15 +11,14 @@ update/bugfix release - various enhancements, including: - add support for easystack file that contains easyconfig filenames + implement parsing of configuration options (#4021) - - skip over unset $EB_PYTHON/$EB_INSTALLPYTHON (#4080) + - skip over unset $EB_PYTHON/$EB_INSTALLPYTHON in eb wrappeer script (#4080) - add GITHUB_RELEASE and GITHUB_LOWER_RELEASE templates (#4084) - add `%(cuda_cc_cmake)s` template (#4087) - various bug fixes, including: - make check_sha256_checksums verify all checksums if they're specified as a dict value (#4076) - - replace use of symlink with copied files in `alt_location` tests (#4083) + - replace use of symlink with copied files in `alt_location` tests to fix failing EasyBuild installation on BeeGFS (#4083) - fix trying to generate RPATH wrappers for Clang (#4088) - - skip GitPython 3.1.28, since it prints 'git' as version which messes up the version check in check_github() (#4090) - - make sure that GitPython version is a proper version before checking minimal required version (#4091) + - make sure that GitPython version is a proper version before checking minimal required version (#4090, #4091) - first look for patch in alt_location when it is specified (#4093) - other changes: - make scripts executable (#4081)