diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 5fbbbd2ef9..a09c9f6859 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -4,6 +4,26 @@ For more detailed information, please see the git log. These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html. +v4.5.1 (December 13th 2021) +--------------------------- + +update/bugfix release + +- various enhancements, including: + - also dump environment to reprod directory (#3374) + - determine which extensions can be skipped in parallel (if --parallel-extensions-install is enabled) (#3890) + - fall back to sequential installation for extensions with unknown dependencies when using --parallel-extensions-install (#3906) + - allow oversubscription in sanity check for OpenMPI-based toolchains (#3909) +- various bug fixes, including: + - don't try to ensure absolute path for path part of repositorypath (#3893, #3899) + - fix typo in EULA agreement error message (#3894) + - only remove lock if it was created in the same EasyBuild session (not if it existed already) (#3889) + - introduce EasyBlock.post_init method to correctly define builddir variable when build-in-installdir mode is enabled in easyconfig or easyblock (#3900) + - also show download progress bar when using --inject-checksums (#3905) + - pick up custom extract_cmd specified for extension (#3907) + - make test_run_cmd_async more robust against fluke failures (#3908) + + v4.5.0 (October 29th 2021) -------------------------- diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 59bf0b7e6a..391f58e3a0 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -52,6 +52,7 @@ from distutils.version import LooseVersion import easybuild.tools.environment as env +import easybuild.tools.toolchain as toolchain from easybuild.base import fancylogger from easybuild.framework.easyconfig import EASYCONFIGS_PKG_SUBDIR from easybuild.framework.easyconfig.easyconfig import ITERATE_OPTIONS, EasyConfig, ActiveMNS, get_easyblock_class @@ -59,7 +60,7 @@ from easybuild.framework.easyconfig.format.format import SANITY_CHECK_PATHS_DIRS, SANITY_CHECK_PATHS_FILES from easybuild.framework.easyconfig.parser import fetch_parameters_from_easyconfig from easybuild.framework.easyconfig.style import MAX_LINE_LENGTH -from easybuild.framework.easyconfig.tools import get_paths_for +from easybuild.framework.easyconfig.tools import dump_env_easyblock, get_paths_for from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP, template_constant_dict from easybuild.framework.extension import Extension, resolve_exts_filter_template from easybuild.tools import config, run @@ -83,7 +84,7 @@ from easybuild.tools.hooks import MODULE_STEP, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTITER_STEP, POSTPROC_STEP from easybuild.tools.hooks import PREPARE_STEP, READY_STEP, SANITYCHECK_STEP, SOURCE_STEP, TEST_STEP, TESTCASES_STEP from easybuild.tools.hooks import MODULE_WRITE, load_hooks, run_hook -from easybuild.tools.run import run_cmd +from easybuild.tools.run import check_async_cmd, run_cmd from easybuild.tools.jenkins import write_to_xml from easybuild.tools.module_generator import ModuleGeneratorLua, ModuleGeneratorTcl, module_generator, dependencies_for from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version @@ -276,6 +277,16 @@ def __init__(self, ec): self.log.info("Init completed for application name %s version %s" % (self.name, self.version)) + def post_init(self): + """ + Run post-initialization tasks. + """ + if self.build_in_installdir: + # self.builddir is set by self.gen_builddir(), + # but needs to be correct if the build is performed in the installation directory + self.log.info("Changing build dir to %s", self.installdir) + self.builddir = self.installdir + # INIT/CLOSE LOG def _init_log(self): """ @@ -572,8 +583,12 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True): if fetch_files: src = self.fetch_source(source, checksums, extension=True) - # copy 'path' entry to 'src' for use with extensions - ext_src.update({'src': src['path']}) + ext_src.update({ + # keep track of custom extract command (if any) + 'extract_cmd': src['cmd'], + # copy 'path' entry to 'src' for use with extensions + 'src': src['path'], + }) else: # use default template for name of source file if none is specified @@ -941,9 +956,6 @@ def make_builddir(self): raise EasyBuildError("self.builddir not set, make sure gen_builddir() is called first!") self.log.debug("Creating the build directory %s (cleanup: %s)", self.builddir, self.cfg['cleanupoldbuild']) else: - self.log.info("Changing build dir to %s" % self.installdir) - self.builddir = self.installdir - self.log.info("Overriding 'cleanupoldinstall' (to False), 'cleanupoldbuild' (to True) " "and 'keeppreviousinstall' because we're building in the installation directory.") # force cleanup before installation @@ -1612,9 +1624,10 @@ def prepare_for_extensions(self): def skip_extensions(self): """ - Called when self.skip is True - - use this to detect existing extensions and to remove them from self.ext_instances - - based on initial R version + Skip already installed extensions, + by removing them from list of Extension instances to install (self.ext_instances). + + This is done in parallel when EasyBuild is configured to install extensions in parallel. """ self.update_exts_progress_bar("skipping installed extensions") @@ -1624,25 +1637,91 @@ def skip_extensions(self): if not exts_filter or len(exts_filter) == 0: raise EasyBuildError("Skipping of extensions, but no exts_filter set in easyconfig") + if build_option('parallel_extensions_install'): + self.skip_extensions_parallel(exts_filter) + else: + self.skip_extensions_sequential(exts_filter) + + def skip_extensions_sequential(self, exts_filter): + """ + Skip already installed extensions (checking sequentially), + by removing them from list of Extension instances to install (self.ext_instances). + """ + print_msg("skipping installed extensions (sequentially)", log=self.log) + exts_cnt = len(self.ext_instances) res = [] for idx, ext_inst in enumerate(self.ext_instances): cmd, stdin = resolve_exts_filter_template(exts_filter, ext_inst) - (cmdstdouterr, ec) = run_cmd(cmd, log_all=False, log_ok=False, simple=False, inp=stdin, regexp=False) - self.log.info("exts_filter result %s %s", cmdstdouterr, ec) - if ec: + (out, ec) = run_cmd(cmd, log_all=False, log_ok=False, simple=False, inp=stdin, + regexp=False, trace=False) + self.log.info("exts_filter result for %s: exit code %s; output: %s", ext_inst.name, ec, out) + if ec == 0: + print_msg("skipping extension %s" % ext_inst.name, silent=self.silent, log=self.log) + else: self.log.info("Not skipping %s", ext_inst.name) - self.log.debug("exit code: %s, stdout/err: %s", ec, cmdstdouterr) res.append(ext_inst) - else: - print_msg("skipping extension %s" % ext_inst.name, silent=self.silent, log=self.log) self.update_exts_progress_bar("skipping installed extensions (%d/%d checked)" % (idx + 1, exts_cnt)) self.ext_instances = res self.update_exts_progress_bar("already installed extensions filtered out", total=len(self.ext_instances)) + def skip_extensions_parallel(self, exts_filter): + """ + Skip already installed extensions (checking in parallel), + by removing them from list of Extension instances to install (self.ext_instances). + """ + self.log.experimental("Skipping installed extensions in parallel") + print_msg("skipping installed extensions (in parallel)", log=self.log) + + async_cmd_info_cache = {} + running_checks_ids = [] + installed_exts_ids = [] + exts_queue = list(enumerate(self.ext_instances[:])) + checked_exts_cnt = 0 + exts_cnt = len(self.ext_instances) + + # asynchronously run checks to see whether extensions are already installed + while exts_queue or running_checks_ids: + + # first handle completed checks + for idx in running_checks_ids[:]: + ext_name = self.ext_instances[idx].name + # don't read any output, just check whether command completed + async_cmd_info = check_async_cmd(*async_cmd_info_cache[idx], output_read_size=0, fail_on_error=False) + if async_cmd_info['done']: + out, ec = async_cmd_info['output'], async_cmd_info['exit_code'] + self.log.info("exts_filter result for %s: exit code %s; output: %s", ext_name, ec, out) + running_checks_ids.remove(idx) + if ec == 0: + print_msg("skipping extension %s" % ext_name, log=self.log) + installed_exts_ids.append(idx) + + checked_exts_cnt += 1 + exts_pbar_label = "skipping installed extensions " + exts_pbar_label += "(%d/%d checked)" % (checked_exts_cnt, exts_cnt) + self.update_exts_progress_bar(exts_pbar_label) + + # start additional checks asynchronously + while exts_queue and len(running_checks_ids) < self.cfg['parallel']: + idx, ext = exts_queue.pop(0) + cmd, stdin = resolve_exts_filter_template(exts_filter, ext) + async_cmd_info_cache[idx] = run_cmd(cmd, log_all=False, log_ok=False, simple=False, inp=stdin, + regexp=False, trace=False, asynchronous=True) + running_checks_ids.append(idx) + + # compose new list of extensions, skip over the ones that are already installed; + # note: original order in extensions list should be preserved! + retained_ext_instances = [] + for idx, ext in enumerate(self.ext_instances): + if idx not in installed_exts_ids: + retained_ext_instances.append(ext) + self.log.info("Not skipping %s", ext.name) + + self.ext_instances = retained_ext_instances + def install_extensions(self, install=True): """ Install extensions. @@ -1785,7 +1864,26 @@ def update_exts_progress_bar_helper(running_exts, progress_size): # check whether extension at top of the queue is ready to install ext = exts_queue.pop(0) - pending_deps = [x for x in ext.required_deps if x not in installed_ext_names] + required_deps = ext.required_deps + if required_deps is None: + pending_deps = None + self.log.info("Required dependencies for %s are unknown!", ext.name) + else: + self.log.info("Required dependencies for %s: %s", ext.name, ', '.join(required_deps)) + pending_deps = [x for x in required_deps if x not in installed_ext_names] + self.log.info("Missing required dependencies for %s: %s", ext.name, ', '.join(pending_deps)) + + # if required dependencies could not be determined, wait until all preceding extensions are installed + if pending_deps is None: + if running_exts: + # add extension back at top of the queue, + # since we need to preverse installation order of extensions; + # break out of for loop since there is no point to keep checking + # until running installations have been completed + exts_queue.insert(0, ext) + break + else: + pending_deps = [] if self.dry_run: tup = (ext.name, ext.version, ext.__class__.__name__) @@ -1798,7 +1896,7 @@ def update_exts_progress_bar_helper(running_exts, progress_size): # make sure all required dependencies are actually going to be installed, # to avoid getting stuck in an infinite loop! - missing_deps = [x for x in ext.required_deps if x not in all_ext_names] + missing_deps = [x for x in required_deps if x not in all_ext_names] if missing_deps: raise EasyBuildError("Missing required dependencies for %s are not going to be installed: %s", ext.name, ', '.join(missing_deps)) @@ -1895,7 +1993,7 @@ def check_accepted_eula(self, name=None, more_info=None): self.log.info("EULA for %s is accepted", name) else: error_lines = [ - "The End User License Argreement (EULA) for %(name)s is currently not accepted!", + "The End User License Agreement (EULA) for %(name)s is currently not accepted!", ] if more_info: error_lines.append("(see %s for more information)" % more_info) @@ -3266,7 +3364,12 @@ def xs2str(xs): if extra_modules: self.log.info("Loading extra modules for sanity check: %s", ', '.join(extra_modules)) - # chdir to installdir (better environment for running tests) + # allow oversubscription of P processes on C cores (P>C) for software installed on top of Open MPI; + # this is useful to avoid failing of sanity check commands that involve MPI + if self.toolchain.mpi_family() and self.toolchain.mpi_family() in toolchain.OPENMPI: + env.setvar('OMPI_MCA_rmaps_base_oversubscribe', '1') + + # change to install directory (better environment for running tests) if os.path.isdir(self.installdir): change_dir(self.installdir) @@ -3756,6 +3859,7 @@ def run_all_steps(self, run_test_cases): ignore_locks = build_option('ignore_locks') + lock_created = False try: if ignore_locks: self.log.info("Ignoring locks...") @@ -3768,6 +3872,10 @@ def run_all_steps(self, run_test_cases): # create lock to avoid that another installation running in parallel messes things up create_lock(lock_name) + lock_created = True + + # run post-initialization tasks first, before running any steps + self.post_init() for step_name, descr, step_methods, skippable in steps: if self.skip_step(step_name, skippable): @@ -3797,7 +3905,8 @@ def run_all_steps(self, run_test_cases): except StopException: pass finally: - if not ignore_locks: + # remove lock, but only if it was created in this session (not if it was there already) + if lock_created: remove_lock(lock_name) stop_progress_bar(PROGRESS_BAR_EASYCONFIG) @@ -3902,6 +4011,14 @@ def build_and_install_one(ecdict, init_env): result = app.run_all_steps(run_test_cases=run_test_cases) if not dry_run: + # Copy over the build environment used during the configuraton + reprod_spec = os.path.join(reprod_dir, app.cfg.filename()) + try: + dump_env_easyblock(app, ec_path=reprod_spec, silent=True) + _log.debug("Created build environment dump for easyconfig %s", reprod_spec) + except EasyBuildError as err: + _log.warning("Failed to create build environment dump for easyconfig %s: %s", reprod_spec, err) + # also add any extension easyblocks used during the build for reproducibility if app.ext_instances: copy_easyblocks_for_reprod(app.ext_instances, reprod_dir) @@ -4080,9 +4197,9 @@ def copy_easyblocks_for_reprod(easyblock_instances, reprod_dir): for easyblock_instance in easyblock_instances: for easyblock_class in inspect.getmro(type(easyblock_instance)): easyblock_path = inspect.getsourcefile(easyblock_class) - # if we reach EasyBlock or ExtensionEasyBlock class, we are done - # (ExtensionEasyblock is hardcoded to avoid a cyclical import) - if easyblock_class.__name__ in [EasyBlock.__name__, 'ExtensionEasyBlock']: + # if we reach EasyBlock, Extension or ExtensionEasyBlock class, we are done + # (Extension and ExtensionEasyblock are hardcoded to avoid a cyclical import) + if easyblock_class.__name__ in [EasyBlock.__name__, 'Extension', 'ExtensionEasyBlock']: break else: easyblock_paths.add(easyblock_path) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 595481d74a..be6ff5fe01 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -557,6 +557,45 @@ def review_pr(paths=None, pr=None, colored=True, branch='develop', testing=False return '\n'.join(lines) +def dump_env_easyblock(app, orig_env=None, ec_path=None, script_path=None, silent=False): + if orig_env is None: + orig_env = copy.deepcopy(os.environ) + if ec_path is None: + raise EasyBuildError("The path to the easyconfig relevant to this environment dump is required") + if script_path is None: + # Assume we are placing it alongside the easyconfig path + script_path = '%s.env' % os.path.splitext(ec_path)[0] + # Compose script + ecfile = os.path.basename(ec_path) + script_lines = [ + "#!/bin/bash", + "# script to set up build environment as defined by EasyBuild v%s for %s" % (EASYBUILD_VERSION, ecfile), + "# usage: source %s" % os.path.basename(script_path), + ] + + script_lines.extend(['', "# toolchain & dependency modules"]) + if app.toolchain.modules: + script_lines.extend(["module load %s" % mod for mod in app.toolchain.modules]) + else: + script_lines.append("# (no modules loaded)") + + script_lines.extend(['', "# build environment"]) + if app.toolchain.vars: + env_vars = sorted(app.toolchain.vars.items()) + script_lines.extend(["export %s='%s'" % (var, val.replace("'", "\\'")) for (var, val) in env_vars]) + else: + script_lines.append("# (no build environment defined)") + + write_file(script_path, '\n'.join(script_lines)) + msg = "Script to set up build environment for %s dumped to %s" % (ecfile, script_path) + if silent: + _log.info(msg) + else: + print_msg(msg, prefix=False) + + restore_env(orig_env) + + def dump_env_script(easyconfigs): """ Dump source scripts that set up build environment for specified easyconfigs. @@ -591,31 +630,8 @@ def dump_env_script(easyconfigs): app.check_readiness_step() app.prepare_step(start_dir=False) - # compose script - ecfile = os.path.basename(ec.path) - script_lines = [ - "#!/bin/bash", - "# script to set up build environment as defined by EasyBuild v%s for %s" % (EASYBUILD_VERSION, ecfile), - "# usage: source %s" % os.path.basename(script_path), - ] - - script_lines.extend(['', "# toolchain & dependency modules"]) - if app.toolchain.modules: - script_lines.extend(["module load %s" % mod for mod in app.toolchain.modules]) - else: - script_lines.append("# (no modules loaded)") - - script_lines.extend(['', "# build environment"]) - if app.toolchain.vars: - env_vars = sorted(app.toolchain.vars.items()) - script_lines.extend(["export %s='%s'" % (var, val.replace("'", "\\'")) for (var, val) in env_vars]) - else: - script_lines.append("# (no build environment defined)") - - write_file(script_path, '\n'.join(script_lines)) - print_msg("Script to set up build environment for %s dumped to %s" % (ecfile, script_path), prefix=False) - - restore_env(orig_env) + # create the environment dump + dump_env_easyblock(app, orig_env=orig_env, ec_path=ec.path, script_path=script_path) def categorize_files_by_type(paths): diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index 5333627c29..e85b94daec 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -119,6 +119,7 @@ def __init__(self, mself, ext, extra_params=None): # list of source/patch files: we use an empty list as default value like in EasyBlock self.src = resolve_template(self.ext.get('src', []), self.cfg.template_values) + self.src_extract_cmd = self.ext.get('extract_cmd', None) self.patches = resolve_template(self.ext.get('patches', []), self.cfg.template_values) self.options = resolve_template(copy.deepcopy(self.ext.get('options', {})), self.cfg.template_values) @@ -225,7 +226,8 @@ def async_cmd_check(self): @property def required_deps(self): """Return list of required dependencies for this extension.""" - raise NotImplementedError("Don't know how to determine required dependencies for extension '%s'" % self.name) + self.log.info("Don't know how to determine required dependencies for extension '%s'", self.name) + return None @property def toolchain(self): diff --git a/easybuild/framework/extensioneasyblock.py b/easybuild/framework/extensioneasyblock.py index c3b5c7a9fb..3c427c5a9f 100644 --- a/easybuild/framework/extensioneasyblock.py +++ b/easybuild/framework/extensioneasyblock.py @@ -118,7 +118,7 @@ def run(self, unpack_src=False): if unpack_src: targetdir = os.path.join(self.master.builddir, remove_unwanted_chars(self.name)) self.ext_dir = extract_file(self.src, targetdir, extra_options=self.unpack_options, - change_into_dir=False) + change_into_dir=False, cmd=self.src_extract_cmd) # setting start dir must be done from unpacked source directory for extension, # because start_dir value is usually a relative path (if it is set) diff --git a/easybuild/main.py b/easybuild/main.py index a6680d83ea..2203636a3b 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -531,7 +531,8 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): dump_env_script(easyconfigs) elif options.inject_checksums: - inject_checksums(ordered_ecs, options.inject_checksums) + with rich_live_cm(): + inject_checksums(ordered_ecs, options.inject_checksums) # cleanup and exit after dry run, searching easyconfigs or submitting regression test stop_options = [options.check_conflicts, dry_run_mode, options.dump_env_script, options.inject_checksums] diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 99546c329d..bde3d1156f 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -456,11 +456,14 @@ def extract_file(fn, dest, cmd=None, extra_options=None, overwrite=False, forced _log.debug("Unpacking %s in directory %s", fn, abs_dest) cwd = change_dir(abs_dest) - if not cmd: - cmd = extract_cmd(fn, overwrite=overwrite) - else: + if cmd: # complete command template with filename cmd = cmd % fn + _log.debug("Using specified command to unpack %s: %s", fn, cmd) + else: + cmd = extract_cmd(fn, overwrite=overwrite) + _log.debug("Using command derived from file extension to unpack %s: %s", fn, cmd) + if not cmd: raise EasyBuildError("Can't extract file %s with unknown filetype", fn) @@ -1366,7 +1369,7 @@ def find_extension(filename): if res: ext = res.group('ext') else: - raise EasyBuildError('Unknown file type for file %s', filename) + raise EasyBuildError("%s has unknown file extension", filename) return ext @@ -1379,7 +1382,9 @@ def extract_cmd(filepath, overwrite=False): ext = find_extension(filename) target = filename[:-len(ext)] + # find_extension will either return an extension listed in EXTRACT_CMDS, or raise an error cmd_tmpl = EXTRACT_CMDS[ext.lower()] + if overwrite: if 'unzip -qq' in cmd_tmpl: cmd_tmpl = cmd_tmpl.replace('unzip -qq', 'unzip -qq -o') diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index e0d7b47324..6d123508a2 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -1067,7 +1067,7 @@ def _postprocess_checks(self): def get_cfg_opt_abs_path(self, opt_name, path): """Get path value of configuration option as absolute path.""" - if os.path.isabs(path): + if os.path.isabs(path) or path.startswith('git@'): abs_path = path else: abs_path = os.path.abspath(path) @@ -1092,22 +1092,20 @@ def _ensure_abs_path(self, opt_name): def _postprocess_config(self): """Postprocessing of configuration options""" - # resolve relative paths for configuration options that specify a location; + # resolve relative paths for configuration options that specify a location, + # to avoid incorrect paths being used when EasyBuild changes the current working directory + # (see https://github.com/easybuilders/easybuild-framework/issues/3619); # ensuring absolute paths for 'robot' is handled separately below, - # because we need to be careful with the argument pass to --robot + # because we need to be careful with the argument pass to --robot; + # note: repositorypath is purposely not listed here, because it's a special case: + # - the value could consist of a 2-tuple (, ); + # - the could also specify the location of a *remote* (Git( repository, + # which can be done in variety of formats (git@:/), https://, etc.) + # (see also https://github.com/easybuilders/easybuild-framework/issues/3892); path_opt_names = ['buildpath', 'containerpath', 'git_working_dirs_path', 'installpath', 'installpath_modules', 'installpath_software', 'prefix', 'packagepath', 'robot_paths', 'sourcepath'] - # repositorypath is a special case: only first part is a path; - # 2nd (optional) part is a relative subdir and should not be resolved to an absolute path! - repositorypath = self.options.repositorypath - if isinstance(repositorypath, (list, tuple)) and len(repositorypath) == 2: - abs_path = self.get_cfg_opt_abs_path('repositorypath', repositorypath[0]) - self.options.repositorypath = (abs_path, repositorypath[1]) - else: - path_opt_names.append('repositorypath') - for opt_name in path_opt_names: self._ensure_abs_path(opt_name) diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index 3aeeffa234..d05df8a975 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.5.0') +VERSION = LooseVersion('4.5.1') UNKNOWN = 'UNKNOWN' diff --git a/test/framework/config.py b/test/framework/config.py index 0c4489a412..cb13d348a5 100644 --- a/test/framework/config.py +++ b/test/framework/config.py @@ -302,7 +302,7 @@ def test_generaloption_config_file(self): self.assertEqual(install_path('mod'), installpath_modules), # via config file self.assertEqual(source_paths(), [testpath2]) # via command line self.assertEqual(build_path(), testpath1) # via config file - self.assertEqual(get_repositorypath(), (os.path.join(topdir, 'ebfiles_repo'), 'somesubdir')) # via config file + self.assertEqual(get_repositorypath(), [os.path.join(topdir, 'ebfiles_repo'), 'somesubdir']) # via config file # hardcoded first entry self.assertEqual(options.robot_paths[0], '/tmp/foo') diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index c62b767069..d0718ac9ff 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1335,8 +1335,7 @@ def test_buildininstalldir(self): self.prep() ec = EasyConfig(self.eb_file) eb = EasyBlock(ec) - eb.gen_builddir() - eb.gen_installdir() + eb.post_init() eb.make_builddir() eb.make_installdir() self.assertEqual(eb.builddir, eb.installdir) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 8c54f5881f..57f5d18975 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -114,6 +114,9 @@ def test_extract_cmd(self): self.assertEqual("unzip -qq -o test.zip", ft.extract_cmd('test.zip', True)) + error_pattern = "test.foo has unknown file extension" + self.assertErrorRegex(EasyBuildError, error_pattern, ft.extract_cmd, 'test.foo') + def test_find_extension(self): """Test find_extension function.""" tests = [ diff --git a/test/framework/options.py b/test/framework/options.py index 4823d16f1e..41b29f0dec 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -4854,8 +4854,8 @@ def test_prefix_option(self): expected = ['buildpath', 'containerpath', 'installpath', 'packagepath', 'prefix', 'repositorypath'] self.assertEqual(sorted(regex.findall(txt)), expected) - def test_dump_env_config(self): - """Test for --dump-env-config.""" + def test_dump_env_script(self): + """Test for --dump-env-script.""" fftw = 'FFTW-3.3.7-gompic-2018a' gcc = 'GCC-4.9.2' @@ -6037,6 +6037,53 @@ def test_sanity_check_only(self): stdout = self.mocked_main(args + ['--trace'], do_build=True, raise_error=True, testing=False) + # check whether %(builddir)s value is correct + # when buildininstalldir is enabled in easyconfig and --sanity-check-only is used + # (see https://github.com/easybuilders/easybuild-framework/issues/3895) + test_ec_txt += '\n' + '\n'.join([ + "buildininstalldir = True", + "sanity_check_commands = [", + # build and install directory should be the same path + " 'test %(builddir)s = %(installdir)s',", + # build/install directory must exist (even though step that creates build dir was never run) + " 'test -d %(builddir)s',", + "]", + ]) + write_file(test_ec, test_ec_txt) + self.eb_main(args, do_build=True, raise_error=True) + + # also check when using easyblock that enables build_in_installdir in its constructor + test_ebs = os.path.join(topdir, 'sandbox', 'easybuild', 'easyblocks') + toy_eb = os.path.join(test_ebs, 't', 'toy.py') + toy_eb_txt = read_file(toy_eb) + + self.assertFalse('self.build_in_installdir = True' in toy_eb_txt) + + regex = re.compile(r'^(\s+)(super\(EB_toy, self\).__init__.*)\n', re.M) + toy_eb_txt = regex.sub(r'\1\2\n\1self.build_in_installdir = True', toy_eb_txt) + self.assertTrue('self.build_in_installdir = True' in toy_eb_txt) + + toy_eb = os.path.join(self.test_prefix, 'toy.py') + write_file(toy_eb, toy_eb_txt) + + test_ec_txt = test_ec_txt.replace('buildininstalldir = True', '') + write_file(test_ec, test_ec_txt) + + orig_local_sys_path = sys.path[:] + args.append('--include-easyblocks=%s' % toy_eb) + self.eb_main(args, do_build=True, raise_error=True) + + # undo import of the toy easyblock, to avoid problems with other tests + del sys.modules['easybuild.easyblocks.toy'] + sys.path = orig_local_sys_path + import easybuild.easyblocks + reload(easybuild.easyblocks) + import easybuild.easyblocks.toy + reload(easybuild.easyblocks.toy) + # need to reload toy_extension, which imports EB_toy, to ensure right EB_toy is picked up in later tests + import easybuild.easyblocks.generic.toy_extension + reload(easybuild.easyblocks.generic.toy_extension) + def test_skip_extensions(self): """Test use of --skip-extensions.""" topdir = os.path.abspath(os.path.dirname(__file__)) @@ -6242,7 +6289,7 @@ def test_accept_eula_for(self): # by default, no EULAs are accepted at all args = [test_ec, '--force'] - error_pattern = r"The End User License Argreement \(EULA\) for toy is currently not accepted!" + error_pattern = r"The End User License Agreement \(EULA\) for toy is currently not accepted!" self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, do_build=True, raise_error=True) toy_modfile = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0') if get_module_syntax() == 'Lua': @@ -6334,7 +6381,7 @@ def test_config_abs_path(self): r"^containerpath\s+\(F\) = /.*/test_topdir/test_middle_dir$", r"^installpath\s+\(E\) = /.*/test_topdir$", r"^prefix\s+\(C\) = /.*/test_topdir/test_middle_dir$", - r"^repositorypath\s+\(F\) = \('/apps/easyconfigs_archive', ' somesubdir'\)$", + r"^repositorypath\s+\(F\) = /apps/easyconfigs_archive,\s+somesubdir$", r"^sourcepath\s+\(C\) = /.*/test_topdir/test_middle_dir/test_subdir$", r"^robot-paths\s+\(E\) = /.*/test_topdir$", ] @@ -6361,6 +6408,25 @@ def test_config_abs_path(self): regex = re.compile(pattern, re.M) self.assertTrue(regex.search(txt), "Pattern '%s' should be found in: %s" % (pattern, txt)) + def test_config_repositorypath(self): + """Test how special repositorypath values are handled.""" + + repositorypath = 'git@github.com:boegel/my_easyconfigs.git' + args = [ + '--repositorypath=%s' % repositorypath, + '--show-config', + ] + txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False, strip=True) + + regex = re.compile(r'repositorypath\s+\(C\) = %s' % repositorypath, re.M) + self.assertTrue(regex.search(txt), "Pattern '%s' should be found in: %s" % (regex.pattern, txt)) + + args[0] = '--repositorypath=%s,some/subdir' % repositorypath + txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False, strip=True) + + regex = re.compile(r"repositorypath\s+\(C\) = %s, some/subdir" % repositorypath, re.M) + self.assertTrue(regex.search(txt), "Pattern '%s' should be found in: %s" % (regex.pattern, txt)) + # end-to-end testing of unknown filename def test_easystack_wrong_read(self): """Test for --easystack when wrong name is provided""" diff --git a/test/framework/run.py b/test/framework/run.py index 6c298890d5..a821a67d3c 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -605,7 +605,9 @@ def test_run_cmd_async(self): self.assertEqual(res, {'done': False, 'exit_code': None, 'output': 'sleeping...\n'}) # 2nd check with default output size (1024) gets full output - res = check_async_cmd(*cmd_info, output=res['output']) + # (keep checking until command is fully done) + while not res['done']: + res = check_async_cmd(*cmd_info, output=res['output']) self.assertEqual(res, {'done': True, 'exit_code': 0, 'output': 'sleeping...\ntest123\n'}) # check asynchronous running of failing command diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 5b75116ee1..3f46f2f4c3 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1362,6 +1362,31 @@ def test_toy_extension_sources(self): write_file(test_ec, test_ec_txt) self.test_toy_build(ec_file=test_ec, raise_error=True) + def test_toy_extension_extract_cmd(self): + """Test for custom extract_cmd specified for an extension.""" + test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') + toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') + toy_ec_txt = read_file(toy_ec) + + test_ec = os.path.join(self.test_prefix, 'test.eb') + test_ec_txt = '\n'.join([ + toy_ec_txt, + 'exts_list = [', + ' ("bar", "0.0", {', + # deliberately incorrect custom extract command, just to verify that it's picked up + ' "sources": [{', + ' "filename": "bar-%(version)s.tar.gz",', + ' "extract_cmd": "unzip %s",', + ' }],', + ' }),', + ']', + ]) + write_file(test_ec, test_ec_txt) + + error_pattern = "unzip .*/bar-0.0.tar.gz.* exited with exit code [1-9]" + self.assertErrorRegex(EasyBuildError, error_pattern, self.test_toy_build, ec_file=test_ec, + raise_error=True, verbose=False) + def test_toy_extension_sources_git_config(self): """Test install toy that includes extensions with 'sources' spec including 'git_config'.""" test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') @@ -1804,7 +1829,7 @@ def test_toy_exts_parallel(self): ]) write_file(test_ec, test_ec_txt) - args = ['--parallel-extensions-install', '--experimental', '--force'] + args = ['--parallel-extensions-install', '--experimental', '--force', '--parallel=3'] stdout, stderr = self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args) self.assertEqual(stderr, '') expected_stdout = '\n'.join([ @@ -1816,6 +1841,46 @@ def test_toy_exts_parallel(self): ]) self.assertEqual(stdout, expected_stdout) + # also test skipping of extensions in parallel + args.append('--skip') + stdout, stderr = self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args) + self.assertEqual(stderr, '') + + # order in which these patterns occur is not fixed, so check them one by one + patterns = [ + r"^== skipping installed extensions \(in parallel\)$", + r"^== skipping extension ls$", + r"^== skipping extension bar$", + r"^== skipping extension barbar$", + r"^== skipping extension toy$", + ] + for pattern in patterns: + regex = re.compile(pattern, re.M) + error_msg = "Expected pattern '%s' should be found in %s'" % (regex.pattern, stdout) + self.assertTrue(regex.search(stdout), error_msg) + + # check behaviour when using Toy_Extension easyblock that doesn't implement required_deps method; + # framework should fall back to installing extensions sequentially + toy_ext_eb = os.path.join(topdir, 'sandbox', 'easybuild', 'easyblocks', 'generic', 'toy_extension.py') + copy_file(toy_ext_eb, self.test_prefix) + toy_ext_eb = os.path.join(self.test_prefix, 'toy_extension.py') + toy_ext_eb_txt = read_file(toy_ext_eb) + toy_ext_eb_txt = toy_ext_eb_txt.replace('def required_deps', 'def xxx_required_deps') + write_file(toy_ext_eb, toy_ext_eb_txt) + + args[-1] = '--include-easyblocks=%s' % toy_ext_eb + stdout, stderr = self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args) + self.assertEqual(stderr, '') + expected_stdout = '\n'.join([ + "== 0 out of 4 extensions installed (3 queued, 1 running: ls)", + "== 1 out of 4 extensions installed (2 queued, 1 running: bar)", + "== 2 out of 4 extensions installed (1 queued, 1 running: barbar)", + "== 3 out of 4 extensions installed (0 queued, 1 running: toy)", + "== 4 out of 4 extensions installed (0 queued, 0 running: )", + '', + ]) + self.assertEqual(stdout, expected_stdout) + def test_backup_modules(self): """Test use of backing up of modules with --module-only.""" @@ -2065,6 +2130,23 @@ def test_reproducibility(self): self.assertTrue(os.path.exists(reprod_ec)) + # Also check that the dumpenv script is placed alongside it + dumpenv_script = '%s.env' % os.path.splitext(reprod_ec)[0] + reprod_dumpenv = os.path.join(reprod_dir, dumpenv_script) + self.assertTrue(os.path.exists(reprod_dumpenv)) + + # Check contents of the dumpenv script + patterns = [ + """#!/bin/bash""", + """# usage: source toy-0.0.env""", + # defining build env + """# (no modules loaded)""", + """# (no build environment defined)""", + ] + env_file = open(reprod_dumpenv, "r").read() + for pattern in patterns: + self.assertTrue(pattern in env_file) + # Check that the toytoy easyblock is recorded in the reprod easyconfig ec = EasyConfig(reprod_ec) self.assertEqual(ec.parser.get_config_dict()['easyblock'], 'EB_toytoy') @@ -2163,6 +2245,24 @@ def test_toy_toy(self): load1_regex = re.compile('load.*toy/0.0-one', re.M) self.assertTrue(load1_regex.search(mod2_txt), "Pattern '%s' found in: %s" % (load1_regex.pattern, mod2_txt)) + # Check the contents of the dumped env in the reprod dir to ensure it contains the dependency load + reprod_dir = os.path.join(self.test_installpath, 'software', 'toy', '0.0-two', 'easybuild', 'reprod') + dumpenv_script = os.path.join(reprod_dir, 'toy-0.0-two.env') + reprod_dumpenv = os.path.join(reprod_dir, dumpenv_script) + self.assertTrue(os.path.exists(reprod_dumpenv)) + + # Check contents of the dumpenv script + patterns = [ + """#!/bin/bash""", + """# usage: source toy-0.0-two.env""", + # defining build env + """module load toy/0.0-one""", + """# (no build environment defined)""", + ] + env_file = open(reprod_dumpenv, "r").read() + for pattern in patterns: + self.assertTrue(pattern in env_file) + def test_toy_sanity_check_commands(self): """Test toy build with extra sanity check commands.""" @@ -3198,6 +3298,13 @@ def test_toy_build_lock(self): error_pattern = "Lock .*_software_toy_0.0.lock already exists, aborting!" self.assertErrorRegex(EasyBuildError, error_pattern, self.test_toy_build, raise_error=True, verbose=False) + # lock should still be there after it was hit + self.assertTrue(os.path.exists(toy_lock_path)) + + # trying again should give same result + self.assertErrorRegex(EasyBuildError, error_pattern, self.test_toy_build, raise_error=True, verbose=False) + self.assertTrue(os.path.exists(toy_lock_path)) + locks_dir = os.path.join(self.test_prefix, 'locks') # no lock in place, so installation proceeds as normal @@ -3216,7 +3323,7 @@ def test_toy_build_lock(self): orig_sigalrm_handler = signal.getsignal(signal.SIGALRM) # define a context manager that remove a lock after a while, so we can check the use of --wait-for-lock - class remove_lock_after(object): + class RemoveLockAfter(object): def __init__(self, seconds, lock_fp): self.seconds = seconds self.lock_fp = lock_fp @@ -3264,7 +3371,7 @@ def __exit__(self, type, value, traceback): all_args = extra_args + opts # use context manager to remove lock after 3 seconds - with remove_lock_after(3, toy_lock_path): + with RemoveLockAfter(3, toy_lock_path): self.mock_stderr(True) self.mock_stdout(True) self.test_toy_build(extra_args=all_args, verify=False, raise_error=True, testing=False) @@ -3332,7 +3439,7 @@ def test_toy_lock_cleanup_signals(self): orig_sigalrm_handler = signal.getsignal(signal.SIGALRM) # context manager which stops the function being called with the specified signal - class wait_and_signal(object): + class WaitAndSignal(object): def __init__(self, seconds, signum): self.seconds = seconds self.signum = signum @@ -3367,7 +3474,7 @@ def __exit__(self, type, value, traceback): # avoid recycling stderr of previous test stderr = '' - with wait_and_signal(1, signum): + with WaitAndSignal(1, signum): # change back to original working directory before each test change_dir(orig_wd)