From 70534b43c9f4b343939a01090bd6b736d6e8405d Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Fri, 3 May 2024 15:28:29 +0200 Subject: [PATCH 01/19] add new method tools.filetools.get_cwd to retrieve current working directory --- easybuild/tools/filetools.py | 14 ++++++++++++++ test/framework/filetools.py | 13 +++++++++++++ 2 files changed, 27 insertions(+) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index ded4d4e361..769fb0c988 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -406,6 +406,20 @@ def remove(paths): else: raise EasyBuildError("Specified path to remove is not an existing file or directory: %s", path) +def get_cwd(must_exist=True): + """ + Retrieve current working directory + """ + try: + cwd = os.getcwd() + except FileNotFoundError as err: + if must_exist is True: + raise EasyBuildError("Working directory does not exist") + else: + _log.debug("Failed to determine current working directory, but proceeding anyway: %s", err) + cwd = None + + return cwd def change_dir(path): """ diff --git a/test/framework/filetools.py b/test/framework/filetools.py index ab1b8fbf7b..bff8b70ba9 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2222,6 +2222,19 @@ def test_copy(self): self.assertTrue(os.path.isfile(os.path.join(self.test_prefix, 'GCC-4.6.3.eb'))) self.assertEqual(txt, '') + def test_get_cwd(self): + """Test get_cwd""" + toy_dir = os.path.join(self.test_prefix, "test_get_cwd_dir") + os.mkdir(toy_dir) + os.chdir(toy_dir) + + self.assertTrue(os.path.samefile(ft.get_cwd(), toy_dir)) + + os.rmdir(toy_dir) + self.assertErrorRegex(EasyBuildError, "Working directory does not exist", ft.get_cwd) + + self.assertEqual(ft.get_cwd(must_exist=False), None) + def test_change_dir(self): """Test change_dir""" From acf02e30dcaa3ccf74674f1be64bf651c4bfc967 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Fri, 3 May 2024 15:29:07 +0200 Subject: [PATCH 02/19] use tools.filetools.get_cwd in tools.filetools.change_dir --- easybuild/tools/filetools.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 769fb0c988..86db5817f8 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -428,19 +428,18 @@ def change_dir(path): :param path: location to change to :return: previous location we were in """ - # determining the current working directory can fail if we're in a non-existing directory - try: - cwd = os.getcwd() - except OSError as err: - _log.debug("Failed to determine current working directory (but proceeding anyway: %s", err) - cwd = None + # determine origin working directory: can fail if non-existent + prev_dir = get_cwd(must_exist=False) try: os.chdir(path) except OSError as err: - raise EasyBuildError("Failed to change from %s to %s: %s", cwd, path, err) + raise EasyBuildError("Failed to change from %s to %s: %s", prev_dir, path, err) - return cwd + # determine final working directory: must exist + get_cwd() + + return prev_dir def extract_file(fn, dest, cmd=None, extra_options=None, overwrite=False, forced=False, change_into_dir=False, From 912fac0c25e24e8297e2b889ad7d0f81c2d19954 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Fri, 3 May 2024 15:36:03 +0200 Subject: [PATCH 03/19] use filetools.get_cwd in filetools.find_base_dir and filetools.parse_http_header_fields_urlpat --- easybuild/tools/filetools.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 86db5817f8..142d048dad 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -684,9 +684,9 @@ def parse_http_header_fields_urlpat(arg, urlpat=None, header=None, urlpat_header if argline == '' or '#' in argline[0]: continue # permit comment lines: ignore them - if os.path.isfile(os.path.join(os.getcwd(), argline)): + if os.path.isfile(os.path.join(get_cwd(), argline)): # expand existing relative path to absolute - argline = os.path.join(os.path.join(os.getcwd(), argline)) + argline = os.path.join(os.path.join(get_cwd(), argline)) if os.path.isfile(argline): # argline is a file path, so read that instead _log.debug('File included in parse_http_header_fields_urlpat: %s' % argline) @@ -1341,14 +1341,14 @@ def get_local_dirs_purged(): # and hidden directories ignoredirs = ["easybuild"] - lst = os.listdir(os.getcwd()) + lst = os.listdir(get_cwd()) lst = [d for d in lst if not d.startswith('.') and d not in ignoredirs] return lst lst = get_local_dirs_purged() - new_dir = os.getcwd() + new_dir = get_cwd() while len(lst) == 1: - new_dir = os.path.join(os.getcwd(), lst[0]) + new_dir = os.path.join(get_cwd(), lst[0]) if not os.path.isdir(new_dir): break From 9ed35ce59089619c3d61a2e3f673c638c88a269a Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Fri, 3 May 2024 15:59:20 +0200 Subject: [PATCH 04/19] use filetools.get_cwd in EasyBlock class --- easybuild/framework/easyblock.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index bf9c939231..d95c1622b7 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -81,8 +81,8 @@ from easybuild.tools.filetools import adjust_permissions, apply_patch, back_up_file, change_dir, check_lock from easybuild.tools.filetools import compute_checksum, convert_name, copy_file, create_lock, create_patch_info from easybuild.tools.filetools import derive_alt_pypi_url, diff_files, dir_contains_files, download_file -from easybuild.tools.filetools import encode_class_name, extract_file -from easybuild.tools.filetools import find_backup_name_candidate, get_source_tarball_from_git, is_alt_pypi_url +from easybuild.tools.filetools import encode_class_name, extract_file, find_backup_name_candidate +from easybuild.tools.filetools import get_cwd, get_source_tarball_from_git, is_alt_pypi_url from easybuild.tools.filetools import is_binary, is_sha256_checksum, mkdir, move_file, move_logs, read_file, remove_dir from easybuild.tools.filetools import remove_file, remove_lock, verify_checksum, weld_paths, write_file, symlink from easybuild.tools.hooks import BUILD_STEP, CLEANUP_STEP, CONFIGURE_STEP, EXTENSIONS_STEP, FETCH_STEP, INSTALL_STEP @@ -151,7 +151,7 @@ def __init__(self, ec): """ # keep track of original working directory, so we can go back there - self.orig_workdir = os.getcwd() + self.orig_workdir = get_cwd() # dict of all hooks (mapping of name to function) self.hooks = load_hooks(build_option('hooks')) @@ -3725,7 +3725,7 @@ def cleanup_step(self): # make sure we're out of the dir we're removing change_dir(self.orig_workdir) - self.log.info("Cleaning up builddir %s (in %s)", self.builddir, os.getcwd()) + self.log.info("Cleaning up builddir %s (in %s)", self.builddir, get_cwd()) try: remove_dir(self.builddir) @@ -4249,7 +4249,7 @@ def build_and_install_one(ecdict, init_env): restore_env(init_env) sanitize_env() - cwd = os.getcwd() + cwd = get_cwd() # load easyblock easyblock = build_option('easyblock') @@ -4554,7 +4554,7 @@ def build_easyconfigs(easyconfigs, output_dir, test_results): instance = get_easyblock_instance(ec) apps.append(instance) - base_dir = os.getcwd() + base_dir = get_cwd() # keep track of environment right before initiating builds # note: may be different from ORIG_OS_ENVIRON, since EasyBuild may have defined additional env vars itself by now From 9ca0937f89f4d9c47f7594eea6060d7dbe36afc2 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Fri, 3 May 2024 16:02:40 +0200 Subject: [PATCH 05/19] use filetools.get_cwd in easyconfig.tools --- easybuild/framework/easyconfig/tools.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index c354bec3a9..2da3dc9d96 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -57,7 +57,7 @@ from easybuild.tools.build_log import EasyBuildError, print_msg, print_warning from easybuild.tools.config import build_option from easybuild.tools.environment import restore_env -from easybuild.tools.filetools import find_easyconfigs, is_patch_file, locate_files +from easybuild.tools.filetools import find_easyconfigs, get_cwd, is_patch_file, locate_files from easybuild.tools.filetools import read_file, resolve_path, which, write_file from easybuild.tools.github import GITHUB_EASYCONFIGS_REPO from easybuild.tools.github import det_pr_labels, det_pr_title, download_repo, fetch_easyconfigs_from_commit @@ -792,14 +792,13 @@ def det_copy_ec_specs(orig_paths, from_pr=None, from_commit=None): target_path, paths = None, [] - # if only one argument is specified, use current directory as target directory if len(orig_paths) == 1: - target_path = os.getcwd() + # if only one argument is specified, use current directory as target directory + target_path = get_cwd() paths = orig_paths[:] - - # if multiple arguments are specified, assume that last argument is target location, - # and remove that from list of paths to copy elif orig_paths: + # if multiple arguments are specified, assume that last argument is target location, + # and remove that from list of paths to copy target_path = orig_paths[-1] paths = orig_paths[:-1] @@ -817,7 +816,7 @@ def det_copy_ec_specs(orig_paths, from_pr=None, from_commit=None): pr_paths.extend(fetch_files_from_pr(pr=pr, path=tmpdir)) # assume that files need to be copied to current working directory for now - target_path = os.getcwd() + target_path = get_cwd() if orig_paths: last_path = orig_paths[-1] @@ -854,7 +853,7 @@ def det_copy_ec_specs(orig_paths, from_pr=None, from_commit=None): commit_paths = fetch_files_from_commit(from_commit, path=tmpdir) # assume that files need to be copied to current working directory for now - target_path = os.getcwd() + target_path = get_cwd() if orig_paths: last_path = orig_paths[-1] From 85ef1dec85f02a383cd1a6320d5cccf21ffbaf92 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Fri, 3 May 2024 16:08:06 +0200 Subject: [PATCH 06/19] use filetools.get_cwd in easyconfig.tools.run --- easybuild/tools/run.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index ca33a94a9c..3fa471dcb2 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -65,6 +65,7 @@ from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_msg, time_str_since from easybuild.tools.config import build_option +from easybuild.tools.filetools import get_cwd from easybuild.tools.hooks import RUN_SHELL_CMD, load_hooks, run_hook from easybuild.tools.utilities import trace_msg @@ -315,7 +316,7 @@ def to_cmd_str(cmd): qa_wait_patterns = [] if work_dir is None: - work_dir = os.getcwd() + work_dir = get_cwd() cmd_str = to_cmd_str(cmd) From a5e2db19c7769861c01f913dc7d7ea413fbdcd71 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Fri, 3 May 2024 16:08:30 +0200 Subject: [PATCH 07/19] use filetools.get_cwd in easybuild.tools.parallelbuild --- easybuild/tools/parallelbuild.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/parallelbuild.py b/easybuild/tools/parallelbuild.py index 3ab31e1efc..b25d8b9bdc 100644 --- a/easybuild/tools/parallelbuild.py +++ b/easybuild/tools/parallelbuild.py @@ -43,6 +43,7 @@ from easybuild.framework.easyconfig.easyconfig import ActiveMNS from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option, get_repository, get_repositorypath +from easybuild.tools.filetools import get_cwd from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version from easybuild.tools.job.backend import job_backend from easybuild.tools.repository.repository import init_repository @@ -126,7 +127,7 @@ def submit_jobs(ordered_ecs, cmd_line_opts, testing=False, prepare_first=True): :param testing: If `True`, skip actual job submission :param prepare_first: prepare by runnning fetch step first for each easyconfig """ - curdir = os.getcwd() + curdir = get_cwd() # regex pattern for options to ignore (help options can't reach here) ignore_opts = re.compile('^--robot$|^--job|^--try-.*$|^--easystack$') From b3c424dfda039d88fb17b4922457b1b1facdb348 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Fri, 3 May 2024 16:10:04 +0200 Subject: [PATCH 08/19] use filetools.get_cwd in easybuild.tools.options --- easybuild/tools/options.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 54e9b81982..8b86cc8fb2 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -84,8 +84,8 @@ from easybuild.tools.docs import avail_toolchain_opts, avail_easyconfig_params, avail_easyconfig_templates from easybuild.tools.docs import list_easyblocks, list_toolchains from easybuild.tools.environment import restore_env, unset_env_vars -from easybuild.tools.filetools import CHECKSUM_TYPE_SHA256, CHECKSUM_TYPES, expand_glob_paths, install_fake_vsc -from easybuild.tools.filetools import move_file, which +from easybuild.tools.filetools import CHECKSUM_TYPE_SHA256, CHECKSUM_TYPES, expand_glob_paths, get_cwd +from easybuild.tools.filetools import install_fake_vsc, move_file, which from easybuild.tools.github import GITHUB_PR_DIRECTION_DESC, GITHUB_PR_ORDER_CREATED from easybuild.tools.github import GITHUB_PR_STATE_OPEN, GITHUB_PR_STATES, GITHUB_PR_ORDERS, GITHUB_PR_DIRECTIONS from easybuild.tools.github import HAVE_GITHUB_API, HAVE_KEYRING, VALID_CLOSE_PR_REASONS @@ -829,7 +829,7 @@ def job_options(self): 'eb-cmd': ("EasyBuild command to use in jobs", 'str', 'store', DEFAULT_JOB_EB_CMD), 'max-jobs': ("Maximum number of concurrent jobs (queued and running, 0 = unlimited)", 'int', 'store', 0), 'max-walltime': ("Maximum walltime for jobs (in hours)", 'int', 'store', 24), - 'output-dir': ("Output directory for jobs (default: current directory)", None, 'store', os.getcwd()), + 'output-dir': ("Output directory for jobs (default: current directory)", None, 'store', get_cwd()), 'polling-interval': ("Interval between polls for status of jobs (in seconds)", float, 'store', 30.0), 'target-resource': ("Target resource for jobs", None, 'store', None), }) From a97d2aea80cee684338a96a334768958b59232ae Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Fri, 3 May 2024 16:10:59 +0200 Subject: [PATCH 09/19] use filetools.get_cwd in easybuild.tools.testing --- easybuild/tools/testing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/testing.py b/easybuild/tools/testing.py index 881788337e..359150ef6c 100644 --- a/easybuild/tools/testing.py +++ b/easybuild/tools/testing.py @@ -47,7 +47,7 @@ from easybuild.framework.easyconfig.tools import skip_available from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option -from easybuild.tools.filetools import find_easyconfigs, mkdir, read_file, write_file +from easybuild.tools.filetools import find_easyconfigs, get_cwd, mkdir, read_file, write_file from easybuild.tools.github import GITHUB_EASYBLOCKS_REPO, GITHUB_EASYCONFIGS_REPO, create_gist, post_comment_in_issue from easybuild.tools.jenkins import aggregate_xml_in_dirs from easybuild.tools.parallelbuild import build_easyconfigs_in_parallel @@ -67,7 +67,7 @@ def regtest(easyconfig_paths, modtool, build_specs=None): :param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...) """ - cur_dir = os.getcwd() + cur_dir = get_cwd() aggregate_regtest = build_option('aggregate_regtest') if aggregate_regtest is not None: From 17ea12a83f8264805ade061b836810029d6cf23f Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Fri, 3 May 2024 16:12:02 +0200 Subject: [PATCH 10/19] use filetools.get_cwd in easybuild.tools.robot --- easybuild/tools/robot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/robot.py b/easybuild/tools/robot.py index 5189769929..a5421b7225 100644 --- a/easybuild/tools/robot.py +++ b/easybuild/tools/robot.py @@ -45,7 +45,7 @@ from easybuild.framework.easyconfig.tools import find_resolved_modules, skip_available from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option -from easybuild.tools.filetools import det_common_path_prefix, search_file +from easybuild.tools.filetools import get_cwd, det_common_path_prefix, search_file from easybuild.tools.module_naming_scheme.easybuild_mns import EasyBuildMNS from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version from easybuild.tools.utilities import flatten, nub @@ -491,7 +491,7 @@ def search_easyconfigs(query, short=False, filename_only=False, terse=False, con """ search_path = build_option('robot_path') if not search_path: - search_path = [os.getcwd()] + search_path = [get_cwd()] extra_search_paths = build_option('search_paths') # If we're returning a list of possible resolutions by the robot, don't include the extra_search_paths if extra_search_paths and consider_extra_paths: From 62fc71d3fd96327f293a3d631ad2b2497d425ca2 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Fri, 3 May 2024 16:13:18 +0200 Subject: [PATCH 11/19] use filetools.get_cwd in easybuild.base.optcomplete --- easybuild/base/optcomplete.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/base/optcomplete.py b/easybuild/base/optcomplete.py index 7a46c49921..ba0f075cf2 100644 --- a/easybuild/base/optcomplete.py +++ b/easybuild/base/optcomplete.py @@ -107,6 +107,7 @@ from optparse import OptionParser, Option from pprint import pformat +from easybuild.tools.filetools import get_cwd from easybuild.tools.utilities import shell_quote debugfn = None # for debugging only @@ -537,7 +538,7 @@ def autocomplete(parser, arg_completer=None, opt_completer=None, subcmd_complete # Note: this will get filtered properly below. completer_kwargs = { - 'pwd': os.getcwd(), + 'pwd': get_cwd(), 'cline': cline, 'cpoint': cpoint, 'prefix': prefix, From 3ebc2f3490ed97bc15d27f5269f86f03f1e6823b Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Fri, 3 May 2024 16:15:17 +0200 Subject: [PATCH 12/19] use filetools.get_cwd in easybuild.tools.jobs.pbs_python --- easybuild/tools/job/pbs_python.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/job/pbs_python.py b/easybuild/tools/job/pbs_python.py index bf90b57063..31199dd1ee 100644 --- a/easybuild/tools/job/pbs_python.py +++ b/easybuild/tools/job/pbs_python.py @@ -39,6 +39,7 @@ from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError, print_msg from easybuild.tools.config import JOB_DEPS_TYPE_ABORT_ON_ERROR, JOB_DEPS_TYPE_ALWAYS_RUN, build_option +from easybuild.tools.filetools import get_cwd from easybuild.tools.job.backend import JobBackend from easybuild.tools.utilities import only_if_module_is_available @@ -320,8 +321,8 @@ def _submit(self): self.log.debug("Job hold attributes: %s" % hold_attributes[0].value) # add a bunch of variables (added by qsub) - # also set PBS_O_WORKDIR to os.getcwd() - os.environ.setdefault('WORKDIR', os.getcwd()) + # also set PBS_O_WORKDIR to current working dir + os.environ.setdefault('WORKDIR', get_cwd()) defvars = ['MAIL', 'HOME', 'PATH', 'SHELL', 'WORKDIR'] pbsvars = ["PBS_O_%s=%s" % (x, os.environ.get(x, 'NOTFOUND_%s' % x)) for x in defvars] From 9273213b96e67709906887c704eb98b8df70b133 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Fri, 3 May 2024 16:15:58 +0200 Subject: [PATCH 13/19] use filetools.get_cwd in easybuild.tools.repository.svnrepo --- easybuild/tools/repository/svnrepo.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/repository/svnrepo.py b/easybuild/tools/repository/svnrepo.py index 79f593b1ee..1e5d980f3b 100644 --- a/easybuild/tools/repository/svnrepo.py +++ b/easybuild/tools/repository/svnrepo.py @@ -46,7 +46,7 @@ from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.filetools import remove_dir +from easybuild.tools.filetools import get_cwd, remove_dir from easybuild.tools.repository.filerepo import FileRepository from easybuild.tools.utilities import only_if_module_is_available @@ -145,7 +145,7 @@ def stage_file(self, path): """ if self.client and not self.client.status(path)[0].is_versioned: # add it to version control - self.log.debug("Going to add %s (working copy: %s, cwd %s)" % (path, self.wc, os.getcwd())) + self.log.debug("Going to add %s (working copy: %s, cwd %s)" % (path, self.wc, get_cwd())) self.client.add(path) def add_easyconfig(self, cfg, name, version, stats, previous_stats): From a2fee23aa81740dd44379cd24d80d0fcbc3eaec8 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Fri, 3 May 2024 16:20:05 +0200 Subject: [PATCH 14/19] fix formatting around tools.filetools.get_cwd --- easybuild/tools/filetools.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 142d048dad..d8990560b1 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -406,6 +406,7 @@ def remove(paths): else: raise EasyBuildError("Specified path to remove is not an existing file or directory: %s", path) + def get_cwd(must_exist=True): """ Retrieve current working directory @@ -421,6 +422,7 @@ def get_cwd(must_exist=True): return cwd + def change_dir(path): """ Change to directory at specified location. From cc902931c0b59172446c952a496cd546421fe5d0 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Fri, 3 May 2024 16:28:51 +0200 Subject: [PATCH 15/19] easyconfig.tools.run is too low to use higher level method from filetools This reverts commit 85ef1dec85f02a383cd1a6320d5cccf21ffbaf92. --- easybuild/tools/run.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 3fa471dcb2..ca33a94a9c 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -65,7 +65,6 @@ from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_msg, time_str_since from easybuild.tools.config import build_option -from easybuild.tools.filetools import get_cwd from easybuild.tools.hooks import RUN_SHELL_CMD, load_hooks, run_hook from easybuild.tools.utilities import trace_msg @@ -316,7 +315,7 @@ def to_cmd_str(cmd): qa_wait_patterns = [] if work_dir is None: - work_dir = get_cwd() + work_dir = os.getcwd() cmd_str = to_cmd_str(cmd) From e6890ca7ee3be8a3ee58176f6beee14ac1533491 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Fri, 3 May 2024 16:38:43 +0200 Subject: [PATCH 16/19] improve error on FileNotFoundError on os.getcwd in run_shell_cmd --- easybuild/tools/run.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index ca33a94a9c..6a3d29972d 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -315,7 +315,10 @@ def to_cmd_str(cmd): qa_wait_patterns = [] if work_dir is None: - work_dir = os.getcwd() + try: + work_dir = os.getcwd() + except FileNotFoundError: + raise EasyBuildError("Working directory does not exist") cmd_str = to_cmd_str(cmd) From 18fef0a0d55f3f2d839910b1e3e4a48bfc815b6a Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Mon, 6 May 2024 12:17:46 +0200 Subject: [PATCH 17/19] define run.CWD_NOTFOUND_ERROR with error message for failures --- easybuild/tools/filetools.py | 11 ++++++----- easybuild/tools/run.py | 6 +++++- test/framework/filetools.py | 2 +- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index d8990560b1..133840efc2 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -66,7 +66,7 @@ from easybuild.tools.config import ERROR, GENERIC_EASYBLOCK_PKG, IGNORE, WARN, build_option, install_path from easybuild.tools.output import PROGRESS_BAR_DOWNLOAD_ONE, start_progress_bar, stop_progress_bar, update_progress_bar from easybuild.tools.hooks import load_source -from easybuild.tools.run import run_shell_cmd +from easybuild.tools.run import CWD_NOTFOUND_ERROR, run_shell_cmd from easybuild.tools.utilities import natural_keys, nub, remove_unwanted_chars, trace_msg try: @@ -415,10 +415,10 @@ def get_cwd(must_exist=True): cwd = os.getcwd() except FileNotFoundError as err: if must_exist is True: - raise EasyBuildError("Working directory does not exist") - else: - _log.debug("Failed to determine current working directory, but proceeding anyway: %s", err) - cwd = None + raise EasyBuildError(CWD_NOTFOUND_ERROR) + + _log.debug("Failed to determine current working directory, but proceeding anyway: %s", err) + cwd = None return cwd @@ -439,6 +439,7 @@ def change_dir(path): raise EasyBuildError("Failed to change from %s to %s: %s", prev_dir, path, err) # determine final working directory: must exist + # stoplight meant to catch filesystems in a faulty state get_cwd() return prev_dir diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 6a3d29972d..b8196940f4 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -83,6 +83,10 @@ "ulimit -u", # used in det_parallelism ) +CWD_NOTFOUND_ERROR = ( + "Current working directory does not exist! It was either unexpectedly removed " + "by an external process to EasyBuild or the filesystem is misbehaving." +) RunShellCmdResult = namedtuple('RunShellCmdResult', ('cmd', 'exit_code', 'output', 'stderr', 'work_dir', 'out_file', 'err_file', 'thread_id', 'task_id')) @@ -318,7 +322,7 @@ def to_cmd_str(cmd): try: work_dir = os.getcwd() except FileNotFoundError: - raise EasyBuildError("Working directory does not exist") + raise EasyBuildError(CWD_NOTFOUND_ERROR) cmd_str = to_cmd_str(cmd) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index bff8b70ba9..c6494aeb2b 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2231,7 +2231,7 @@ def test_get_cwd(self): self.assertTrue(os.path.samefile(ft.get_cwd(), toy_dir)) os.rmdir(toy_dir) - self.assertErrorRegex(EasyBuildError, "Working directory does not exist", ft.get_cwd) + self.assertErrorRegex(EasyBuildError, ft.CWD_NOTFOUND_ERROR, ft.get_cwd) self.assertEqual(ft.get_cwd(must_exist=False), None) From c3af7a8b04867c045f806fc9d42e19625f137825 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Mon, 6 May 2024 12:24:01 +0200 Subject: [PATCH 18/19] fix order of imports from filetools in tools.robot Co-authored-by: Kenneth Hoste --- easybuild/tools/robot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/robot.py b/easybuild/tools/robot.py index a5421b7225..edef824fbc 100644 --- a/easybuild/tools/robot.py +++ b/easybuild/tools/robot.py @@ -45,7 +45,7 @@ from easybuild.framework.easyconfig.tools import find_resolved_modules, skip_available from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option -from easybuild.tools.filetools import get_cwd, det_common_path_prefix, search_file +from easybuild.tools.filetools import det_common_path_prefix, get_cwd, search_file from easybuild.tools.module_naming_scheme.easybuild_mns import EasyBuildMNS from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version from easybuild.tools.utilities import flatten, nub From 27526c7dfad2d1cad2d4e42fc6af43b970f9aa15 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 22 May 2024 09:21:07 +0200 Subject: [PATCH 19/19] move CWD_NOTFOUND_ERROR constant to tools.build_log --- easybuild/tools/build_log.py | 5 +++++ easybuild/tools/filetools.py | 4 ++-- easybuild/tools/run.py | 7 +------ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/easybuild/tools/build_log.py b/easybuild/tools/build_log.py index 2e2bf726c0..4880415151 100644 --- a/easybuild/tools/build_log.py +++ b/easybuild/tools/build_log.py @@ -61,6 +61,11 @@ DRY_RUN_SOFTWARE_INSTALL_DIR = None DRY_RUN_MODULES_INSTALL_DIR = None +CWD_NOTFOUND_ERROR = ( + "Current working directory does not exist! It was either unexpectedly removed " + "by an external process to EasyBuild or the filesystem is misbehaving." +) + DEVEL_LOG_LEVEL = logging.DEBUG - 1 logging.addLevelName(DEVEL_LOG_LEVEL, 'DEVEL') diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 133840efc2..e20b876196 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -62,11 +62,11 @@ from easybuild.base import fancylogger # import build_log must stay, to use of EasyBuildLog -from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_msg, print_warning +from easybuild.tools.build_log import EasyBuildError, CWD_NOTFOUND_ERROR, dry_run_msg, print_msg, print_warning from easybuild.tools.config import ERROR, GENERIC_EASYBLOCK_PKG, IGNORE, WARN, build_option, install_path from easybuild.tools.output import PROGRESS_BAR_DOWNLOAD_ONE, start_progress_bar, stop_progress_bar, update_progress_bar from easybuild.tools.hooks import load_source -from easybuild.tools.run import CWD_NOTFOUND_ERROR, run_shell_cmd +from easybuild.tools.run import run_shell_cmd from easybuild.tools.utilities import natural_keys, nub, remove_unwanted_chars, trace_msg try: diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index b8196940f4..b8d3eb310e 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -63,7 +63,7 @@ from threading import get_ident as get_thread_id from easybuild.base import fancylogger -from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_msg, time_str_since +from easybuild.tools.build_log import EasyBuildError, CWD_NOTFOUND_ERROR, dry_run_msg, print_msg, time_str_since from easybuild.tools.config import build_option from easybuild.tools.hooks import RUN_SHELL_CMD, load_hooks, run_hook from easybuild.tools.utilities import trace_msg @@ -83,11 +83,6 @@ "ulimit -u", # used in det_parallelism ) -CWD_NOTFOUND_ERROR = ( - "Current working directory does not exist! It was either unexpectedly removed " - "by an external process to EasyBuild or the filesystem is misbehaving." -) - RunShellCmdResult = namedtuple('RunShellCmdResult', ('cmd', 'exit_code', 'output', 'stderr', 'work_dir', 'out_file', 'err_file', 'thread_id', 'task_id'))