From 26016cd5e93aa41b61cc6557ec03b6f9cdd934f9 Mon Sep 17 00:00:00 2001 From: James Frost Date: Thu, 2 Apr 2026 13:26:53 +0100 Subject: [PATCH 1/2] Include full path in filename for linked filesystem files The path separator is replaced with a ")", which should be fairly clear in most cases, and isn't generally a restricted character on filesystems. By doing this we are able to handle files from different directories with the same name. Fixes #1939 --- src/CSET/cset_workflow/app/fetch_fcst/bin/fetch_data.py | 8 +++++--- tests/workflow_utils/test_fetch_data.py | 6 +++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/CSET/cset_workflow/app/fetch_fcst/bin/fetch_data.py b/src/CSET/cset_workflow/app/fetch_fcst/bin/fetch_data.py index acf999a7c..c2fdb16b9 100644 --- a/src/CSET/cset_workflow/app/fetch_fcst/bin/fetch_data.py +++ b/src/CSET/cset_workflow/app/fetch_fcst/bin/fetch_data.py @@ -110,10 +110,12 @@ def get_file(self, file_path: str, output_dir: str) -> bool: logging.warning("file_path does not match any files: %s", file_path) any_files_copied = False for f in file_paths: - file = Path(f) + file = Path(f).absolute() try: - # We know file exists from glob. - os.symlink(file.absolute(), f"{output_dir}/{file.name}") + # We know file exists from glob. Save to a filename derived from + # the full path, to differentiate similarly named files from + # different directories. + os.symlink(file, f"{output_dir}/{str(file).replace('/', ')')}") any_files_copied = True except OSError as err: logging.warning("Failed to copy %s, error: %s", file, err) diff --git a/tests/workflow_utils/test_fetch_data.py b/tests/workflow_utils/test_fetch_data.py index f9415bcf5..50574b53c 100644 --- a/tests/workflow_utils/test_fetch_data.py +++ b/tests/workflow_utils/test_fetch_data.py @@ -237,10 +237,10 @@ def test_FilesystemFileRetriever(tmp_path): # Correct return value. assert files_found # Symlinks created. - assert (tmp_path / "exeter_em01.nc").exists(follow_symlinks=False) - assert (tmp_path / "exeter_em02.nc").exists(follow_symlinks=False) + retrieved_files = sorted(tmp_path.glob("*exeter_em0?.nc")) + assert len(retrieved_files) == 2 # Check symlink points to correct file. - with open((tmp_path / "exeter_em01.nc"), "rb") as fp: + with open(retrieved_files[0], "rb") as fp: digest = hashlib.file_digest(fp, "sha256").hexdigest() assert digest == "23761fd2456b2dbbb18f35fa4909561569af0851ebda6e3705e70e366c86ac09" From 2dc39d2e4d123b200c3addae41458d758597cf9e Mon Sep 17 00:00:00 2001 From: James Frost Date: Tue, 7 Apr 2026 09:43:45 +0100 Subject: [PATCH 2/2] Update comment and join file parts more cleanly --- src/CSET/cset_workflow/app/fetch_fcst/bin/fetch_data.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/CSET/cset_workflow/app/fetch_fcst/bin/fetch_data.py b/src/CSET/cset_workflow/app/fetch_fcst/bin/fetch_data.py index c2fdb16b9..fb5453b98 100644 --- a/src/CSET/cset_workflow/app/fetch_fcst/bin/fetch_data.py +++ b/src/CSET/cset_workflow/app/fetch_fcst/bin/fetch_data.py @@ -112,10 +112,11 @@ def get_file(self, file_path: str, output_dir: str) -> bool: for f in file_paths: file = Path(f).absolute() try: - # We know file exists from glob. Save to a filename derived from - # the full path, to differentiate similarly named files from - # different directories. - os.symlink(file, f"{output_dir}/{str(file).replace('/', ')')}") + # Save to a filename derived from the full path, to + # differentiate similarly named files from different + # directories. + # `)` replaces `/` as it can be in file names. + os.symlink(file, f"{output_dir}/{')'.join(file.parts)}") any_files_copied = True except OSError as err: logging.warning("Failed to copy %s, error: %s", file, err)