From 4b3d8e79e04b388fce1267d8f1c4fa530f5df779 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Tue, 25 Nov 2025 15:55:23 -0700 Subject: [PATCH 01/74] Initial isolate command --- lib/pavilion/commands/isolate.py | 107 +++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 lib/pavilion/commands/isolate.py diff --git a/lib/pavilion/commands/isolate.py b/lib/pavilion/commands/isolate.py new file mode 100644 index 000000000..c298a3a09 --- /dev/null +++ b/lib/pavilion/commands/isolate.py @@ -0,0 +1,107 @@ +from argparse import ArgumentParser, Namespace, Action +from pathlib import Path +import shutil +from typing import Dict, Any, Optional + +from pavilion import output +from pavilion.config import PavConfig +from pavilion.test_ids import TestID +from pavilion.cmd_utils import get_last_test_id +from pavilion.utils import copytree +from .base_classes import Command + + +class ArchiveOptionsAction(Action) + def __call__( + parser: ArgumentParser, + namespace: Namespace, + values: Dict[str, Any] + option_string: Optional[str] = None) -> None: + if not getattr(namespace, "archive") and geattr(namespace, "zip"): + parser.error("--archive must be specified to use --zip.") + + +class IsolateCommand(Command): + """Isolates an existing test run in a form that can be run without Pavilion.""" + + def __init__(self): + super().__init__( + "isolate", + "Isolate an existing test run.", + short_help="Isolate a test run." + ) + + def _setup_arguments(self, parser: ArgumentParser) -> None: + parser.add_argument( + "test_id", + type=TestID, + nargs="?", + help="test ID" + ) + + parser.add_argument( + "path", + type=Path, + help="isolation path" + ) + + parser.add_argument( + "-a", + "--archive", + action="store_true", + default=False, + help="archive the test" + ) + + parser.add_argument( + "-z", + "--zip", + action="store_true", + default=False, + help="compress the test archive", + action=ArchiveOptionsAction + ) + + def run(self, pav_cfg: PavConfig, args: Namespace) -> int: + test_id = args.test_id + + if args.test_id is None: + test_id = get_last_test_id(pav_cfg, self.errfile) + + if test_id is None: + output.fprint(self.errfile, "No last test found.", color=output.RED) + + return 1 + + tests = cmd_utils.get_tests_by_id(pav_cfg, [test_id], self.errfile) + + if len(tests) == 0: + output.fprint(self.errfile, "Could not find test '{}'".format(test_id)) + + return 2 + + elif len(tests) > 1: + output.fprint( + self.errfile, "Matched multiple tests. Printing file contents for first " + "test only (test {})".format(tests[0].full_id), + color=output.YELLOW) + + return 3 + + test = next(tests) + + if not test.path.is_dir(): + output.fprint(sys.stderr, "Directory '{}' does not exist." + .format(test.path.as_posix()), color=output.RED) + + return 4 + + if args.archive: + if args.zip: + archive_format = "gztar" + else: + archive_format = "tar" + + shutil.make_archive(args.path, archive_format, test.path) + else: + copytree(test.path, args.path) \ No newline at end of file From 845c6a174dc8c71cc07bdb7060d19ba867867f57 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Tue, 25 Nov 2025 16:43:49 -0700 Subject: [PATCH 02/74] Fix get_last_test_id --- lib/pavilion/cmd_utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/pavilion/cmd_utils.py b/lib/pavilion/cmd_utils.py index 10acb3727..92455eb76 100644 --- a/lib/pavilion/cmd_utils.py +++ b/lib/pavilion/cmd_utils.py @@ -523,18 +523,18 @@ def get_last_test_id(pav_cfg: "PavConfig", errfile: TextIO) -> Optional[TestID]: if last_series is None: return None - test_ids = list(last_series.tests.keys()) + id_pairs = list(last_series.tests.keys()) - if len(test_ids) == 0: + if len(id_pairs) == 0: output.fprint( errfile, f"Most recent series contains no tests.") return None - if len(test_ids) > 1: + if len(id_pairs) > 1: output.fprint( errfile, f"Multiple tests exist in last series. Could not unambiguously identify last test.") return None - return TestID(test_ids[0]) + return TestID(str(id_pairs[0][1])) From cd9d3d4986f5a9ba13828f1c89f1f53d875171dc Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Tue, 25 Nov 2025 16:44:34 -0700 Subject: [PATCH 03/74] Add isolate to built-in commands --- lib/pavilion/commands/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pavilion/commands/__init__.py b/lib/pavilion/commands/__init__.py index 54309f773..7c76e2815 100644 --- a/lib/pavilion/commands/__init__.py +++ b/lib/pavilion/commands/__init__.py @@ -23,6 +23,7 @@ 'config': ('config', 'ConfigCommand'), 'graph': ('graph', 'GraphCommand'), 'group': ('group', 'GroupCommand'), + 'isolate': ('isolate', 'IsolateCommand'), 'list': ('list_cmd', 'ListCommand'), 'log': ('log', 'LogCommand'), 'ls': ('ls', 'LSCommand'), From 9ec331b65449bee1a3baf88894895a9cdd9cd0d4 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Tue, 25 Nov 2025 16:44:46 -0700 Subject: [PATCH 04/74] Tweak parser action --- lib/pavilion/commands/isolate.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/lib/pavilion/commands/isolate.py b/lib/pavilion/commands/isolate.py index c298a3a09..2ec2ae9a3 100644 --- a/lib/pavilion/commands/isolate.py +++ b/lib/pavilion/commands/isolate.py @@ -6,19 +6,21 @@ from pavilion import output from pavilion.config import PavConfig from pavilion.test_ids import TestID -from pavilion.cmd_utils import get_last_test_id +from pavilion.cmd_utils import get_last_test_id, get_tests_by_id from pavilion.utils import copytree from .base_classes import Command -class ArchiveOptionsAction(Action) +class ArchiveOptionsAction(Action): def __call__( parser: ArgumentParser, namespace: Namespace, - values: Dict[str, Any] + values: Dict[str, Any], option_string: Optional[str] = None) -> None: - if not getattr(namespace, "archive") and geattr(namespace, "zip"): + if not getattr(namespace, "archive"): parser.error("--archive must be specified to use --zip.") + else: + setattr(namespace, "zip", True) class IsolateCommand(Command): @@ -56,7 +58,6 @@ def _setup_arguments(self, parser: ArgumentParser) -> None: parser.add_argument( "-z", "--zip", - action="store_true", default=False, help="compress the test archive", action=ArchiveOptionsAction @@ -73,7 +74,7 @@ def run(self, pav_cfg: PavConfig, args: Namespace) -> int: return 1 - tests = cmd_utils.get_tests_by_id(pav_cfg, [test_id], self.errfile) + tests = get_tests_by_id(pav_cfg, [test_id], self.errfile) if len(tests) == 0: output.fprint(self.errfile, "Could not find test '{}'".format(test_id)) @@ -88,7 +89,7 @@ def run(self, pav_cfg: PavConfig, args: Namespace) -> int: return 3 - test = next(tests) + test = next(iter(tests)) if not test.path.is_dir(): output.fprint(sys.stderr, "Directory '{}' does not exist." @@ -104,4 +105,5 @@ def run(self, pav_cfg: PavConfig, args: Namespace) -> int: shutil.make_archive(args.path, archive_format, test.path) else: + print("Copying directory tree...") copytree(test.path, args.path) \ No newline at end of file From 41cc611846072ac7cfb67e654d66f1b8d41e5f8f Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Tue, 25 Nov 2025 16:45:21 -0700 Subject: [PATCH 05/74] Add first isolate command unit test --- test/tests/isolate_cmd_tests.py | 35 +++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 test/tests/isolate_cmd_tests.py diff --git a/test/tests/isolate_cmd_tests.py b/test/tests/isolate_cmd_tests.py new file mode 100644 index 000000000..1fa518888 --- /dev/null +++ b/test/tests/isolate_cmd_tests.py @@ -0,0 +1,35 @@ +import tempfile +from pathlib import Path + +from pavilion import commands +from pavilion import arguments +from pavilion.unittest import PavTestCase + + +class IsolateCmdTests(PavTestCase): + + def test_no_archive(self): + run_cmd = commands.get_command("run") + isolate_cmd = commands.get_command("isolate") + + run_cmd.silence() + isolate_cmd.silence() + + parser = arguments.get_parser() + run_args = ["run", "-H", "this", "hello_world.hello"] + + run_cmd.run(self.pav_cfg, parser.parse_args(run_args)) + last_test = next(iter(run_cmd.last_tests)) + + with tempfile.TemporaryDirectory() as dir: + isolate_args = parser.parse_args(["isolate", str(Path(dir) / "dest")]) + + self.assertEqual(isolate_cmd.run(self.pav_cfg, isolate_args), 0) + + source_files = set(map(lambda x: x.name, last_test.path.listdir())) + dest_files = set(map(lambda x: x.name, (Path(dir) / "dest").listdir())) + + self.assertEqual(source_files, dest_files) + + def test_zip_arcive(self): + ... \ No newline at end of file From f842155213628b0cf904b68385bb68fbd4cac4b5 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Tue, 25 Nov 2025 17:01:50 -0700 Subject: [PATCH 06/74] Ignore series and job directories when isolating --- lib/pavilion/commands/isolate.py | 6 +++--- test/tests/isolate_cmd_tests.py | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/pavilion/commands/isolate.py b/lib/pavilion/commands/isolate.py index 2ec2ae9a3..13ef94c86 100644 --- a/lib/pavilion/commands/isolate.py +++ b/lib/pavilion/commands/isolate.py @@ -7,7 +7,6 @@ from pavilion.config import PavConfig from pavilion.test_ids import TestID from pavilion.cmd_utils import get_last_test_id, get_tests_by_id -from pavilion.utils import copytree from .base_classes import Command @@ -105,5 +104,6 @@ def run(self, pav_cfg: PavConfig, args: Namespace) -> int: shutil.make_archive(args.path, archive_format, test.path) else: - print("Copying directory tree...") - copytree(test.path, args.path) \ No newline at end of file + shutil.copytree(test.path, args.path, ignore=lambda x, y: ("series", "job")) + + return 0 \ No newline at end of file diff --git a/test/tests/isolate_cmd_tests.py b/test/tests/isolate_cmd_tests.py index 1fa518888..d772d2be8 100644 --- a/test/tests/isolate_cmd_tests.py +++ b/test/tests/isolate_cmd_tests.py @@ -26,10 +26,10 @@ def test_no_archive(self): self.assertEqual(isolate_cmd.run(self.pav_cfg, isolate_args), 0) - source_files = set(map(lambda x: x.name, last_test.path.listdir())) - dest_files = set(map(lambda x: x.name, (Path(dir) / "dest").listdir())) + source_files = set(map(lambda x: x.name, last_test.path.iterdir())) + dest_files = set(map(lambda x: x.name, (Path(dir) / "dest").iterdir())) - self.assertEqual(source_files, dest_files) + self.assertEqual({f for f in source_files if f not in ("series", "job")}, dest_files) - def test_zip_arcive(self): + def test_zip_archive(self): ... \ No newline at end of file From 03cd02dfbf87dafef40a4ffe4bd9d37f54fbd5bf Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Wed, 26 Nov 2025 09:36:00 -0700 Subject: [PATCH 07/74] Add argument validation for isolate --- lib/pavilion/commands/isolate.py | 43 +++++++++++++++----------------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/lib/pavilion/commands/isolate.py b/lib/pavilion/commands/isolate.py index 13ef94c86..c531aee11 100644 --- a/lib/pavilion/commands/isolate.py +++ b/lib/pavilion/commands/isolate.py @@ -10,18 +10,6 @@ from .base_classes import Command -class ArchiveOptionsAction(Action): - def __call__( - parser: ArgumentParser, - namespace: Namespace, - values: Dict[str, Any], - option_string: Optional[str] = None) -> None: - if not getattr(namespace, "archive"): - parser.error("--archive must be specified to use --zip.") - else: - setattr(namespace, "zip", True) - - class IsolateCommand(Command): """Isolates an existing test run in a form that can be run without Pavilion.""" @@ -59,10 +47,15 @@ def _setup_arguments(self, parser: ArgumentParser) -> None: "--zip", default=False, help="compress the test archive", - action=ArchiveOptionsAction + action="store_true" ) def run(self, pav_cfg: PavConfig, args: Namespace) -> int: + if args.zip and not args.archive: + output.fprint(self.errfile, "--archive must be specified to use --zip.") + + return 1 + test_id = args.test_id if args.test_id is None: @@ -71,14 +64,14 @@ def run(self, pav_cfg: PavConfig, args: Namespace) -> int: if test_id is None: output.fprint(self.errfile, "No last test found.", color=output.RED) - return 1 + return 2 tests = get_tests_by_id(pav_cfg, [test_id], self.errfile) if len(tests) == 0: output.fprint(self.errfile, "Could not find test '{}'".format(test_id)) - return 2 + return 3 elif len(tests) > 1: output.fprint( @@ -86,24 +79,28 @@ def run(self, pav_cfg: PavConfig, args: Namespace) -> int: "test only (test {})".format(tests[0].full_id), color=output.YELLOW) - return 3 + return 4 test = next(iter(tests)) - if not test.path.is_dir(): + return self._isolate(test.path, args.path, args.archive, args.zip) + + @staticmethod + def _isolate(test_path: Path, dest: Path, archive: bool, zip: bool) -> int: + if not test_path.is_dir(): output.fprint(sys.stderr, "Directory '{}' does not exist." - .format(test.path.as_posix()), color=output.RED) + .format(test_path.as_posix()), color=output.RED) - return 4 + return 5 - if args.archive: - if args.zip: + if archive: + if zip: archive_format = "gztar" else: archive_format = "tar" - shutil.make_archive(args.path, archive_format, test.path) + shutil.make_archive(dest, archive_format, test_path) else: - shutil.copytree(test.path, args.path, ignore=lambda x, y: ("series", "job")) + shutil.copytree(test_path, dest, ignore=lambda x, y: ("series", "job")) return 0 \ No newline at end of file From ee06027769038c838d2c415c3bcbdd29ee05ca54 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Wed, 26 Nov 2025 09:56:33 -0700 Subject: [PATCH 08/74] Add error handling --- lib/pavilion/commands/isolate.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/lib/pavilion/commands/isolate.py b/lib/pavilion/commands/isolate.py index c531aee11..6f8081ad5 100644 --- a/lib/pavilion/commands/isolate.py +++ b/lib/pavilion/commands/isolate.py @@ -99,8 +99,18 @@ def _isolate(test_path: Path, dest: Path, archive: bool, zip: bool) -> int: else: archive_format = "tar" - shutil.make_archive(dest, archive_format, test_path) + try: + shutil.make_archive(dest, archive_format, test_path) + except OSError: + output.fprint(f"Unable to isolate test {test.id} at {dest}.") + + return 6 else: - shutil.copytree(test_path, dest, ignore=lambda x, y: ("series", "job")) + try: + shutil.copytree(test_path, dest, ignore=lambda x, y: ("series", "job")) + except OSError: + output.fprint(f"Unable to isolate test {test.id} at {dest}.") + + return 6 return 0 \ No newline at end of file From 27e535596e0fa03623a6324d871d4d5aff5ecdfb Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Wed, 26 Nov 2025 12:09:13 -0700 Subject: [PATCH 09/74] Add unit test for isolating with archives --- lib/pavilion/commands/isolate.py | 6 +++- test/tests/isolate_cmd_tests.py | 47 +++++++++++++++++++++++++++++++- 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/lib/pavilion/commands/isolate.py b/lib/pavilion/commands/isolate.py index 6f8081ad5..3005b2142 100644 --- a/lib/pavilion/commands/isolate.py +++ b/lib/pavilion/commands/isolate.py @@ -100,7 +100,11 @@ def _isolate(test_path: Path, dest: Path, archive: bool, zip: bool) -> int: archive_format = "tar" try: - shutil.make_archive(dest, archive_format, test_path) + shutil.make_archive( + dest, + archive_format, + root_dir=test_path.parent, + base_dir=test_path.name) except OSError: output.fprint(f"Unable to isolate test {test.id} at {dest}.") diff --git a/test/tests/isolate_cmd_tests.py b/test/tests/isolate_cmd_tests.py index d772d2be8..7507854d8 100644 --- a/test/tests/isolate_cmd_tests.py +++ b/test/tests/isolate_cmd_tests.py @@ -1,5 +1,8 @@ import tempfile +import os from pathlib import Path +import subprocess as sp +from typing import Iterator from pavilion import commands from pavilion import arguments @@ -32,4 +35,46 @@ def test_no_archive(self): self.assertEqual({f for f in source_files if f not in ("series", "job")}, dest_files) def test_zip_archive(self): - ... \ No newline at end of file + run_cmd = commands.get_command("run") + isolate_cmd = commands.get_command("isolate") + + run_cmd.silence() + isolate_cmd.silence() + + parser = arguments.get_parser() + run_args = ["run", "-H", "this", "hello_world.hello"] + + run_cmd.run(self.pav_cfg, parser.parse_args(run_args)) + last_test = next(iter(run_cmd.last_tests)) + + with tempfile.TemporaryDirectory() as dir: + isolate_args = parser.parse_args(["isolate", + str(Path(dir) / "dest"), + "--archive", + "--zip"]) + + self.assertEqual(isolate_cmd.run(self.pav_cfg, isolate_args), 0) + + res = sp.run( + ["tar", "-tf", str(Path(dir) / "dest.tar.gz")], + stdout=sp.PIPE, + stderr=sp.PIPE, + universal_newlines=True) + + def list_files(path: Path) -> Iterator[Path]: + for root, dirs, files in os.walk(path): + yield root + + for f in files: + yield Path(root) / f + for d in dirs: + yield Path(root) / d + + source_files = set(map( + lambda x: Path(x).relative_to(last_test.path.parent), + list_files(last_test.path))) + dest_files = set(map(Path, res.stdout.splitlines())) + + self.assertEqual( + {f for f in source_files if f.name not in ("series", "job")}, + dest_files) \ No newline at end of file From a7f71f067a908519aa2fcd82a36206d59375fe44 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Wed, 26 Nov 2025 12:40:41 -0700 Subject: [PATCH 10/74] Ignore job and series dirs on tarball creation --- lib/pavilion/cmd_utils.py | 14 ++++++++ lib/pavilion/commands/isolate.py | 55 +++++++++++++++++++++----------- test/tests/isolate_cmd_tests.py | 16 +++------- 3 files changed, 55 insertions(+), 30 deletions(-) diff --git a/lib/pavilion/cmd_utils.py b/lib/pavilion/cmd_utils.py index 92455eb76..5d0d89210 100644 --- a/lib/pavilion/cmd_utils.py +++ b/lib/pavilion/cmd_utils.py @@ -7,6 +7,7 @@ import logging import sys import time +import os from pathlib import Path from typing import List, TextIO, Union, Iterator, Optional from collections import defaultdict @@ -538,3 +539,16 @@ def get_last_test_id(pav_cfg: "PavConfig", errfile: TextIO) -> Optional[TestID]: return None return TestID(str(id_pairs[0][1])) + + +def list_files(path: Path, include_root: bool = False) -> Iterator[Path]: + """Recursively list all files in a directory, optionally including the directory itself.""" + + for root, dirs, files in os.walk(path): + if include_root: + yield Path(root) + + for f in files: + yield Path(root) / f + for d in dirs: + yield Path(root) / d \ No newline at end of file diff --git a/lib/pavilion/commands/isolate.py b/lib/pavilion/commands/isolate.py index 3005b2142..3e53da65e 100644 --- a/lib/pavilion/commands/isolate.py +++ b/lib/pavilion/commands/isolate.py @@ -1,18 +1,23 @@ from argparse import ArgumentParser, Namespace, Action from pathlib import Path import shutil +import tarfile +import sys from typing import Dict, Any, Optional from pavilion import output from pavilion.config import PavConfig +from pavilion.test_run import TestRun from pavilion.test_ids import TestID -from pavilion.cmd_utils import get_last_test_id, get_tests_by_id +from pavilion.cmd_utils import get_last_test_id, get_tests_by_id, list_files from .base_classes import Command class IsolateCommand(Command): """Isolates an existing test run in a form that can be run without Pavilion.""" + IGNORE_FILES = ("series", "job") + def __init__(self): super().__init__( "isolate", @@ -83,16 +88,24 @@ def run(self, pav_cfg: PavConfig, args: Namespace) -> int: test = next(iter(tests)) - return self._isolate(test.path, args.path, args.archive, args.zip) + return self._isolate(test, args.path, args.archive, args.zip) - @staticmethod - def _isolate(test_path: Path, dest: Path, archive: bool, zip: bool) -> int: - if not test_path.is_dir(): + @classmethod + def _isolate(cls, test: TestRun, dest: Path, archive: bool, zip: bool) -> int: + if not test.path.is_dir(): output.fprint(sys.stderr, "Directory '{}' does not exist." - .format(test_path.as_posix()), color=output.RED) + .format(test.path.as_posix()), color=output.RED) return 5 + if dest.exists(): + output.fprint( + sys.stderr, + f"Unable to isolate test {test.id}. Destination {dest} already exists.", + color=output.RED) + + return 6 + if archive: if zip: archive_format = "gztar" @@ -100,21 +113,27 @@ def _isolate(test_path: Path, dest: Path, archive: bool, zip: bool) -> int: archive_format = "tar" try: - shutil.make_archive( - dest, - archive_format, - root_dir=test_path.parent, - base_dir=test_path.name) - except OSError: - output.fprint(f"Unable to isolate test {test.id} at {dest}.") - - return 6 + with tarfile.open(dest, "w:gz") as tf: + for f in list_files(test.path, include_root=True): + if f.name not in cls.IGNORE_FILES: + print(f.relative_to(test.path.parent)) + tf.add(f, arcname=f.relative_to(test.path.parent), recursive=False) + except Exception as err: + output.fprint( + sys.stderr, + f"Unable to isolate test {test.id} at {dest}: {err}", + color=output.RED) + + return 7 else: try: - shutil.copytree(test_path, dest, ignore=lambda x, y: ("series", "job")) + shutil.copytree(test.path, dest, ignore=lambda x, y: cls.IGNORE_FILES) except OSError: - output.fprint(f"Unable to isolate test {test.id} at {dest}.") + output.fprint( + sys.stderr, + f"Unable to isolate test {test.id} at {dest}.", + color=output.RED) - return 6 + return 8 return 0 \ No newline at end of file diff --git a/test/tests/isolate_cmd_tests.py b/test/tests/isolate_cmd_tests.py index 7507854d8..c57bf3262 100644 --- a/test/tests/isolate_cmd_tests.py +++ b/test/tests/isolate_cmd_tests.py @@ -7,6 +7,7 @@ from pavilion import commands from pavilion import arguments from pavilion.unittest import PavTestCase +from pavilion.cmd_utils import list_files class IsolateCmdTests(PavTestCase): @@ -49,30 +50,21 @@ def test_zip_archive(self): with tempfile.TemporaryDirectory() as dir: isolate_args = parser.parse_args(["isolate", - str(Path(dir) / "dest"), + str(Path(dir) / "dest.tgz"), "--archive", "--zip"]) self.assertEqual(isolate_cmd.run(self.pav_cfg, isolate_args), 0) res = sp.run( - ["tar", "-tf", str(Path(dir) / "dest.tar.gz")], + ["tar", "-tf", str(Path(dir) / "dest.tgz")], stdout=sp.PIPE, stderr=sp.PIPE, universal_newlines=True) - def list_files(path: Path) -> Iterator[Path]: - for root, dirs, files in os.walk(path): - yield root - - for f in files: - yield Path(root) / f - for d in dirs: - yield Path(root) / d - source_files = set(map( lambda x: Path(x).relative_to(last_test.path.parent), - list_files(last_test.path))) + list_files(last_test.path, include_root=True))) dest_files = set(map(Path, res.stdout.splitlines())) self.assertEqual( From d91f1d62bddfd92bc1b6a978e0e922ca357c7ea1 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Wed, 26 Nov 2025 12:51:24 -0700 Subject: [PATCH 11/74] Remove stray print function --- lib/pavilion/commands/isolate.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/pavilion/commands/isolate.py b/lib/pavilion/commands/isolate.py index 3e53da65e..029a0888f 100644 --- a/lib/pavilion/commands/isolate.py +++ b/lib/pavilion/commands/isolate.py @@ -116,7 +116,6 @@ def _isolate(cls, test: TestRun, dest: Path, archive: bool, zip: bool) -> int: with tarfile.open(dest, "w:gz") as tf: for f in list_files(test.path, include_root=True): if f.name not in cls.IGNORE_FILES: - print(f.relative_to(test.path.parent)) tf.add(f, arcname=f.relative_to(test.path.parent), recursive=False) except Exception as err: output.fprint( From 216cc9f55a16989e765e32f0b904e821d8b08af4 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Wed, 26 Nov 2025 12:56:51 -0700 Subject: [PATCH 12/74] Handle archive suffixes --- lib/pavilion/commands/isolate.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/pavilion/commands/isolate.py b/lib/pavilion/commands/isolate.py index 029a0888f..975c69ea1 100644 --- a/lib/pavilion/commands/isolate.py +++ b/lib/pavilion/commands/isolate.py @@ -108,8 +108,13 @@ def _isolate(cls, test: TestRun, dest: Path, archive: bool, zip: bool) -> int: if archive: if zip: + if len(dest.suffixes) == 0: + dest = dest.with_suffix(".tgz") + archive_format = "gztar" else: + if len(dest.suffixes) == 0: + dest = dest.with_suffix(".tar") archive_format = "tar" try: From 6558190a528958a856764c305c62356e21e108e5 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Wed, 26 Nov 2025 13:31:55 -0700 Subject: [PATCH 13/74] Check for symlinks in isolate command unit tests --- lib/pavilion/commands/isolate.py | 5 +++-- test/tests/isolate_cmd_tests.py | 34 ++++++++++++++++++-------------- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/lib/pavilion/commands/isolate.py b/lib/pavilion/commands/isolate.py index 975c69ea1..265716a68 100644 --- a/lib/pavilion/commands/isolate.py +++ b/lib/pavilion/commands/isolate.py @@ -115,6 +115,7 @@ def _isolate(cls, test: TestRun, dest: Path, archive: bool, zip: bool) -> int: else: if len(dest.suffixes) == 0: dest = dest.with_suffix(".tar") + archive_format = "tar" try: @@ -122,10 +123,10 @@ def _isolate(cls, test: TestRun, dest: Path, archive: bool, zip: bool) -> int: for f in list_files(test.path, include_root=True): if f.name not in cls.IGNORE_FILES: tf.add(f, arcname=f.relative_to(test.path.parent), recursive=False) - except Exception as err: + except: output.fprint( sys.stderr, - f"Unable to isolate test {test.id} at {dest}: {err}", + f"Unable to isolate test {test.id} at {dest}.", color=output.RED) return 7 diff --git a/test/tests/isolate_cmd_tests.py b/test/tests/isolate_cmd_tests.py index c57bf3262..3cef0678a 100644 --- a/test/tests/isolate_cmd_tests.py +++ b/test/tests/isolate_cmd_tests.py @@ -1,5 +1,6 @@ import tempfile import os +import tarfile from pathlib import Path import subprocess as sp from typing import Iterator @@ -30,9 +31,10 @@ def test_no_archive(self): self.assertEqual(isolate_cmd.run(self.pav_cfg, isolate_args), 0) - source_files = set(map(lambda x: x.name, last_test.path.iterdir())) - dest_files = set(map(lambda x: x.name, (Path(dir) / "dest").iterdir())) + source_files = set(list_files(last_test.path)) + dest_files = set(list_files(Path(dir) / "dest")) + self.assertFalse(any(map(lambda x: x.is_symlink(), dest_files))) self.assertEqual({f for f in source_files if f not in ("series", "job")}, dest_files) def test_zip_archive(self): @@ -50,23 +52,25 @@ def test_zip_archive(self): with tempfile.TemporaryDirectory() as dir: isolate_args = parser.parse_args(["isolate", - str(Path(dir) / "dest.tgz"), + str(Path(dir) / "dest"), "--archive", "--zip"]) self.assertEqual(isolate_cmd.run(self.pav_cfg, isolate_args), 0) - res = sp.run( - ["tar", "-tf", str(Path(dir) / "dest.tgz")], - stdout=sp.PIPE, - stderr=sp.PIPE, - universal_newlines=True) + with tempfile.TemporaryDirectory() as extract_dir: + with tarfile.open(Path(dir) / "dest.tgz", "r:gz") as tf: + tf.extractall(extract_dir) - source_files = set(map( - lambda x: Path(x).relative_to(last_test.path.parent), - list_files(last_test.path, include_root=True))) - dest_files = set(map(Path, res.stdout.splitlines())) + dest_files = list_files(Path(extract_dir)) - self.assertEqual( - {f for f in source_files if f.name not in ("series", "job")}, - dest_files) \ No newline at end of file + self.assertFalse(any(map(lambda x: x.is_symlink(), dest_files))) + + source_files = set(map( + lambda x: Path(x).relative_to(last_test.path.parent), + list_files(last_test.path, include_root=True))) + dest_files = set(dest_files) + + self.assertEqual( + {f for f in source_files if f.name not in ("series", "job")}, + dest_files) \ No newline at end of file From 6ea4849712569d4542d7776b9deaa65dc6586b94 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Wed, 26 Nov 2025 15:51:53 -0700 Subject: [PATCH 14/74] Add copytree_resolved function --- lib/pavilion/utils.py | 44 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/lib/pavilion/utils.py b/lib/pavilion/utils.py index 877384803..c4378770f 100644 --- a/lib/pavilion/utils.py +++ b/lib/pavilion/utils.py @@ -13,8 +13,7 @@ import textwrap import zipfile from pathlib import Path -from typing import Iterator, Union, TextIO -from typing import List, Dict +from typing import Iterator, Union, TextIO, List, Dict, Optional, Set class WrappedFormatter(argparse.HelpFormatter): @@ -208,6 +207,47 @@ def copytree(src, dst, symlinks=False, ignore=None, copy_function=shutil.copy2, raise shutil.Error(errors) return dst +def copytree_resolved( + src: Path, + dest: Path, + seen_files: Optional[Set] = None, + flatten: bool = False) -> Path: + """Copy a directory tree to another location, such that the resulting directory contains + the targets of all symlinks. If flatten is specified, replace all symlinks with + their targets.""" + + seen_files = seen_files or set() + + if src in seen_files: + return src + + seen_files.add(src) + + if src.is_symlink(): + target = src.resolve() + + if target not in seen_files: + if flatten: + copytree_resolved(target, dest, seen_files) + else: + # Retain the symlink but copy its target to the present directory + copytree_resolved(target, dest.parent / target.name, seen_files) + dest.symlink_to(dest.parent / target.name) + else: + dest.symlink_to(target) + + return dest + + elif src.is_file(): + return shutil.copy(src, dest, seen_files) + elif src.is_dir(): + files = src.iterdir() + + for f in files: + copytree(f, dest / f.name, seen_files) + + return dest + def path_is_external(path: Path): """Returns True if a path contains enough back 'up-references' to escape From ffc001ee9f39c10fe1b5cad5505d5b65e15a9067 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Wed, 26 Nov 2025 15:54:18 -0700 Subject: [PATCH 15/74] Fix copytree_resolved --- lib/pavilion/utils.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/pavilion/utils.py b/lib/pavilion/utils.py index c4378770f..c526f5153 100644 --- a/lib/pavilion/utils.py +++ b/lib/pavilion/utils.py @@ -239,16 +239,15 @@ def copytree_resolved( return dest elif src.is_file(): - return shutil.copy(src, dest, seen_files) + return shutil.copy(src, dest) elif src.is_dir(): files = src.iterdir() for f in files: - copytree(f, dest / f.name, seen_files) + copytree_resolved(f, dest / f.name, seen_files) return dest - def path_is_external(path: Path): """Returns True if a path contains enough back 'up-references' to escape the base directory.""" From b39160efd0e7b09575dcd019846d86422a755b58 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Wed, 26 Nov 2025 16:16:52 -0700 Subject: [PATCH 16/74] Add simple unit test for copytree_resolved --- lib/pavilion/utils.py | 1 + test/tests/utils_tests.py | 18 +++++++++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/lib/pavilion/utils.py b/lib/pavilion/utils.py index c526f5153..2ff08d260 100644 --- a/lib/pavilion/utils.py +++ b/lib/pavilion/utils.py @@ -241,6 +241,7 @@ def copytree_resolved( elif src.is_file(): return shutil.copy(src, dest) elif src.is_dir(): + dest.mkdir(exist_ok=True) files = src.iterdir() for f in files: diff --git a/test/tests/utils_tests.py b/test/tests/utils_tests.py index fe0706492..299d2b2fd 100644 --- a/test/tests/utils_tests.py +++ b/test/tests/utils_tests.py @@ -155,4 +155,20 @@ def test_copytree_dotfiles(self): utils.copytree(src, dest) names = (map(lambda x: x.name, dest.iterdir())) - self.assertIn(".dotfile", names) \ No newline at end of file + self.assertIn(".dotfile", names) + + def test_copytree_resolved(self): + with tempfile.TemporaryDirectory() as src: + src = Path(src) + + with tempfile.TemporaryDirectory() as dest: + dest = Path(dest) + + (src / "foo").mkdir() + (src / "bar").touch() + (src / "foo" / "baz").touch() + + utils.copytree_resolved(src, dest) + + dest_files = set(map(lambda x: x.relative_to(src), src.iterdir())) + dest_files = set(map(lambda x: x.relative_to(dest), dest.iterdir())) From 4780b62b93ae26019c92b05daec87e02e44c0e74 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Mon, 1 Dec 2025 12:51:57 -0700 Subject: [PATCH 17/74] Progress towards isolate --- test/tests/cmd_util_tests.py | 8 ++++++++ test/tests/isolate_cmd_tests.py | 7 ++++++- test/tests/utils_tests.py | 23 ++++++++++++++++++++++- 3 files changed, 36 insertions(+), 2 deletions(-) diff --git a/test/tests/cmd_util_tests.py b/test/tests/cmd_util_tests.py index 395332a4d..295e2ffb9 100644 --- a/test/tests/cmd_util_tests.py +++ b/test/tests/cmd_util_tests.py @@ -3,6 +3,7 @@ import io import json import shutil +import tempfile from pathlib import Path from pavilion import dir_db @@ -62,4 +63,11 @@ def test_arg_filtered_tests(self): self.assertEqual(len(cmd_utils.arg_filtered_tests(self.pav_cfg, args).paths), count) + def test_copytree_resolved(self): + with tempfile.TemporaryDirectory() as src: + with tempfile.TemporaryDirectory() as dest: + (Path(src) / "foo").touch() + + cmd_utils.copytree_resolved(Path(src), Path(dest)) + # TODO: We really need to add unit tests for each of the cmd utils functions. diff --git a/test/tests/isolate_cmd_tests.py b/test/tests/isolate_cmd_tests.py index 3cef0678a..99752aa7a 100644 --- a/test/tests/isolate_cmd_tests.py +++ b/test/tests/isolate_cmd_tests.py @@ -64,6 +64,11 @@ def test_zip_archive(self): dest_files = list_files(Path(extract_dir)) + for df in dest_files: + if df.is_symlink(): + import pdb; pdb.set_trace() + + self.assertFalse(any(map(lambda x: x.is_symlink(), dest_files))) source_files = set(map( @@ -73,4 +78,4 @@ def test_zip_archive(self): self.assertEqual( {f for f in source_files if f.name not in ("series", "job")}, - dest_files) \ No newline at end of file + dest_files) diff --git a/test/tests/utils_tests.py b/test/tests/utils_tests.py index 299d2b2fd..525b18aed 100644 --- a/test/tests/utils_tests.py +++ b/test/tests/utils_tests.py @@ -170,5 +170,26 @@ def test_copytree_resolved(self): utils.copytree_resolved(src, dest) - dest_files = set(map(lambda x: x.relative_to(src), src.iterdir())) + src_files = set(map(lambda x: x.relative_to(src), src.iterdir())) dest_files = set(map(lambda x: x.relative_to(dest), dest.iterdir())) + + self.assertEqual(src_files, dest_files) + + with tempfile.TemporaryDirectory() as src: + src = Path(src) + + with tempfile.TemporaryDirectory() as dest: + dest = Path(dest) + + (src / "foo").mkdir() + (src / "bar").touch() + + (src / "foo" / "baz").symlink_to(src / "bar") + + utils.copytree_resolved(src / "foo", dest, flatten=True) + + dest_files = set(map(lambda x: x.relative_to(dest), dest.iterdir())) + expected = set([Path("foo"), Path("foo") / "baz"]) + + self.assertEqual(dest_files, expected) + self.assertFalse((dest / "foo" / "baz").is_symlink()) From 42f141db810e2af9ad591bf095d75409bba1b0ff Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Mon, 1 Dec 2025 12:59:56 -0700 Subject: [PATCH 18/74] Add more unit tests for copytree_resolved --- test/tests/utils_tests.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/test/tests/utils_tests.py b/test/tests/utils_tests.py index 525b18aed..bf973e5b3 100644 --- a/test/tests/utils_tests.py +++ b/test/tests/utils_tests.py @@ -193,3 +193,38 @@ def test_copytree_resolved(self): self.assertEqual(dest_files, expected) self.assertFalse((dest / "foo" / "baz").is_symlink()) + + with tempfile.TemporaryDirectory() as src: + src = Path(src) + + with tempfile.TemporaryDirectory() as dest: + dest = Path(dest) + + (src / "foo").mkdir() + 18 (src / "bar").touch() + 17 + 16 (src / "foo" / "baz").symlink_to(src / "bar") + 15 + 14 utils.copytree_resolved(src / "foo", dest, flatten=False) + 13 + 12 dest_files = set(map(lambda x: x.relative_to(dest), dest.iterdir())) + 11 expected = set([Path("foo"), Path("foo") / "baz", Path("foo") / "bar"]) + 10 + 9 self.assertEqual(dest_files, expected) + 8 self.assertTrue((dest / "foo" / "baz").is_symlink()) + + with tempfile.TemporaryDirectory() as src: + src = Path(src) + + with tempfile.TemporaryDirectory() as dest: + dest = Path(dest) + + (src / "foo").symlink_to(src / "bar") + 18 (src / "bar").symlink_to(src / "foo") + 17 + 14 utils.copytree_resolved(src / "foo", dest, flatten=False) + 13 + 12 dest_files = set(map(lambda x: x.relative_to(dest), dest.iterdir())) + 11 expected = set([Path("foo"), Path("bar")]) + 10 + 9 self.assertEqual(dest_files, expected) From 3014c2979b0be5fc615cc736f34e9c8f700d9a0f Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Mon, 1 Dec 2025 13:05:29 -0700 Subject: [PATCH 19/74] Remove garbage --- test/tests/utils_tests.py | 40 +++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/test/tests/utils_tests.py b/test/tests/utils_tests.py index bf973e5b3..8f8d56856 100644 --- a/test/tests/utils_tests.py +++ b/test/tests/utils_tests.py @@ -199,19 +199,19 @@ def test_copytree_resolved(self): with tempfile.TemporaryDirectory() as dest: dest = Path(dest) - + (src / "foo").mkdir() - 18 (src / "bar").touch() - 17 - 16 (src / "foo" / "baz").symlink_to(src / "bar") - 15 - 14 utils.copytree_resolved(src / "foo", dest, flatten=False) - 13 - 12 dest_files = set(map(lambda x: x.relative_to(dest), dest.iterdir())) - 11 expected = set([Path("foo"), Path("foo") / "baz", Path("foo") / "bar"]) - 10 - 9 self.assertEqual(dest_files, expected) - 8 self.assertTrue((dest / "foo" / "baz").is_symlink()) + (src / "bar").touch() + + (src / "foo" / "baz").symlink_to(src / "bar") + + utils.copytree_resolved(src / "foo", dest, flatten=False) + + dest_files = set(map(lambda x: x.relative_to(dest), dest.iterdir())) + expected = set([Path("foo"), Path("foo") / "baz", Path("foo") / "bar"]) + + self.assertEqual(dest_files, expected) + self.assertTrue((dest / "foo" / "baz").is_symlink()) with tempfile.TemporaryDirectory() as src: src = Path(src) @@ -220,11 +220,11 @@ def test_copytree_resolved(self): dest = Path(dest) (src / "foo").symlink_to(src / "bar") - 18 (src / "bar").symlink_to(src / "foo") - 17 - 14 utils.copytree_resolved(src / "foo", dest, flatten=False) - 13 - 12 dest_files = set(map(lambda x: x.relative_to(dest), dest.iterdir())) - 11 expected = set([Path("foo"), Path("bar")]) - 10 - 9 self.assertEqual(dest_files, expected) + (src / "bar").symlink_to(src / "foo") + + utils.copytree_resolved(src / "foo", dest, flatten=False) + + dest_files = set(map(lambda x: x.relative_to(dest), dest.iterdir())) + expected = set([Path("foo"), Path("bar")]) + + self.assertEqual(dest_files, expected) From e80a6e59d20b62abb6b78f4ec63348a2aa7a3e42 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Mon, 1 Dec 2025 13:55:26 -0700 Subject: [PATCH 20/74] Improve copytree_resolved unit test --- test/tests/utils_tests.py | 151 ++++++++++++++++++++++---------------- 1 file changed, 89 insertions(+), 62 deletions(-) diff --git a/test/tests/utils_tests.py b/test/tests/utils_tests.py index 8f8d56856..fd9f9727c 100644 --- a/test/tests/utils_tests.py +++ b/test/tests/utils_tests.py @@ -158,73 +158,100 @@ def test_copytree_dotfiles(self): self.assertIn(".dotfile", names) def test_copytree_resolved(self): - with tempfile.TemporaryDirectory() as src: - src = Path(src) - - with tempfile.TemporaryDirectory() as dest: - dest = Path(dest) - - (src / "foo").mkdir() - (src / "bar").touch() - (src / "foo" / "baz").touch() - - utils.copytree_resolved(src, dest) - - src_files = set(map(lambda x: x.relative_to(src), src.iterdir())) - dest_files = set(map(lambda x: x.relative_to(dest), dest.iterdir())) - - self.assertEqual(src_files, dest_files) - - with tempfile.TemporaryDirectory() as src: - src = Path(src) - - with tempfile.TemporaryDirectory() as dest: - dest = Path(dest) - - (src / "foo").mkdir() - (src / "bar").touch() - - (src / "foo" / "baz").symlink_to(src / "bar") - - utils.copytree_resolved(src / "foo", dest, flatten=True) - - dest_files = set(map(lambda x: x.relative_to(dest), dest.iterdir())) - expected = set([Path("foo"), Path("foo") / "baz"]) - - self.assertEqual(dest_files, expected) - self.assertFalse((dest / "foo" / "baz").is_symlink()) - - with tempfile.TemporaryDirectory() as src: - src = Path(src) - - with tempfile.TemporaryDirectory() as dest: - dest = Path(dest) - - (src / "foo").mkdir() - (src / "bar").touch() - - (src / "foo" / "baz").symlink_to(src / "bar") - - utils.copytree_resolved(src / "foo", dest, flatten=False) + examples = [ + { + "flatten": False, + "files": [ + {"name": "foo", "dir": True, "target": None}, + {"name": "bar", "dir": False, "target": None}, + {"name": "foo/baz", "dir": False, "target": None}, + ], + "expected": [ + {"name": "foo", "dir": True, "target": None}, + {"name": "bar", "dir": False, "target": None}, + {"name": "foo/baz", "dir": False, "target": None}, + ] + }, + { + "flatten": True, + "files": [ + {"name": "foo", "dir": True, "target": None}, + {"name": "bar", "dir": False, "target": None}, + {"name": "foo/baz", "dir": False, "target": "bar"}, + ], + "expected": [ + {"name": "foo", "dir": True, "target": None}, + {"name": "foo/baz", "dir": False, "target": None}, + ] + }, + { + "flatten": False, + "files": [ + {"name": "foo", "dir": True, "target": None}, + {"name": "bar", "dir": False, "target": None}, + {"name": "foo/baz", "dir": False, "target": "bar"}, + ], + "expected": [ + {"name": "foo", "dir": True, "target": None}, + {"name": "foo/baz", "dir": False, "target": "foo/bar"}, + {"name": "foo/bar", "dir": False, "target": None}, + ] + }, + { + "flatten": False, + "files": [ + {"name": "foo", "dir": False, "target": "bar"}, + {"name": "bar", "dir": False, "target": "foo"}, + ], + "expected": [ + {"name": "foo", "dir": False, "target": "bar"}, + {"name": "bar", "dir": False, "target": "foo"}, + ] + }, + { + "flatten": True, + "files": [ + {"name": "foo", "dir": False, "target": "bar"}, + {"name": "bar", "dir": False, "target": "foo"}, + ], + "expected": [ + {"name": "foo", "dir": False, "target": "bar"}, + {"name": "bar", "dir": False, "target": "foo"}, + ] + }, + ] - dest_files = set(map(lambda x: x.relative_to(dest), dest.iterdir())) - expected = set([Path("foo"), Path("foo") / "baz", Path("foo") / "bar"]) + for ex in examples: + with tempfile.TemporaryDirectory() as src: + src = Path(src) - self.assertEqual(dest_files, expected) - self.assertTrue((dest / "foo" / "baz").is_symlink()) + with tempfile.TemporaryDirectory() as dest: + dest = Path(dest) - with tempfile.TemporaryDirectory() as src: - src = Path(src) + # Create the files. + for f in ex["files"]: + file_path = src / f["name"] - with tempfile.TemporaryDirectory() as dest: - dest = Path(dest) + if f["dir"]: + file_path.mkdir(parents=True) + else: + if f["target"] is None: + file_path.touch() + else: + target_path = src / f["target"] + file_path.symlink_to(target_path) - (src / "foo").symlink_to(src / "bar") - (src / "bar").symlink_to(src / "foo") + utils.copytree_resolved(src, dest, flatten=ex["flatten"]) - utils.copytree_resolved(src / "foo", dest, flatten=False) + expected = set(Path(f["name"]) for f in ex["expected"]) + actual = set(p.relative_to(dest) for p in dest.rglob('*')) - dest_files = set(map(lambda x: x.relative_to(dest), dest.iterdir())) - expected = set([Path("foo"), Path("bar")]) + self.assertEqual(expected, actual) - self.assertEqual(dest_files, expected) + for f in ex["expected"]: + for g in actual: + if Path(f["name"]) == g: + if f["target"] is None: + self.assertFalse((dest / g).is_symlink()) + else: + self.assertTrue((dest / g).is_symlink()) \ No newline at end of file From da3d3bde2d4edc0862f2ab1f733724ef847c068a Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Mon, 1 Dec 2025 13:58:56 -0700 Subject: [PATCH 21/74] Fix style issue --- lib/pavilion/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pavilion/utils.py b/lib/pavilion/utils.py index 2ff08d260..ef169a856 100644 --- a/lib/pavilion/utils.py +++ b/lib/pavilion/utils.py @@ -244,8 +244,8 @@ def copytree_resolved( dest.mkdir(exist_ok=True) files = src.iterdir() - for f in files: - copytree_resolved(f, dest / f.name, seen_files) + for fl in files: + copytree_resolved(fl, dest / fl.name, seen_files) return dest From 930243e2caad9e1bd35129994749743387b0d49d Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Mon, 1 Dec 2025 14:06:26 -0700 Subject: [PATCH 22/74] Fix copytree_resolve unit tests --- test/tests/utils_tests.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/test/tests/utils_tests.py b/test/tests/utils_tests.py index fd9f9727c..579648731 100644 --- a/test/tests/utils_tests.py +++ b/test/tests/utils_tests.py @@ -161,6 +161,7 @@ def test_copytree_resolved(self): examples = [ { "flatten": False, + "copy_root": None, "files": [ {"name": "foo", "dir": True, "target": None}, {"name": "bar", "dir": False, "target": None}, @@ -174,6 +175,7 @@ def test_copytree_resolved(self): }, { "flatten": True, + "copy_root": "foo", "files": [ {"name": "foo", "dir": True, "target": None}, {"name": "bar", "dir": False, "target": None}, @@ -186,6 +188,7 @@ def test_copytree_resolved(self): }, { "flatten": False, + "copy_root": "foo", "files": [ {"name": "foo", "dir": True, "target": None}, {"name": "bar", "dir": False, "target": None}, @@ -199,6 +202,7 @@ def test_copytree_resolved(self): }, { "flatten": False, + "copt_root": None, "files": [ {"name": "foo", "dir": False, "target": "bar"}, {"name": "bar", "dir": False, "target": "foo"}, @@ -210,6 +214,7 @@ def test_copytree_resolved(self): }, { "flatten": True, + "copy_root": None, "files": [ {"name": "foo", "dir": False, "target": "bar"}, {"name": "bar", "dir": False, "target": "foo"}, @@ -241,7 +246,10 @@ def test_copytree_resolved(self): target_path = src / f["target"] file_path.symlink_to(target_path) - utils.copytree_resolved(src, dest, flatten=ex["flatten"]) + if ex["copy_root"] is None: + utils.copytree_resolved(src, dest, flatten=ex["flatten"]) + else: + utils.copytree_resolved(src / ex["copy_root"], dest, flatten=ex["flatten"]) expected = set(Path(f["name"]) for f in ex["expected"]) actual = set(p.relative_to(dest) for p in dest.rglob('*')) From a868e94a5406a9707401d3876c372be63d99598a Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Mon, 1 Dec 2025 14:10:17 -0700 Subject: [PATCH 23/74] Fix bad variable names --- lib/pavilion/cmd_utils.py | 8 ++++---- lib/pavilion/commands/isolate.py | 8 ++++---- lib/pavilion/utils.py | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/pavilion/cmd_utils.py b/lib/pavilion/cmd_utils.py index 5d0d89210..b0fc979d8 100644 --- a/lib/pavilion/cmd_utils.py +++ b/lib/pavilion/cmd_utils.py @@ -548,7 +548,7 @@ def list_files(path: Path, include_root: bool = False) -> Iterator[Path]: if include_root: yield Path(root) - for f in files: - yield Path(root) / f - for d in dirs: - yield Path(root) / d \ No newline at end of file + for fname in files: + yield Path(root) / fname + for dname in dirs: + yield Path(root) / dname \ No newline at end of file diff --git a/lib/pavilion/commands/isolate.py b/lib/pavilion/commands/isolate.py index 265716a68..42b83939f 100644 --- a/lib/pavilion/commands/isolate.py +++ b/lib/pavilion/commands/isolate.py @@ -119,10 +119,10 @@ def _isolate(cls, test: TestRun, dest: Path, archive: bool, zip: bool) -> int: archive_format = "tar" try: - with tarfile.open(dest, "w:gz") as tf: - for f in list_files(test.path, include_root=True): - if f.name not in cls.IGNORE_FILES: - tf.add(f, arcname=f.relative_to(test.path.parent), recursive=False) + with tarfile.open(dest, "w:gz") as tarf: + for fname in list_files(test.path, include_root=True): + if fname.name not in cls.IGNORE_FILES: + tarf.add(f, arcname=f.relative_to(test.path.parent), recursive=False) except: output.fprint( sys.stderr, diff --git a/lib/pavilion/utils.py b/lib/pavilion/utils.py index ef169a856..9048315c8 100644 --- a/lib/pavilion/utils.py +++ b/lib/pavilion/utils.py @@ -244,8 +244,8 @@ def copytree_resolved( dest.mkdir(exist_ok=True) files = src.iterdir() - for fl in files: - copytree_resolved(fl, dest / fl.name, seen_files) + for fname in files: + copytree_resolved(fname, dest / fname.name, seen_files) return dest From 0bbbea4983257ca0083a6e83f0b24e167edf360a Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Mon, 1 Dec 2025 14:25:21 -0700 Subject: [PATCH 24/74] Fix more style issues --- lib/pavilion/cmd_utils.py | 2 +- lib/pavilion/commands/isolate.py | 16 +++++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/lib/pavilion/cmd_utils.py b/lib/pavilion/cmd_utils.py index b0fc979d8..d6c343328 100644 --- a/lib/pavilion/cmd_utils.py +++ b/lib/pavilion/cmd_utils.py @@ -551,4 +551,4 @@ def list_files(path: Path, include_root: bool = False) -> Iterator[Path]: for fname in files: yield Path(root) / fname for dname in dirs: - yield Path(root) / dname \ No newline at end of file + yield Path(root) / dname diff --git a/lib/pavilion/commands/isolate.py b/lib/pavilion/commands/isolate.py index 42b83939f..76cfa9784 100644 --- a/lib/pavilion/commands/isolate.py +++ b/lib/pavilion/commands/isolate.py @@ -3,7 +3,6 @@ import shutil import tarfile import sys -from typing import Dict, Any, Optional from pavilion import output from pavilion.config import PavConfig @@ -111,19 +110,22 @@ def _isolate(cls, test: TestRun, dest: Path, archive: bool, zip: bool) -> int: if len(dest.suffixes) == 0: dest = dest.with_suffix(".tgz") - archive_format = "gztar" + modestr = "w:gz" else: if len(dest.suffixes) == 0: dest = dest.with_suffix(".tar") - archive_format = "tar" + modestr = "w:" try: - with tarfile.open(dest, "w:gz") as tarf: + with tarfile.open(dest, modestr) as tarf: for fname in list_files(test.path, include_root=True): if fname.name not in cls.IGNORE_FILES: - tarf.add(f, arcname=f.relative_to(test.path.parent), recursive=False) - except: + tarf.add( + fname, + arcname=fname.relative_to(test.path.parent), + recursive=False) + except (tarfile.TarError, OSError): output.fprint( sys.stderr, f"Unable to isolate test {test.id} at {dest}.", @@ -141,4 +143,4 @@ def _isolate(cls, test: TestRun, dest: Path, archive: bool, zip: bool) -> int: return 8 - return 0 \ No newline at end of file + return 0 From dc028156f3f6e0cc005a4963f57eed9015e6f39c Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Mon, 1 Dec 2025 16:03:37 -0700 Subject: [PATCH 25/74] Complete copytree_resolve unit tests --- lib/pavilion/utils.py | 8 ++++---- test/tests/cmd_util_tests.py | 7 ------- test/tests/utils_tests.py | 17 ++++++++--------- 3 files changed, 12 insertions(+), 20 deletions(-) diff --git a/lib/pavilion/utils.py b/lib/pavilion/utils.py index 9048315c8..f0e9d98ba 100644 --- a/lib/pavilion/utils.py +++ b/lib/pavilion/utils.py @@ -224,14 +224,14 @@ def copytree_resolved( seen_files.add(src) if src.is_symlink(): - target = src.resolve() + target = Path(os.readlink(src)) if target not in seen_files: if flatten: - copytree_resolved(target, dest, seen_files) + copytree_resolved(target, dest, seen_files, flatten=flatten) else: # Retain the symlink but copy its target to the present directory - copytree_resolved(target, dest.parent / target.name, seen_files) + copytree_resolved(target, dest.parent / target.name, seen_files, flatten=flatten) dest.symlink_to(dest.parent / target.name) else: dest.symlink_to(target) @@ -245,7 +245,7 @@ def copytree_resolved( files = src.iterdir() for fname in files: - copytree_resolved(fname, dest / fname.name, seen_files) + copytree_resolved(fname, dest / fname.name, seen_files, flatten=flatten) return dest diff --git a/test/tests/cmd_util_tests.py b/test/tests/cmd_util_tests.py index 295e2ffb9..22f532100 100644 --- a/test/tests/cmd_util_tests.py +++ b/test/tests/cmd_util_tests.py @@ -63,11 +63,4 @@ def test_arg_filtered_tests(self): self.assertEqual(len(cmd_utils.arg_filtered_tests(self.pav_cfg, args).paths), count) - def test_copytree_resolved(self): - with tempfile.TemporaryDirectory() as src: - with tempfile.TemporaryDirectory() as dest: - (Path(src) / "foo").touch() - - cmd_utils.copytree_resolved(Path(src), Path(dest)) - # TODO: We really need to add unit tests for each of the cmd utils functions. diff --git a/test/tests/utils_tests.py b/test/tests/utils_tests.py index 579648731..5953ff04f 100644 --- a/test/tests/utils_tests.py +++ b/test/tests/utils_tests.py @@ -9,6 +9,7 @@ from pavilion import unittest from pavilion import utils +from pavilion.cmd_utils import list_files class UtilsTests(unittest.PavTestCase): @@ -182,8 +183,7 @@ def test_copytree_resolved(self): {"name": "foo/baz", "dir": False, "target": "bar"}, ], "expected": [ - {"name": "foo", "dir": True, "target": None}, - {"name": "foo/baz", "dir": False, "target": None}, + {"name": "baz", "dir": False, "target": None}, ] }, { @@ -195,14 +195,13 @@ def test_copytree_resolved(self): {"name": "foo/baz", "dir": False, "target": "bar"}, ], "expected": [ - {"name": "foo", "dir": True, "target": None}, - {"name": "foo/baz", "dir": False, "target": "foo/bar"}, - {"name": "foo/bar", "dir": False, "target": None}, + {"name": "baz", "dir": False, "target": "bar"}, + {"name": "bar", "dir": False, "target": None}, ] }, { "flatten": False, - "copt_root": None, + "copy_root": None, "files": [ {"name": "foo", "dir": False, "target": "bar"}, {"name": "bar", "dir": False, "target": "foo"}, @@ -220,7 +219,6 @@ def test_copytree_resolved(self): {"name": "bar", "dir": False, "target": "foo"}, ], "expected": [ - {"name": "foo", "dir": False, "target": "bar"}, {"name": "bar", "dir": False, "target": "foo"}, ] }, @@ -241,6 +239,7 @@ def test_copytree_resolved(self): file_path.mkdir(parents=True) else: if f["target"] is None: + print(f"Making file {file_path}") file_path.touch() else: target_path = src / f["target"] @@ -252,7 +251,7 @@ def test_copytree_resolved(self): utils.copytree_resolved(src / ex["copy_root"], dest, flatten=ex["flatten"]) expected = set(Path(f["name"]) for f in ex["expected"]) - actual = set(p.relative_to(dest) for p in dest.rglob('*')) + actual = set(p.relative_to(dest) for p in list_files(dest)) self.assertEqual(expected, actual) @@ -262,4 +261,4 @@ def test_copytree_resolved(self): if f["target"] is None: self.assertFalse((dest / g).is_symlink()) else: - self.assertTrue((dest / g).is_symlink()) \ No newline at end of file + self.assertTrue((dest / g).is_symlink()) From 73cef316adaf38e0caace4841ea0fa23df6f067c Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Mon, 1 Dec 2025 16:15:36 -0700 Subject: [PATCH 26/74] Use copytree_resolved in isolate cmd --- lib/pavilion/commands/isolate.py | 22 ++++++++++++---------- lib/pavilion/utils.py | 6 +++++- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/lib/pavilion/commands/isolate.py b/lib/pavilion/commands/isolate.py index 76cfa9784..4a5ce1c1e 100644 --- a/lib/pavilion/commands/isolate.py +++ b/lib/pavilion/commands/isolate.py @@ -117,21 +117,23 @@ def _isolate(cls, test: TestRun, dest: Path, archive: bool, zip: bool) -> int: modestr = "w:" - try: - with tarfile.open(dest, modestr) as tarf: - for fname in list_files(test.path, include_root=True): - if fname.name not in cls.IGNORE_FILES: + with tempfile.TemporaryDirectory() as tmp: + utils.copytree_resolved(test.path, tmp, ignore_files=cls.IGNORE_FILES) + + try: + with tarfile.open(dest, modestr) as tarf: + for fname in list_files(tmp): tarf.add( fname, arcname=fname.relative_to(test.path.parent), recursive=False) - except (tarfile.TarError, OSError): - output.fprint( - sys.stderr, - f"Unable to isolate test {test.id} at {dest}.", - color=output.RED) + except (tarfile.TarError, OSError): + output.fprint( + sys.stderr, + f"Unable to isolate test {test.id} at {dest}.", + color=output.RED) - return 7 + return 7 else: try: shutil.copytree(test.path, dest, ignore=lambda x, y: cls.IGNORE_FILES) diff --git a/lib/pavilion/utils.py b/lib/pavilion/utils.py index f0e9d98ba..a20510cb5 100644 --- a/lib/pavilion/utils.py +++ b/lib/pavilion/utils.py @@ -211,11 +211,15 @@ def copytree_resolved( src: Path, dest: Path, seen_files: Optional[Set] = None, - flatten: bool = False) -> Path: + flatten: bool = False, + ignore_files: List[str], bool]) -> Path: """Copy a directory tree to another location, such that the resulting directory contains the targets of all symlinks. If flatten is specified, replace all symlinks with their targets.""" + if src.name in ignore_files: + return src + seen_files = seen_files or set() if src in seen_files: From fd7c38984c88f1ddbb6771eef4d5f75860d7ea75 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Mon, 1 Dec 2025 16:21:01 -0700 Subject: [PATCH 27/74] Fix bad type annotation --- lib/pavilion/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pavilion/utils.py b/lib/pavilion/utils.py index a20510cb5..5b4ae7224 100644 --- a/lib/pavilion/utils.py +++ b/lib/pavilion/utils.py @@ -212,7 +212,7 @@ def copytree_resolved( dest: Path, seen_files: Optional[Set] = None, flatten: bool = False, - ignore_files: List[str], bool]) -> Path: + ignore_files: List[str]) -> Path: """Copy a directory tree to another location, such that the resulting directory contains the targets of all symlinks. If flatten is specified, replace all symlinks with their targets.""" From 5944ca299f1a81fb768df610b541ad643e90c4fa Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Mon, 1 Dec 2025 16:22:16 -0700 Subject: [PATCH 28/74] Fix ignore_files argument --- lib/pavilion/utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/pavilion/utils.py b/lib/pavilion/utils.py index 5b4ae7224..bcea3b9bf 100644 --- a/lib/pavilion/utils.py +++ b/lib/pavilion/utils.py @@ -212,11 +212,13 @@ def copytree_resolved( dest: Path, seen_files: Optional[Set] = None, flatten: bool = False, - ignore_files: List[str]) -> Path: + ignore_files: List[str] = None) -> Path: """Copy a directory tree to another location, such that the resulting directory contains the targets of all symlinks. If flatten is specified, replace all symlinks with their targets.""" + ignore_files = ignore_files or [] + if src.name in ignore_files: return src From d6bb72c97d57264feb426b0123935b2c1665eb16 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Tue, 2 Dec 2025 10:19:09 -0700 Subject: [PATCH 29/74] Output environment to file --- lib/pavilion/test_run/test_run.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/pavilion/test_run/test_run.py b/lib/pavilion/test_run/test_run.py index a3b5eec6c..e977fd818 100644 --- a/lib/pavilion/test_run/test_run.py +++ b/lib/pavilion/test_run/test_run.py @@ -1191,8 +1191,6 @@ def _write_script(self, stype: str, path: Path, config: dict, module_wrappers: d script.comment('List all the module modules for posterity') script.command("module -t list") script.newline() - script.comment('Output the environment for posterity') - script.command("declare -p") if self.spack_enabled(): script.command(f'echo "(pav) Setting up Spack."') @@ -1229,6 +1227,9 @@ def _write_script(self, stype: str, path: Path, config: dict, module_wrappers: d script.command('spack load {} || exit 1' .format(package)) + script.comment('Output the environment for posterity') + script.command(f'declare -p | tee > {stype}.env.sh') + script.command(f'echo "(pav) Executing {stype} commands."') script.newline() cmds = config.get('cmds', []) From 44eb42cd1acb71536fb3118cf941a33773cde0a2 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Tue, 2 Dec 2025 10:32:56 -0700 Subject: [PATCH 30/74] Fix environment file path --- lib/pavilion/test_run/test_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pavilion/test_run/test_run.py b/lib/pavilion/test_run/test_run.py index e977fd818..2d77af2e9 100644 --- a/lib/pavilion/test_run/test_run.py +++ b/lib/pavilion/test_run/test_run.py @@ -1228,7 +1228,7 @@ def _write_script(self, stype: str, path: Path, config: dict, module_wrappers: d .format(package)) script.comment('Output the environment for posterity') - script.command(f'declare -p | tee > {stype}.env.sh') + script.command(f'declare -p | tee > {path.parent / stype}.env.sh') script.command(f'echo "(pav) Executing {stype} commands."') script.newline() From 04959a28c84fb746fa835b36d12801e110f8dc14 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Tue, 2 Dec 2025 12:20:05 -0700 Subject: [PATCH 31/74] Fix verbose output in run script --- lib/pavilion/test_run/test_run.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/pavilion/test_run/test_run.py b/lib/pavilion/test_run/test_run.py index 2d77af2e9..f4bc0c5e9 100644 --- a/lib/pavilion/test_run/test_run.py +++ b/lib/pavilion/test_run/test_run.py @@ -1227,9 +1227,15 @@ def _write_script(self, stype: str, path: Path, config: dict, module_wrappers: d script.command('spack load {} || exit 1' .format(package)) + script.newline() script.comment('Output the environment for posterity') - script.command(f'declare -p | tee > {path.parent / stype}.env.sh') + if verbose: + script.command(f'declare -p | tee > {path.parent / stype}.env.sh') + else: + script.command(f'declare -p > {path.parent / stype}.env.sh') + + script.newline() script.command(f'echo "(pav) Executing {stype} commands."') script.newline() cmds = config.get('cmds', []) From 25f4a7ddc40a55a5a5c8d3b6fe8460c949aae510 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Tue, 2 Dec 2025 13:05:54 -0700 Subject: [PATCH 32/74] Write kickoff script to isolated test --- lib/pavilion/commands/isolate.py | 92 ++++++++++++++++++++++---------- 1 file changed, 64 insertions(+), 28 deletions(-) diff --git a/lib/pavilion/commands/isolate.py b/lib/pavilion/commands/isolate.py index 4a5ce1c1e..d2f52c7b1 100644 --- a/lib/pavilion/commands/isolate.py +++ b/lib/pavilion/commands/isolate.py @@ -9,6 +9,7 @@ from pavilion.test_run import TestRun from pavilion.test_ids import TestID from pavilion.cmd_utils import get_last_test_id, get_tests_by_id, list_files +from pavilion.schedulers.config import validate_config from .base_classes import Command @@ -16,6 +17,7 @@ class IsolateCommand(Command): """Isolates an existing test run in a form that can be run without Pavilion.""" IGNORE_FILES = ("series", "job") + KICKOFF_FN = "kickoff.isolated" def __init__(self): super().__init__( @@ -106,34 +108,8 @@ def _isolate(cls, test: TestRun, dest: Path, archive: bool, zip: bool) -> int: return 6 if archive: - if zip: - if len(dest.suffixes) == 0: - dest = dest.with_suffix(".tgz") - - modestr = "w:gz" - else: - if len(dest.suffixes) == 0: - dest = dest.with_suffix(".tar") - - modestr = "w:" - - with tempfile.TemporaryDirectory() as tmp: - utils.copytree_resolved(test.path, tmp, ignore_files=cls.IGNORE_FILES) - - try: - with tarfile.open(dest, modestr) as tarf: - for fname in list_files(tmp): - tarf.add( - fname, - arcname=fname.relative_to(test.path.parent), - recursive=False) - except (tarfile.TarError, OSError): - output.fprint( - sys.stderr, - f"Unable to isolate test {test.id} at {dest}.", - color=output.RED) - - return 7 + self._write_tarball(test.id, test.path, dest, zip, cls.IGNORE_FILES) + else: try: shutil.copytree(test.path, dest, ignore=lambda x, y: cls.IGNORE_FILES) @@ -145,4 +121,64 @@ def _isolate(cls, test: TestRun, dest: Path, archive: bool, zip: bool) -> int: return 8 + self._write_kickoff_script(pav_cfg, test.id, dest / cls.KICKOFF_FN) + return 0 + + def _write_tarball(self, pav_cfg: PavConfig, test_id: TestID, src: Path, dest: Path, zip: bool, + ignore_files) -> None: + if zip: + if len(dest.suffixes) == 0: + dest = dest.with_suffix(".tgz") + + modestr = "w:gz" + else: + if len(dest.suffixes) == 0: + dest = dest.with_suffix(".tar") + + modestr = "w:" + + with tempfile.TemporaryDirectory() as tmp: + utils.copytree_resolved(src, tmp, ignore_files=ignore_files) + self._write_kickoff_script(pav_cfg, test_id, tmp / self.KICKOFF_FN) + + try: + with tarfile.open(dest, modestr) as tarf: + for fname in list_files(tmp): + tarf.add( + fname, + arcname=fname.relative_to(src.parent), + recursive=False) + except (tarfile.TarError, OSError): + output.fprint( + sys.stderr, + f"Unable to isolate test {test_id} at {dest}.", + color=output.RED) + + return 7 + + def _write_kickoff_script(self, pav_cfg: PavConfig, test_id: TestID, script_path: Path) -> None: + """Write a special kickoff script that can be used to run the given test independently of + Pavilion.""" + + test = TestRun.load_from_raw_id(pav_cfg, test_id) + + try: + sched = schedulers.get_plugin(test.scheduler) + except SchedulerPluginError: + output.fprint( + sys.stderr, + f"Unable to generate kickoff script for test {test_id}: unable to load scheduler" + f" {test.scheduler}." + ) + return 9 + + script = sched._get_kickoff_script_header( + job_name="pav_{test.name}_isolated", + sched_config=validate_config(test.config['schedule']), + nodes=None, + node_range=None, + shebang=test.shebang + ) + + script.write(script_path) \ No newline at end of file From 84bc915fd1005bb13d49c1c4030869cd257674b4 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Tue, 2 Dec 2025 13:10:38 -0700 Subject: [PATCH 33/74] Fix isolate command class methods --- lib/pavilion/commands/isolate.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/pavilion/commands/isolate.py b/lib/pavilion/commands/isolate.py index d2f52c7b1..ac5ab9869 100644 --- a/lib/pavilion/commands/isolate.py +++ b/lib/pavilion/commands/isolate.py @@ -121,11 +121,12 @@ def _isolate(cls, test: TestRun, dest: Path, archive: bool, zip: bool) -> int: return 8 - self._write_kickoff_script(pav_cfg, test.id, dest / cls.KICKOFF_FN) + cls._write_kickoff_script(pav_cfg, test.id, dest / cls.KICKOFF_FN) return 0 - def _write_tarball(self, pav_cfg: PavConfig, test_id: TestID, src: Path, dest: Path, zip: bool, + @classmethod + def _write_tarball(cls, pav_cfg: PavConfig, test_id: TestID, src: Path, dest: Path, zip: bool, ignore_files) -> None: if zip: if len(dest.suffixes) == 0: @@ -140,7 +141,7 @@ def _write_tarball(self, pav_cfg: PavConfig, test_id: TestID, src: Path, dest: P with tempfile.TemporaryDirectory() as tmp: utils.copytree_resolved(src, tmp, ignore_files=ignore_files) - self._write_kickoff_script(pav_cfg, test_id, tmp / self.KICKOFF_FN) + cls._write_kickoff_script(pav_cfg, test_id, tmp / cls.KICKOFF_FN) try: with tarfile.open(dest, modestr) as tarf: @@ -157,7 +158,8 @@ def _write_tarball(self, pav_cfg: PavConfig, test_id: TestID, src: Path, dest: P return 7 - def _write_kickoff_script(self, pav_cfg: PavConfig, test_id: TestID, script_path: Path) -> None: + @classmethod + def _write_kickoff_script(cls, pav_cfg: PavConfig, test_id: TestID, script_path: Path) -> None: """Write a special kickoff script that can be used to run the given test independently of Pavilion.""" From bd4ac92cc6fcbf9f3590612837e5adbd6c607485 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Tue, 2 Dec 2025 13:12:24 -0700 Subject: [PATCH 34/74] Fix _isolate method --- lib/pavilion/commands/isolate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pavilion/commands/isolate.py b/lib/pavilion/commands/isolate.py index ac5ab9869..c75772d22 100644 --- a/lib/pavilion/commands/isolate.py +++ b/lib/pavilion/commands/isolate.py @@ -89,10 +89,10 @@ def run(self, pav_cfg: PavConfig, args: Namespace) -> int: test = next(iter(tests)) - return self._isolate(test, args.path, args.archive, args.zip) + return self._isolate(pav_cfg, test, args.path, args.archive, args.zip) @classmethod - def _isolate(cls, test: TestRun, dest: Path, archive: bool, zip: bool) -> int: + def _isolate(cls, pav_cfg: PavConfig, test: TestRun, dest: Path, archive: bool, zip: bool) -> int: if not test.path.is_dir(): output.fprint(sys.stderr, "Directory '{}' does not exist." .format(test.path.as_posix()), color=output.RED) From f4939c3b2e938f92af04227cc58b1aeaaa9b487d Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Tue, 2 Dec 2025 13:19:25 -0700 Subject: [PATCH 35/74] Fix isolate command ID parsing --- lib/pavilion/commands/isolate.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/pavilion/commands/isolate.py b/lib/pavilion/commands/isolate.py index c75772d22..9fc4dd66c 100644 --- a/lib/pavilion/commands/isolate.py +++ b/lib/pavilion/commands/isolate.py @@ -121,7 +121,7 @@ def _isolate(cls, pav_cfg: PavConfig, test: TestRun, dest: Path, archive: bool, return 8 - cls._write_kickoff_script(pav_cfg, test.id, dest / cls.KICKOFF_FN) + cls._write_kickoff_script(pav_cfg, test, dest / cls.KICKOFF_FN) return 0 @@ -159,12 +159,10 @@ def _write_tarball(cls, pav_cfg: PavConfig, test_id: TestID, src: Path, dest: Pa return 7 @classmethod - def _write_kickoff_script(cls, pav_cfg: PavConfig, test_id: TestID, script_path: Path) -> None: + def _write_kickoff_script(cls, pav_cfg: PavConfig, test: TestRun, script_path: Path) -> None: """Write a special kickoff script that can be used to run the given test independently of Pavilion.""" - test = TestRun.load_from_raw_id(pav_cfg, test_id) - try: sched = schedulers.get_plugin(test.scheduler) except SchedulerPluginError: From 94e21536b709754b6b610c6ae4430b0d3134521d Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Tue, 2 Dec 2025 13:20:15 -0700 Subject: [PATCH 36/74] Fix missing import --- lib/pavilion/commands/isolate.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pavilion/commands/isolate.py b/lib/pavilion/commands/isolate.py index 9fc4dd66c..7417e4d7e 100644 --- a/lib/pavilion/commands/isolate.py +++ b/lib/pavilion/commands/isolate.py @@ -5,6 +5,7 @@ import sys from pavilion import output +from pavilion import schedulers from pavilion.config import PavConfig from pavilion.test_run import TestRun from pavilion.test_ids import TestID From f398fa147e9e554396f5897973ec4b94fd7b3aa9 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Tue, 2 Dec 2025 13:27:39 -0700 Subject: [PATCH 37/74] Temporarily un-ignore job directory --- lib/pavilion/commands/isolate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pavilion/commands/isolate.py b/lib/pavilion/commands/isolate.py index 7417e4d7e..30a3c1f49 100644 --- a/lib/pavilion/commands/isolate.py +++ b/lib/pavilion/commands/isolate.py @@ -17,7 +17,7 @@ class IsolateCommand(Command): """Isolates an existing test run in a form that can be run without Pavilion.""" - IGNORE_FILES = ("series", "job") + IGNORE_FILES = ("series") KICKOFF_FN = "kickoff.isolated" def __init__(self): From 8b7965e5486b6af517b48cf0f68de2d5776cecc0 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Tue, 2 Dec 2025 13:28:49 -0700 Subject: [PATCH 38/74] Fix missing comma --- lib/pavilion/commands/isolate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pavilion/commands/isolate.py b/lib/pavilion/commands/isolate.py index 30a3c1f49..5f9eaad2d 100644 --- a/lib/pavilion/commands/isolate.py +++ b/lib/pavilion/commands/isolate.py @@ -17,7 +17,7 @@ class IsolateCommand(Command): """Isolates an existing test run in a form that can be run without Pavilion.""" - IGNORE_FILES = ("series") + IGNORE_FILES = ("series",) KICKOFF_FN = "kickoff.isolated" def __init__(self): From 95ed87365c2f20d7c626128d6e94177f00f4f979 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Tue, 2 Dec 2025 13:37:54 -0700 Subject: [PATCH 39/74] Fix scheduler node min and max --- lib/pavilion/schedulers/scheduler.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/pavilion/schedulers/scheduler.py b/lib/pavilion/schedulers/scheduler.py index ed08420b1..ba8d8dcd0 100644 --- a/lib/pavilion/schedulers/scheduler.py +++ b/lib/pavilion/schedulers/scheduler.py @@ -49,8 +49,13 @@ def __init__(self, job_name: str, sched_config: dict, if nodes is None: self._include_nodes = self._config['include_nodes'] self._exclude_nodes = self._config['exclude_nodes'] - self._node_min = node_range[0] - self._node_max = node_range[1] + + if node_range is not None: + self._node_min = node_range[0] + self._node_max = node_range[1] + else: + self._node_min = None + self._node_max = None else: self._include_nodes = nodes # Any nodes in the exclude list will have already been filtered out. From 3ec0bb0ff653c034427730c3d37a61fcad715f2f Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Tue, 2 Dec 2025 14:03:17 -0700 Subject: [PATCH 40/74] Remove flatten option from copytree_resolved --- lib/pavilion/commands/isolate.py | 10 +++++----- lib/pavilion/utils.py | 13 +++---------- 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/lib/pavilion/commands/isolate.py b/lib/pavilion/commands/isolate.py index 5f9eaad2d..8c6513caf 100644 --- a/lib/pavilion/commands/isolate.py +++ b/lib/pavilion/commands/isolate.py @@ -1,6 +1,5 @@ from argparse import ArgumentParser, Namespace, Action from pathlib import Path -import shutil import tarfile import sys @@ -10,6 +9,7 @@ from pavilion.test_run import TestRun from pavilion.test_ids import TestID from pavilion.cmd_utils import get_last_test_id, get_tests_by_id, list_files +from pavilion.utils import copytree_resolved from pavilion.schedulers.config import validate_config from .base_classes import Command @@ -113,11 +113,11 @@ def _isolate(cls, pav_cfg: PavConfig, test: TestRun, dest: Path, archive: bool, else: try: - shutil.copytree(test.path, dest, ignore=lambda x, y: cls.IGNORE_FILES) - except OSError: + copytree_resolved(test.path, dest, ignore_files=cls.IGNORE_FILES) + except OSError as err: output.fprint( sys.stderr, - f"Unable to isolate test {test.id} at {dest}.", + f"Unable to isolate test {test.id} at {dest}: {err}", color=output.RED) return 8 @@ -182,4 +182,4 @@ def _write_kickoff_script(cls, pav_cfg: PavConfig, test: TestRun, script_path: P shebang=test.shebang ) - script.write(script_path) \ No newline at end of file + script.write(script_path) diff --git a/lib/pavilion/utils.py b/lib/pavilion/utils.py index bcea3b9bf..a31b1502e 100644 --- a/lib/pavilion/utils.py +++ b/lib/pavilion/utils.py @@ -211,11 +211,9 @@ def copytree_resolved( src: Path, dest: Path, seen_files: Optional[Set] = None, - flatten: bool = False, ignore_files: List[str] = None) -> Path: """Copy a directory tree to another location, such that the resulting directory contains - the targets of all symlinks. If flatten is specified, replace all symlinks with - their targets.""" + the targets of all symlinks.""" ignore_files = ignore_files or [] @@ -233,12 +231,7 @@ def copytree_resolved( target = Path(os.readlink(src)) if target not in seen_files: - if flatten: - copytree_resolved(target, dest, seen_files, flatten=flatten) - else: - # Retain the symlink but copy its target to the present directory - copytree_resolved(target, dest.parent / target.name, seen_files, flatten=flatten) - dest.symlink_to(dest.parent / target.name) + copytree_resolved(target, dest, seen_files) else: dest.symlink_to(target) @@ -251,7 +244,7 @@ def copytree_resolved( files = src.iterdir() for fname in files: - copytree_resolved(fname, dest / fname.name, seen_files, flatten=flatten) + copytree_resolved(fname, dest / fname.name, seen_files) return dest From 6fd4d9006e410a734eca55397eaa9f04d3489dfc Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Tue, 2 Dec 2025 14:06:09 -0700 Subject: [PATCH 41/74] Fix isolate kickoff script --- lib/pavilion/commands/isolate.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/pavilion/commands/isolate.py b/lib/pavilion/commands/isolate.py index 8c6513caf..929776ca1 100644 --- a/lib/pavilion/commands/isolate.py +++ b/lib/pavilion/commands/isolate.py @@ -174,7 +174,7 @@ def _write_kickoff_script(cls, pav_cfg: PavConfig, test: TestRun, script_path: P ) return 9 - script = sched._get_kickoff_script_header( + header = sched._get_kickoff_script_header( job_name="pav_{test.name}_isolated", sched_config=validate_config(test.config['schedule']), nodes=None, @@ -182,4 +182,5 @@ def _write_kickoff_script(cls, pav_cfg: PavConfig, test: TestRun, script_path: P shebang=test.shebang ) + script = ScriptComposer(header=header) script.write(script_path) From ddd741120fb4e445317e79e774467eb15ed0588e Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Tue, 2 Dec 2025 14:10:52 -0700 Subject: [PATCH 42/74] Fix isolate command job name --- lib/pavilion/commands/isolate.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/pavilion/commands/isolate.py b/lib/pavilion/commands/isolate.py index 929776ca1..e39244a7e 100644 --- a/lib/pavilion/commands/isolate.py +++ b/lib/pavilion/commands/isolate.py @@ -10,6 +10,7 @@ from pavilion.test_ids import TestID from pavilion.cmd_utils import get_last_test_id, get_tests_by_id, list_files from pavilion.utils import copytree_resolved +from pavilion.scriptcomposer import ScriptComposer from pavilion.schedulers.config import validate_config from .base_classes import Command @@ -175,7 +176,7 @@ def _write_kickoff_script(cls, pav_cfg: PavConfig, test: TestRun, script_path: P return 9 header = sched._get_kickoff_script_header( - job_name="pav_{test.name}_isolated", + job_name=f"pav_{test.name}_isolated", sched_config=validate_config(test.config['schedule']), nodes=None, node_range=None, From 8470e5b2de32d62d225322a48970759a8ff893d4 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Tue, 2 Dec 2025 14:18:42 -0700 Subject: [PATCH 43/74] Add run script to kickoff for pav isolate --- lib/pavilion/commands/isolate.py | 9 ++++++++- lib/pavilion/test_run/test_run.py | 19 +++++++++++++++---- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/lib/pavilion/commands/isolate.py b/lib/pavilion/commands/isolate.py index e39244a7e..1cc84d2e1 100644 --- a/lib/pavilion/commands/isolate.py +++ b/lib/pavilion/commands/isolate.py @@ -184,4 +184,11 @@ def _write_kickoff_script(cls, pav_cfg: PavConfig, test: TestRun, script_path: P ) script = ScriptComposer(header=header) - script.write(script_path) + script.newline() + + test._write_script( + script, + 'run', + script_path, + test.config['run'], + test.config.get('module_wrappers', {})) diff --git a/lib/pavilion/test_run/test_run.py b/lib/pavilion/test_run/test_run.py index f4bc0c5e9..217d3ec35 100644 --- a/lib/pavilion/test_run/test_run.py +++ b/lib/pavilion/test_run/test_run.py @@ -319,8 +319,12 @@ def save(self) -> None: self.status.set(STATES.CREATED, "Test directory and status file created.") + header = scriptcomposer.ScriptHeader(shebang=self.shebang) + script = scriptcomposer.ScriptComposer(header=header) + if self._build_needed(): self._write_script( + script, 'build', path=self.build_script_path, config=self.config.get('build', {}), @@ -334,6 +338,7 @@ def save(self) -> None: self._build_trivial() self._write_script( + script, 'run', path=self.run_tmpl_path, config=self.config.get('run', {}), @@ -521,7 +526,11 @@ def finalize(self, new_vars: VariableSetManager): self.save_attributes() + header = scriptcomposer.ScriptHeader(shebang=self.shebang) + script = scriptcomposer.ScriptComposer(header=header) + self._write_script( + script, 'run', self.run_script_path, self.config['run'], @@ -1119,7 +1128,12 @@ def complete_time(self): .format(run_complete_path.as_posix(), err)) return None - def _write_script(self, stype: str, path: Path, config: dict, module_wrappers: dict): + def _write_script(self, + script: scriptcomposer.ScriptComposer, + stype: str, + path: Path, + config: dict, + module_wrappers: dict): """Write a build or run script or template. The formats for each are mostly identical. :param stype: The type of script (run or build). @@ -1128,9 +1142,6 @@ def _write_script(self, stype: str, path: Path, config: dict, module_wrappers: d :param module_wrappers: The module wrappers definition. """ - header = scriptcomposer.ScriptHeader(shebang=self.shebang) - script = scriptcomposer.ScriptComposer(header=header) - verbose = config.get('verbose', 'false').lower() == 'true' if verbose: From 2ee5194d217c2c15fa5c5cf256743eea07af0902 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Tue, 2 Dec 2025 14:36:09 -0700 Subject: [PATCH 44/74] Isolate run script --- lib/pavilion/commands/isolate.py | 23 ++++++++++++++++------- lib/pavilion/test_run/test_run.py | 28 ++++++++++++++++++---------- 2 files changed, 34 insertions(+), 17 deletions(-) diff --git a/lib/pavilion/commands/isolate.py b/lib/pavilion/commands/isolate.py index 1cc84d2e1..15546f975 100644 --- a/lib/pavilion/commands/isolate.py +++ b/lib/pavilion/commands/isolate.py @@ -18,8 +18,9 @@ class IsolateCommand(Command): """Isolates an existing test run in a form that can be run without Pavilion.""" - IGNORE_FILES = ("series",) + IGNORE_FILES = ("series", "job") KICKOFF_FN = "kickoff.isolated" + PAV_LIB_FN = "pav-lib.bash" def __init__(self): super().__init__( @@ -123,6 +124,9 @@ def _isolate(cls, pav_cfg: PavConfig, test: TestRun, dest: Path, archive: bool, return 8 + pav_lib_bash = pav_cfg.pav_root / 'bin' / cls.PAV_LIB_FN + shutil.copyfile(pav_lib_bash, dest / cls.PAV_LIB_FN) + cls._write_kickoff_script(pav_cfg, test, dest / cls.KICKOFF_FN) return 0 @@ -143,6 +147,11 @@ def _write_tarball(cls, pav_cfg: PavConfig, test_id: TestID, src: Path, dest: Pa with tempfile.TemporaryDirectory() as tmp: utils.copytree_resolved(src, tmp, ignore_files=ignore_files) + + # Copy Pavilion bash library into tarball + pav_lib_bash = pav_cfg.pav_root / 'bin' / cls.PAV_LIB_FN + shutil.copyfile(pav_lib_bash, tmp / cls.PAV_LIB_FN) + cls._write_kickoff_script(pav_cfg, test_id, tmp / cls.KICKOFF_FN) try: @@ -184,11 +193,11 @@ def _write_kickoff_script(cls, pav_cfg: PavConfig, test: TestRun, script_path: P ) script = ScriptComposer(header=header) - script.newline() test._write_script( - script, - 'run', - script_path, - test.config['run'], - test.config.get('module_wrappers', {})) + script=script, + stype='run', + path=script_path, + config=test.config['run'], + module_wrappers=test.config.get('module_wrappers', {}), + isolate=True) diff --git a/lib/pavilion/test_run/test_run.py b/lib/pavilion/test_run/test_run.py index 217d3ec35..9673249a8 100644 --- a/lib/pavilion/test_run/test_run.py +++ b/lib/pavilion/test_run/test_run.py @@ -1133,7 +1133,8 @@ def _write_script(self, stype: str, path: Path, config: dict, - module_wrappers: dict): + module_wrappers: dict, + isolate: bool = False) -> None: """Write a build or run script or template. The formats for each are mostly identical. :param stype: The type of script (run or build). @@ -1149,17 +1150,23 @@ def _write_script(self, script.command('set -v') script.newline() - pav_lib_bash = self._pav_cfg.pav_root/'bin'/'pav-lib.bash' + if isolate: + pav_lib_bash = path.parent / "pav-lib.bash" + else: + pav_lib_bash = self._pav_cfg.pav_root/'bin'/'pav-lib.bash' script.command(f'echo "(pav) Starting {stype} script"') # If we include this directly, it breaks build hashing. script.comment('The first (and only) argument of the build script is ' 'the test id.') - script.env_change({ - 'TEST_ID': '${1:-0}', # Default to test id 0 if one isn't given. - 'PAV_CONFIG_FILE': self._pav_cfg['pav_cfg_file'] - }) + + env = {'TEST_ID': '${1:-0}'} # Default to test id 0 if one isn't given. + + if not isolate: + env["PAV_CONFIG_FILE"] = self._pav_cfg['pav_cfg_file'] + + script.env_change(env) script.command('source {}'.format(pav_lib_bash)) if config.get('preamble', []): @@ -1241,10 +1248,11 @@ def _write_script(self, script.newline() script.comment('Output the environment for posterity') - if verbose: - script.command(f'declare -p | tee > {path.parent / stype}.env.sh') - else: - script.command(f'declare -p > {path.parent / stype}.env.sh') + if not isolate: + if verbose: + script.command(f'declare -p | tee > {path.parent / stype}.env.sh') + else: + script.command(f'declare -p > {path.parent / stype}.env.sh') script.newline() script.command(f'echo "(pav) Executing {stype} commands."') From bdf08b9a7ca5030501b810c9a1cad7bc8cdf75cd Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Tue, 2 Dec 2025 14:36:54 -0700 Subject: [PATCH 45/74] Fix missing import --- lib/pavilion/commands/isolate.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pavilion/commands/isolate.py b/lib/pavilion/commands/isolate.py index 15546f975..55742baec 100644 --- a/lib/pavilion/commands/isolate.py +++ b/lib/pavilion/commands/isolate.py @@ -2,6 +2,7 @@ from pathlib import Path import tarfile import sys +import shutil from pavilion import output from pavilion import schedulers From 2f54c8e8927bebb7e7ac998c2e5375bf44743628 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Tue, 2 Dec 2025 14:41:04 -0700 Subject: [PATCH 46/74] Add newlines to build/run script --- lib/pavilion/test_run/test_run.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/pavilion/test_run/test_run.py b/lib/pavilion/test_run/test_run.py index 9673249a8..475e402e8 100644 --- a/lib/pavilion/test_run/test_run.py +++ b/lib/pavilion/test_run/test_run.py @@ -1178,6 +1178,7 @@ def _write_script(self, if stype == 'build' and not self.build_local: script.comment('To be built in an allocation.') + script.newline() script.command(f'echo "(pav) Setting up {stype} environment."') purge = utils.str_bool(config.get("purge_modules")) @@ -1270,6 +1271,7 @@ def _write_script(self, else: script.comment('No commands given for this script.') + script.newline() script.command(f'echo "(pav) Test {stype} commands completed without error."') script.write(path) From 3ab3ec890142e60cebd85527550722f12812c131 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Tue, 2 Dec 2025 14:47:21 -0700 Subject: [PATCH 47/74] Fix ignore_files in copytree_resolved --- lib/pavilion/commands/isolate.py | 11 ++++++++--- lib/pavilion/utils.py | 12 ++++++------ 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/lib/pavilion/commands/isolate.py b/lib/pavilion/commands/isolate.py index 55742baec..1cdd3040b 100644 --- a/lib/pavilion/commands/isolate.py +++ b/lib/pavilion/commands/isolate.py @@ -2,7 +2,6 @@ from pathlib import Path import tarfile import sys -import shutil from pavilion import output from pavilion import schedulers @@ -112,7 +111,13 @@ def _isolate(cls, pav_cfg: PavConfig, test: TestRun, dest: Path, archive: bool, return 6 if archive: - self._write_tarball(test.id, test.path, dest, zip, cls.IGNORE_FILES) + self._write_tarball( + pav_cfg, + test.id, + test.path, + dest, + zip, + cls.IGNORE_FILES) else: try: @@ -134,7 +139,7 @@ def _isolate(cls, pav_cfg: PavConfig, test: TestRun, dest: Path, archive: bool, @classmethod def _write_tarball(cls, pav_cfg: PavConfig, test_id: TestID, src: Path, dest: Path, zip: bool, - ignore_files) -> None: + ignore_files: Iterable[str]) -> None: if zip: if len(dest.suffixes) == 0: dest = dest.with_suffix(".tgz") diff --git a/lib/pavilion/utils.py b/lib/pavilion/utils.py index a31b1502e..033e7ea1e 100644 --- a/lib/pavilion/utils.py +++ b/lib/pavilion/utils.py @@ -13,7 +13,7 @@ import textwrap import zipfile from pathlib import Path -from typing import Iterator, Union, TextIO, List, Dict, Optional, Set +from typing import Iterator, Union, TextIO, List, Dict, Optional, Set, Iterable class WrappedFormatter(argparse.HelpFormatter): @@ -211,9 +211,9 @@ def copytree_resolved( src: Path, dest: Path, seen_files: Optional[Set] = None, - ignore_files: List[str] = None) -> Path: - """Copy a directory tree to another location, such that the resulting directory contains - the targets of all symlinks.""" + ignore_files: Optional[Iterable[str]] = None) -> Path: + """Copy a directory tree to another location, such that the only symlinks that remain are + symlinks internal to the directory.""" ignore_files = ignore_files or [] @@ -231,7 +231,7 @@ def copytree_resolved( target = Path(os.readlink(src)) if target not in seen_files: - copytree_resolved(target, dest, seen_files) + copytree_resolved(target, dest, seen_files, ignore_files) else: dest.symlink_to(target) @@ -244,7 +244,7 @@ def copytree_resolved( files = src.iterdir() for fname in files: - copytree_resolved(fname, dest / fname.name, seen_files) + copytree_resolved(fname, dest / fname.name, seen_files, ignore_files) return dest From a1c0a18c9cbddb840518bd93e70b9411a34225da Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Tue, 2 Dec 2025 14:48:11 -0700 Subject: [PATCH 48/74] Fix missing import --- lib/pavilion/commands/isolate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pavilion/commands/isolate.py b/lib/pavilion/commands/isolate.py index 1cdd3040b..ec412b8e7 100644 --- a/lib/pavilion/commands/isolate.py +++ b/lib/pavilion/commands/isolate.py @@ -2,6 +2,7 @@ from pathlib import Path import tarfile import sys +from typing import Iterable from pavilion import output from pavilion import schedulers @@ -111,8 +112,7 @@ def _isolate(cls, pav_cfg: PavConfig, test: TestRun, dest: Path, archive: bool, return 6 if archive: - self._write_tarball( - pav_cfg, + self._write_tarball(pav_cfg, test.id, test.path, dest, From fbb18dc738a18da284240fad96a5e2705d4b2fbe Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Tue, 2 Dec 2025 14:48:53 -0700 Subject: [PATCH 49/74] Fix missing import --- lib/pavilion/commands/isolate.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pavilion/commands/isolate.py b/lib/pavilion/commands/isolate.py index ec412b8e7..dc3ed55d1 100644 --- a/lib/pavilion/commands/isolate.py +++ b/lib/pavilion/commands/isolate.py @@ -2,6 +2,7 @@ from pathlib import Path import tarfile import sys +import shutil from typing import Iterable from pavilion import output From 6132c039f687b32592b0aae3cc04bf8897774cba Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Tue, 2 Dec 2025 15:13:34 -0700 Subject: [PATCH 50/74] Fix copytree_resolved symlink behavior --- lib/pavilion/utils.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/lib/pavilion/utils.py b/lib/pavilion/utils.py index 033e7ea1e..12b80aff4 100644 --- a/lib/pavilion/utils.py +++ b/lib/pavilion/utils.py @@ -210,11 +210,15 @@ def copytree(src, dst, symlinks=False, ignore=None, copy_function=shutil.copy2, def copytree_resolved( src: Path, dest: Path, + src_root: Optional[Path] = None, + dest_root: Optional[Path] = None, seen_files: Optional[Set] = None, ignore_files: Optional[Iterable[str]] = None) -> Path: """Copy a directory tree to another location, such that the only symlinks that remain are symlinks internal to the directory.""" + src_root = src_root or src + dest_root = dest_root or dest ignore_files = ignore_files or [] if src.name in ignore_files: @@ -230,10 +234,22 @@ def copytree_resolved( if src.is_symlink(): target = Path(os.readlink(src)) - if target not in seen_files: - copytree_resolved(target, dest, seen_files, ignore_files) + # Only recreate symlinks if they are internal to the source directory + if target.is_relative_to(src_root): + target = target.relative_to(src_root) + + # Don't create the symlink if it points inside a directory we're ignoring + skip_link = False + + for pt in target.parts: + if pt in ignore_files + skip_link = True + break + + if not skip_link: + dest.symlink_to(dest_root / target) else: - dest.symlink_to(target) + copytree_resolved(target.resolve(), dest, src_root, dest_root, seen_files, ignore_files) return dest @@ -244,7 +260,8 @@ def copytree_resolved( files = src.iterdir() for fname in files: - copytree_resolved(fname, dest / fname.name, seen_files, ignore_files) + copytree_resolved(fname, dest / fname.name, src_root, dest_root, + seen_files, ignore_files) return dest From 6e7cff32bcc957ad855d584c76861ed81425630d Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Tue, 2 Dec 2025 15:14:17 -0700 Subject: [PATCH 51/74] Fix syntax error --- lib/pavilion/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pavilion/utils.py b/lib/pavilion/utils.py index 12b80aff4..16fa3b6fa 100644 --- a/lib/pavilion/utils.py +++ b/lib/pavilion/utils.py @@ -242,7 +242,7 @@ def copytree_resolved( skip_link = False for pt in target.parts: - if pt in ignore_files + if pt in ignore_files: skip_link = True break From 0cb55347bd411c396a07d6016466fdf28fba044b Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Tue, 2 Dec 2025 15:17:25 -0700 Subject: [PATCH 52/74] Fix copytree_resolved --- lib/pavilion/utils.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/lib/pavilion/utils.py b/lib/pavilion/utils.py index 16fa3b6fa..7e35f4512 100644 --- a/lib/pavilion/utils.py +++ b/lib/pavilion/utils.py @@ -235,19 +235,24 @@ def copytree_resolved( target = Path(os.readlink(src)) # Only recreate symlinks if they are internal to the source directory - if target.is_relative_to(src_root): - target = target.relative_to(src_root) + relative = True + try: + rel_target = target.relative_to(src_root) + except ValueError: + relative = False + + if relative: # Don't create the symlink if it points inside a directory we're ignoring skip_link = False - for pt in target.parts: + for pt in rel_target.parts: if pt in ignore_files: skip_link = True break if not skip_link: - dest.symlink_to(dest_root / target) + dest.symlink_to(dest_root / rel_target) else: copytree_resolved(target.resolve(), dest, src_root, dest_root, seen_files, ignore_files) From 589da500ce96d28e0c06a51bb9fc2dede4905c2d Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Tue, 2 Dec 2025 15:44:26 -0700 Subject: [PATCH 53/74] Add more examples to copytree_resolved unit test --- test/tests/utils_tests.py | 76 +++++++++++++++++++-------------------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/test/tests/utils_tests.py b/test/tests/utils_tests.py index 5953ff04f..d4b93ff4f 100644 --- a/test/tests/utils_tests.py +++ b/test/tests/utils_tests.py @@ -74,7 +74,7 @@ def test_owner(self): def test_relative_to(self): """Check relative path calculations.""" - # base, target, answer + # base, "target", answer tests = [ # Outside 'base' (self.PAV_LIB_DIR, @@ -96,11 +96,11 @@ def test_relative_to(self): def test_repair_symlinks(self): """Check symlink repairing.""" - # (File, target, answer) - # A target of None means to create a regular file with the filename + # (File, "target", answer) + # A "target" of None means to create a regular file with the filename # as the contents. - # An answer of None means the target won't exist. - # An answer of '*' means we can't know the target's contents (but it + # An answer of None means the "target" won't exist. + # An answer of '*' means we can't know the "target"'s contents (but it # should exist). test_files = ( ('t1/A', None, 'A'), @@ -161,67 +161,67 @@ def test_copytree_dotfiles(self): def test_copytree_resolved(self): examples = [ { - "flatten": False, - "copy_root": None, + "copy_root": "foo", "files": [ {"name": "foo", "dir": True, "target": None}, {"name": "bar", "dir": False, "target": None}, - {"name": "foo/baz", "dir": False, "target": None}, + {"name": "foo/baz", "dir": False, "target": "bar"}, ], "expected": [ - {"name": "foo", "dir": True, "target": None}, - {"name": "bar", "dir": False, "target": None}, - {"name": "foo/baz", "dir": False, "target": None}, + {"name": "baz", "dir": False, "target": None}, ] }, { - "flatten": True, - "copy_root": "foo", + "copy_root": None, "files": [ - {"name": "foo", "dir": True, "target": None}, - {"name": "bar", "dir": False, "target": None}, - {"name": "foo/baz", "dir": False, "target": "bar"}, + {"name": "foo", "dir": False, "target": "bar"}, + {"name": "bar", "dir": False, "target": "foo"}, ], - "expected": [ - {"name": "baz", "dir": False, "target": None}, - ] + "expected": [] }, { - "flatten": False, "copy_root": "foo", "files": [ - {"name": "foo", "dir": True, "target": None}, - {"name": "bar", "dir": False, "target": None}, - {"name": "foo/baz", "dir": False, "target": "bar"}, + {"name": "foo", "dir": True, "target": None}, + {"name": "foo/bar", "dir": False, "target": "foobar"}, + {"name": "foo/baz", "dir": False, "target": None}, + {"name": "foobar", "dir": False, "target": "foo/baz"} ], "expected": [ - {"name": "baz", "dir": False, "target": "bar"}, - {"name": "bar", "dir": False, "target": None}, + {"name": "foo", "dir": True, "target": None}, + {"name": "foo/bar", "dir": False, "target": "foo/baz"}, + {"name": "foo/baz", "dir": False, "target": None}, ] }, { - "flatten": False, "copy_root": None, "files": [ - {"name": "foo", "dir": False, "target": "bar"}, - {"name": "bar", "dir": False, "target": "foo"}, + {"name": "foo", "dir": True, "target": None}, + {"name": "bar", "dir": False, "target": None}, + {"name": "foo/baz", "dir": False, "target": "bar"}, + {"name": "foobar", "dir": False, "target": "bar"}, ], "expected": [ - {"name": "foo", "dir": False, "target": "bar"}, - {"name": "bar", "dir": False, "target": "foo"}, + {"name": "foo", "dir": True, "target": None}, + {"name": "bar", "dir": False, "target": None}, + {"name": "foo/baz", "dir": False, "target": "bar"}, + {"name": "foobar", "dir": False, "target": "bar"}, ] }, { - "flatten": True, - "copy_root": None, + "copy_root": "foo", "files": [ - {"name": "foo", "dir": False, "target": "bar"}, - {"name": "bar", "dir": False, "target": "foo"}, + {"name": "foo", "dir": True, "target": None}, + {"name": "bar", "dir": False, "target": None}, + {"name": "foo/baz", "dir": False, "target": "bar"}, + {"name": "foo/foobar", "dir": False, "target": "bar"}, ], "expected": [ - {"name": "bar", "dir": False, "target": "foo"}, + {"name": "foo", "dir": True, "target": None}, + {"name": "foo/baz", "dir": False, "target": None}, + {"name": "foo/foobar", "dir": False, "target": "foo/baz"}, ] - }, + } ] for ex in examples: @@ -246,9 +246,9 @@ def test_copytree_resolved(self): file_path.symlink_to(target_path) if ex["copy_root"] is None: - utils.copytree_resolved(src, dest, flatten=ex["flatten"]) + utils.copytree_resolved(src, dest) else: - utils.copytree_resolved(src / ex["copy_root"], dest, flatten=ex["flatten"]) + utils.copytree_resolved(src / ex["copy_root"], dest) expected = set(Path(f["name"]) for f in ex["expected"]) actual = set(p.relative_to(dest) for p in list_files(dest)) From a4ec4e98f9128bb0448b6642c6ffc75dca0505e1 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Tue, 2 Dec 2025 15:54:31 -0700 Subject: [PATCH 54/74] Fix copytree_resolved unit tests --- test/tests/utils_tests.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/test/tests/utils_tests.py b/test/tests/utils_tests.py index d4b93ff4f..336d060ac 100644 --- a/test/tests/utils_tests.py +++ b/test/tests/utils_tests.py @@ -177,7 +177,10 @@ def test_copytree_resolved(self): {"name": "foo", "dir": False, "target": "bar"}, {"name": "bar", "dir": False, "target": "foo"}, ], - "expected": [] + "expected": [ + {"name": "foo", "dir": False, "target": "bar"}, + {"name": "bar", "dir": False, "target": "foo"}, + ] }, { "copy_root": "foo", @@ -188,9 +191,8 @@ def test_copytree_resolved(self): {"name": "foobar", "dir": False, "target": "foo/baz"} ], "expected": [ - {"name": "foo", "dir": True, "target": None}, - {"name": "foo/bar", "dir": False, "target": "foo/baz"}, - {"name": "foo/baz", "dir": False, "target": None}, + {"name": "bar", "dir": False, "target": "baz"}, + {"name": "baz", "dir": False, "target": None}, ] }, { @@ -217,9 +219,8 @@ def test_copytree_resolved(self): {"name": "foo/foobar", "dir": False, "target": "bar"}, ], "expected": [ - {"name": "foo", "dir": True, "target": None}, - {"name": "foo/baz", "dir": False, "target": None}, - {"name": "foo/foobar", "dir": False, "target": "foo/baz"}, + {"name": "baz", "dir": False, "target": None}, + {"name": "foobar", "dir": False, "target": "baz"}, ] } ] From cfda806db8bd394f6b2f961897fbe8563acc4e50 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Tue, 2 Dec 2025 17:14:16 -0700 Subject: [PATCH 55/74] Fix copytree_resolved --- lib/pavilion/utils.py | 73 ++++++++++++++++++++++----------------- test/tests/utils_tests.py | 8 ++--- 2 files changed, 43 insertions(+), 38 deletions(-) diff --git a/lib/pavilion/utils.py b/lib/pavilion/utils.py index 7e35f4512..150120347 100644 --- a/lib/pavilion/utils.py +++ b/lib/pavilion/utils.py @@ -13,7 +13,7 @@ import textwrap import zipfile from pathlib import Path -from typing import Iterator, Union, TextIO, List, Dict, Optional, Set, Iterable +from typing import Iterator, Union, TextIO, List, Dict, Optional, Iterable class WrappedFormatter(argparse.HelpFormatter): @@ -212,7 +212,7 @@ def copytree_resolved( dest: Path, src_root: Optional[Path] = None, dest_root: Optional[Path] = None, - seen_files: Optional[Set] = None, + symlinks: Optional[Dict[Path, Path]] = None, ignore_files: Optional[Iterable[str]] = None) -> Path: """Copy a directory tree to another location, such that the only symlinks that remain are symlinks internal to the directory.""" @@ -221,52 +221,61 @@ def copytree_resolved( dest_root = dest_root or dest ignore_files = ignore_files or [] - if src.name in ignore_files: - return src + if symlinks is None: + symlinks = {} - seen_files = seen_files or set() - - if src in seen_files: + if src.name in ignore_files: return src - seen_files.add(src) - if src.is_symlink(): - target = Path(os.readlink(src)) - - # Only recreate symlinks if they are internal to the source directory - relative = True - try: - rel_target = target.relative_to(src_root) - except ValueError: - relative = False - - if relative: - # Don't create the symlink if it points inside a directory we're ignoring - skip_link = False + resolved = src.resolve() + except RuntimeError: + # There is a circular symlink + return src - for pt in rel_target.parts: - if pt in ignore_files: - skip_link = True - break - - if not skip_link: - dest.symlink_to(dest_root / rel_target) + if resolved in symlinks: + dest.symlink_to(symlinks.get(resolved)) else: - copytree_resolved(target.resolve(), dest, src_root, dest_root, seen_files, ignore_files) + # Only recreate symlinks if they are internal to the source directory + target_in_tree = True + + try: + rel_target = resolved.relative_to(src_root) + except ValueError: + target_in_tree = False + + if target_in_tree: + # Don't create the symlink if it points inside a directory we're ignoring + skip_link = False + + for pt in rel_target.parts: + if pt in ignore_files: + skip_link = True + break + + if not skip_link: + dest.symlink_to(dest_root / rel_target) + symlinks[resolved] = dest_root / rel_target + else: + symlinks[resolved] = dest + copytree_resolved(resolved, dest, src_root, dest_root, symlinks, ignore_files) return dest elif src.is_file(): - return shutil.copy(src, dest) + ret = shutil.copy(src, dest) + + return ret elif src.is_dir(): dest.mkdir(exist_ok=True) - files = src.iterdir() + + # Sort for reproduceability + files = sorted(src.iterdir(), key=lambda p: p.name) for fname in files: copytree_resolved(fname, dest / fname.name, src_root, dest_root, - seen_files, ignore_files) + symlinks, ignore_files) return dest diff --git a/test/tests/utils_tests.py b/test/tests/utils_tests.py index 336d060ac..59e214651 100644 --- a/test/tests/utils_tests.py +++ b/test/tests/utils_tests.py @@ -177,10 +177,7 @@ def test_copytree_resolved(self): {"name": "foo", "dir": False, "target": "bar"}, {"name": "bar", "dir": False, "target": "foo"}, ], - "expected": [ - {"name": "foo", "dir": False, "target": "bar"}, - {"name": "bar", "dir": False, "target": "foo"}, - ] + "expected": [] }, { "copy_root": "foo", @@ -225,7 +222,7 @@ def test_copytree_resolved(self): } ] - for ex in examples: + for i, ex in enumerate(examples): with tempfile.TemporaryDirectory() as src: src = Path(src) @@ -240,7 +237,6 @@ def test_copytree_resolved(self): file_path.mkdir(parents=True) else: if f["target"] is None: - print(f"Making file {file_path}") file_path.touch() else: target_path = src / f["target"] From 96f306dec51010bb4e64ff5ddd5f8f9047b1530d Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Tue, 2 Dec 2025 17:34:51 -0700 Subject: [PATCH 56/74] Use relative paths when creating symlinks --- lib/pavilion/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pavilion/utils.py b/lib/pavilion/utils.py index 150120347..e45185aa0 100644 --- a/lib/pavilion/utils.py +++ b/lib/pavilion/utils.py @@ -235,7 +235,7 @@ def copytree_resolved( return src if resolved in symlinks: - dest.symlink_to(symlinks.get(resolved)) + dest.symlink_to(Path(os.path.relpath(symlinks.get(resolved), dest.resolve().parent))) else: # Only recreate symlinks if they are internal to the source directory target_in_tree = True @@ -255,7 +255,7 @@ def copytree_resolved( break if not skip_link: - dest.symlink_to(dest_root / rel_target) + dest.symlink_to(Path(os.path.relpath(dest_root / rel_target, dest.resolve().parent))) symlinks[resolved] = dest_root / rel_target else: symlinks[resolved] = dest From 51f969e9bb959c8e65718df2b53cd1b87b4744b5 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Tue, 2 Dec 2025 17:37:41 -0700 Subject: [PATCH 57/74] Style --- lib/pavilion/utils.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/lib/pavilion/utils.py b/lib/pavilion/utils.py index e45185aa0..5c55e67fc 100644 --- a/lib/pavilion/utils.py +++ b/lib/pavilion/utils.py @@ -235,7 +235,12 @@ def copytree_resolved( return src if resolved in symlinks: - dest.symlink_to(Path(os.path.relpath(symlinks.get(resolved), dest.resolve().parent))) + # Create a relative symlink + dest.symlink_to( + Path( + os.path.relpath( + symlinks.get(resolved), + dest.resolve().parent))) else: # Only recreate symlinks if they are internal to the source directory target_in_tree = True @@ -255,7 +260,12 @@ def copytree_resolved( break if not skip_link: - dest.symlink_to(Path(os.path.relpath(dest_root / rel_target, dest.resolve().parent))) + # Create a relative symlink + dest.symlink_to( + Path( + os.path.relpath( + dest_root / rel_target, + dest.resolve().parent))) symlinks[resolved] = dest_root / rel_target else: symlinks[resolved] = dest From 9f7fec56f5d9467ae641c64c0a7f707ecdd3c92c Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Tue, 2 Dec 2025 18:15:04 -0700 Subject: [PATCH 58/74] Remove unnecessary return value --- lib/pavilion/utils.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/pavilion/utils.py b/lib/pavilion/utils.py index 5c55e67fc..3682bba94 100644 --- a/lib/pavilion/utils.py +++ b/lib/pavilion/utils.py @@ -213,7 +213,7 @@ def copytree_resolved( src_root: Optional[Path] = None, dest_root: Optional[Path] = None, symlinks: Optional[Dict[Path, Path]] = None, - ignore_files: Optional[Iterable[str]] = None) -> Path: + ignore_files: Optional[Iterable[str]] = None) -> None: """Copy a directory tree to another location, such that the only symlinks that remain are symlinks internal to the directory.""" @@ -225,14 +225,14 @@ def copytree_resolved( symlinks = {} if src.name in ignore_files: - return src + return if src.is_symlink(): try: resolved = src.resolve() except RuntimeError: # There is a circular symlink - return src + return if resolved in symlinks: # Create a relative symlink @@ -271,12 +271,12 @@ def copytree_resolved( symlinks[resolved] = dest copytree_resolved(resolved, dest, src_root, dest_root, symlinks, ignore_files) - return dest + return elif src.is_file(): - ret = shutil.copy(src, dest) + shutil.copy(src, dest) - return ret + return elif src.is_dir(): dest.mkdir(exist_ok=True) @@ -287,7 +287,7 @@ def copytree_resolved( copytree_resolved(fname, dest / fname.name, src_root, dest_root, symlinks, ignore_files) - return dest + return def path_is_external(path: Path): """Returns True if a path contains enough back 'up-references' to escape From 5a2cd427957485858d20235fc08f263eb50fd888 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Tue, 2 Dec 2025 18:52:46 -0700 Subject: [PATCH 59/74] Refine isolate command unit tests --- lib/pavilion/commands/isolate.py | 22 +++++++++++++--------- test/tests/isolate_cmd_tests.py | 32 +++++++++++++++----------------- 2 files changed, 28 insertions(+), 26 deletions(-) diff --git a/lib/pavilion/commands/isolate.py b/lib/pavilion/commands/isolate.py index dc3ed55d1..bd08d9c7b 100644 --- a/lib/pavilion/commands/isolate.py +++ b/lib/pavilion/commands/isolate.py @@ -3,6 +3,7 @@ import tarfile import sys import shutil +import tempfile from typing import Iterable from pavilion import output @@ -13,6 +14,7 @@ from pavilion.cmd_utils import get_last_test_id, get_tests_by_id, list_files from pavilion.utils import copytree_resolved from pavilion.scriptcomposer import ScriptComposer +from pavilion.errors import SchedulerPluginError from pavilion.schedulers.config import validate_config from .base_classes import Command @@ -113,9 +115,8 @@ def _isolate(cls, pav_cfg: PavConfig, test: TestRun, dest: Path, archive: bool, return 6 if archive: - self._write_tarball(pav_cfg, - test.id, - test.path, + cls._write_tarball(pav_cfg, + test, dest, zip, cls.IGNORE_FILES) @@ -139,7 +140,7 @@ def _isolate(cls, pav_cfg: PavConfig, test: TestRun, dest: Path, archive: bool, return 0 @classmethod - def _write_tarball(cls, pav_cfg: PavConfig, test_id: TestID, src: Path, dest: Path, zip: bool, + def _write_tarball(cls, pav_cfg: PavConfig, test: TestRun, dest: Path, zip: bool, ignore_files: Iterable[str]) -> None: if zip: if len(dest.suffixes) == 0: @@ -153,25 +154,28 @@ def _write_tarball(cls, pav_cfg: PavConfig, test_id: TestID, src: Path, dest: Pa modestr = "w:" with tempfile.TemporaryDirectory() as tmp: - utils.copytree_resolved(src, tmp, ignore_files=ignore_files) + tmp = Path(tmp) + tmp_dest = tmp / dest.stem + tmp_dest.mkdir() + copytree_resolved(test.path, tmp_dest, ignore_files=ignore_files) # Copy Pavilion bash library into tarball pav_lib_bash = pav_cfg.pav_root / 'bin' / cls.PAV_LIB_FN - shutil.copyfile(pav_lib_bash, tmp / cls.PAV_LIB_FN) + shutil.copyfile(pav_lib_bash, tmp_dest / cls.PAV_LIB_FN) - cls._write_kickoff_script(pav_cfg, test_id, tmp / cls.KICKOFF_FN) + cls._write_kickoff_script(pav_cfg, test, tmp_dest / cls.KICKOFF_FN) try: with tarfile.open(dest, modestr) as tarf: for fname in list_files(tmp): tarf.add( fname, - arcname=fname.relative_to(src.parent), + arcname=fname.relative_to(tmp), recursive=False) except (tarfile.TarError, OSError): output.fprint( sys.stderr, - f"Unable to isolate test {test_id} at {dest}.", + f"Unable to isolate test {test.id} at {dest}.", color=output.RED) return 7 diff --git a/test/tests/isolate_cmd_tests.py b/test/tests/isolate_cmd_tests.py index 99752aa7a..e4ab548cf 100644 --- a/test/tests/isolate_cmd_tests.py +++ b/test/tests/isolate_cmd_tests.py @@ -31,11 +31,16 @@ def test_no_archive(self): self.assertEqual(isolate_cmd.run(self.pav_cfg, isolate_args), 0) - source_files = set(list_files(last_test.path)) - dest_files = set(list_files(Path(dir) / "dest")) - - self.assertFalse(any(map(lambda x: x.is_symlink(), dest_files))) - self.assertEqual({f for f in source_files if f not in ("series", "job")}, dest_files) + source_files = set(map( + lambda x: x.relative_to(last_test.path), + list_files(last_test.path))) + dest_files = set(map( + lambda x: x.relative_to(Path(dir) / "dest"), + list_files(Path(dir) / "dest"))) + + self.assertEqual( + {f for f in source_files if f.name not in ("series", "job")}, + {f for f in dest_files if f.name not in ("pav-lib.bash", "kickoff.isolated")}) def test_zip_archive(self): run_cmd = commands.get_command("run") @@ -62,20 +67,13 @@ def test_zip_archive(self): with tarfile.open(Path(dir) / "dest.tgz", "r:gz") as tf: tf.extractall(extract_dir) - dest_files = list_files(Path(extract_dir)) - - for df in dest_files: - if df.is_symlink(): - import pdb; pdb.set_trace() - - - self.assertFalse(any(map(lambda x: x.is_symlink(), dest_files))) - source_files = set(map( - lambda x: Path(x).relative_to(last_test.path.parent), + lambda x: Path(x).relative_to(last_test.path), list_files(last_test.path, include_root=True))) - dest_files = set(dest_files) + dest_files = set(map( + lambda x: x.relative_to(Path(extract_dir) / "dest"), + list_files(Path(extract_dir)))) self.assertEqual( {f for f in source_files if f.name not in ("series", "job")}, - dest_files) + {f for f in dest_files if f.name not in ("pav-lib.bash", "kickoff.isolated")}) From bbda33dedb6f1522a1389cfebfb5a0bdf95fd714 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Tue, 2 Dec 2025 19:14:12 -0700 Subject: [PATCH 60/74] Fix isolate command unit tests --- test/tests/isolate_cmd_tests.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/test/tests/isolate_cmd_tests.py b/test/tests/isolate_cmd_tests.py index e4ab548cf..0ce4d069c 100644 --- a/test/tests/isolate_cmd_tests.py +++ b/test/tests/isolate_cmd_tests.py @@ -40,7 +40,10 @@ def test_no_archive(self): self.assertEqual( {f for f in source_files if f.name not in ("series", "job")}, - {f for f in dest_files if f.name not in ("pav-lib.bash", "kickoff.isolated")}) + {f for f in dest_files if f.name not in \ + ("pav-lib.bash", "kickoff.isolated") and \ + # Have to exclude build_origin because of how os.walk handles symlinks + f.parent.name != "build_origin"}) def test_zip_archive(self): run_cmd = commands.get_command("run") @@ -76,4 +79,7 @@ def test_zip_archive(self): self.assertEqual( {f for f in source_files if f.name not in ("series", "job")}, - {f for f in dest_files if f.name not in ("pav-lib.bash", "kickoff.isolated")}) + {f for f in dest_files if f.name not in \ + ("pav-lib.bash", "kickoff.isolated") and \ + # Have to exclude build_origin because of how os.walk handles symlinks + f.parent.name != "build_origin"}) From bf05bc8b1f2d3aae51a0cc84b4989cab2c24f87d Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Tue, 2 Dec 2025 19:15:17 -0700 Subject: [PATCH 61/74] Add docstrings to unit tests --- test/tests/isolate_cmd_tests.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/tests/isolate_cmd_tests.py b/test/tests/isolate_cmd_tests.py index 0ce4d069c..c49dcff4f 100644 --- a/test/tests/isolate_cmd_tests.py +++ b/test/tests/isolate_cmd_tests.py @@ -14,6 +14,8 @@ class IsolateCmdTests(PavTestCase): def test_no_archive(self): + """Test that isolating without archiving works correctly.""" + run_cmd = commands.get_command("run") isolate_cmd = commands.get_command("isolate") @@ -46,6 +48,8 @@ def test_no_archive(self): f.parent.name != "build_origin"}) def test_zip_archive(self): + """Test that isolating using a compressed archive works correctly.""" + run_cmd = commands.get_command("run") isolate_cmd = commands.get_command("isolate") From c0dc04c457fa3039b5522cad1ee29ea1400ec898 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Tue, 2 Dec 2025 19:17:30 -0700 Subject: [PATCH 62/74] Add docstrings to isolate command --- lib/pavilion/commands/isolate.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/pavilion/commands/isolate.py b/lib/pavilion/commands/isolate.py index bd08d9c7b..feb270c7b 100644 --- a/lib/pavilion/commands/isolate.py +++ b/lib/pavilion/commands/isolate.py @@ -34,6 +34,8 @@ def __init__(self): ) def _setup_arguments(self, parser: ArgumentParser) -> None: + """Setup the argument parser for the isolate command.""" + parser.add_argument( "test_id", type=TestID, @@ -64,6 +66,8 @@ def _setup_arguments(self, parser: ArgumentParser) -> None: ) def run(self, pav_cfg: PavConfig, args: Namespace) -> int: + """Run the isolate command.""" + if args.zip and not args.archive: output.fprint(self.errfile, "--archive must be specified to use --zip.") @@ -99,7 +103,11 @@ def run(self, pav_cfg: PavConfig, args: Namespace) -> int: return self._isolate(pav_cfg, test, args.path, args.archive, args.zip) @classmethod - def _isolate(cls, pav_cfg: PavConfig, test: TestRun, dest: Path, archive: bool, zip: bool) -> int: + def _isolate(cls, pav_cfg: PavConfig, test: TestRun, dest: Path, archive: bool, + zip: bool) -> int: + """Given a test run and a destination path, isolate that test run, optionally + creating a tarball.""" + if not test.path.is_dir(): output.fprint(sys.stderr, "Directory '{}' does not exist." .format(test.path.as_posix()), color=output.RED) From 6806f3af3b73856324d9e0d2504f836021d8711a Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Tue, 2 Dec 2025 19:32:48 -0700 Subject: [PATCH 63/74] Incorporate node_range into kickoff.isolated script --- lib/pavilion/commands/isolate.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/lib/pavilion/commands/isolate.py b/lib/pavilion/commands/isolate.py index feb270c7b..5b82061f7 100644 --- a/lib/pavilion/commands/isolate.py +++ b/lib/pavilion/commands/isolate.py @@ -15,7 +15,7 @@ from pavilion.utils import copytree_resolved from pavilion.scriptcomposer import ScriptComposer from pavilion.errors import SchedulerPluginError -from pavilion.schedulers.config import validate_config +from pavilion.schedulers.config import validate_config, calc_node_range from .base_classes import Command @@ -150,6 +150,9 @@ def _isolate(cls, pav_cfg: PavConfig, test: TestRun, dest: Path, archive: bool, @classmethod def _write_tarball(cls, pav_cfg: PavConfig, test: TestRun, dest: Path, zip: bool, ignore_files: Iterable[str]) -> None: + """Given a test run object, create a tarball of its run directory in the specified + location.""" + if zip: if len(dest.suffixes) == 0: dest = dest.with_suffix(".tgz") @@ -203,11 +206,14 @@ def _write_kickoff_script(cls, pav_cfg: PavConfig, test: TestRun, script_path: P ) return 9 + sched_config = validate_config(test.config['schedule']) + node_range = calc_node_range(test.config, sched_config['cluster_info']['node_count']) + header = sched._get_kickoff_script_header( job_name=f"pav_{test.name}_isolated", - sched_config=validate_config(test.config['schedule']), + sched_config=sched_config, nodes=None, - node_range=None, + node_range=node_range, shebang=test.shebang ) From 5802b248771d3ede808c249674cf7c9bc92c5052 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Tue, 2 Dec 2025 19:56:08 -0700 Subject: [PATCH 64/74] Fix pav-lib.bash path --- lib/pavilion/commands/isolate.py | 2 +- lib/pavilion/test_run/test_run.py | 17 +++++++++-------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/lib/pavilion/commands/isolate.py b/lib/pavilion/commands/isolate.py index 5b82061f7..0213c1456 100644 --- a/lib/pavilion/commands/isolate.py +++ b/lib/pavilion/commands/isolate.py @@ -207,7 +207,7 @@ def _write_kickoff_script(cls, pav_cfg: PavConfig, test: TestRun, script_path: P return 9 sched_config = validate_config(test.config['schedule']) - node_range = calc_node_range(test.config, sched_config['cluster_info']['node_count']) + node_range = calc_node_range(sched_config, sched_config['cluster_info']['node_count']) header = sched._get_kickoff_script_header( job_name=f"pav_{test.name}_isolated", diff --git a/lib/pavilion/test_run/test_run.py b/lib/pavilion/test_run/test_run.py index 475e402e8..0b9e5224b 100644 --- a/lib/pavilion/test_run/test_run.py +++ b/lib/pavilion/test_run/test_run.py @@ -1150,11 +1150,6 @@ def _write_script(self, script.command('set -v') script.newline() - if isolate: - pav_lib_bash = path.parent / "pav-lib.bash" - else: - pav_lib_bash = self._pav_cfg.pav_root/'bin'/'pav-lib.bash' - script.command(f'echo "(pav) Starting {stype} script"') # If we include this directly, it breaks build hashing. @@ -1167,6 +1162,12 @@ def _write_script(self, env["PAV_CONFIG_FILE"] = self._pav_cfg['pav_cfg_file'] script.env_change(env) + + if isolate: + pav_lib_bash = '$( dirname -- "${BASH_SOURCE[0]}" )/pav-lib.bash' + else: + pav_lib_bash = self._pav_cfg.pav_root/'bin'/'pav-lib.bash' + script.command('source {}'.format(pav_lib_bash)) if config.get('preamble', []): @@ -1246,10 +1247,10 @@ def _write_script(self, script.command('spack load {} || exit 1' .format(package)) - script.newline() - script.comment('Output the environment for posterity') - if not isolate: + script.newline() + script.comment('Output the environment for posterity') + if verbose: script.command(f'declare -p | tee > {path.parent / stype}.env.sh') else: From 6d3ecadff123747bdb16bb286de00ab6340586d7 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Tue, 2 Dec 2025 20:03:55 -0700 Subject: [PATCH 65/74] Remove accidents --- test/tests/cmd_util_tests.py | 1 - test/tests/utils_tests.py | 10 +++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/test/tests/cmd_util_tests.py b/test/tests/cmd_util_tests.py index 22f532100..395332a4d 100644 --- a/test/tests/cmd_util_tests.py +++ b/test/tests/cmd_util_tests.py @@ -3,7 +3,6 @@ import io import json import shutil -import tempfile from pathlib import Path from pavilion import dir_db diff --git a/test/tests/utils_tests.py b/test/tests/utils_tests.py index 59e214651..877f3c453 100644 --- a/test/tests/utils_tests.py +++ b/test/tests/utils_tests.py @@ -74,7 +74,7 @@ def test_owner(self): def test_relative_to(self): """Check relative path calculations.""" - # base, "target", answer + # base, target, answer tests = [ # Outside 'base' (self.PAV_LIB_DIR, @@ -96,11 +96,11 @@ def test_relative_to(self): def test_repair_symlinks(self): """Check symlink repairing.""" - # (File, "target", answer) - # A "target" of None means to create a regular file with the filename + # (File, target, answer) + # A target of None means to create a regular file with the filename # as the contents. - # An answer of None means the "target" won't exist. - # An answer of '*' means we can't know the "target"'s contents (but it + # An answer of None means the target won't exist. + # An answer of '*' means we can't know the target's contents (but it # should exist). test_files = ( ('t1/A', None, 'A'), From 99241a1d7dfa7e3f1177c545b2c1362e4768c0b8 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Tue, 2 Dec 2025 20:08:52 -0700 Subject: [PATCH 66/74] Fix style issue --- lib/pavilion/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pavilion/utils.py b/lib/pavilion/utils.py index 3682bba94..c140126d2 100644 --- a/lib/pavilion/utils.py +++ b/lib/pavilion/utils.py @@ -254,8 +254,8 @@ def copytree_resolved( # Don't create the symlink if it points inside a directory we're ignoring skip_link = False - for pt in rel_target.parts: - if pt in ignore_files: + for part in rel_target.parts: + if part in ignore_files: skip_link = True break From ce3c112413a29e8823c71c80624c0a6fe660abf1 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Wed, 3 Dec 2025 09:55:15 -0700 Subject: [PATCH 67/74] Add create_kickoff_script method to scheduler class --- lib/pavilion/commands/isolate.py | 23 ++---- lib/pavilion/schedulers/advanced.py | 105 +++++++++++++++++---------- lib/pavilion/schedulers/basic.py | 68 +++++++++++++---- lib/pavilion/schedulers/scheduler.py | 35 ++++++++- lib/pavilion/test_run/test_run.py | 48 +++++------- 5 files changed, 177 insertions(+), 102 deletions(-) diff --git a/lib/pavilion/commands/isolate.py b/lib/pavilion/commands/isolate.py index 0213c1456..a1eafc5b9 100644 --- a/lib/pavilion/commands/isolate.py +++ b/lib/pavilion/commands/isolate.py @@ -209,20 +209,9 @@ def _write_kickoff_script(cls, pav_cfg: PavConfig, test: TestRun, script_path: P sched_config = validate_config(test.config['schedule']) node_range = calc_node_range(sched_config, sched_config['cluster_info']['node_count']) - header = sched._get_kickoff_script_header( - job_name=f"pav_{test.name}_isolated", - sched_config=sched_config, - nodes=None, - node_range=node_range, - shebang=test.shebang - ) - - script = ScriptComposer(header=header) - - test._write_script( - script=script, - stype='run', - path=script_path, - config=test.config['run'], - module_wrappers=test.config.get('module_wrappers', {}), - isolate=True) + script = sched.create_kickoff_script( + pav_cfg, + test, + isolate=True) + + script.write(script_path) \ No newline at end of file diff --git a/lib/pavilion/schedulers/advanced.py b/lib/pavilion/schedulers/advanced.py index 35f673b47..6371c4098 100644 --- a/lib/pavilion/schedulers/advanced.py +++ b/lib/pavilion/schedulers/advanced.py @@ -4,11 +4,14 @@ import collections import pprint from abc import ABC -from typing import Tuple, List, Any, Union, Dict, FrozenSet, NewType +from pathlib import Path +from typing import Tuple, List, Any, Union, Dict, FrozenSet, NewType, Optional +from pavilion.config import PavConfig from pavilion.jobs import Job, JobError from pavilion.status_file import STATES from pavilion.test_run import TestRun +from pavilion.scriptcomposer import ScriptComposer from pavilion.types import NodeInfo, Nodes, NodeList, NodeSet, NodeRange from .config import validate_config, AVAILABLE, BACKFILL, calc_node_range from .scheduler import SchedulerPlugin @@ -542,19 +545,7 @@ def _schedule_shared(self, pav_cfg, tests: List[TestRun], node_range: NodeRange, # Clear the node range - it's only used for flexible scheduling. node_range = None - - job_name = 'pav_{}'.format(','.join(test.name for test in tests[:4])) - if len(tests) > 4: - job_name += ' ...' - script = self._create_kickoff_script_stub(pav_cfg, job_name, job.kickoff_log, - base_sched_config, nodes=picked_nodes, - node_range=node_range, - shebang=base_test.shebang) - - # Run each test via pavilion - script.command('echo "Starting {} tests - $(date)"'.format(len(tests))) - script.command('pav _run {}'.format(" ".join(test.full_id for test in tests))) - + script = self.create_kickoff_script(pav_cfg, tests, job.kickoff_log, nodes=picked_nodes) script.write(job.kickoff_path) # Create symlinks for each test to the one test with the kickoff script and @@ -567,7 +558,7 @@ def _schedule_shared(self, pav_cfg, tests: List[TestRun], node_range: NodeRange, pav_cfg=pav_cfg, job=job, sched_config=base_sched_config, - job_name=job_name, + job_name=self._job_name(tests), nodes=picked_nodes, node_range=node_range) except SchedulerPluginError as err: @@ -608,16 +599,7 @@ def _schedule_indi_flex(self, pav_cfg, tests: List[TestRun], node_range = calc_node_range(sched_config, len(chunk)) - job_name = 'pav_{}'.format(test.name) - script = self._create_kickoff_script_stub( - pav_cfg=pav_cfg, - job_name=job_name, - log_path=job.kickoff_log, - sched_config=sched_config, - node_range=node_range, - shebang=test.shebang) - - script.command('pav _run {t.full_id}'.format(t=test)) + script = self.create_kickoff_script(pav_cfg, test, job.kickoff_log) script.write(job.kickoff_path) test.job = job @@ -627,7 +609,7 @@ def _schedule_indi_flex(self, pav_cfg, tests: List[TestRun], pav_cfg=pav_cfg, job=job, sched_config=sched_config, - job_name=job_name, + job_name=self._job_name(test), node_range=node_range, ) except SchedulerPluginError as err: @@ -703,16 +685,7 @@ def _schedule_indi_chunk(self, pav_cfg, tests: List[TestRun], prior_error=err, tests=[test])) continue - job_name = 'pav_{}'.format(test.name) - script = self._create_kickoff_script_stub( - pav_cfg=pav_cfg, - job_name=job_name, - log_path=job.kickoff_log, - sched_config=sched_config, - nodes=picked_nodes, - shebang=test.shebang) - - script.command('pav _run {t.full_id}'.format(t=test)) + script = self.create_kickoff_script(pav_cfg, test, job.kickoff_log, nodes=picked_nodes) script.write(job.kickoff_path) test.job = job @@ -722,7 +695,7 @@ def _schedule_indi_chunk(self, pav_cfg, tests: List[TestRun], pav_cfg=pav_cfg, job=job, sched_config=sched_config, - job_name=job_name, + job_name=self._job_name(test), nodes=picked_nodes) except SchedulerPluginError as err: return [self._make_kickoff_error(err, [test])] @@ -737,3 +710,61 @@ def _schedule_indi_chunk(self, pav_cfg, tests: List[TestRun], .format(self.name, len(test_chunk))) return errors + + def create_kickoff_script(self, + pav_cfg: PavConfig, + tests: Union[TestRun, List[TestRun]], + log_path: Optional[Path] = None, + nodes: Optional[NodeSet] = None, + isolate: bool = False) -> ScriptComposer: + """Create the kickoff script.""" + + if not isinstance(tests, list): + tests = [tests] + + sched_config = validate_config(tests[0].config['schedule']) + sched_config["time_limit"] = max(map( + lambda x: x.config["schedule"]["time_limit"], + tests)) + + job_name = self._job_name(tests) + + if isolate: + job_name = job_name + "_isolated" + + node_range = calc_node_range( + sched_config, + sched_config['cluster_info']['node_count']) + + script = self._create_kickoff_script_stub( + pav_cfg=pav_cfg, + job_name=job_name, + log_path=log_path, + sched_config=sched_config, + node_range=node_range, + nodes=nodes, + shebang=tests[0].shebang) + + test_ids = ' '.join(test.full_id for test in tests) + + script.command('echo "Starting {} tests - $(date)"'.format(len(tests))) + + if isolate: + script = tests[0].make_script(script, "run", isolate=True) + else: + script.command('pav _run {}'.format(test_ids)) + + return script + + def _job_name(self, tests: Union[TestRun, List[TestRun]]) -> str: + """Given a test, get the name of the job.""" + + if not isinstance(tests, list): + tests = [tests] + + job_name = 'pav_{}'.format(','.join(test.name for test in tests[:4])) + + if len(tests) > 4: + job_name += ' ...' + + return job_name \ No newline at end of file diff --git a/lib/pavilion/schedulers/basic.py b/lib/pavilion/schedulers/basic.py index 381023666..29dddadeb 100644 --- a/lib/pavilion/schedulers/basic.py +++ b/lib/pavilion/schedulers/basic.py @@ -3,12 +3,15 @@ from abc import ABC from collections import defaultdict -from typing import List +from pathlib import Path +from typing import List, Dict, Tuple, Optional, Union +from pavilion.config import PavConfig from pavilion.jobs import Job, JobError from pavilion.status_file import STATES from pavilion.test_run import TestRun -from pavilion.types import NodeInfo, Nodes +from pavilion.scriptcomposer import ScriptComposer +from pavilion.types import NodeInfo, Nodes, NodeSet from .config import validate_config, calc_node_range from .scheduler import SchedulerPlugin from ..errors import SchedulerPluginError @@ -92,21 +95,10 @@ def schedule_tests(self, pav_cfg, tests: List[TestRun]) -> List[SchedulerPluginE prior_error=err, tests=[test])) continue - job_name = 'pav-{}-{}-runs'.format(self.name, test_bin[0].series) - for test in test_bin: test.job = job - script = self._create_kickoff_script_stub( - pav_cfg=pav_cfg, - job_name=job_name, - log_path=job.kickoff_log, - sched_config=sched_config, - node_range=node_range, - shebang=test.shebang) - - test_ids = ' '.join(test.full_id for test in tests) - script.command('pav _run {}'.format(test_ids)) + script = self.create_kickoff_script(pav_cfg, test_bin, job.kickoff_log) script.write(job.kickoff_path) try: @@ -114,7 +106,7 @@ def schedule_tests(self, pav_cfg, tests: List[TestRun]) -> List[SchedulerPluginE pav_cfg=pav_cfg, job=job, sched_config=sched_config, - job_name=job_name, + job_name=self._job_name(test_bin), node_range=node_range) except SchedulerPluginError as err: errors.append(self._make_kickoff_error(err, [test])) @@ -131,3 +123,49 @@ def schedule_tests(self, pav_cfg, tests: List[TestRun]) -> List[SchedulerPluginE "Test kicked off with the {} scheduler".format(self.name)) return errors + + def _job_name(self, tests: List[TestRun]) -> str: + """Given a test, get the name of the job.""" + + return 'pav-{}-{}-runs'.format(self.name, tests[0].series) + + def create_kickoff_script(self, + pav_cfg: PavConfig, + tests: Union[TestRun, List[TestRun]], + log_path: Optional[Path] = None, + nodes: Optional[NodeSet] = None, + isolate: bool = False) -> ScriptComposer: + """Create the kickoff script.""" + + if not isinstance(tests, list): + tests = [tests] + + sched_config = validate_config(tests[0].config['schedule']) + node_range = calc_node_range( + sched_config, + sched_config['cluster_info']['node_count']) + + job_name = self._job_name(tests) + + if isolate: + job_name = job_name + "_isolated" + + script = self._create_kickoff_script_stub( + pav_cfg=pav_cfg, + job_name=job_name, + log_path=log_path, + sched_config=sched_config, + node_range=node_range, + nodes=nodes, + shebang=tests[0].shebang) + + test_ids = ' '.join(test.full_id for test in tests) + + script.command('echo "Starting {} tests - $(date)"'.format(len(tests))) + + if isolate: + script = tests[0].make_script(script, "run", isolate=True) + else: + script.command('pav _run {}'.format(test_ids)) + + return script diff --git a/lib/pavilion/schedulers/scheduler.py b/lib/pavilion/schedulers/scheduler.py index ba8d8dcd0..423316c95 100644 --- a/lib/pavilion/schedulers/scheduler.py +++ b/lib/pavilion/schedulers/scheduler.py @@ -5,10 +5,12 @@ import inspect import os import time +from abc import abstractmethod from pathlib import Path -from typing import List, Union, Dict, NewType, Tuple, Type +from typing import List, Union, Dict, NewType, Tuple, Type, Optional import yaml_config as yc +from pavilion.config import PavConfig from pavilion.jobs import JobError, JobInfo, Job from pavilion.scriptcomposer import ScriptHeader, ScriptComposer from pavilion.status_file import STATES, TestStatusInfo @@ -99,6 +101,8 @@ class SchedulerPlugin(IPlugin.IPlugin): KICKOFF_FN = None """If the kickoff script requires a special filename, set it here.""" + KICKOFF_LOG_DEFAULT_FN = "kickoff.log" + VAR_CLASS = SchedulerVariables # type: Type[SchedulerVariables] """The scheduler's variable class.""" @@ -422,8 +426,22 @@ def _get_config_elems(self) -> Tuple[List[yc.ConfigElement], dict, dict]: return [], {}, {} - def _create_kickoff_script_stub(self, pav_cfg, job_name: str, log_path: Path, + @abstractmethod + def create_kickoff_script(self, + pav_cfg: PavConfig, + tests: Union[TestRun, List[TestRun]], + log_path: Optional[Path] = None, + nodes: Optional = None, + isolate: bool = False) -> ScriptComposer: + """Create the kickoff script.""" + + raise NotImplementedError + + def _create_kickoff_script_stub(self, + pav_cfg: PavConfig, + job_name: str, sched_config: dict, + log_path: Optional[Path] = None, nodes: Union[NodeList, None] = None, node_range: Union[Tuple[int, int], None] = None, shebang: str = None)\ @@ -450,7 +468,12 @@ def _create_kickoff_script_stub(self, pav_cfg, job_name: str, log_path: Path, script = ScriptComposer(header=header) script.comment("Redirect all output to the kickoff log.") - script.command("exec >{} 2>&1".format(log_path.as_posix())) + + if log_path is not None: + script.command(f"exec >{log_path.as_posix()} 2>&1") + else: + script.command( + f'exec > $(dirname -- ${{BASH_SOURCE[0]}})/{self.KICKOFF_LOG_DEFAULT_FN} 2>&1') # Make sure the pavilion spawned env_changes = { @@ -577,6 +600,12 @@ def convert_lists_to_tuples(obj): return tuple(key_parts) + @abstractmethod + def _job_name(self, tests: Union[TestRun, List[TestRun]]) -> str: + """Given a test, get the name of the job.""" + + raise NotImplementedError + def __reset(): """This exists for testing purposes only.""" diff --git a/lib/pavilion/test_run/test_run.py b/lib/pavilion/test_run/test_run.py index 0b9e5224b..0c98e71b8 100644 --- a/lib/pavilion/test_run/test_run.py +++ b/lib/pavilion/test_run/test_run.py @@ -323,12 +323,8 @@ def save(self) -> None: script = scriptcomposer.ScriptComposer(header=header) if self._build_needed(): - self._write_script( - script, - 'build', - path=self.build_script_path, - config=self.config.get('build', {}), - module_wrappers=self.config.get('module_wrappers', {})) + script = self.make_script(script, 'build') + script.write(self.build_script_path) self.builder = self._make_builder() self.build_name = self.builder.name @@ -337,12 +333,8 @@ def save(self) -> None: # process of creating and using a builder. self._build_trivial() - self._write_script( - script, - 'run', - path=self.run_tmpl_path, - config=self.config.get('run', {}), - module_wrappers=self.config.get('module_wrappers', {})) + script = self.make_script(script, 'run') + script.write(self.run_tmpl_path) self.save_attributes() self.status.set(STATES.CREATED, "Test directory setup complete.") @@ -529,13 +521,8 @@ def finalize(self, new_vars: VariableSetManager): header = scriptcomposer.ScriptHeader(shebang=self.shebang) script = scriptcomposer.ScriptComposer(header=header) - self._write_script( - script, - 'run', - self.run_script_path, - self.config['run'], - self.config.get('module_wrappers', {}) - ) + script = self.make_script(script, 'run') + script.write(self.run_script_path) self.status.set(STATES.FINALIZED, "Test Run Finalized.") @@ -1128,13 +1115,10 @@ def complete_time(self): .format(run_complete_path.as_posix(), err)) return None - def _write_script(self, - script: scriptcomposer.ScriptComposer, - stype: str, - path: Path, - config: dict, - module_wrappers: dict, - isolate: bool = False) -> None: + def make_script(self, + script: scriptcomposer.ScriptComposer, + stype: str, + isolate: bool = False) -> scriptcomposer.ScriptComposer: """Write a build or run script or template. The formats for each are mostly identical. :param stype: The type of script (run or build). @@ -1143,6 +1127,8 @@ def _write_script(self, :param module_wrappers: The module wrappers definition. """ + config = self.config[stype] + module_wrappers = self.config.get('module_wrappers', {}) verbose = config.get('verbose', 'false').lower() == 'true' if verbose: @@ -1163,8 +1149,10 @@ def _write_script(self, script.env_change(env) + script.command('this_dir=$( dirname -- "${BASH_SOURCE[0]}" )') + if isolate: - pav_lib_bash = '$( dirname -- "${BASH_SOURCE[0]}" )/pav-lib.bash' + pav_lib_bash = '${this_dir}/pav-lib.bash' else: pav_lib_bash = self._pav_cfg.pav_root/'bin'/'pav-lib.bash' @@ -1252,9 +1240,9 @@ def _write_script(self, script.comment('Output the environment for posterity') if verbose: - script.command(f'declare -p | tee > {path.parent / stype}.env.sh') + script.command(f'declare -p | tee > $(this_dir)/{stype}.env.sh') else: - script.command(f'declare -p > {path.parent / stype}.env.sh') + script.command(f'declare -p > $(this_dir)/{stype}.env.sh') script.newline() script.command(f'echo "(pav) Executing {stype} commands."') @@ -1275,7 +1263,7 @@ def _write_script(self, script.newline() script.command(f'echo "(pav) Test {stype} commands completed without error."') - script.write(path) + return script def __repr__(self): return "TestRun({s.name}-{s.full_id})".format(s=self) From 64f6c6d0665a63fdaf8643a7f7b8841eba16df81 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Wed, 3 Dec 2025 09:57:11 -0700 Subject: [PATCH 68/74] Fix style issues --- lib/pavilion/commands/isolate.py | 2 +- lib/pavilion/schedulers/advanced.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pavilion/commands/isolate.py b/lib/pavilion/commands/isolate.py index a1eafc5b9..1ae96a5b7 100644 --- a/lib/pavilion/commands/isolate.py +++ b/lib/pavilion/commands/isolate.py @@ -214,4 +214,4 @@ def _write_kickoff_script(cls, pav_cfg: PavConfig, test: TestRun, script_path: P test, isolate=True) - script.write(script_path) \ No newline at end of file + script.write(script_path) diff --git a/lib/pavilion/schedulers/advanced.py b/lib/pavilion/schedulers/advanced.py index 6371c4098..a3c4147f8 100644 --- a/lib/pavilion/schedulers/advanced.py +++ b/lib/pavilion/schedulers/advanced.py @@ -767,4 +767,4 @@ def _job_name(self, tests: Union[TestRun, List[TestRun]]) -> str: if len(tests) > 4: job_name += ' ...' - return job_name \ No newline at end of file + return job_name From de6497161290957ac5253629f154cb7238660fbd Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Wed, 3 Dec 2025 10:15:37 -0700 Subject: [PATCH 69/74] Pass some unit tests --- lib/pavilion/schedulers/advanced.py | 17 +++++++++++------ lib/pavilion/schedulers/basic.py | 10 +++++++--- lib/pavilion/test_run/test_run.py | 4 ++-- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/lib/pavilion/schedulers/advanced.py b/lib/pavilion/schedulers/advanced.py index a3c4147f8..78dd69b88 100644 --- a/lib/pavilion/schedulers/advanced.py +++ b/lib/pavilion/schedulers/advanced.py @@ -723,18 +723,23 @@ def create_kickoff_script(self, tests = [tests] sched_config = validate_config(tests[0].config['schedule']) - sched_config["time_limit"] = max(map( - lambda x: x.config["schedule"]["time_limit"], - tests)) + time_limits = [t.config["schedule"]["time_limit"] for t in tests \ + if t.config["schedule"]["time_limit"] is not None] + + if len(time_limits) > 0: + sched_config["time_limit"] = max(time_limits) job_name = self._job_name(tests) if isolate: job_name = job_name + "_isolated" - node_range = calc_node_range( - sched_config, - sched_config['cluster_info']['node_count']) + if nodes is None: + node_range = calc_node_range( + sched_config, + sched_config['cluster_info']['node_count']) + else: + node_range = None script = self._create_kickoff_script_stub( pav_cfg=pav_cfg, diff --git a/lib/pavilion/schedulers/basic.py b/lib/pavilion/schedulers/basic.py index 29dddadeb..c86881bc4 100644 --- a/lib/pavilion/schedulers/basic.py +++ b/lib/pavilion/schedulers/basic.py @@ -141,9 +141,13 @@ def create_kickoff_script(self, tests = [tests] sched_config = validate_config(tests[0].config['schedule']) - node_range = calc_node_range( - sched_config, - sched_config['cluster_info']['node_count']) + + if nodes is None: + node_range = calc_node_range( + sched_config, + sched_config['cluster_info']['node_count']) + else: + node_range = None job_name = self._job_name(tests) diff --git a/lib/pavilion/test_run/test_run.py b/lib/pavilion/test_run/test_run.py index 0c98e71b8..5dc6c0f4a 100644 --- a/lib/pavilion/test_run/test_run.py +++ b/lib/pavilion/test_run/test_run.py @@ -1240,9 +1240,9 @@ def make_script(self, script.comment('Output the environment for posterity') if verbose: - script.command(f'declare -p | tee > $(this_dir)/{stype}.env.sh') + script.command(f'declare -p | tee > ${{this_dir}}/{stype}.env.sh') else: - script.command(f'declare -p > $(this_dir)/{stype}.env.sh') + script.command(f'declare -p > ${{this_dir}}/{stype}.env.sh') script.newline() script.command(f'echo "(pav) Executing {stype} commands."') From 1bf874f418070d7fc1edc8067496f2dce9156b08 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Wed, 3 Dec 2025 10:34:04 -0700 Subject: [PATCH 70/74] Pass logging unit test --- lib/pavilion/schedulers/basic.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/pavilion/schedulers/basic.py b/lib/pavilion/schedulers/basic.py index c86881bc4..c12f750e8 100644 --- a/lib/pavilion/schedulers/basic.py +++ b/lib/pavilion/schedulers/basic.py @@ -165,7 +165,9 @@ def create_kickoff_script(self, test_ids = ' '.join(test.full_id for test in tests) - script.command('echo "Starting {} tests - $(date)"'.format(len(tests))) + # This is commented out for consistency with prior behavior and with expected output for + # logging unit tests. We may want to consider adding it. — HW + # script.command('echo "Starting {} tests - $(date)"'.format(len(tests))) if isolate: script = tests[0].make_script(script, "run", isolate=True) From e22d5983f84aceecd2571bc0e6c1932f1adc9fb1 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Wed, 3 Dec 2025 11:08:42 -0700 Subject: [PATCH 71/74] Tweak written scripts slightly --- lib/pavilion/schedulers/advanced.py | 3 +++ lib/pavilion/schedulers/basic.py | 2 ++ lib/pavilion/schedulers/scheduler.py | 24 +++++++++++++----------- lib/pavilion/test_run/test_run.py | 11 ++++++----- 4 files changed, 24 insertions(+), 16 deletions(-) diff --git a/lib/pavilion/schedulers/advanced.py b/lib/pavilion/schedulers/advanced.py index 78dd69b88..dbbb3a478 100644 --- a/lib/pavilion/schedulers/advanced.py +++ b/lib/pavilion/schedulers/advanced.py @@ -752,8 +752,11 @@ def create_kickoff_script(self, test_ids = ' '.join(test.full_id for test in tests) + script.newline() script.command('echo "Starting {} tests - $(date)"'.format(len(tests))) + script.newline() + if isolate: script = tests[0].make_script(script, "run", isolate=True) else: diff --git a/lib/pavilion/schedulers/basic.py b/lib/pavilion/schedulers/basic.py index c12f750e8..35bbde331 100644 --- a/lib/pavilion/schedulers/basic.py +++ b/lib/pavilion/schedulers/basic.py @@ -169,6 +169,8 @@ def create_kickoff_script(self, # logging unit tests. We may want to consider adding it. — HW # script.command('echo "Starting {} tests - $(date)"'.format(len(tests))) + script.newline() + if isolate: script = tests[0].make_script(script, "run", isolate=True) else: diff --git a/lib/pavilion/schedulers/scheduler.py b/lib/pavilion/schedulers/scheduler.py index 423316c95..ba5b258d5 100644 --- a/lib/pavilion/schedulers/scheduler.py +++ b/lib/pavilion/schedulers/scheduler.py @@ -444,8 +444,8 @@ def _create_kickoff_script_stub(self, log_path: Optional[Path] = None, nodes: Union[NodeList, None] = None, node_range: Union[Tuple[int, int], None] = None, - shebang: str = None)\ - -> ScriptComposer: + shebang: str = None, + isolate: bool = False) -> ScriptComposer: """Generate the kickoff script essentials preamble common to all scheduled tests. @@ -475,15 +475,17 @@ def _create_kickoff_script_stub(self, script.command( f'exec > $(dirname -- ${{BASH_SOURCE[0]}})/{self.KICKOFF_LOG_DEFAULT_FN} 2>&1') - # Make sure the pavilion spawned - env_changes = { - 'PATH': '{}:${{PATH}}'.format(pav_cfg.pav_root / 'bin'), - 'PAV_CONFIG_FILE': str(pav_cfg.pav_cfg_file), - } - if 'PAV_CONFIG_DIR' in os.environ: - env_changes['PAV_CONFIG_DIR'] = os.environ['PAV_CONFIG_DIR'] - - script.env_change(env_changes) + if not isolate: + script.newline() + # Make sure the pavilion spawned + env_changes = { + 'PATH': '{}:${{PATH}}'.format(pav_cfg.pav_root / 'bin'), + 'PAV_CONFIG_FILE': str(pav_cfg.pav_cfg_file), + } + if 'PAV_CONFIG_DIR' in os.environ: + env_changes['PAV_CONFIG_DIR'] = os.environ['PAV_CONFIG_DIR'] + + script.env_change(env_changes) # Run Kickoff Env setup commands for command in pav_cfg.env_setup: diff --git a/lib/pavilion/test_run/test_run.py b/lib/pavilion/test_run/test_run.py index 5dc6c0f4a..b8138c14d 100644 --- a/lib/pavilion/test_run/test_run.py +++ b/lib/pavilion/test_run/test_run.py @@ -1138,6 +1138,8 @@ def make_script(self, script.command(f'echo "(pav) Starting {stype} script"') + script.newline() + # If we include this directly, it breaks build hashing. script.comment('The first (and only) argument of the build script is ' 'the test id.') @@ -1149,10 +1151,8 @@ def make_script(self, script.env_change(env) - script.command('this_dir=$( dirname -- "${BASH_SOURCE[0]}" )') - if isolate: - pav_lib_bash = '${this_dir}/pav-lib.bash' + pav_lib_bash = '$( dirname -- "${BASH_SOURCE[0]}" )/pav-lib.bash' else: pav_lib_bash = self._pav_cfg.pav_root/'bin'/'pav-lib.bash' @@ -1240,9 +1240,10 @@ def make_script(self, script.comment('Output the environment for posterity') if verbose: - script.command(f'declare -p | tee > ${{this_dir}}/{stype}.env.sh') + script.command( + f'declare -p | tee > $( dirname -- "${{BASH_SOURCE[0]}}" )/{stype}.env.sh') else: - script.command(f'declare -p > ${{this_dir}}/{stype}.env.sh') + script.command(f'declare -p > $( dirname -- "${{BASH_SOURCE[0]}}" )/{stype}.env.sh') script.newline() script.command(f'echo "(pav) Executing {stype} commands."') From e33af189c958631339fbd7e18b32aa2d57787aed Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Wed, 3 Dec 2025 11:14:43 -0700 Subject: [PATCH 72/74] Remove pavilion environment variables from kickoff script when isolating --- lib/pavilion/schedulers/advanced.py | 5 +++-- lib/pavilion/schedulers/basic.py | 5 +++-- lib/pavilion/schedulers/scheduler.py | 2 ++ 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/pavilion/schedulers/advanced.py b/lib/pavilion/schedulers/advanced.py index dbbb3a478..ee1227895 100644 --- a/lib/pavilion/schedulers/advanced.py +++ b/lib/pavilion/schedulers/advanced.py @@ -732,7 +732,7 @@ def create_kickoff_script(self, job_name = self._job_name(tests) if isolate: - job_name = job_name + "_isolated" + job_name = job_name + self.ISOLATE_KICKOFF_SUFFIX if nodes is None: node_range = calc_node_range( @@ -748,7 +748,8 @@ def create_kickoff_script(self, sched_config=sched_config, node_range=node_range, nodes=nodes, - shebang=tests[0].shebang) + shebang=tests[0].shebang, + isolate=isolate) test_ids = ' '.join(test.full_id for test in tests) diff --git a/lib/pavilion/schedulers/basic.py b/lib/pavilion/schedulers/basic.py index 35bbde331..de2408cfb 100644 --- a/lib/pavilion/schedulers/basic.py +++ b/lib/pavilion/schedulers/basic.py @@ -152,7 +152,7 @@ def create_kickoff_script(self, job_name = self._job_name(tests) if isolate: - job_name = job_name + "_isolated" + job_name = job_name + self.ISOLATE_KICKOFF_SUFFIX script = self._create_kickoff_script_stub( pav_cfg=pav_cfg, @@ -161,7 +161,8 @@ def create_kickoff_script(self, sched_config=sched_config, node_range=node_range, nodes=nodes, - shebang=tests[0].shebang) + shebang=tests[0].shebang, + isolate=isolate) test_ids = ' '.join(test.full_id for test in tests) diff --git a/lib/pavilion/schedulers/scheduler.py b/lib/pavilion/schedulers/scheduler.py index ba5b258d5..c938efe49 100644 --- a/lib/pavilion/schedulers/scheduler.py +++ b/lib/pavilion/schedulers/scheduler.py @@ -103,6 +103,8 @@ class SchedulerPlugin(IPlugin.IPlugin): KICKOFF_LOG_DEFAULT_FN = "kickoff.log" + ISOLATE_KICKOFF_SUFFIX = "_isolated" + VAR_CLASS = SchedulerVariables # type: Type[SchedulerVariables] """The scheduler's variable class.""" From 9901811a3dce09613227b28a9d1021b443a56fd1 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Wed, 3 Dec 2025 11:26:39 -0700 Subject: [PATCH 73/74] Refine isolate command unit tests --- lib/pavilion/commands/isolate.py | 5 ++--- lib/pavilion/test_run/test_run.py | 7 +++++-- test/tests/isolate_cmd_tests.py | 11 ++++++++++- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/lib/pavilion/commands/isolate.py b/lib/pavilion/commands/isolate.py index 1ae96a5b7..5d02a76b4 100644 --- a/lib/pavilion/commands/isolate.py +++ b/lib/pavilion/commands/isolate.py @@ -24,7 +24,6 @@ class IsolateCommand(Command): IGNORE_FILES = ("series", "job") KICKOFF_FN = "kickoff.isolated" - PAV_LIB_FN = "pav-lib.bash" def __init__(self): super().__init__( @@ -140,8 +139,8 @@ def _isolate(cls, pav_cfg: PavConfig, test: TestRun, dest: Path, archive: bool, return 8 - pav_lib_bash = pav_cfg.pav_root / 'bin' / cls.PAV_LIB_FN - shutil.copyfile(pav_lib_bash, dest / cls.PAV_LIB_FN) + pav_lib_bash = pav_cfg.pav_root / 'bin' / TestRun.PAV_LIB_FN + shutil.copyfile(pav_lib_bash, dest / TestRun.PAV_LIB_FN) cls._write_kickoff_script(pav_cfg, test, dest / cls.KICKOFF_FN) diff --git a/lib/pavilion/test_run/test_run.py b/lib/pavilion/test_run/test_run.py index b8138c14d..ffb7322e2 100644 --- a/lib/pavilion/test_run/test_run.py +++ b/lib/pavilion/test_run/test_run.py @@ -100,6 +100,9 @@ class TestRun(TestAttributes): BUILD_TEMPLATE_DIR = 'templates' """Directory that holds build templates.""" + PAV_LIB_FN = "pav-lib.bash" + """Pavilion bash utilities""" + def __init__(self, pav_cfg: PavConfig, config: Dict, var_man: VariableSetManager = None, _id: int = None, rebuild: bool = False, build_only: bool = False): """Create an new TestRun object. If loading an existing test @@ -1152,9 +1155,9 @@ def make_script(self, script.env_change(env) if isolate: - pav_lib_bash = '$( dirname -- "${BASH_SOURCE[0]}" )/pav-lib.bash' + pav_lib_bash = f'$( dirname -- "${{BASH_SOURCE[0]}}" )/{self.PAV_LIB_FN}' else: - pav_lib_bash = self._pav_cfg.pav_root/'bin'/'pav-lib.bash' + pav_lib_bash = self._pav_cfg.pav_root / 'bin' / self.PAV_LIB_FN script.command('source {}'.format(pav_lib_bash)) diff --git a/test/tests/isolate_cmd_tests.py b/test/tests/isolate_cmd_tests.py index c49dcff4f..eed65e98a 100644 --- a/test/tests/isolate_cmd_tests.py +++ b/test/tests/isolate_cmd_tests.py @@ -9,6 +9,7 @@ from pavilion import arguments from pavilion.unittest import PavTestCase from pavilion.cmd_utils import list_files +from pavilion.test_run import TestRun class IsolateCmdTests(PavTestCase): @@ -28,6 +29,8 @@ def test_no_archive(self): run_cmd.run(self.pav_cfg, parser.parse_args(run_args)) last_test = next(iter(run_cmd.last_tests)) + last_test.wait(timeout=10) + with tempfile.TemporaryDirectory() as dir: isolate_args = parser.parse_args(["isolate", str(Path(dir) / "dest")]) @@ -40,10 +43,13 @@ def test_no_archive(self): lambda x: x.relative_to(Path(dir) / "dest"), list_files(Path(dir) / "dest"))) + self.assertIn(TestRun.PAV_LIB_FN, dest_files) + self.assertIn(isolate_cmd.KICKOFF_FN, dest_files) + self.assertEqual( {f for f in source_files if f.name not in ("series", "job")}, {f for f in dest_files if f.name not in \ - ("pav-lib.bash", "kickoff.isolated") and \ + (TestRun.PAV_LIB_FN, isolate_cmd.KICKOFF_FN) and \ # Have to exclude build_origin because of how os.walk handles symlinks f.parent.name != "build_origin"}) @@ -81,6 +87,9 @@ def test_zip_archive(self): lambda x: x.relative_to(Path(extract_dir) / "dest"), list_files(Path(extract_dir)))) + self.assertIn("pav_lib.bash", dest_files) + self.assertIn(isolate_cmd.KICKOFF_FN, dest_files) + self.assertEqual( {f for f in source_files if f.name not in ("series", "job")}, {f for f in dest_files if f.name not in \ From 629bd6e047083a79dae6ea1473a7ef367254078e Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Wed, 3 Dec 2025 11:28:55 -0700 Subject: [PATCH 74/74] Fix isolate command unit tests --- lib/pavilion/commands/isolate.py | 4 ++-- test/tests/isolate_cmd_tests.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/pavilion/commands/isolate.py b/lib/pavilion/commands/isolate.py index 5d02a76b4..8e7c9ceb9 100644 --- a/lib/pavilion/commands/isolate.py +++ b/lib/pavilion/commands/isolate.py @@ -170,8 +170,8 @@ def _write_tarball(cls, pav_cfg: PavConfig, test: TestRun, dest: Path, zip: bool copytree_resolved(test.path, tmp_dest, ignore_files=ignore_files) # Copy Pavilion bash library into tarball - pav_lib_bash = pav_cfg.pav_root / 'bin' / cls.PAV_LIB_FN - shutil.copyfile(pav_lib_bash, tmp_dest / cls.PAV_LIB_FN) + pav_lib_bash = pav_cfg.pav_root / 'bin' / TestRun.PAV_LIB_FN + shutil.copyfile(pav_lib_bash, tmp_dest / TestRun.PAV_LIB_FN) cls._write_kickoff_script(pav_cfg, test, tmp_dest / cls.KICKOFF_FN) diff --git a/test/tests/isolate_cmd_tests.py b/test/tests/isolate_cmd_tests.py index eed65e98a..e55944f3f 100644 --- a/test/tests/isolate_cmd_tests.py +++ b/test/tests/isolate_cmd_tests.py @@ -43,8 +43,8 @@ def test_no_archive(self): lambda x: x.relative_to(Path(dir) / "dest"), list_files(Path(dir) / "dest"))) - self.assertIn(TestRun.PAV_LIB_FN, dest_files) - self.assertIn(isolate_cmd.KICKOFF_FN, dest_files) + self.assertIn(Path(TestRun.PAV_LIB_FN), dest_files) + self.assertIn(Path(isolate_cmd.KICKOFF_FN), dest_files) self.assertEqual( {f for f in source_files if f.name not in ("series", "job")}, @@ -87,8 +87,8 @@ def test_zip_archive(self): lambda x: x.relative_to(Path(extract_dir) / "dest"), list_files(Path(extract_dir)))) - self.assertIn("pav_lib.bash", dest_files) - self.assertIn(isolate_cmd.KICKOFF_FN, dest_files) + self.assertIn(Path(TestRun.PAV_LIB_FN), dest_files) + self.assertIn(Path(isolate_cmd.KICKOFF_FN), dest_files) self.assertEqual( {f for f in source_files if f.name not in ("series", "job")},