From dcf0b2927af3cddc462251c0cfabcae94e3a966e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sat, 12 Jul 2025 15:43:23 +0200 Subject: [PATCH 1/2] Provide the log_obj_size option in every Executor --- executorlib/executor/flux.py | 5 +++++ executorlib/executor/slurm.py | 5 +++++ executorlib/standalone/inputcheck.py | 14 ++++++++++++++ tests/test_standalone_inputcheck.py | 5 +++++ 4 files changed, 29 insertions(+) diff --git a/executorlib/executor/flux.py b/executorlib/executor/flux.py index a6066c11..3fe1bbab 100644 --- a/executorlib/executor/flux.py +++ b/executorlib/executor/flux.py @@ -3,6 +3,7 @@ from executorlib.executor.base import BaseExecutor from executorlib.standalone.inputcheck import ( check_command_line_argument_lst, + check_log_obj_size, check_init_function, check_oversubscribe, check_plot_dependency_graph, @@ -246,6 +247,7 @@ class FluxClusterExecutor(BaseExecutor): plot_dependency_graph (bool): Plot the dependencies of multiple future objects without executing them. For debugging purposes and to get an overview of the specified dependencies. plot_dependency_graph_filename (str): Name of the file to store the plotted graph in. + log_obj_size (bool): Enable debug mode which reports the size of the communicated objects. Examples: ``` @@ -282,6 +284,7 @@ def __init__( refresh_rate: float = 0.01, plot_dependency_graph: bool = False, plot_dependency_graph_filename: Optional[str] = None, + log_obj_size: bool = False, ): """ The executorlib.FluxClusterExecutor leverages either the message passing interface (MPI), the SLURM workload @@ -323,6 +326,7 @@ def __init__( plot_dependency_graph (bool): Plot the dependencies of multiple future objects without executing them. For debugging purposes and to get an overview of the specified dependencies. plot_dependency_graph_filename (str): Name of the file to store the plotted graph in. + log_obj_size (bool): Enable debug mode which reports the size of the communicated objects. """ default_resource_dict: dict = { @@ -338,6 +342,7 @@ def __init__( resource_dict.update( {k: v for k, v in default_resource_dict.items() if k not in resource_dict} ) + check_log_obj_size(log_obj_size=log_obj_size) if not plot_dependency_graph: import pysqa # noqa diff --git a/executorlib/executor/slurm.py b/executorlib/executor/slurm.py index 3f429e3b..bf3e48c7 100644 --- a/executorlib/executor/slurm.py +++ b/executorlib/executor/slurm.py @@ -3,6 +3,7 @@ from executorlib.executor.base import BaseExecutor from executorlib.standalone.inputcheck import ( check_init_function, + check_log_obj_size, check_plot_dependency_graph, check_refresh_rate, validate_number_of_cores, @@ -58,6 +59,7 @@ class SlurmClusterExecutor(BaseExecutor): plot_dependency_graph (bool): Plot the dependencies of multiple future objects without executing them. For debugging purposes and to get an overview of the specified dependencies. plot_dependency_graph_filename (str): Name of the file to store the plotted graph in. + log_obj_size (bool): Enable debug mode which reports the size of the communicated objects. Examples: ``` @@ -94,6 +96,7 @@ def __init__( refresh_rate: float = 0.01, plot_dependency_graph: bool = False, plot_dependency_graph_filename: Optional[str] = None, + log_obj_size: bool = False, ): """ The executorlib.SlurmClusterExecutor leverages either the message passing interface (MPI), the SLURM workload @@ -135,6 +138,7 @@ def __init__( plot_dependency_graph (bool): Plot the dependencies of multiple future objects without executing them. For debugging purposes and to get an overview of the specified dependencies. plot_dependency_graph_filename (str): Name of the file to store the plotted graph in. + log_obj_size (bool): Enable debug mode which reports the size of the communicated objects. """ default_resource_dict: dict = { @@ -150,6 +154,7 @@ def __init__( resource_dict.update( {k: v for k, v in default_resource_dict.items() if k not in resource_dict} ) + check_log_obj_size(log_obj_size=log_obj_size) if not plot_dependency_graph: import pysqa # noqa diff --git a/executorlib/standalone/inputcheck.py b/executorlib/standalone/inputcheck.py index cd368903..6e681f0e 100644 --- a/executorlib/standalone/inputcheck.py +++ b/executorlib/standalone/inputcheck.py @@ -194,7 +194,21 @@ def validate_number_of_cores( def check_file_exists(file_name: Optional[str]): + """ + Check if file exists and raise a ValueError if it does not or file_name is None. + """ if file_name is None: raise ValueError("file_name is not set.") if not os.path.exists(file_name): raise ValueError("file_name is not written to the file system.") + + +def check_log_obj_size(log_obj_size: bool) -> None: + """ + Check if log_obj_size is True and raise a ValueError if it is. + """ + if log_obj_size: + raise ValueError( + "log_obj_size is not supported for the executorlib.SlurmClusterExecutor and executorlib.FluxClusterExecutor." + "Please use log_obj_size=False instead of log_obj_size=True." + ) \ No newline at end of file diff --git a/tests/test_standalone_inputcheck.py b/tests/test_standalone_inputcheck.py index d35a9611..95ee6d51 100644 --- a/tests/test_standalone_inputcheck.py +++ b/tests/test_standalone_inputcheck.py @@ -18,6 +18,7 @@ check_hostname_localhost, check_pysqa_config_directory, check_file_exists, + check_log_obj_size, validate_number_of_cores, ) @@ -119,3 +120,7 @@ def test_validate_number_of_cores(self): ), int, ) + + def test_check_log_obj_size(self): + with self.assertRaises(ValueError): + check_log_obj_size(log_obj_size=True) \ No newline at end of file From 144b16e29b7301c4837fd53b5ebf80acc90fb709 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 12 Jul 2025 13:43:45 +0000 Subject: [PATCH 2/2] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- executorlib/executor/flux.py | 2 +- executorlib/standalone/inputcheck.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/executorlib/executor/flux.py b/executorlib/executor/flux.py index 3fe1bbab..145ccc73 100644 --- a/executorlib/executor/flux.py +++ b/executorlib/executor/flux.py @@ -3,8 +3,8 @@ from executorlib.executor.base import BaseExecutor from executorlib.standalone.inputcheck import ( check_command_line_argument_lst, - check_log_obj_size, check_init_function, + check_log_obj_size, check_oversubscribe, check_plot_dependency_graph, check_pmi, diff --git a/executorlib/standalone/inputcheck.py b/executorlib/standalone/inputcheck.py index 6e681f0e..2ebe6808 100644 --- a/executorlib/standalone/inputcheck.py +++ b/executorlib/standalone/inputcheck.py @@ -211,4 +211,4 @@ def check_log_obj_size(log_obj_size: bool) -> None: raise ValueError( "log_obj_size is not supported for the executorlib.SlurmClusterExecutor and executorlib.FluxClusterExecutor." "Please use log_obj_size=False instead of log_obj_size=True." - ) \ No newline at end of file + )