Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Validate SRUN variables when launching in SLURM #15011

Merged
merged 21 commits into from
Oct 19, 2022
Merged
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions src/lightning_lite/plugins/environments/slurm_environment.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@
import signal
from typing import Optional

from lightning_utilities.core.rank_zero import rank_zero_warn

from lightning_lite.plugins.environments.cluster_environment import ClusterEnvironment
from lightning_lite.utilities.imports import _IS_WINDOWS

Expand All @@ -40,6 +42,7 @@ def __init__(self, auto_requeue: bool = True, requeue_signal: Optional[signal.Si
if requeue_signal is None and not _IS_WINDOWS:
requeue_signal = signal.SIGUSR1
self.requeue_signal = requeue_signal
self._validate_srun_variables()

@property
def creates_processes_externally(self) -> bool:
Expand Down Expand Up @@ -141,3 +144,19 @@ def resolve_root_node_address(nodes: str) -> str:
nodes = re.sub(r"\[(.*?)[,-].*\]", "\\1", nodes) # Take the first node of every node range
nodes = re.sub(r"\[(.*?)\]", "\\1", nodes) # handle special case where node range is single number
return nodes.split(" ")[0].split(",")[0]

@staticmethod
awaelchli marked this conversation as resolved.
Show resolved Hide resolved
def _validate_srun_variables():
"""Checks for conflicting or incorrectly set variables set through `srun` and raises a useful error
message.

Right now, we only check for the most commond user errors. See `the srun docs
<https://slurm.schedmd.com/srun.html>`_ for a complete list of supported srun variables.
"""

ntasks = int(os.environ.get("SLURM_NTASKS", "1"))
if ntasks > 1:
raise rank_zero_warn(
f"You set `--ntasks={ntasks}` in your SLURM bash script, but this variable is not supported. Please use"
f" `--ntasks-per-node={ntasks}` instead."
)
awaelchli marked this conversation as resolved.
Show resolved Hide resolved