Skip to content

Commit

Permalink
Add retry for SSHException due to paramiko/paramiko#1508 (#327)
Browse files Browse the repository at this point in the history
As mentioned paramiko/paramiko#1508, there
might be a possibility that SSH tunnel open fail and dpdispatcher run
just down. As a temperary solution, add retry to catch the issue might
take effect aiming at a more robust run.
  • Loading branch information
zjgemi committed Apr 18, 2023
2 parents 4a5564e + 877975f commit 3736b9c
Showing 1 changed file with 8 additions and 3 deletions.
11 changes: 8 additions & 3 deletions dpdispatcher/ssh_context.py
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ def _check_alive(self):
# transport = self.ssh.get_transport()
# transport.set_keepalive(60)

@retry(max_retry=3, sleep=1)
@retry(max_retry=6, sleep=1)
def _setup_ssh(self):
# machine = self.machine
self.ssh = paramiko.SSHClient()
Expand Down Expand Up @@ -199,7 +199,7 @@ def _setup_ssh(self):
ts.auth_interactive(self.username, self.inter_handler)
except paramiko.ssh_exception.AuthenticationException:
# since the asynchrony of interactive authentication, one addtional try is added
# retry for up to 3 times
# retry for up to 6 times
raise RetrySignal("Authentication failed")
elif key_ok:
pass
Expand All @@ -213,7 +213,12 @@ def _setup_ssh(self):
raise RuntimeError("Please provide at least one form of authentication")
assert ts.is_active()
# Opening a session creates a channel along the socket to the server
ts.open_session(timeout=self.timeout)
try:
ts.open_session(timeout=self.timeout)
except paramiko.ssh_exception.SSHException:
# retry for up to 6 times
# ref: https://github.com/paramiko/paramiko/issues/1508
raise RetrySignal("Opening session failed")
ts.set_keepalive(60)
self.ssh._transport = ts # type: ignore
# reset sftp
Expand Down

0 comments on commit 3736b9c

Please sign in to comment.