Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Ssh retry #2359

Closed
wants to merge 2 commits into from
Closed
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
64 changes: 45 additions & 19 deletions lib/ansible/runner/connection_plugins/paramiko_ssh.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import pipes
import socket
import random
import time
from ansible.callbacks import vvv
from ansible import errors
from ansible import utils
Expand All @@ -39,11 +40,10 @@
SSH_CONNECTION_CACHE = {}
SFTP_CONNECTION_CACHE = {}


class Connection(object):
''' SSH based connections with Paramiko '''

def __init__(self, runner, host, port, user, password):
def __init__(self, runner, host, port, user, password, paramiko_tcp_timeout=60):

self.ssh = None
self.sftp = None
Expand Down Expand Up @@ -78,23 +78,49 @@ def _connect_uncached(self):
allow_agent = True
if self.password is not None:
allow_agent = False
try:
if self.runner.private_key_file:
key_filename = os.path.expanduser(self.runner.private_key_file)
else:
key_filename = None
ssh.connect(self.host, username=self.user, allow_agent=allow_agent, look_for_keys=True,
key_filename=key_filename, password=self.password,
timeout=self.runner.timeout, port=self.port)
except Exception, e:
msg = str(e)
if "PID check failed" in msg:
raise errors.AnsibleError("paramiko version issue, please upgrade paramiko on the machine running ansible")
elif "Private key file is encrypted" in msg:
msg = 'ssh %s@%s:%s : %s\nTo connect as a different user, use -u <username>.' % (
self.user, self.host, self.port, msg)
raise errors.AnsibleConnectionFailed(msg)
else:
if self.runner.private_key_file:
key_filename = os.path.expanduser(self.runner.private_key_file)
else:
key_filename = None
# retry logic for cases where a host has been recently booted and
# the ssh service isn't yet prepared. Happens with cloud-init
# putting private keys in place
retry_timeout = time.time() + self.runner.timeout
while time.time() < retry_timeout:
try:
ssh.connect(self.host, username=self.user, allow_agent=allow_agent, look_for_keys=True,
key_filename=key_filename, password=self.password,
timeout=self.runner.timeout, port=self.port)
break
except (paramiko.AuthenticationException, paramiko.ChannelException, socket.error) as err:
vvv("got %s establishing connection for: %s@%s:%s; retrying" %
(err, self.host, self.user, self.port))
continue
except paramiko.SSHException as err:
msg = str(e)
if "PID check failed" in msg:
raise errors.AnsibleError("paramiko version issue, please upgrade paramiko on the machine running ansible")
else:
raise errors.AnsibleConnectionFailed(msg)
else:
# timeout happened---last try
# give the user some hints, too
try:
ssh.connect(self.host, username=self.user, allow_agent=allow_agent, look_for_keys=True,
key_filename=key_filename, password=self.password,
timeout=self.runner.timeout, port=self.port)
except paramiko.BadAuthenticationType as err:
raise errors.AnsibleError(
"Bad authentication type provided for %s@%s:%s. Password given when public-key required? (%s)" %
(self.user, self.host, self.port, err)
)
except paramiko.PasswordRequiredException as err:
msg = "Password required for " + str(self.user) +"@"+ str(self.host) + ":" + str(self.port) + ". " + \
"Private key may be password-protected; try running with an ssh-agent. " + \
"To connect as a different user, use -u <username>."
raise errors.AnsibleError(msg)
except Exception as err:
msg = str(err)
raise errors.AnsibleConnectionFailed(msg)

return ssh
Expand Down