diff --git a/conf/minion b/conf/minion index abe6abcd6dde..7bc61ccf3724 100644 --- a/conf/minion +++ b/conf/minion @@ -66,6 +66,11 @@ # Set to zero if the minion should shutdown and not retry. # retry_dns: 30 +# Set the number of times to attempt to resolve +# the master hostname if name resolution fails. Defaults to None, +# which will attempt the resolution indefinitely. +# retry_dns_count: 3 + # Set the port used by the master reply and authentication server. #master_port: 4506 diff --git a/doc/ref/configuration/minion.rst b/doc/ref/configuration/minion.rst index d38b8cb19b4d..06bfbeea41dd 100644 --- a/doc/ref/configuration/minion.rst +++ b/doc/ref/configuration/minion.rst @@ -307,6 +307,23 @@ Set to zero if the minion should shutdown and not retry. retry_dns: 30 +.. conf_minion:: retry_dns_count + +``retry_dns_count`` +------------------- + +.. versionadded:: 2018.3.4 + +Default: ``None`` + +Set the number of attempts to perform when resolving +the master hostname if name resolution fails. +By default the minion will retry indefinitely. + +.. code-block:: yaml + + retry_dns_count: 3 + .. conf_minion:: master_port ``master_port`` diff --git a/salt/config/__init__.py b/salt/config/__init__.py index 9c4e6bd7c4f0..b3692cccc876 100644 --- a/salt/config/__init__.py +++ b/salt/config/__init__.py @@ -518,6 +518,7 @@ def _gather_buffer_space(): # The number of seconds to sleep between retrying an attempt to resolve the hostname of a # salt master 'retry_dns': float, + 'retry_dns_count': (type(None), int), # In the case when the resolve of the salt master hostname fails, fall back to localhost 'resolve_dns_fallback': bool, @@ -1397,6 +1398,7 @@ def _gather_buffer_space(): 'update_url': False, 'update_restart_services': [], 'retry_dns': 30, + 'retry_dns_count': None, 'resolve_dns_fallback': True, 'recon_max': 10000, 'recon_default': 1000, diff --git a/salt/exceptions.py b/salt/exceptions.py index 63774c640b2d..e0add6c3781d 100644 --- a/salt/exceptions.py +++ b/salt/exceptions.py @@ -96,6 +96,12 @@ class SaltSyndicMasterError(SaltException): ''' +class SaltMasterUnresolvableError(SaltException): + ''' + Problem resolving the name of the Salt master + ''' + + class MasterExit(SystemExit): ''' Rise when the master exits diff --git a/salt/minion.py b/salt/minion.py index 273c1dcbd2ee..ebe96fabd79b 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -115,6 +115,7 @@ SaltSystemExit, SaltDaemonNotRunning, SaltException, + SaltMasterUnresolvableError ) @@ -154,8 +155,13 @@ def resolve_dns(opts, fallback=True): True, opts['ipv6']) except SaltClientError: + retry_dns_count = opts.get('retry_dns_count', None) if opts['retry_dns']: while True: + if retry_dns_count is not None: + if retry_dns_count == 0: + raise SaltMasterUnresolvableError + retry_dns_count -= 1 import salt.log msg = ('Master hostname: \'{0}\' not found or not responsive. ' 'Retrying in {1} seconds').format(opts['master'], opts['retry_dns']) @@ -962,7 +968,17 @@ def _create_minion_object(self, opts, timeout, safe, loaded_base_name=loaded_base_name, jid_queue=jid_queue) - def _spawn_minions(self): + def _check_minions(self): + ''' + Check the size of self.minions and raise an error if it's empty + ''' + if not self.minions: + err = ('Minion unable to successfully connect to ' + 'a Salt Master. Exiting.') + log.error(err) + raise SaltSystemExit(code=42, msg=err) + + def _spawn_minions(self, timeout=60): ''' Spawn all the coroutines which will sign in to masters ''' @@ -981,8 +997,9 @@ def _spawn_minions(self): loaded_base_name='salt.loader.{0}'.format(s_opts['master']), jid_queue=self.jid_queue, ) - self.minions.append(minion) - self.io_loop.spawn_callback(self._connect_minion, minion) + + self._connect_minion(minion) + self.io_loop.call_later(timeout, self._check_minions) @tornado.gen.coroutine def _connect_minion(self, minion): @@ -1000,6 +1017,7 @@ def _connect_minion(self, minion): minion.setup_scheduler(before_connect=True) yield minion.connect_master(failed=failed) minion.tune_in(start=False) + self.minions.append(minion) break except SaltClientError as exc: failed = True @@ -1011,6 +1029,11 @@ def _connect_minion(self, minion): if auth_wait < self.max_auth_wait: auth_wait += self.auth_wait yield tornado.gen.sleep(auth_wait) # TODO: log? + except SaltMasterUnresolvableError: + err = 'Master address: \'{0}\' could not be resolved. Invalid or unresolveable address. ' \ + 'Set \'master\' value in minion config.'.format(minion.opts['master']) + log.error(err) + break except Exception as e: failed = True log.critical( diff --git a/tests/unit/test_minion.py b/tests/unit/test_minion.py index f2cf8258b304..f889e4cedb6d 100644 --- a/tests/unit/test_minion.py +++ b/tests/unit/test_minion.py @@ -16,7 +16,7 @@ # Import salt libs import salt.minion import salt.utils.event as event -from salt.exceptions import SaltSystemExit +from salt.exceptions import SaltSystemExit, SaltMasterUnresolvableError import salt.syspaths import tornado import tornado.testing @@ -282,6 +282,17 @@ def test_scheduler_before_connect(self): finally: minion.destroy() + def test_minion_retry_dns_count(self): + ''' + Tests that the resolve_dns will retry dns look ups for a maximum of + 3 times before raising a SaltMasterUnresolvableError exception. + ''' + with patch.dict(__opts__, {'ipv6': False, 'master': 'dummy', + 'master_port': '4555', + 'retry_dns': 1, 'retry_dns_count': 3}): + self.assertRaises(SaltMasterUnresolvableError, + salt.minion.resolve_dns, __opts__) + @skipIf(NO_MOCK, NO_MOCK_REASON) class MinionAsyncTestCase(TestCase, AdaptedConfigurationTestCaseMixin, tornado.testing.AsyncTestCase):