From d1f3b86205e2a056be3dae8f7209bdd0e47d659b Mon Sep 17 00:00:00 2001 From: Mike Place Date: Tue, 10 Mar 2015 15:38:32 -0600 Subject: [PATCH 01/10] Catch all returner exceptions --- salt/client/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/client/__init__.py b/salt/client/__init__.py index aa7f37cfa436..6edb22d0db87 100644 --- a/salt/client/__init__.py +++ b/salt/client/__init__.py @@ -39,6 +39,7 @@ import salt.utils.event import salt.utils.minions import salt.utils.verify +import salt.utils.jid import salt.syspaths as syspaths from salt.exceptions import ( EauthAuthenticationError, SaltInvocationError, SaltReqTimeoutError @@ -852,7 +853,7 @@ def get_iter_returns( yield {} # stop the iteration, since the jid is invalid raise StopIteration() - except salt.exceptions.SaltMasterError as exc: + except Exception as exc: log.warning('Returner unavailable: {exc}'.format(exc=exc)) # Wait for the hosts to check in syndic_wait = 0 From faf580ec8241ca18974aad6a65f54db0afc3f9de Mon Sep 17 00:00:00 2001 From: Mike Place Date: Tue, 10 Mar 2015 15:52:06 -0600 Subject: [PATCH 02/10] More exception handling for returners --- salt/client/__init__.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/salt/client/__init__.py b/salt/client/__init__.py index 6edb22d0db87..bea60372efb6 100644 --- a/salt/client/__init__.py +++ b/salt/client/__init__.py @@ -1014,9 +1014,13 @@ def get_returns( found = set() ret = {} # Check to see if the jid is real, if not return the empty dict - if self.returners['{0}.get_load'.format(self.opts['master_job_cache'])](jid) == {}: - log.warning('jid does not exist') - return ret + try: + if self.returners['{0}.get_load'.format(self.opts['master_job_cache'])](jid) == {}: + log.warning('jid does not exist') + return ret + except Exception as exc: + raise SaltClientError('Master job cache returner [{0}] failed to verify jid. ' + 'Exception details: {1}'.format(self.opts['master_job_cache'], exc)) # Wait for the hosts to check in while True: @@ -1101,7 +1105,13 @@ def get_cache_returns(self, jid): ''' ret = {} - data = self.returners['{0}.get_jid'.format(self.opts['master_job_cache'])](jid) + try: + data = self.returners['{0}.get_jid'.format(self.opts['master_job_cache'])](jid) + except Exception: + raise SaltClientError('Could not examine master job cache. ' + 'Error occured in {0} returner. ' + 'Exception details: {1}'.format(self.opts['master_job_cache'], + exc)) for minion in data: m_data = {} if u'return' in data[minion]: From 1e5d40584db65b068c06b3c6bb6081b1dcc5b68f Mon Sep 17 00:00:00 2001 From: Mike Place Date: Tue, 10 Mar 2015 16:30:12 -0600 Subject: [PATCH 03/10] Finish returner exception handling --- salt/client/__init__.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/salt/client/__init__.py b/salt/client/__init__.py index bea60372efb6..f7f007903def 100644 --- a/salt/client/__init__.py +++ b/salt/client/__init__.py @@ -1062,7 +1062,13 @@ def get_full_returns(self, jid, minions, timeout=None): # create the iterator-- since we want to get anyone in the middle event_iter = self.get_event_iter_returns(jid, minions, timeout=timeout) - data = self.returners['{0}.get_jid'.format(self.opts['master_job_cache'])](jid) + try: + data = self.returners['{0}.get_jid'.format(self.opts['master_job_cache'])](jid) + except Exception as exc: + raise SaltClientError('Returner {0} could not fetch jid data. ' + 'Exception details: {1}'.format( + self.opts['master_job_cache'], + exc)) for minion in data: m_data = {} if u'return' in data[minion]: @@ -1157,9 +1163,15 @@ def get_cli_static_event_returns( found = set() ret = {} # Check to see if the jid is real, if not return the empty dict - if self.returners['{0}.get_load'.format(self.opts['master_job_cache'])](jid) == {}: - log.warning('jid does not exist') - return ret + try: + if self.returners['{0}.get_load'.format(self.opts['master_job_cache'])](jid) == {}: + log.warning('jid does not exist') + return ret + except Exception as exc: + raise SaltClientError('Load could not be retreived from ' + 'returner {0}. Exception details: {1}'.format( + self.opts['master_job_cache'], + exc)) # Wait for the hosts to check in while True: # Process events until timeout is reached or all minions have returned From aa76f2f3b81e7aa7af6c42b0d203c3e52d0b050a Mon Sep 17 00:00:00 2001 From: Mike Place Date: Tue, 10 Mar 2015 16:31:34 -0600 Subject: [PATCH 04/10] Lint --- salt/client/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/client/__init__.py b/salt/client/__init__.py index f7f007903def..1a7aa10956bb 100644 --- a/salt/client/__init__.py +++ b/salt/client/__init__.py @@ -42,7 +42,7 @@ import salt.utils.jid import salt.syspaths as syspaths from salt.exceptions import ( - EauthAuthenticationError, SaltInvocationError, SaltReqTimeoutError + EauthAuthenticationError, SaltInvocationError, SaltReqTimeoutError, SaltClientError ) import salt.ext.six as six @@ -1113,7 +1113,7 @@ def get_cache_returns(self, jid): try: data = self.returners['{0}.get_jid'.format(self.opts['master_job_cache'])](jid) - except Exception: + except Exception as exc: raise SaltClientError('Could not examine master job cache. ' 'Error occured in {0} returner. ' 'Exception details: {1}'.format(self.opts['master_job_cache'], From fc68ea9dffb228cc36ef29c19ea50eba3cbb2211 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Tue, 10 Mar 2015 17:51:00 -0500 Subject: [PATCH 05/10] Add a versionadded directive for new function --- salt/modules/win_status.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/modules/win_status.py b/salt/modules/win_status.py index 43f72d62823c..42126f722876 100644 --- a/salt/modules/win_status.py +++ b/salt/modules/win_status.py @@ -102,6 +102,8 @@ def _get_process_owner(process): def master(master=None, connected=True): ''' + .. versionadded:: 2015.2.0 + Fire an event if the minion gets disconnected from its master. This function is meant to be run via a scheduled job from the minion. If master_ip is an FQDN/Hostname, is must be resolvable to a valid IPv4 From 8c682fb7f601e1c030297291349b290af18b8038 Mon Sep 17 00:00:00 2001 From: Seth House Date: Tue, 10 Mar 2015 19:13:13 -0400 Subject: [PATCH 06/10] Formatted the Django auth backend docstrings --- salt/auth/django.py | 95 ++++++++++++++++++++++++++------------------- 1 file changed, 55 insertions(+), 40 deletions(-) diff --git a/salt/auth/django.py b/salt/auth/django.py index e18bd5a1ec53..b03c9c2f7976 100644 --- a/salt/auth/django.py +++ b/salt/auth/django.py @@ -2,42 +2,49 @@ ''' Provide authentication using Django Web Framework -Django authentication depends on the presence of the django -framework in the PYTHONPATH, the django project's settings.py file being in -the PYTHONPATH and accessible via the DJANGO_SETTINGS_MODULE environment -variable. This can be hard to debug. +:depends: - Django Web Framework -django auth can be defined like any other eauth module: +Django authentication depends on the presence of the django framework in the +``PYTHONPATH``, the Django project's ``settings.py`` file being in the +``PYTHONPATH`` and accessible via the ``DJANGO_SETTINGS_MODULE`` environment +variable. -external_auth: - django: - fred: - - .* - - '@runner' +Django auth can be defined like any other eauth module: -This will authenticate Fred via django and allow him to run any -execution module and all runners. +.. code-block:: yaml -The details of the django auth can also be located inside the django database. The -relevant entry in the models.py file would look like this: + external_auth: + django: + fred: + - .* + - '@runner' -class SaltExternalAuthModel(models.Model): +This will authenticate Fred via Django and allow him to run any execution +module and all runners. - user_fk = models.ForeignKey(auth.User) - minion_matcher = models.CharField() - minion_fn = models.CharField() +The authorization details can optionally be located inside the Django database. +The relevant entry in the ``models.py`` file would look like this: -Then, in the master's config file the external_auth clause should look like +.. code-block:: python -external_auth: - django: - ^model: + class SaltExternalAuthModel(models.Model): + user_fk = models.ForeignKey(auth.User) + minion_matcher = models.CharField() + minion_fn = models.CharField() -When a user attempts to authenticate via Django, Salt will import the package -indicated via the keyword '^model'. That model must have the fields -indicated above, though the model DOES NOT have to be named 'SaltExternalAuthModel'. +The :conf_master:`external_auth` clause in the master config would then look +like this: -:depends: - Django Web Framework +.. code-block:: yaml + + external_auth: + django: + ^model: + +When a user attempts to authenticate via Django, Salt will import the package +indicated via the keyword ``^model``. That model must have the fields +indicated above, though the model DOES NOT have to be named +'SaltExternalAuthModel'. ''' # Import python libs @@ -117,30 +124,38 @@ def retrieve_auth_entries(u=None): :param django_auth_class: Reference to the django model class for auth :param u: Username to filter for - :return: Dictionary that can be slotted into the __opts__ structure for eauth that designates the - user and his or her ACL + :return: Dictionary that can be slotted into the ``__opts__`` structure for + eauth that designates the user associated ACL + + Database records such as: + =========== ==================== ========= username minion_or_fn_matcher minion_fn + =========== ==================== ========= fred test.ping fred server1 network.interfaces fred server1 raid.list fred server2 .* guru .* smartadmin server1 .* + =========== ==================== ========= + + Should result in an eauth config such as: + + .. code-block:: yaml - Should result in - fred: - - test.ping - - server1: - - network.interfaces - - raid.list - - server2: + fred: + - test.ping + - server1: + - network.interfaces + - raid.list + - server2: + - .* + guru: - .* - guru: - - .* - smartadmin: - - server1: - - .* + smartadmin: + - server1: + - .* ''' global django_auth_class From a9d2a384e367db17d2a611d84ba90679effa58f4 Mon Sep 17 00:00:00 2001 From: Fred Reimer Date: Tue, 10 Mar 2015 22:15:50 -0400 Subject: [PATCH 07/10] Add multi-master-pki capability to cloud deploy. Note that salt-bootstrap needs to be updated to copy the master_sign.pub file from the temp directory to the pki minion directory for this to work with Linux. Verified for Windows hosts. --- salt/utils/cloud.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/salt/utils/cloud.py b/salt/utils/cloud.py index 35a79ce8ca74..1abbb5aad60d 100644 --- a/salt/utils/cloud.py +++ b/salt/utils/cloud.py @@ -345,6 +345,8 @@ def bootstrap(vm_, opts): 'conf_file': opts['conf_file'], 'minion_pem': vm_['priv_key'], 'minion_pub': vm_['pub_key'], + 'master_sign_pub_file': salt.config.get_cloud_config_value( + 'master_sign_pub_file', vm_, opts, default=None), 'keep_tmp': opts['keep_tmp'], 'sudo': salt.config.get_cloud_config_value( 'sudo', vm_, opts, default=(ssh_username != 'root') @@ -804,6 +806,7 @@ def deploy_windows(host, master=None, tmp_dir='C:\\salttmp', opts=None, + master_sign_pub_file=None, **kwargs): ''' Copy the install files to a remote Windows box, and execute them @@ -842,6 +845,15 @@ def deploy_windows(host, if minion_pem: salt.utils.smb.put_str(minion_pem, 'salt\\conf\\pki\\minion\\minion.pem', conn=smb_conn) + if master_sign_pub_file: + # Read master-sign.pub file + log.debug("Copying master_sign.pub file from {0} to minion".format(master_sign_pub_file)) + try: + with salt.utils.fopen(master_sign_pub_file, 'rb') as master_sign_fh: + smb_conn.putFile('C$', 'salt\\conf\\pki\\minion\\master_sign.pub', master_sign_fh.read) + except Exception as e: + log.debug("Exception copying master_sign.pub file {0} to minion".format(master_sign_pub_file)) + # Copy over win_installer # win_installer refers to a file such as: # /root/Salt-Minion-0.17.0-win32-Setup.exe @@ -951,6 +963,7 @@ def deploy_script(host, opts=None, tmp_dir='/tmp/.saltcloud', file_map=None, + master_sign_pub_file=None, **kwargs): ''' Copy a deploy script to a remote server, execute it, and remove it @@ -1073,6 +1086,9 @@ def deploy_script(host, if minion_pub: sftp_file('{0}/minion.pub'.format(tmp_dir), minion_pub, ssh_kwargs) + if master_sign_pub_file: + sftp_file('{0}/master_sign.pub'.format(tmp_dir), kwargs=ssh_kwargs, local_file=master_sign_pub_file) + if minion_conf: if not isinstance(minion_conf, dict): # Let's not just fail regarding this change, specially @@ -1296,6 +1312,10 @@ def deploy_script(host, root_cmd('rm -f {0}/minion'.format(tmp_dir), tty, sudo, **ssh_kwargs) log.debug('Removed {0}/minion'.format(tmp_dir)) + if master_sign_pub_file: + root_cmd('rm -f {0}/master_sign.pub'.format(tmp_dir), + tty, sudo, **ssh_kwargs) + log.debug('Removed {0}/master_sign.pub'.format(tmp_dir)) # Remove master configuration if master_pub: From 7f632902639ddcf1d17659d241e37f691b6a050a Mon Sep 17 00:00:00 2001 From: Loren Gordon Date: Wed, 11 Mar 2015 10:32:39 -0400 Subject: [PATCH 08/10] Update `new_file` if append or prepend required Fixes saltstack#21093 --- salt/modules/file.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/modules/file.py b/salt/modules/file.py index ee2bce9bf41a..e627892759d0 100644 --- a/salt/modules/file.py +++ b/salt/modules/file.py @@ -1292,7 +1292,8 @@ def replace(path, # Keep track of show_changes here, in case the file isn't # modified - if show_changes: + if show_changes or append_if_not_found or \ + prepend_if_not_found: orig_file.append(line) new_file.append(result) From 049975a1ca64f79e5b5904ca02116130a7a95749 Mon Sep 17 00:00:00 2001 From: Seth House Date: Wed, 11 Mar 2015 11:53:59 -0400 Subject: [PATCH 09/10] Added versionadded for ext_pillar_first --- doc/ref/configuration/master.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/ref/configuration/master.rst b/doc/ref/configuration/master.rst index 00111609423c..905ccb6e6c27 100644 --- a/doc/ref/configuration/master.rst +++ b/doc/ref/configuration/master.rst @@ -1832,6 +1832,8 @@ There are additional details at :ref:`salt-pillars` ``ext_pillar_first`` -------------------- +.. versionadded:: 2015.2 + The ext_pillar_first option allows for external pillar sources to populate before file system pillar. This allows for targeting file system pillar from ext_pillar. From 3f90ba9dc39837780574e300676c7a6ddb791840 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 11 Mar 2015 19:07:21 +0000 Subject: [PATCH 10/10] Update the bootstrap script to latest stable, v2015.03.15 * Add multi-master support. Thanks Fred Reimer(freimer). saltstack/salt-boostrap#555 --- salt/cloud/deploy/bootstrap-salt.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/salt/cloud/deploy/bootstrap-salt.sh b/salt/cloud/deploy/bootstrap-salt.sh index a35afb1a894c..5ed75c50f635 100755 --- a/salt/cloud/deploy/bootstrap-salt.sh +++ b/salt/cloud/deploy/bootstrap-salt.sh @@ -17,7 +17,7 @@ # CREATED: 10/15/2012 09:49:37 PM WEST #====================================================================================================================== set -o nounset # Treat unset variables as an error -__ScriptVersion="2015.03.04" +__ScriptVersion="2015.03.15" __ScriptName="bootstrap-salt.sh" #====================================================================================================================== @@ -4510,6 +4510,12 @@ config_salt() { chmod 664 "$_PKI_DIR/minion/minion.pub" || return 1 CONFIGURED_ANYTHING=$BS_TRUE fi + # For multi-master-pki, copy the master_sign public key if found + if [ -f "$_TEMP_CONFIG_DIR/master_sign.pub" ]; then + movefile "$_TEMP_CONFIG_DIR/master_sign.pub" "$_PKI_DIR/minion/" || return 1 + chmod 664 "$_PKI_DIR/minion/master_sign.pub" || return 1 + CONFIGURED_ANYTHING=$BS_TRUE + fi fi