Write lists of minions targeted by syndic masters to job cache

doc/ref/modules/all/salt.modules.slsutil.rst

@@ -0,0 +1,6 @@
+====================
+salt.modules.slsutil
+====================
+
+.. automodule:: salt.modules.slsutil
+    :members:

doc/ref/returners/all/salt.returners.cassandra_cql_return.rst

@@ -3,4 +3,5 @@ salt.returners.cassandra_cql_return
 ===================================
 
 .. automodule:: salt.returners.cassandra_cql_return
-    :members:
+    :members:
+    :exclude-members: save_minions

doc/ref/returners/all/salt.returners.couchdb_return.rst

@@ -3,4 +3,5 @@ salt.returners.couchdb_return
 =============================
 
 .. automodule:: salt.returners.couchdb_return
-    :members:
+    :members:
+    :exclude-members: save_minions

doc/ref/returners/all/salt.returners.etcd_return.rst

@@ -3,4 +3,5 @@ salt.returners.etcd_return
 ==========================
 
 .. automodule:: salt.returners.etcd_return
-    :members:
+    :members:
+    :exclude-members: save_minions

doc/ref/returners/all/salt.returners.influxdb_return.rst

@@ -4,3 +4,4 @@ salt.returners.influxdb_return
 
 .. automodule:: salt.returners.influxdb_return
     :members:
+    :exclude-members: save_minions

doc/ref/returners/all/salt.returners.memcache_return.rst

@@ -3,4 +3,5 @@ salt.returners.memcache_return
 ==============================
 
 .. automodule:: salt.returners.memcache_return
-    :members:
+    :members:
+    :exclude-members: save_minions

doc/ref/returners/all/salt.returners.mongo_future_return.rst

@@ -3,4 +3,5 @@ salt.returners.mongo_future_return
 ==================================
 
 .. automodule:: salt.returners.mongo_future_return
-    :members:
+    :members:
+    :exclude-members: save_minions

doc/ref/returners/all/salt.returners.mongo_return.rst

@@ -3,4 +3,5 @@ salt.returners.mongo_return
 ===========================
 
 .. automodule:: salt.returners.mongo_return
-    :members:
+    :members:
+    :exclude-members: save_minions

doc/ref/returners/all/salt.returners.multi_returner.rst

@@ -3,4 +3,5 @@ salt.returners.multi_returner
 =============================
 
 .. automodule:: salt.returners.multi_returner
-    :members:
+    :members:
+    :exclude-members: save_minions

doc/ref/returners/all/salt.returners.mysql.rst

@@ -3,4 +3,5 @@ salt.returners.mysql
 ====================
 
 .. automodule:: salt.returners.mysql
-    :members:
+    :members:
+    :exclude-members: save_minions

doc/ref/returners/all/salt.returners.odbc.rst

@@ -3,4 +3,5 @@ salt.returners.odbc
 ===================
 
 .. automodule:: salt.returners.odbc
-    :members:
+    :members:
+    :exclude-members: save_minions

doc/ref/returners/all/salt.returners.pgjsonb.rst

@@ -4,3 +4,4 @@ salt.returners.pgjsonb
 
 .. automodule:: salt.returners.pgjsonb
     :members:
+    :exclude-members: save_minions

doc/ref/returners/all/salt.returners.postgres.rst

@@ -3,4 +3,5 @@ salt.returners.postgres
 =======================
 
 .. automodule:: salt.returners.postgres
-    :members:
+    :members:
+    :exclude-members: save_minions

doc/ref/returners/all/salt.returners.postgres_local_cache.rst

@@ -4,3 +4,4 @@ salt.returners.postgres_local_cache
 
 .. automodule:: salt.returners.postgres_local_cache
     :members:
+    :exclude-members: save_minions

doc/ref/returners/all/salt.returners.redis_return.rst

@@ -3,4 +3,5 @@ salt.returners.redis_return
 ===========================
 
 .. automodule:: salt.returners.redis_return
-    :members:
+    :members:
+    :exclude-members: save_minions

doc/ref/returners/all/salt.returners.sqlite3_return.rst

@@ -3,4 +3,5 @@ salt.returners.sqlite3
 =======================
 
 .. automodule:: salt.returners.sqlite3_return
-    :members:
+    :members:
+    :exclude-members: save_minions

salt/exceptions.py

@@ -6,8 +6,14 @@
 
 # Import python libs
 import copy
+import logging
+import time
+
+# Import salt libs
 import salt.defaults.exitcodes
 
+log = logging.getLogger(__name__)
+
 
 class SaltException(Exception):
     '''
@@ -98,6 +104,23 @@ class FileserverConfigError(SaltException):
     '''
 
 
+class FileLockError(SaltException):
+    '''
+    Used when an error occurs obtaining a file lock
+    '''
+    def __init__(self, msg, time_start=None, *args, **kwargs):
+        super(FileLockError, self).__init__(msg, *args, **kwargs)
+        if time_start is None:
+            log.warning(
+                'time_start should be provided when raising a FileLockError. '
+                'Defaulting to current time as a fallback, but this may '
+                'result in an inaccurate timeout.'
+            )
+            self.time_start = time.time()
+        else:
+            self.time_start = time_start
+
+
 class SaltInvocationError(SaltException, TypeError):
     '''
     Used when the wrong number of arguments are sent to modules or invalid

salt/master.py

@@ -1186,6 +1186,27 @@ def _handle_minion_event(self, load):
             log.error('Received minion error from [{minion}]: '
                       '{data}'.format(minion=load['id'],
                                       data=load['data']['message']))
+        for event in load.get('events', []):
+            event_data = event.get('data', {})
+            if 'minions' in event_data:
+                jid = event_data.get('jid')
+                if not jid:
+                    # Should not happen but if it ever does we definitely want
+                    # to know about it.
+                    log.warning('No jid in event: %s', event_data)
+                else:
+                    minions = event_data['minions']
+                    try:
+                        salt.utils.job.store_minions(
+                            self.opts,
+                            jid,
+                            minions,
+                            mminion=self.mminion)
+                    except (KeyError, salt.exceptions.SaltCacheError) as exc:
+                        log.error(
+                            'Could not add minion(s) {0} for job {1}: {2}'
+                            .format(minions, jid, exc)
+                        )
 
     def _return(self, load):
         '''

salt/returners/cassandra_cql_return.py

@@ -260,6 +260,13 @@ def save_load(jid, load):
         raise
 
 
+def save_minions(jid, minions):  # pylint: disable=unused-argument
+    '''
+    Included for API consistency
+    '''
+    pass
+
+
 # salt-run jobs.list_jobs FAILED
 def get_load(jid):
     '''

salt/returners/couchbase_return.py

@@ -124,7 +124,8 @@ def _get_ttl():
 def prep_jid(nocache=False, passed_jid=None):
     '''
     Return a job id and prepare the job id directory
-    This is the function responsible for making sure jids don't collide (unless its passed a jid)
+    This is the function responsible for making sure jids don't collide (unless
+    its passed a jid)
     So do what you have to do to make sure that stays the case
     '''
     if passed_jid is None:
@@ -195,6 +196,9 @@ def save_load(jid, clear_load):
         log.warning('Could not write job cache file for jid: {0}'.format(jid))
         return False
 
+    jid_doc.value['load'] = clear_load
+    cb_.replace(str(jid), jid_doc.value, cas=jid_doc.cas, ttl=_get_ttl())
+
     # if you have a tgt, save that for the UI etc
     if 'tgt' in clear_load:
         ckminions = salt.utils.minions.CkMinions(__opts__)
@@ -203,16 +207,24 @@ def save_load(jid, clear_load):
             clear_load['tgt'],
             clear_load.get('tgt_type', 'glob')
             )
-        # save the minions to a cache so we can see in the UI
-        jid_doc.value['minions'] = minions
+        save_minions(jid, minions)
 
-    jid_doc.value['load'] = clear_load
 
-    cb_.replace(str(jid),
-               jid_doc.value,
-               cas=jid_doc.cas,
-               ttl=_get_ttl()
-               )
+def save_minions(jid, minions):
+    '''
+    Save/update the minion list for a given jid
+    '''
+    cb_ = _get_connection()
+
+    try:
+        jid_doc = cb_.get(str(jid))
+    except couchbase.exceptions.NotFoundError:
+        log.warning('Could not write job cache file for jid: {0}'.format(jid))
+        return False
+
+    # save the minions to a cache so we can see in the UI
+    jid_doc.value['minions'] = minions
+    cb_.replace(str(jid), jid_doc.value, cas=jid_doc.cas, ttl=_get_ttl())
 
 
 def get_load(jid):

salt/returners/couchdb_return.py

@@ -367,3 +367,10 @@ def prep_jid(nocache=False, passed_jid=None):  # pylint: disable=unused-argument
     Do any work necessary to prepare a JID, including sending a custom id
     '''
     return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid()
+
+
+def save_minions(jid, minions):  # pylint: disable=unused-argument
+    '''
+    Included for API consistency
+    '''
+    pass

salt/returners/etcd_return.py

@@ -113,6 +113,13 @@ def save_load(jid, load):
     )
 
 
+def save_minions(jid, minions):  # pylint: disable=unused-argument
+    '''
+    Included for API consistency
+    '''
+    pass
+
+
 def get_load(jid):
     '''
     Return the load data that marks a specified jid

salt/returners/influxdb_return.py

@@ -149,6 +149,13 @@ def save_load(jid, load):
         log.critical('Failed to store load with InfluxDB returner: {0}'.format(ex))
 
 
+def save_minions(jid, minions):  # pylint: disable=unused-argument
+    '''
+    Included for API consistency
+    '''
+    pass
+
+
 def get_load(jid):
     '''
     Return the load data that marks a specified jid