Merge remote-tracking branch 'upstream/develop' into develop

* upstream/develop: (35 commits)
  Typo mine not mind
  Refactor command construction for a better readibility
  Revert "Py3 for integration test"
  Py3 for integration test
  Match overriden method
  Immediate shutdown
  Increase the speed of test suite startup
  Support remote port forwarding with salt-ssh
  Esxi dvs (saltstack#34352)
  suggest a sane default interval
  need a format without spaces
  change to use a ctime string
  network_settings.py: fix documentation
  Beacon network docs (saltstack#34386)
  Enable all privileges
  Handle users that aren't admin
  Fix runas function for System Account
  NamespacedDictWrapper: implement human readable representation
  Be more explicit with salt.utils imports and usage
  Add salt.utils import to wheel.key.py
  ...

doc/ref/beacons/all/index.rst

@@ -19,6 +19,7 @@ beacon modules
     load
     memusage
     network_info
+    network_settings
     pkg
     proxy_example
     ps

doc/ref/beacons/all/salt.beacons.network_settings.rst

@@ -0,0 +1,6 @@
+=============================
+salt.beacons.network_settings
+=============================
+
+.. automodule:: salt.beacons.network_settings
+    :members:

doc/topics/thorium/index.rst

@@ -46,31 +46,38 @@ looks exactly like the Salt Configuration Management state tree, it is just
 located in the `thorium_roots_dir` instead of the `file_roots_dir`. The default
 location for the `thorium_roots_dir` is `/srv/thorium`.
 
-This VERY simple example maintains a file on the master with all minion logins:
+This example uses thorium to detect when a minion has disappeared and then
+deletes the key from the master when the minion has been gone for 60 seconds:
 
 
 .. code-block:: yaml
 
-    failed_logins:
-      reg.list:
-        - add:
-            - user
-            - id
-        - match: /salt/beacon/\*/btmp*
+    startreg:
+      status.reg
 
-    save_reg:
-      file.save:
-        - reg: failed_logins
+    keydel:
+      key.timeout:
+        - require:
+          - status: statreg
+        - delete: 60
 
 Remember to set up a top file so Thorium knows which sls files to use!!
 
 .. code-block:: yaml
 
     base:
       '*':
-        - logins
+        - key_clean
 
-The Reg/Check/Act Pattern
-=========================
+Thorium Links to Beacons
+========================
+
+The above example was added in the Carbon release of Salt and makes use of the
+`status` beacon also added in the Carbon release. For the above Thorium state
+to function properly you will also need to enable the `status` beacon:
 
+.. code-block:: yaml
 
+    beacons:
+      status:
+        interval: 10

salt/beacons/network_settings.py

@@ -2,6 +2,8 @@
 '''
 Beacon to monitor network adapter setting changes on Linux
 
+.. versionadded:: 2016.3.0
+
 '''
 from __future__ import absolute_import
 # Import third party libs

salt/beacons/status.py

@@ -0,0 +1,32 @@
+# -*- coding: utf-8 -*-
+'''
+The status beacon is intended to send a basic health check event up to the
+master, this allows for event driven routines based on presence to be set up.
+
+The intention of this beacon is to add the config options to add monitoring
+stats to the health beacon making it a one stop shop for gathering systems
+health and status data
+
+.. versionadded:: Carbon
+'''
+
+# Import python libs
+from __future__ import absolute_import
+import datetime
+
+
+def validate(config):
+    '''
+    Validate the the config is a dict
+    '''
+    if not isinstance(config, dict):
+        return False, ('Configuration for status beacon must be a dictionary.')
+    return True, 'Valid beacon configuration'
+
+
+def beacon(config):
+    '''
+    Just say that we are ok!
+    '''
+    ctime = datetime.datetime.utcnow().isoformat()
+    return [{'tag': ctime}]

salt/client/mixins.py

@@ -15,18 +15,16 @@
 import salt.exceptions
 import salt.minion
 import salt.utils
+import salt.utils.doc
+import salt.utils.error
 import salt.utils.event
 import salt.utils.jid
 import salt.utils.job
+import salt.utils.lazy
+import salt.utils.process
 import salt.transport
 import salt.log.setup
 import salt.ext.six as six
-from salt.utils.error import raise_error
-from salt.utils.event import tagify
-from salt.utils.doc import strip_rst as _strip_rst
-from salt.utils.lazy import verify_fun
-from salt.utils.process import default_signals, SignalHandlingMultiprocessingProcess
-from salt.utils import warn_until
 
 # Import 3rd-party libs
 import tornado.stack_context
@@ -136,7 +134,7 @@ def master_call(self, **kwargs):
         ret = channel.send(load)
         if isinstance(ret, collections.Mapping):
             if 'error' in ret:
-                raise_error(**ret['error'])
+                salt.utils.error.raise_error(**ret['error'])
         return ret
 
     def cmd_sync(self, low, timeout=None):
@@ -250,7 +248,7 @@ def low(self, fun, low):
             low['kwarg'] = low.pop('kwargs')
 
         if msg:
-            warn_until('Oxygen', ' '.join(msg))
+            salt.utils.warn_until('Oxygen', ' '.join(msg))
 
         return self._low(fun, low)
 
@@ -272,7 +270,7 @@ def _low(self, fun, low):
         # if we have a high debug level.
         self.mminion  # pylint: disable=W0104
         jid = low.get('__jid__', salt.utils.jid.gen_jid())
-        tag = low.get('__tag__', tagify(jid, prefix=self.tag_prefix))
+        tag = low.get('__tag__', salt.utils.event.tagify(jid, prefix=self.tag_prefix))
 
         data = {'fun': '{0}.{1}'.format(self.client, fun),
                 'jid': jid,
@@ -306,7 +304,7 @@ def _low(self, fun, low):
         func_globals['__jid_event__'].fire_event(data, 'new')
 
         try:
-            verify_fun(self.functions, fun)
+            salt.utils.lazy.verify_fun(self.functions, fun)
 
             # Inject some useful globals to *all* the function's global
             # namespace only once per module-- not per func
@@ -408,7 +406,7 @@ def get_docs(self, arg=None):
             docs = [(fun, self.functions[fun].__doc__)
                     for fun in sorted(self.functions)]
         docs = dict(docs)
-        return _strip_rst(docs)
+        return salt.utils.doc.strip_rst(docs)
 
 
 class AsyncClientMixin(object):
@@ -462,7 +460,7 @@ def cmd_async(self, low):
     def _gen_async_pub(self, jid=None):
         if jid is None:
             jid = salt.utils.jid.gen_jid()
-        tag = tagify(jid, prefix=self.tag_prefix)
+        tag = salt.utils.event.tagify(jid, prefix=self.tag_prefix)
         return {'tag': tag, 'jid': jid}
 
     def async(self, fun, low, user='UNKNOWN'):
@@ -472,10 +470,10 @@ def async(self, fun, low, user='UNKNOWN'):
         '''
         async_pub = self._gen_async_pub()
 
-        proc = SignalHandlingMultiprocessingProcess(
+        proc = salt.utils.process.SignalHandlingMultiprocessingProcess(
                 target=self._proc_function,
                 args=(fun, low, user, async_pub['tag'], async_pub['jid']))
-        with default_signals(signal.SIGINT, signal.SIGTERM):
+        with salt.utils.process.default_signals(signal.SIGINT, signal.SIGTERM):
             # Reset current signals before starting the process in
             # order not to inherit the current signal handlers
             proc.start()

salt/client/ssh/__init__.py

@@ -280,6 +280,9 @@ def __init__(self, opts):
                 'ssh_identities_only',
                 salt.config.DEFAULT_MASTER_OPTS['ssh_identities_only']
             ),
+            'remote_port_forwards': self.opts.get(
+                'ssh_remote_port_forwards'
+            ),
         }
         if self.opts.get('rand_thin_dir'):
             self.defaults['thin_dir'] = os.path.join(
@@ -666,6 +669,7 @@ def __init__(
             mine=False,
             minion_opts=None,
             identities_only=False,
+            remote_port_forwards=None,
             **kwargs):
         # Get mine setting and mine_functions if defined in kwargs (from roster)
         self.mine = mine
@@ -720,7 +724,8 @@ def __init__(
                 'sudo': sudo,
                 'tty': tty,
                 'mods': self.mods,
-                'identities_only': identities_only}
+                'identities_only': identities_only,
+                'remote_port_forwards': remote_port_forwards}
         # Pre apply changeable defaults
         self.minion_opts = {
                     'grains_cache': True,
@@ -914,7 +919,7 @@ def run_wfunc(self):
         self.wfuncs = salt.loader.ssh_wrapper(opts, wrapper, self.context)
         wrapper.wfuncs = self.wfuncs
 
-        # We're running in the mind, need to fetch the arguments from the
+        # We're running in the mine, need to fetch the arguments from the
         # roster, pillar, master config (in that order)
         if self.mine:
             mine_args = None

salt/client/ssh/shell.py

@@ -62,7 +62,8 @@ def __init__(
             sudo=False,
             tty=False,
             mods=None,
-            identities_only=False):
+            identities_only=False,
+            remote_port_forwards=None):
         self.opts = opts
         self.host = host
         self.user = user
@@ -74,6 +75,7 @@ def __init__(
         self.tty = tty
         self.mods = mods
         self.identities_only = identities_only
+        self.remote_port_forwards = remote_port_forwards
 
     def get_error(self, errstr):
         '''
@@ -215,20 +217,19 @@ def _cmd_str(self, cmd, ssh='ssh'):
         # TODO: if tty, then our SSH_SHIM cannot be supplied from STDIN Will
         # need to deliver the SHIM to the remote host and execute it there
 
-        opts = ''
-        tty = self.tty
-        if ssh != 'ssh':
-            tty = False
-        if self.passwd:
-            opts = self._passwd_opts()
-        if self.priv:
-            opts = self._key_opts()
-        return "{0} {1} {2} {3} {4}".format(
-                ssh,
-                '' if ssh == 'scp' else self.host,
-                '-t -t' if tty else '',
-                opts,
-                cmd)
+        command = [ssh]
+        if ssh != 'scp':
+            command.append(self.host)
+        if self.tty and ssh == 'ssh':
+            command.append('-t -t')
+        if self.passwd or self.priv:
+            command.append(self.priv and self._key_opts() or self._passwd_opts())
+        if ssh != 'scp' and self.remote_port_forwards:
+            command.append(' '.join(['-R {0}'.format(item)
+                                     for item in self.remote_port_forwards.split(',')]))
+        command.append(cmd)
+
+        return ' '.join(command)
 
     def _old_run_cmd(self, cmd):
         '''

salt/cloud/clouds/lxc.py

@@ -299,13 +299,18 @@ def list_nodes(conn=None, call=None):
             # so we hide the running vm from being seen as already installed
             # do not also mask half configured nodes which are explicitly asked
             # to be acted on, on the command line
-            if (
-                (call in ['full'] or not hide) and (
-                    (lxcc in names and call in ['action']) or (
-                        call in ['full'])
-                )
-            ):
-                nodes[lxcc] = info
+            if (call in ['full'] or not hide) and ((lxcc in names and call in ['action']) or call in ['full']):
+                nodes[lxcc] = {
+                    'id': lxcc,
+                    'name': lxcc,  # required for cloud cache
+                    'image': None,
+                    'size': linfos['size'],
+                    'state': state.lower(),
+                    'public_ips': linfos['public_ips'],
+                    'private_ips': linfos['private_ips'],
+                }
+            else:
+                nodes[lxcc] = {'id': lxcc, 'state': state.lower()}
     return nodes
 
 

salt/grains/core.py

@@ -1414,8 +1414,8 @@ def os_data():
             uname_v = os.uname()[3]
             grains['os'] = grains['osfullname'] = 'SmartOS'
             grains['osrelease'] = uname_v[uname_v.index('_')+1:]
-        elif salt.utils.is_smartos_globalzone():
-            grains.update(_smartos_computenode_data())
+            if salt.utils.is_smartos_globalzone():
+                grains.update(_smartos_computenode_data())
         elif os.path.isfile('/etc/release'):
             with salt.utils.fopen('/etc/release', 'r') as fp_:
                 rel_data = fp_.read()