Merge remote-tracking branch 'upstream/develop' into develop

* upstream/develop: (153 commits) Increase all run_script timeouts to 30s (saltstack#34956) Fix grains integration tests for Windows (saltstack#34941) GCE Cloud tests (saltstack#34871) Create regdata before try/except (saltstack#34931) Add prune option to reg.list (saltstack#34934) Skip inode test in Windows (saltstack#34938) Remove use_carrier bond for Ubuntu Xenial later (saltstack#34779) Add jid to salt.function orchestration events [bashcompletion] replace ticks with bash subshell [bashcompletion] ignore salt-key headers Add jid to orchestration returns Avoid UnboundLocalError in beacons module [bash-completion] hide timeout in salt-output Master performance improvement (saltstack#34916) Update service_rh provider to exclude XenServer >= 7. (saltstack#34915) Lint fixes for saltstack#34923 pv_present should execute pvcreate with -y if existing filesystem signature detected Handle exception when no Slack API key was provided zone absent with tests added zone_present state check ...
gitebra · Jul 26, 2016 · 4a1dd11 · 4a1dd11
2 parents bd238de + 0c08724
commit 4a1dd11
Show file tree

Hide file tree

Showing 79 changed files with 2,014 additions and 577 deletions.
diff --git a/conf/master b/conf/master
@@ -309,6 +309,11 @@
 # running any commands. It would also blacklist any use of the "cmd"
 # module. This is completely disabled by default.
 #
+#
+# Check the list of configured users in client ACL against users on the
+# system and throw errors if they do not exist.
+#client_acl_verify: True
+#
 #publisher_acl_blacklist:
 #  users:
 #    - root

diff --git a/doc/topics/releases/carbon.rst b/doc/topics/releases/carbon.rst
@@ -20,6 +20,13 @@ Features
   SSL can be enabled by setting ``ssl_options`` for the returner.
   Also added support for specifying ``protocol_version`` when establishing
   cluster connection.
+- The ``mode`` parameter in the :py:mod:`file.managed
+  <salt.states.file.managed>` state, and the ``file_mode`` parameter in the
+  :py:mod:`file.managed <salt.states.file.managed>`, can both now be set to
+  ``keep`` and the minion will keep the mode of the file from the Salt
+  fileserver. This works only with files coming from sources prefixed with
+  ``salt://``, or files local to the minion (i.e. those which are absolute
+  paths, or are prefixed with ``file://``).
 
 Config Changes
 ==============

diff --git a/doc/topics/troubleshooting/yaml_idiosyncrasies.rst b/doc/topics/troubleshooting/yaml_idiosyncrasies.rst
@@ -113,6 +113,20 @@ Pillar data. Make sure that your Pillars which need to use the string versions
 of these values are enclosed in quotes.  Pillars will be parsed twice by salt,
 so you'll need to wrap your values in multiple quotes, for example '"false"'.
 
+The '%' Sign
+============
+
+The `%` symbol has a special meaning in YAML, it needs to be passed as a
+string literal:
+
+.. code-block:: yaml
+
+    cheese:
+      ssh_auth.present:
+        - user: tbortels
+        - source: salt://ssh_keys/chease.pub
+        - config: '%h/.ssh/authorized_keys'
+
 Integers are Parsed as Integers
 ===============================
 

diff --git a/pkg/salt.bash b/pkg/salt.bash
@@ -16,18 +16,24 @@ _salt_get_grains(){
     if [ "$1" = 'local' ] ; then
         salt-call --out=txt -- grains.ls | sed  's/^.*\[//' | tr -d ",']" |sed 's:\([a-z0-9]\) :\1\: :g'
     else
-      salt '*' --timeout 2 --out=txt -- grains.ls | sed  's/^.*\[//' | tr -d ",']" |sed 's:\([a-z0-9]\) :\1\: :g'
+      salt '*' --timeout 2 --hide-timeout --out=txt -- grains.ls | sed  's/^.*\[//' | tr -d ",']" |sed 's:\([a-z0-9]\) :\1\: :g'
     fi
 }
 
 _salt_get_grain_values(){
     if [ "$1" = 'local' ] ; then
         salt-call --out=txt -- grains.item $1 |sed 's/^\S*:\s//' |grep -v '^\s*$'
     else
-        salt '*' --timeout 2 --out=txt -- grains.item $1 |sed 's/^\S*:\s//' |grep -v '^\s*$'
+        salt '*' --timeout 2 --hide-timeout --out=txt -- grains.item $1 |sed 's/^\S*:\s//' |grep -v '^\s*$'
     fi
 }
 
+_salt_get_keys(){
+    for type in $*; do
+      # remove header from data:
+      salt-key --no-color -l $type | tail -n+2
+    done
+}
 
 _salt(){
     local cur prev opts _salt_grains _salt_coms pprev ppprev
@@ -86,11 +92,11 @@ _salt(){
         return 0
         ;;
      salt)
-        COMPREPLY=($(compgen -W "\'*\' ${opts} `salt-key --no-color -l acc`" -- ${cur}))
+        COMPREPLY=($(compgen -W "\'*\' ${opts} $(_salt_get_keys acc)" -- ${cur}))
         return 0
         ;;
      -E|--pcre)
-        COMPREPLY=($(compgen -W "`salt-key --no-color -l acc`" -- ${cur}))
+        COMPREPLY=($(compgen -W "$(_salt_get_keys acc)" -- ${cur}))
         return 0
         ;;
      -G|--grain|--grain-pcre)
@@ -116,7 +122,7 @@ _salt(){
      ;;
     esac
 
-    _salt_coms="$(salt '*' --timeout 2 --out=txt -- sys.list_functions | sed 's/^.*\[//' | tr -d ",']" )"
+    _salt_coms="$(salt '*' --timeout 2 --hide-timeout --out=txt -- sys.list_functions | sed 's/^.*\[//' | tr -d ",']" )"
     all="${opts} ${_salt_coms}"
     COMPREPLY=( $(compgen -W "${all}" -- ${cur}) )
 
@@ -158,15 +164,15 @@ _saltkey(){
 
     case "${prev}" in
      -a|--accept)
-        COMPREPLY=($(compgen -W "$(salt-key -l un --no-color; salt-key -l rej --no-color)" -- ${cur}))
+        COMPREPLY=($(compgen -W "$(_salt_get_keys un rej)" -- ${cur}))
         return 0
       ;;
      -r|--reject)
-        COMPREPLY=($(compgen -W "$(salt-key -l acc --no-color)" -- ${cur}))
+        COMPREPLY=($(compgen -W "$(_salt_get_keys acc)" -- ${cur}))
         return 0
         ;;
      -d|--delete)
-        COMPREPLY=($(compgen -W "$(salt-key -l acc --no-color; salt-key -l un --no-color; salt-key -l rej --no-color)" -- ${cur}))
+        COMPREPLY=($(compgen -W "$(_salt_get_keys acc un rej)" -- ${cur}))
         return 0
         ;;
      -c|--config)
@@ -185,7 +191,7 @@ _saltkey(){
         return 0
         ;;
      -p|--print)
-        COMPREPLY=($(compgen -W "$(salt-key -l acc --no-color; salt-key -l un --no-color; salt-key -l rej --no-color)" -- ${cur}))
+        COMPREPLY=($(compgen -W "$(_salt_get_keys acc un rej)" -- ${cur}))
         return 0
      ;;
      -l|--list)
@@ -280,7 +286,7 @@ _saltcp(){
 
     case ${prev} in
  	salt-cp)
-	    COMPREPLY=($(compgen -W "${opts} `salt-key -l acc --no-color`" -- ${cur}))
+	    COMPREPLY=($(compgen -W "${opts} $(_salt_get_keys acc)" -- ${cur}))
 	    return 0
 	;;
         -t|--timeout)
@@ -289,15 +295,15 @@ _saltcp(){
 	    return 0
         ;;
 	-E|--pcre)
-            COMPREPLY=($(compgen -W "`salt-key -l acc --no-color`" -- ${cur}))
+            COMPREPLY=($(compgen -W "$(_salt_get_keys acc)" -- ${cur}))
             return 0
 	;;
 	-L|--list)
 	    # IMPROVEMENTS ARE WELCOME
 	    prefpart="${cur%,*},"
 	    postpart=${cur##*,}
 	    filt="^\($(echo ${cur}| sed 's:,:\\|:g')\)$"
-            helper=($(salt-key -l acc --no-color | grep -v "${filt}" | sed "s/^/${prefpart}/"))
+            helper=($(_salt_get_keys acc | grep -v "${filt}" | sed "s/^/${prefpart}/"))
 	    COMPREPLY=($(compgen -W "${helper[*]}" -- ${cur}))
 
 	    return 0

diff --git a/salt/beacons/__init__.py b/salt/beacons/__init__.py
@@ -23,6 +23,7 @@ class Beacon(object):
     '''
     def __init__(self, opts, functions):
         self.opts = opts
+        self.functions = functions
         self.beacons = salt.loader.beacons(opts, functions)
         self.interval_map = dict()
 
@@ -187,7 +188,8 @@ def list_beacons(self):
         '''
         # Fire the complete event back along with the list of beacons
         evt = salt.utils.event.get_event('minion', opts=self.opts)
-        evt.fire_event({'complete': True, 'beacons': self.opts['beacons']},
+        b_conf = self.functions['config.merge']('beacons')
+        evt.fire_event({'complete': True, 'beacons': b_conf},
                        tag='/salt/minion/minion_beacons_list_complete')
 
         return True

diff --git a/salt/beacons/avahi_announce.py b/salt/beacons/avahi_announce.py
@@ -0,0 +1,117 @@
+# -*- coding: utf-8 -*-
+'''
+ Beacon to announce via avahi (zeroconf)
+
+'''
+# Import Python libs
+from __future__ import absolute_import
+import logging
+
+# Import 3rd Party libs
+try:
+    import avahi
+    HAS_PYAVAHI = True
+except ImportError:
+    HAS_PYAVAHI = False
+import dbus
+
+log = logging.getLogger(__name__)
+
+__virtualname__ = 'avahi_announce'
+
+LAST_GRAINS = {}
+BUS = dbus.SystemBus()
+SERVER = dbus.Interface(BUS.get_object(avahi.DBUS_NAME, avahi.DBUS_PATH_SERVER),
+                        avahi.DBUS_INTERFACE_SERVER)
+GROUP = dbus.Interface(BUS.get_object(avahi.DBUS_NAME, SERVER.EntryGroupNew()),
+                       avahi.DBUS_INTERFACE_ENTRY_GROUP)
+
+
+def __virtual__():
+    if HAS_PYAVAHI:
+        return __virtualname__
+    return False
+
+
+def validate(config):
+    '''
+    Validate the beacon configuration
+    '''
+    if not isinstance(config, dict):
+        return False, ('Configuration for avahi_announcement '
+                       'beacon must be a dictionary')
+    elif not all(x in list(config.keys()) for x in ('servicetype', 'port', 'txt')):
+        return False, ('Configuration for avahi_announce beacon '
+                       'must contain servicetype, port and txt items')
+    return True, 'Valid beacon configuration'
+
+
+def beacon(config):
+    '''
+    Broadcast values via zeroconf
+
+    If the announced values are static, it is adviced to set run_once: True
+    (do not poll) on the beacon configuration. Grains can be used to define
+    txt values using the syntax: grains.<grain_name>
+
+    The default servicename its the hostname grain value.
+
+    Example Config
+
+    .. code-block:: yaml
+
+       beacons:
+          avahi_announce:
+             run_once: True
+             servicetype: _demo._tcp
+             txt:
+                ProdName: grains.productname
+                SerialNo: grains.serialnumber
+                Comments: 'this is a test'
+    '''
+    ret = []
+    changes = {}
+    txt = {}
+
+    global LAST_GRAINS
+
+    _validate = validate(config)
+    if not _validate[0]:
+        log.warning('Beacon {0} configuration invalid, '
+                    'not adding. {1}'.format(__virtualname__, _validate[1]))
+        return ret
+
+    if 'servicename' in config:
+        servicename = config['servicename']
+    else:
+        servicename = __grains__['host']
+
+    for item in config['txt']:
+        if config['txt'][item].startswith('grains.'):
+            grain = config['txt'][item][7:]
+            txt[item] = __grains__[grain]
+            if LAST_GRAINS and (LAST_GRAINS[grain] != __grains__[grain]):
+                changes[str('txt.' + item)] = txt[item]
+        else:
+            txt[item] = config['txt'][item]
+
+        if not LAST_GRAINS:
+            changes[str('txt.' + item)] = txt[item]
+
+    if changes:
+        if not LAST_GRAINS:
+            changes['servicename'] = servicename
+            changes['servicetype'] = config['servicetype']
+            changes['port'] = config['port']
+        else:
+            GROUP.Reset()
+        GROUP.AddService(avahi.IF_UNSPEC, avahi.PROTO_UNSPEC, dbus.UInt32(0),
+                         servicename, config['servicetype'], '', '',
+                         dbus.UInt16(config['port']), avahi.dict_to_txt_array(txt))
+        GROUP.Commit()
+
+        ret.append({'tag': 'result', 'changes': changes})
+
+    LAST_GRAINS = __grains__
+
+    return ret
diff --git a/salt/cli/daemons.py b/salt/cli/daemons.py
@@ -46,7 +46,6 @@
 try:
     from salt.utils import parsers, ip_bracket
     from salt.utils.verify import check_user, verify_env, verify_socket
-    from salt.utils.verify import verify_files
 except ImportError as exc:
     if exc.args[0] != 'No module named _msgpack':
         raise
@@ -162,12 +161,6 @@ def prepare(self):
                     permissive=self.config['permissive_pki_access'],
                     pki_dir=self.config['pki_dir'],
                 )
-                logfile = self.config['log_file']
-                if logfile is not None and not logfile.startswith(('tcp://', 'udp://', 'file://')):
-                    # Logfile is not using Syslog, verify
-                    current_umask = os.umask(0o027)
-                    verify_files([logfile], self.config['user'])
-                    os.umask(current_umask)
                 # Clear out syndics from cachedir
                 for syndic_file in os.listdir(self.config['syndic_dir']):
                     os.remove(os.path.join(self.config['syndic_dir'], syndic_file))
@@ -288,12 +281,6 @@ def prepare(self):
                     permissive=self.config['permissive_pki_access'],
                     pki_dir=self.config['pki_dir'],
                 )
-                logfile = self.config['log_file']
-                if logfile is not None and not logfile.startswith(('tcp://', 'udp://', 'file://')):
-                    # Logfile is not using Syslog, verify
-                    current_umask = os.umask(0o027)
-                    verify_files([logfile], self.config['user'])
-                    os.umask(current_umask)
         except OSError as error:
             self.environment_failure(error)
 
@@ -464,14 +451,6 @@ def prepare(self):
                     permissive=self.config['permissive_pki_access'],
                     pki_dir=self.config['pki_dir'],
                 )
-
-                logfile = self.config.get('proxy_log') or self.config['log_file']
-                if logfile is not None and not logfile.startswith(('tcp://', 'udp://', 'file://')):
-                    # Logfile is not using Syslog, verify
-                    current_umask = os.umask(0o027)
-                    verify_files([logfile], self.config['user'])
-                    os.umask(current_umask)
-
         except OSError as error:
             self.environment_failure(error)
 
@@ -569,12 +548,6 @@ def prepare(self):
                     permissive=self.config['permissive_pki_access'],
                     pki_dir=self.config['pki_dir'],
                 )
-                logfile = self.config['log_file']
-                if logfile is not None and not logfile.startswith(('tcp://', 'udp://', 'file://')):
-                    # Logfile is not using Syslog, verify
-                    current_umask = os.umask(0o027)
-                    verify_files([logfile], self.config['user'])
-                    os.umask(current_umask)
         except OSError as error:
             self.environment_failure(error)
 

diff --git a/salt/client/__init__.py b/salt/client/__init__.py
@@ -933,6 +933,8 @@ def get_iter_returns(
                         ret[raw['data']['id']]['out'] = raw['data']['out']
                     if 'retcode' in raw['data']:
                         ret[raw['data']['id']]['retcode'] = raw['data']['retcode']
+                    if 'jid' in raw['data']:
+                        ret[raw['data']['id']]['jid'] = raw['data']['jid']
                     if kwargs.get('_cmd_meta', False):
                         ret[raw['data']['id']].update(raw['data'])
                     log.debug('jid {0} return from {1}'.format(jid, raw['data']['id']))

diff --git a/salt/client/ssh/__init__.py b/salt/client/ssh/__init__.py
@@ -738,7 +738,8 @@ def __init__(
                     'root_dir': os.path.join(self.thin_dir, 'running_data'),
                     'id': self.id,
                     'sock_dir': '/',
-                    'log_file': 'salt-call.log'
+                    'log_file': 'salt-call.log',
+                    'fileserver_list_cache_time': 3,
                 })
         self.minion_config = salt.serializers.yaml.serialize(self.minion_opts)
         self.target = kwargs
@@ -1299,6 +1300,10 @@ def mod_data(fsclient):
                     ret[ref] = mods_data
     if not ret:
         return {}
+
+    if six.PY3:
+        ver_base = salt.utils.to_bytes(ver_base)
+
     ver = hashlib.sha1(ver_base).hexdigest()
     ext_tar_path = os.path.join(
             fsclient.opts['cachedir'],

diff --git a/salt/client/ssh/wrapper/config.py b/salt/client/ssh/wrapper/config.py
@@ -78,12 +78,10 @@ def manage_mode(mode):
 
         salt '*' config.manage_mode
     '''
-    if mode is None:
-        return None
-    ret = str(mode).lstrip('0').zfill(4)
-    if ret[0] != '0':
-        return '0{0}'.format(ret)
-    return ret
+    # config.manage_mode should no longer be invoked from the __salt__ dunder
+    # in Salt code, this function is only being left here for backwards
+    # compatibility.
+    return salt.utils.normalize_mode(mode)
 
 
 def valid_fileproto(uri):