Merge pull request #51232 from Ch3LL/merge-2019.2

[2019.2] Merge forward from 2018.3 to 2019.2

salt/auth/ldap.py

@@ -338,7 +338,14 @@ def groups(username, **kwargs):
 
     '''
     group_list = []
-    bind = auth(username, kwargs.get('password', None))
+
+    # If bind credentials are configured, use them instead of user's
+    if _config('binddn', mandatory=False) and _config('bindpw', mandatory=False):
+        bind = _bind_for_search(anonymous=_config('anonymous', mandatory=False))
+    else:
+        bind = _bind(username, kwargs.get('password', ''),
+                     anonymous=_config('auth_by_group_membership_only', mandatory=False)
+                     and _config('anonymous', mandatory=False))
 
     if bind:
         log.debug('ldap bind to determine group membership succeeded!')

salt/crypt.py

@@ -9,6 +9,7 @@
 # the Array class, which has incompatibilities with it.
 from __future__ import absolute_import, print_function
 import os
+import random
 import sys
 import copy
 import time
@@ -740,6 +741,10 @@ def sign_in(self, timeout=60, safe=True, tries=1, channel=None):
                             'minion.\nOr restart the Salt Master in open mode to '
                             'clean out the keys. The Salt Minion will now exit.'
                         )
+                        # Add a random sleep here for systems that are using a
+                        # a service manager to immediately restart the service
+                        # to avoid overloading the system
+                        time.sleep(random.randint(10, 20))
                         sys.exit(salt.defaults.exitcodes.EX_NOPERM)
                 # has the master returned that its maxed out with minions?
                 elif payload['load']['ret'] == 'full':

salt/fileserver/roots.py

@@ -303,8 +303,8 @@ def _file_lists(load, form):
         except os.error:
             log.critical('Unable to make cachedir %s', list_cachedir)
             return []
-    list_cache = os.path.join(list_cachedir, '{0}.p'.format(load['saltenv']))
-    w_lock = os.path.join(list_cachedir, '.{0}.w'.format(load['saltenv']))
+    list_cache = os.path.join(list_cachedir, '{0}.p'.format(salt.utils.files.safe_filename_leaf(load['saltenv'])))
+    w_lock = os.path.join(list_cachedir, '.{0}.w'.format(salt.utils.files.safe_filename_leaf(load['saltenv'])))
     cache_match, refresh_cache, save_cache = \
         salt.fileserver.check_file_list_cache(
             __opts__, form, list_cache, w_lock

salt/grains/core.py

@@ -1007,10 +1007,11 @@ def _virtual(osdata):
             if 'QEMU Virtual CPU' in model:
                 grains['virtual'] = 'kvm'
     elif osdata['kernel'] == 'OpenBSD':
-        if osdata['manufacturer'] in ['QEMU', 'Red Hat']:
-            grains['virtual'] = 'kvm'
-        if osdata['manufacturer'] == 'OpenBSD':
-            grains['virtual'] = 'vmm'
+        if 'manufacturer' in osdata:
+            if osdata['manufacturer'] in ['QEMU', 'Red Hat', 'Joyent']:
+                grains['virtual'] = 'kvm'
+            if osdata['manufacturer'] == 'OpenBSD':
+                grains['virtual'] = 'vmm'
     elif osdata['kernel'] == 'SunOS':
         if grains['virtual'] == 'LDOM':
             roles = []

salt/loader.py

@@ -1725,7 +1725,7 @@ def _load(self, key):
         if not isinstance(key, six.string_types):
             raise KeyError('The key must be a string.')
         if '.' not in key:
-            raise KeyError('The key \'%s\' should contain a \'.\'', key)
+            raise KeyError('The key \'{0}\' should contain a \'.\''.format(key))
         mod_name, _ = key.split('.', 1)
         with self._lock:
             # It is possible that the key is in the dictionary after

salt/log/handlers/sentry_mod.py

@@ -136,9 +136,7 @@ def setup_handlers():
             transport_registry = TransportRegistry(default_transports)
             url = urlparse(dsn)
             if not transport_registry.supported_scheme(url.scheme):
-                raise ValueError(
-                    'Unsupported Sentry DSN scheme: %s', url.scheme
-                )
+                raise ValueError('Unsupported Sentry DSN scheme: {0}'.format(url.scheme))
         except ValueError as exc:
             log.info(
                 'Raven failed to parse the configuration provided DSN: %s', exc

salt/minion.py

@@ -25,6 +25,8 @@
 # Import Salt Libs
 # pylint: disable=import-error,no-name-in-module,redefined-builtin
 from salt.ext import six
+from salt._compat import ipaddress
+from salt.utils.network import parse_host_port
 from salt.ext.six.moves import range
 from salt.utils.zeromq import zmq, ZMQDefaultLoop, install_zmq, ZMQ_VERSION_INFO
 import salt.transport.client
@@ -238,27 +240,29 @@ def resolve_dns(opts, fallback=True):
 
 
 def prep_ip_port(opts):
+    '''
+    parse host:port values from opts['master'] and return valid:
+        master: ip address or hostname as a string
+        master_port: (optional) master returner port as integer
+
+    e.g.:
+      - master: 'localhost:1234' -> {'master': 'localhost', 'master_port': 1234}
+      - master: '127.0.0.1:1234' -> {'master': '127.0.0.1', 'master_port' :1234}
+      - master: '[::1]:1234' -> {'master': '::1', 'master_port': 1234}
+      - master: 'fe80::a00:27ff:fedc:ba98' -> {'master': 'fe80::a00:27ff:fedc:ba98'}
+    '''
     ret = {}
     # Use given master IP if "ip_only" is set or if master_ip is an ipv6 address without
     # a port specified. The is_ipv6 check returns False if brackets are used in the IP
     # definition such as master: '[::1]:1234'.
-    if opts['master_uri_format'] == 'ip_only' or salt.utils.network.is_ipv6(opts['master']):
-        ret['master'] = opts['master']
+    if opts['master_uri_format'] == 'ip_only':
+        ret['master'] = ipaddress.ip_address(opts['master'])
     else:
-        ip_port = opts['master'].rsplit(':', 1)
-        if len(ip_port) == 1:
-            # e.g. master: mysaltmaster
-            ret['master'] = ip_port[0]
-        else:
-            # e.g. master: localhost:1234
-            # e.g. master: 127.0.0.1:1234
-            # e.g. master: [::1]:1234
-            # Strip off brackets for ipv6 support
-            ret['master'] = ip_port[0].strip('[]')
-
-            # Cast port back to an int! Otherwise a TypeError is thrown
-            # on some of the socket calls elsewhere in the minion and utils code.
-            ret['master_port'] = int(ip_port[1])
+        host, port = parse_host_port(opts['master'])
+        ret = {'master': host}
+        if port:
+            ret.update({'master_port': port})
+
     return ret
 
 

salt/modules/debian_ip.py

@@ -1899,9 +1899,11 @@ def get_network_settings():
 
         hostname = _parse_hostname()
         domainname = _parse_domainname()
+        searchdomain = _parse_searchdomain()
 
         settings['hostname'] = hostname
         settings['domainname'] = domainname
+        settings['searchdomain'] = searchdomain
 
     else:
         settings = _parse_current_network_settings()

salt/modules/mysql.py

@@ -81,34 +81,49 @@
     'ALL PRIVILEGES',
     'ALTER',
     'ALTER ROUTINE',
+    'BACKUP_ADMIN',
+    'BINLOG_ADMIN',
+    'CONNECTION_ADMIN',
     'CREATE',
+    'CREATE ROLE',
     'CREATE ROUTINE',
     'CREATE TABLESPACE',
     'CREATE TEMPORARY TABLES',
     'CREATE USER',
     'CREATE VIEW',
     'DELETE',
     'DROP',
+    'DROP ROLE',
+    'ENCRYPTION_KEY_ADMIN',
     'EVENT',
     'EXECUTE',
     'FILE',
     'GRANT OPTION',
+    'GROUP_REPLICATION_ADMIN',
     'INDEX',
     'INSERT',
     'LOCK TABLES',
+    'PERSIST_RO_VARIABLES_ADMIN',
     'PROCESS',
     'REFERENCES',
     'RELOAD',
     'REPLICATION CLIENT',
     'REPLICATION SLAVE',
+    'REPLICATION_SLAVE_ADMIN',
+    'RESOURCE_GROUP_ADMIN',
+    'RESOURCE_GROUP_USER',
+    'ROLE_ADMIN',
     'SELECT',
+    'SET_USER_ID',
     'SHOW DATABASES',
     'SHOW VIEW',
     'SHUTDOWN',
     'SUPER',
+    'SYSTEM_VARIABLES_ADMIN',
     'TRIGGER',
     'UPDATE',
-    'USAGE'
+    'USAGE',
+    'XA_RECOVER_ADMIN'
 ]
 
 __ssl_options_parameterized__ = [
@@ -121,6 +136,52 @@
     'X509'
 ]
 
+__all_privileges__ = [
+    'ALTER',
+    'ALTER ROUTINE',
+    'BACKUP_ADMIN',
+    'BINLOG_ADMIN',
+    'CONNECTION_ADMIN',
+    'CREATE',
+    'CREATE ROLE',
+    'CREATE ROUTINE',
+    'CREATE TABLESPACE',
+    'CREATE TEMPORARY TABLES',
+    'CREATE USER',
+    'CREATE VIEW',
+    'DELETE',
+    'DROP',
+    'DROP ROLE',
+    'ENCRYPTION_KEY_ADMIN',
+    'EVENT',
+    'EXECUTE',
+    'FILE',
+    'GROUP_REPLICATION_ADMIN',
+    'INDEX',
+    'INSERT',
+    'LOCK TABLES',
+    'PERSIST_RO_VARIABLES_ADMIN',
+    'PROCESS',
+    'REFERENCES',
+    'RELOAD',
+    'REPLICATION CLIENT',
+    'REPLICATION SLAVE',
+    'REPLICATION_SLAVE_ADMIN',
+    'RESOURCE_GROUP_ADMIN',
+    'RESOURCE_GROUP_USER',
+    'ROLE_ADMIN',
+    'SELECT',
+    'SET_USER_ID',
+    'SHOW DATABASES',
+    'SHOW VIEW',
+    'SHUTDOWN',
+    'SUPER',
+    'SYSTEM_VARIABLES_ADMIN',
+    'TRIGGER',
+    'UPDATE',
+    'XA_RECOVER_ADMIN'
+]
+
 r'''
 DEVELOPER NOTE: ABOUT arguments management, escapes, formats, arguments and
 security of SQL.
@@ -1787,12 +1848,12 @@ def user_grants(user,
 
 
 def grant_exists(grant,
-                database,
-                user,
-                host='localhost',
-                grant_option=False,
-                escape=True,
-                **connection_args):
+                 database,
+                 user,
+                 host='localhost',
+                 grant_option=False,
+                 escape=True,
+                 **connection_args):
     '''
     Checks to see if a grant exists in the database
 
@@ -1803,6 +1864,14 @@ def grant_exists(grant,
         salt '*' mysql.grant_exists \
              'SELECT,INSERT,UPDATE,...' 'database.*' 'frank' 'localhost'
     '''
+
+    server_version = version(**connection_args)
+    if 'ALL' in grant:
+        if salt.utils.versions.version_cmp(server_version, '8.0') >= 0:
+            grant = ','.join([i for i in __all_privileges__])
+        else:
+            grant = 'ALL PRIVILEGES'
+
     try:
         target = __grant_generate(
             grant, database, user, host, grant_option, escape
@@ -1818,23 +1887,35 @@ def grant_exists(grant,
                   'this could also indicate a connection error. Check your configuration.')
         return False
 
-    target_tokens = None
+    # Combine grants that match the same database
+    _grants = {}
     for grant in grants:
-        try:
-            if not target_tokens:  # Avoid the overhead of re-calc in loop
-                target_tokens = _grant_to_tokens(target)
-            grant_tokens = _grant_to_tokens(grant)
+        grant_token = _grant_to_tokens(grant)
+        if grant_token['database'] not in _grants:
+            _grants[grant_token['database']] = {'user': grant_token['user'],
+                                                'database': grant_token['database'],
+                                                'host': grant_token['host'],
+                                                'grant': grant_token['grant']}
+        else:
+            _grants[grant_token['database']]['grant'].extend(grant_token['grant'])
 
+    target_tokens = _grant_to_tokens(target)
+    for database, grant_tokens in _grants.items():
+        try:
             _grant_tokens = {}
             _target_tokens = {}
+
+            _grant_matches = [True if i in grant_tokens['grant']
+                              else False for i in target_tokens['grant']]
+
             for item in ['user', 'database', 'host']:
                 _grant_tokens[item] = grant_tokens[item].replace('"', '').replace('\\', '').replace('`', '')
                 _target_tokens[item] = target_tokens[item].replace('"', '').replace('\\', '').replace('`', '')
 
             if _grant_tokens['user'] == _target_tokens['user'] and \
                     _grant_tokens['database'] == _target_tokens['database'] and \
                     _grant_tokens['host'] == _target_tokens['host'] and \
-                    set(grant_tokens['grant']) >= set(target_tokens['grant']):
+                    all(_grant_matches):
                 return True
             else:
                 log.debug('grants mismatch \'%s\'<>\'%s\'', grant_tokens, target_tokens)

salt/modules/opsgenie.py

@@ -29,7 +29,7 @@
 import salt.exceptions
 import salt.utils.json
 
-API_ENDPOINT = "https://api.opsgenie.com/v1/json/saltstack?apiKey="
+API_ENDPOINT = "https://api.opsgenie.com/v2/alerts"
 
 log = logging.getLogger(__name__)
 
@@ -68,14 +68,14 @@ def post_data(api_key=None, name='OpsGenie Execution Module', reason=None,
         functionality you must provide name field for both states like in
         this case.
     '''
-    if api_key is None or reason is None or action_type is None:
+    if api_key is None or reason is None:
         raise salt.exceptions.SaltInvocationError(
-            'API Key or Reason or Action Type cannot be None.')
+            'API Key or Reason cannot be None.')
 
     data = dict()
-    data['name'] = name
-    data['reason'] = reason
-    data['actionType'] = action_type
+    data['alias'] = name
+    data['message'] = reason
+    # data['actions'] = action_type
     data['cpuModel'] = __grains__['cpu_model']
     data['cpuArch'] = __grains__['cpuarch']
     data['fqdn'] = __grains__['fqdn']
@@ -93,8 +93,17 @@ def post_data(api_key=None, name='OpsGenie Execution Module', reason=None,
     log.debug('Below data will be posted:\n%s', data)
     log.debug('API Key: %s \t API Endpoint: %s', api_key, API_ENDPOINT)
 
-    response = requests.post(
-        url=API_ENDPOINT + api_key,
-        data=salt.utils.json.dumps(data),
-        headers={'Content-Type': 'application/json'})
+    if action_type == "Create":
+        response = requests.post(
+            url=API_ENDPOINT,
+            data=salt.utils.json.dumps(data),
+            headers={'Content-Type': 'application/json',
+                 'Authorization': 'GenieKey ' + api_key})
+    else:
+        response = requests.post(
+            url=API_ENDPOINT + "/" + name + "/close?identifierType=alias",
+            data=salt.utils.json.dumps(data),
+            headers={'Content-Type': 'application/json',
+                     'Authorization': 'GenieKey ' + api_key})
+
     return response.status_code, response.text

salt/modules/snapper.py

@@ -412,7 +412,7 @@ def create_snapshot(config='root', snapshot_type='single', pre_number=None,
                                                 cleanup_algorithm, userdata)
         else:
             raise CommandExecutionError(
-                "Invalid snapshot type '{0}'", format(snapshot_type))
+                "Invalid snapshot type '{0}'".format(snapshot_type))
     except dbus.DBusException as exc:
         raise CommandExecutionError(
             'Error encountered while listing changed files: {0}'

salt/modules/vsphere.py

@@ -357,6 +357,9 @@ def _gets_service_instance_via_proxy(*args, **kwargs):
                         local_service_instance = \
                                 salt.utils.vmware.get_service_instance(
                                     *connection_details)
+                        # Tuples are immutable, so if we want to change what
+                        # was passed in, we need to first convert to a list.
+                        args = list(args)
                         args[idx] = local_service_instance
                 else:
                     # case 2: Not enough positional parameters so