Merge pull request #48809 from cro/matcher_in_loader2

Matcher in loader, take 4

doc/topics/matchers/index.rst

@@ -0,0 +1,81 @@
+.. _matchers:
+
+========
+Matchers
+========
+
+.. versionadded:: Flourine
+
+Matchers are modules that provide Salt's targeting abilities.  As of the
+Flourine release, matchers can be dynamically loaded.  Currently new matchers
+cannot be created because the required plumbing for the CLI does not exist yet.
+Existing matchers may have their functionality altered or extended.
+
+For details of targeting methods, see the :ref:`Targeting <targeting>` topic.
+
+A matcher module must have a function called ``match()``. This function ends up
+becoming a method on the Matcher class.  All matcher functions require at least
+two arguments, ``self`` (because the function will be turned into a method), and
+``tgt``, which is the actual target string.  The grains and pillar matchers also
+take a ``delimiter`` argument and should default to ``DEFAULT_TARGET_DELIM``.
+
+Like other Salt loadable modules, modules that override built-in functionality
+can be placed in ``file_roots`` in a special directory and then copied to the
+minion through the normal sync process.  :py:func:`saltutil.sync_all <salt.modules.saltutil.sync_all>`
+will transfer all loadable modules, and the Flourine release introduces
+:py:func:`saltutil.sync_matchers <salt.modules.saltutil.sync_matchers>`.  For matchers, the directory is
+``/srv/salt/_matchers`` (assuming your ``file_roots`` is set to the default
+``/srv/salt``).
+
+As an example, let's modify the ``list`` matcher to have the separator be a
+'``/``' instead of the default '``,``'.
+
+
+.. code-block:: python
+
+    from __future__ import absolute_import, print_function, unicode_literals
+    from salt.ext import six  # pylint: disable=3rd-party-module-not-gated
+
+    def match(self, tgt):
+        '''
+        Determines if this host is on the list
+        '''
+        if isinstance(tgt, six.string_types):
+            # The stock matcher splits on `,`.  Change to `/` below.
+            tgt = tgt.split('/')
+        return bool(self.opts['id'] in tgt)
+
+
+Place this code in a file called ``list_matcher.py`` in ``_matchers`` in your
+``file_roots``. Sync this down to your minions with
+:py:func:`saltutil.sync_matchers <salt.modules.saltutil.sync_matchers>`.
+Then attempt to match with the following, replacing ``minionX`` with three of your minions.
+
+.. code-block:: shell
+
+   salt -L 'minion1/minion2/minion3' test.ping
+
+
+Three of your minions should respond.
+
+The current supported matchers and associated filenames are
+
+===============  ======================  ===================
+Salt CLI Switch  Match Type              Filename
+===============  ======================  ===================
+<none>           Glob                    glob_match.py
+-C               Compound                compound_match.py
+-E               Perl-Compatible         pcre_match.py
+                 Regular Expressions
+-L               List                    list_match.py
+-G               Grain                   grain_match.py
+-P               Grain Perl-Compatible   grain_pcre_match.py
+                 Regular Expressions
+-N               Nodegroup               nodegroup_match.py
+-R               Range                   range_match.py
+-I               Pillar                  pillar_match.py
+-J               Pillar Perl-Compatible  pillar_pcre.py
+                 Regular Expressions    
+-S               IP-Classless Internet   ipcidr_match.py
+                 Domain Routing
+===============  ======================  ===================

doc/topics/targeting/index.rst

@@ -111,3 +111,19 @@ There are many ways to target individual minions or groups of minions in Salt:
     nodegroups
     batch
     range
+
+
+Loadable Matchers
+=================
+
+.. versionadded:: Flourine
+
+Internally targeting is implemented with chunks of code called Matchers.  As of
+the Flourine release, matchers can be loaded dynamically.  Currently new matchers
+cannot be created, but existing matchers can have their functionality altered or
+extended.  For more information on Matchers see
+
+.. toctree::
+    :maxdepth: 2
+
+    Loadable Matchers <../matchers/index.rst>

salt/client/ssh/state.py

@@ -74,7 +74,7 @@ def __init__(self, opts, pillar=None, wrapper=None, fsclient=None):
         self.client = fsclient
         salt.state.BaseHighState.__init__(self, opts)
         self.state = SSHState(opts, pillar, wrapper)
-        self.matcher = salt.minion.Matcher(self.opts)
+        self.matchers = salt.loader.matchers(self.opts)
         self.tops = salt.loader.tops(self.opts)
 
         self._pydsl_all_decls = {}

salt/loader.py

@@ -294,6 +294,17 @@ def raw_mod(opts, name, functions, mod='modules'):
     return dict(loader._dict)  # return a copy of *just* the funcs for `name`
 
 
+def matchers(opts):
+    '''
+    Return the matcher services plugins
+    '''
+    return LazyLoader(
+        _module_dirs(opts, 'matchers'),
+        opts,
+        tag='matchers'
+    )
+
+
 def engines(opts, functions, runners, utils, proxy=None):
     '''
     Return the master services plugins

salt/matchers/__init__.py

@@ -0,0 +1,97 @@
+# -*- coding: utf-8 -*-
+'''
+Salt package
+'''
+
+# Import Python libs
+from __future__ import absolute_import, print_function, unicode_literals
+import warnings
+
+# All salt related deprecation warnings should be shown once each!
+warnings.filterwarnings(
+    'once',  # Show once
+    '',  # No deprecation message match
+    DeprecationWarning,  # This filter is for DeprecationWarnings
+    r'^(salt|salt\.(.*))$'  # Match module(s) 'salt' and 'salt.<whatever>'
+)
+
+# While we are supporting Python2.6, hide nested with-statements warnings
+warnings.filterwarnings(
+    'ignore',
+    'With-statements now directly support multiple context managers',
+    DeprecationWarning
+)
+
+# Filter the backports package UserWarning about being re-imported
+warnings.filterwarnings(
+    'ignore',
+    '^Module backports was already imported from (.*), but (.*) is being added to sys.path$',
+    UserWarning
+)
+
+
+def __define_global_system_encoding_variable__():
+    import sys
+    # This is the most trustworthy source of the system encoding, though, if
+    # salt is being imported after being daemonized, this information is lost
+    # and reset to None
+    encoding = None
+
+    if not sys.platform.startswith('win') and sys.stdin is not None:
+        # On linux we can rely on sys.stdin for the encoding since it
+        # most commonly matches the filesystem encoding. This however
+        # does not apply to windows
+        encoding = sys.stdin.encoding
+
+    if not encoding:
+        # If the system is properly configured this should return a valid
+        # encoding. MS Windows has problems with this and reports the wrong
+        # encoding
+        import locale
+        try:
+            encoding = locale.getdefaultlocale()[-1]
+        except ValueError:
+            # A bad locale setting was most likely found:
+            #   https://github.com/saltstack/salt/issues/26063
+            pass
+
+        # This is now garbage collectable
+        del locale
+        if not encoding:
+            # This is most likely ascii which is not the best but we were
+            # unable to find a better encoding. If this fails, we fall all
+            # the way back to ascii
+            encoding = sys.getdefaultencoding()
+        if not encoding:
+            if sys.platform.startswith('darwin'):
+                # Mac OS X uses UTF-8
+                encoding = 'utf-8'
+            elif sys.platform.startswith('win'):
+                # Windows uses a configurable encoding; on Windows, Python uses the name “mbcs”
+                # to refer to whatever the currently configured encoding is.
+                encoding = 'mbcs'
+            else:
+                # On linux default to ascii as a last resort
+                encoding = 'ascii'
+
+    # We can't use six.moves.builtins because these builtins get deleted sooner
+    # than expected. See:
+    #    https://github.com/saltstack/salt/issues/21036
+    if sys.version_info[0] < 3:
+        import __builtin__ as builtins  # pylint: disable=incompatible-py3-code
+    else:
+        import builtins  # pylint: disable=import-error
+
+    # Define the detected encoding as a built-in variable for ease of use
+    setattr(builtins, '__salt_system_encoding__', encoding)
+
+    # This is now garbage collectable
+    del sys
+    del builtins
+    del encoding
+
+
+__define_global_system_encoding_variable__()
+
+# This is now garbage collectable
+del __define_global_system_encoding_variable__

salt/matchers/cache_match.py

@@ -0,0 +1,30 @@
+# -*- coding: utf-8 -*-
+'''
+This is the default cache matcher function.  It only exists for the master,
+this is why there is only a ``mmatch()`` but not ``match()``.
+'''
+from __future__ import absolute_import, print_function, unicode_literals
+import logging
+
+import salt.utils.data     # pylint: disable=3rd-party-module-not-gated
+import salt.utils.minions  # pylint: disable=3rd-party-module-not-gated
+
+log = logging.getLogger(__name__)
+
+
+def mmatch(expr,
+           delimiter,
+           greedy,
+           search_type,
+           regex_match=False,
+           exact_match=False):
+    '''
+    Helper function to search for minions in master caches
+    If 'greedy' return accepted minions that matched by the condition or absent in the cache.
+    If not 'greedy' return the only minions have cache data and matched by the condition.
+    '''
+    ckminions = salt.utils.minions.CkMinions(__opts__)
+
+    return ckminions._check_cache_minions(expr, delimiter, greedy,
+                                          search_type, regex_match=regex_match,
+                                          exact_match=exact_match)

salt/matchers/compound_match.py

@@ -0,0 +1,112 @@
+# -*- coding: utf-8 -*-
+'''
+This is the default compound matcher function.
+'''
+from __future__ import absolute_import, print_function, unicode_literals
+
+import logging
+from salt.ext import six  # pylint: disable=3rd-party-module-not-gated
+import salt.loader
+import salt.utils.minions  # pylint: disable=3rd-party-module-not-gated
+
+HAS_RANGE = False
+try:
+    import seco.range  # pylint: disable=unused-import
+    HAS_RANGE = True
+except ImportError:
+    pass
+
+log = logging.getLogger(__name__)
+
+
+def match(tgt):
+    '''
+    Runs the compound target check
+    '''
+    nodegroups = __opts__.get('nodegroups', {})
+    matchers = salt.loader.matchers(__opts__)
+
+    if not isinstance(tgt, six.string_types) and not isinstance(tgt, (list, tuple)):
+        log.error('Compound target received that is neither string, list nor tuple')
+        return False
+    log.debug('compound_match: %s ? %s', __opts__['id'], tgt)
+    ref = {'G': 'grain',
+           'P': 'grain_pcre',
+           'I': 'pillar',
+           'J': 'pillar_pcre',
+           'L': 'list',
+           'N': None,      # Nodegroups should already be expanded
+           'S': 'ipcidr',
+           'E': 'pcre'}
+    if HAS_RANGE:
+        ref['R'] = 'range'
+
+    results = []
+    opers = ['and', 'or', 'not', '(', ')']
+
+    if isinstance(tgt, six.string_types):
+        words = tgt.split()
+    else:
+        # we make a shallow copy in order to not affect the passed in arg
+        words = tgt[:]
+
+    while words:
+        word = words.pop(0)
+        target_info = salt.utils.minions.parse_target(word)
+
+        # Easy check first
+        if word in opers:
+            if results:
+                if results[-1] == '(' and word in ('and', 'or'):
+                    log.error('Invalid beginning operator after "(": %s', word)
+                    return False
+                if word == 'not':
+                    if not results[-1] in ('and', 'or', '('):
+                        results.append('and')
+                results.append(word)
+            else:
+                # seq start with binary oper, fail
+                if word not in ['(', 'not']:
+                    log.error('Invalid beginning operator: %s', word)
+                    return False
+                results.append(word)
+
+        elif target_info and target_info['engine']:
+            if 'N' == target_info['engine']:
+                # if we encounter a node group, just evaluate it in-place
+                decomposed = salt.utils.minions.nodegroup_comp(target_info['pattern'], nodegroups)
+                if decomposed:
+                    words = decomposed + words
+                continue
+
+            engine = ref.get(target_info['engine'])
+            if not engine:
+                # If an unknown engine is called at any time, fail out
+                log.error(
+                    'Unrecognized target engine "%s" for target '
+                    'expression "%s"', target_info['engine'], word
+                )
+                return False
+
+            engine_args = [target_info['pattern']]
+            engine_kwargs = {}
+            if target_info['delimiter']:
+                engine_kwargs['delimiter'] = target_info['delimiter']
+
+            results.append(
+                six.text_type(matchers['{0}_match.match'.format(engine)](*engine_args, **engine_kwargs))
+            )
+
+        else:
+            # The match is not explicitly defined, evaluate it as a glob
+            results.append(six.text_type(matchers['glob_match.match'](word)))
+
+    results = ' '.join(results)
+    log.debug('compound_match %s ? "%s" => "%s"', __opts__['id'], tgt, results)
+    try:
+        return eval(results)  # pylint: disable=W0123
+    except Exception:
+        log.error(
+            'Invalid compound target: %s for results: %s', tgt, results)
+        return False
+    return False

salt/matchers/compound_pillar_exact_match.py

@@ -0,0 +1,23 @@
+# -*- coding: utf-8 -*-
+'''
+This is the default pillar exact matcher for compound matches.
+
+There is no minion-side equivalent for this, so consequently there is no ``match()``
+function below, only an ``mmatch()``
+'''
+from __future__ import absolute_import, print_function, unicode_literals
+
+import logging
+
+import salt.utils.minions  # pylint: disable=3rd-party-module-not-gated
+
+log = logging.getLogger(__name__)
+
+
+def mmatch(expr, delimiter, greedy):
+    '''
+    Return the minions found by looking via pillar
+    '''
+    ckminions = salt.utils.minions.CkMinions(__opts__)
+    return ckminions._check_compound_minions(expr, delimiter, greedy,
+                                             pillar_exact=True)