Merge pull request #45269 from MarloweW/pillar_wildcard_include

Fix #22063: pillar wildcard support include

salt/pillar/__init__.py

@@ -754,13 +754,25 @@ def render_pstate(self, sls, saltenv, mods, defaults=None):
                     else:
                         # render included state(s)
                         include_states = []
+
+                        matched_pstates = []
                         for sub_sls in state.pop('include'):
                             if isinstance(sub_sls, dict):
                                 sub_sls, v = next(six.iteritems(sub_sls))
                                 defaults = v.get('defaults', {})
                                 key = v.get('key', None)
                             else:
                                 key = None
+
+                            try:
+                                matched_pstates += fnmatch.filter(self.avail[saltenv], sub_sls)
+                            except KeyError:
+                                errors.extend(
+                                    ['No matching pillar environment for environment '
+                                     '\'{0}\' found'.format(saltenv)]
+                                )
+
+                        for sub_sls in set(matched_pstates):
                             if sub_sls not in mods:
                                 nstate, mods, err = self.render_pstate(
                                         sub_sls,

tests/unit/test_pillar.py

@@ -296,7 +296,8 @@ def test_ext_pillar_with_extra_minion_data_val_elem(self):
             'mocked-minion', 'fake_pillar', 'bar',
             extra_minion_data={'fake_key': 'foo'})
 
-    def test_malformed_pillar_sls(self):
+    @patch('salt.fileclient.Client.list_states')
+    def test_malformed_pillar_sls(self, mock_list_states):
         with patch('salt.pillar.compile_template') as compile_template:
             opts = {
                 'renderer': 'json',
@@ -315,6 +316,8 @@ def test_malformed_pillar_sls(self):
                 'osrelease': '13.04',
                 'kernel': 'Linux'
             }
+
+            mock_list_states.return_value = ['foo', 'blah']
             pillar = salt.pillar.Pillar(opts, grains, 'mocked-minion', 'base')
             # Mock getting the proper template files
             pillar.client.get_state = MagicMock(
@@ -506,6 +509,35 @@ def get_state(sls, env):
 
         client.get_state.side_effect = get_state
 
+    def test_include(self):
+        with patch('salt.pillar.salt.fileclient.get_file_client', autospec=True) as get_file_client, \
+            patch('salt.pillar.salt.minion.Matcher') as Matcher:  # autospec=True disabled due to py3 mock bug
+            opts = {
+                'renderer': 'yaml',
+                'renderer_blacklist': [],
+                'renderer_whitelist': [],
+                'state_top': '',
+                'pillar_roots': [],
+                'extension_modules': '',
+                'saltenv': 'base',
+                'file_roots': [],
+            }
+            grains = {
+                'os': 'Ubuntu',
+                'os_family': 'Debian',
+                'oscodename': 'raring',
+                'osfullname': 'Ubuntu',
+                'osrelease': '13.04',
+                'kernel': 'Linux'
+            }
+
+            self._setup_test_include_mocks(Matcher, get_file_client)
+            pillar = salt.pillar.Pillar(opts, grains, 'minion', 'base')
+            compiled_pillar = pillar.compile_pillar()
+            self.assertEqual(compiled_pillar['foo_wildcard'], 'bar_wildcard')
+            self.assertEqual(compiled_pillar['foo1'], 'bar1')
+            self.assertEqual(compiled_pillar['foo2'], 'bar2')
+
     def _setup_test_include_mocks(self, Matcher, get_file_client):
         self.top_file = top_file = tempfile.NamedTemporaryFile(dir=TMP, delete=False)
         top_file.write(b'''
@@ -522,39 +554,43 @@ def _setup_test_include_mocks(self, Matcher, get_file_client):
         init_sls.write(b'''
 include:
    - test.sub1
-   - test.sub2
+   - test.sub_wildcard*
 ''')
         init_sls.flush()
         self.sub1_sls = sub1_sls = tempfile.NamedTemporaryFile(dir=TMP, delete=False)
         sub1_sls.write(b'''
-p1:
-   - value1_1
-   - value1_2
+foo1:
+  bar1
 ''')
         sub1_sls.flush()
         self.sub2_sls = sub2_sls = tempfile.NamedTemporaryFile(dir=TMP, delete=False)
         sub2_sls.write(b'''
-p1:
-   - value1_3
-p2:
-   - value2_1
-   - value2_2
+foo2:
+  bar2 
 ''')
         sub2_sls.flush()
 
+        self.sub_wildcard_1_sls = sub_wildcard_1_sls = tempfile.NamedTemporaryFile(dir=TMP, delete=False)
+        sub_wildcard_1_sls.write(b'''
+foo_wildcard:
+   bar_wildcard
+''')
+        sub_wildcard_1_sls.flush()
         # Setup Matcher mock
         matcher = Matcher.return_value
         matcher.confirm_top.return_value = True
 
         # Setup fileclient mock
         client = get_file_client.return_value
         client.cache_file.return_value = self.top_file.name
+        client.list_states.return_value = ['top', 'test.init', 'test.sub1', 'test.sub2', 'test.sub_wildcard_1']
 
         def get_state(sls, env):
             return {
                 'test': {'path': '', 'dest': init_sls.name},
                 'test.sub1': {'path': '', 'dest': sub1_sls.name},
                 'test.sub2': {'path': '', 'dest': sub2_sls.name},
+                'test.sub_wildcard_1': {'path': '', 'dest': sub_wildcard_1_sls.name},
             }[sls]
 
         client.get_state.side_effect = get_state