Support querying for JSON data in external sql pillar.

changelog/60905.added

@@ -0,0 +1 @@
+Support querying for JSON data in SQL external pillar

salt/pillar/sql_base.py

@@ -136,6 +136,33 @@
 The range for with_lists is 1 to number_of_fields, inclusive.
 Numbers outside this range are ignored.
 
+If you specify `as_json: True` in the mapping expression and query only for
+single value, returned data are considered in JSON format and will be merged
+directly.
+
+.. code-block:: yaml
+
+  ext_pillar:
+    - sql_base:
+        - query: "SELECT json_pillar FROM pillars WHERE minion_id = %s"
+          as_json: True
+
+The processed JSON entries are recursively merged in a single dictionary.
+Additionnaly if `as_list` is set to `True` the lists will be merged in case of collision.
+
+For instance the following rows:
+
+    {"a": {"b": [1, 2]}, "c": 3}
+    {"a": {"b": [1, 3]}, "d": 4}
+
+will result in the following pillar with `as_list=False`
+
+    {"a": {"b": [1, 3], "c": 3, "d": 4}
+
+and in with `as_list=True`
+
+    {"a": {"b": [1, 2, 3], "c": 3, "d": 4}
+
 Finally, if you pass the queries in via a mapping, the key will be the
 first level name where as passing them in as a list will place them in the
 root.  This isolates the query results into their own subtrees.
@@ -147,6 +174,9 @@
 
 Configuration of the connection depends on the adapter in use.
 
+.. versionadded:: 3005
+   The *as_json* parameter.
+
 More complete example for MySQL (to also show configuration)
 ============================================================
 
@@ -171,6 +201,7 @@
 import abc  # Added in python2.6 so always available
 import logging
 
+from salt.utils.dictupdate import update
 from salt.utils.odict import OrderedDict
 
 # Please don't strip redundant parentheses from this file.
@@ -200,6 +231,7 @@ class SqlBaseExtPillar(metaclass=abc.ABCMeta):
     num_fields = 0
     depth = 0
     as_list = False
+    as_json = False
     with_lists = None
     ignore_null = False
 
@@ -259,6 +291,7 @@ def extract_queries(self, args, kwargs):
                 "query": "",
                 "depth": 0,
                 "as_list": False,
+                "as_json": False,
                 "with_lists": None,
                 "ignore_null": False,
             }
@@ -314,6 +347,13 @@ def process_results(self, rows):
         for ret in rows:
             # crd is the Current Return Data level, to make this non-recursive.
             crd = self.focus
+
+            # We have just one field without any key, assume returned row is already a dict
+            # aka JSON storage
+            if self.as_json and self.num_fields == 1:
+                crd = update(crd, ret[0], merge_lists=self.as_list)
+                continue
+
             # Walk and create dicts above the final layer
             for i in range(0, self.depth - 1):
                 # At the end we'll use listify to find values to make a list of
@@ -433,6 +473,7 @@ def fetch(self, minion_id, pillar, *args, **kwargs):  # pylint: disable=W0613
                 )
                 self.enter_root(root)
                 self.as_list = details["as_list"]
+                self.as_json = details["as_json"]
                 if details["with_lists"]:
                     self.with_lists = details["with_lists"]
                 else:

tests/pytests/unit/pillar/test_sql_base.py

@@ -0,0 +1,43 @@
+import pytest
+import salt.pillar.sql_base as sql_base
+from tests.support.mock import MagicMock
+
+
+class FakeExtPillar(sql_base.SqlBaseExtPillar):
+    """
+    Mock SqlBaseExtPillar implementation for testing purpose
+    """
+
+    @classmethod
+    def _db_name(cls):
+        return "fake"
+
+    def _get_cursor(self):
+        return MagicMock()
+
+
+@pytest.mark.parametrize("as_list", [True, False])
+def test_process_results_as_json(as_list):
+    """
+    Validates merging of dict values returned from JSON datatype.
+    """
+    return_data = FakeExtPillar()
+    return_data.as_list = as_list
+    return_data.as_json = True
+    return_data.with_lists = None
+    return_data.enter_root(None)
+    return_data.process_fields(["json_data"], 0)
+    test_dicts = [
+        ({"a": [1]},),
+        ({"b": [2, 3]},),
+        ({"a": [4]},),
+        ({"c": {"d": [4, 5], "e": 6}},),
+        ({"f": [{"g": 7, "h": "test"}], "c": {"g": 8}},),
+    ]
+    return_data.process_results(test_dicts)
+    assert return_data.result == {
+        "a": [1, 4] if as_list else [4],
+        "b": [2, 3],
+        "c": {"d": [4, 5], "e": 6, "g": 8},
+        "f": [{"g": 7, "h": "test"}],
+    }