From 5a3f97b0f913bc9cb69bc1ea39142914890ad296 Mon Sep 17 00:00:00 2001 From: Wayne Werner Date: Wed, 29 Jan 2020 16:37:47 -0600 Subject: [PATCH 1/8] Test that options are passed --- salt/utils/jinja.py | 4 ++-- tests/unit/utils/test_jinja.py | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/salt/utils/jinja.py b/salt/utils/jinja.py index c5a30ba04eb7..6a9cfc769873 100644 --- a/salt/utils/jinja.py +++ b/salt/utils/jinja.py @@ -386,8 +386,8 @@ def indent(s, width=4, first=False, blank=False, indentfirst=None): return rv -@jinja_filter("tojson") -def tojson(val, indent=None): +@jinja_filter('tojson') +def tojson(val, indent=None, **options): """ Implementation of tojson filter (only present in Jinja 2.9 and later). If Jinja 2.9 or later is installed, then the upstream version of this filter diff --git a/tests/unit/utils/test_jinja.py b/tests/unit/utils/test_jinja.py index 8c3ec2a11107..59d2bb399f68 100644 --- a/tests/unit/utils/test_jinja.py +++ b/tests/unit/utils/test_jinja.py @@ -74,6 +74,13 @@ def test_indent(self): expected = Markup('foo:\n "bar"') assert result == expected, result + def test_tojson_should_ascii_sort_keys_when_told(self): + data = {'z': 'zzz', 'y': 'yyy', 'x': 'xxx'} + expected = '{"x": "xxx", "y": "yyy", "z": "zzz"}' + + actual = tojson(data, sort_keys=True) + assert actual == expected + class MockFileClient: """ From 1a99a5c0ebe1b3ad2c67e90d8d5fc953d4cf31d4 Mon Sep 17 00:00:00 2001 From: Wayne Werner Date: Wed, 29 Jan 2020 16:38:50 -0600 Subject: [PATCH 2/8] Use passed in options --- salt/utils/jinja.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/utils/jinja.py b/salt/utils/jinja.py index 6a9cfc769873..94383d2cd4bb 100644 --- a/salt/utils/jinja.py +++ b/salt/utils/jinja.py @@ -393,7 +393,7 @@ def tojson(val, indent=None, **options): Jinja 2.9 or later is installed, then the upstream version of this filter will be used. """ - options = {"ensure_ascii": True} + options.setdefault('ensure_ascii', True) if indent is not None: options["indent"] = indent return ( From c71d943e3fe904d7c2679fcd008cae3035cca199 Mon Sep 17 00:00:00 2001 From: Wayne Werner Date: Wed, 29 Jan 2020 17:10:52 -0600 Subject: [PATCH 3/8] Test that our tojson is used over jinja's Closes #55911 --- tests/unit/utils/test_templates.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/unit/utils/test_templates.py b/tests/unit/utils/test_templates.py index d46ef9d39780..705b317bffbf 100644 --- a/tests/unit/utils/test_templates.py +++ b/tests/unit/utils/test_templates.py @@ -2,6 +2,7 @@ Unit tests for salt.utils.templates.py """ + import logging import os import sys @@ -12,6 +13,7 @@ import salt.utils.templates from tests.support import mock from tests.support.helpers import with_tempdir +from tests.support.mock import patch from tests.support.unit import TestCase, skipIf try: @@ -56,6 +58,24 @@ def test_render_jinja_variable(self): res = salt.utils.templates.render_jinja_tmpl(tmpl, ctx) self.assertEqual(res, "OK") + def test_render_jinja_tojson_sorted(self): + templ = """thing: {{ var|tojson(sort_keys=False) }}""" + expected = """thing: {"z": "zzz", "y": "yyy", "x": "xxx"}""" + + with patch.dict(self.context, {"var": {"z": "zzz", "y": "yyy", "x": "xxx"}}): + res = salt.utils.templates.render_jinja_tmpl(templ, self.context) + + assert res == expected + + def test_render_jinja_tojson_unsorted(self): + templ = """thing: {{ var|tojson(sort_keys=False) }}""" + expected = """thing: {"z": "zzz", "y": "yyy", "x": "xxx"}""" + + with patch.dict(self.context, {"var": {"z": "zzz", "y": "yyy", "x": "xxx"}}): + res = salt.utils.templates.render_jinja_tmpl(templ, self.context) + + assert res == expected + ### Tests for mako template def test_render_mako_sanity(self): tmpl = """OK""" From 5189321ee9ac98d60ddba8c4cd6a08b3f7fc3e12 Mon Sep 17 00:00:00 2001 From: Wayne Werner Date: Wed, 29 Jan 2020 17:46:04 -0600 Subject: [PATCH 4/8] Update tojson docs. --- salt/utils/jinja.py | 10 +++++----- tests/unit/utils/test_jinja.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/salt/utils/jinja.py b/salt/utils/jinja.py index 94383d2cd4bb..68a8646474fd 100644 --- a/salt/utils/jinja.py +++ b/salt/utils/jinja.py @@ -386,14 +386,14 @@ def indent(s, width=4, first=False, blank=False, indentfirst=None): return rv -@jinja_filter('tojson') +@jinja_filter("tojson") def tojson(val, indent=None, **options): """ - Implementation of tojson filter (only present in Jinja 2.9 and later). If - Jinja 2.9 or later is installed, then the upstream version of this filter - will be used. + Implementation of tojson filter (only present in Jinja 2.9 and later). + Unlike the Jinja built-in filter, this allows arbitrary options to be + passed in to the underlying JSON library. """ - options.setdefault('ensure_ascii', True) + options.setdefault("ensure_ascii", True) if indent is not None: options["indent"] = indent return ( diff --git a/tests/unit/utils/test_jinja.py b/tests/unit/utils/test_jinja.py index 59d2bb399f68..ec05eec5c5a4 100644 --- a/tests/unit/utils/test_jinja.py +++ b/tests/unit/utils/test_jinja.py @@ -75,7 +75,7 @@ def test_indent(self): assert result == expected, result def test_tojson_should_ascii_sort_keys_when_told(self): - data = {'z': 'zzz', 'y': 'yyy', 'x': 'xxx'} + data = {"z": "zzz", "y": "yyy", "x": "xxx"} expected = '{"x": "xxx", "y": "yyy", "z": "zzz"}' actual = tojson(data, sort_keys=True) From 33a7e151ff0af86231b950d39eeebd895327194f Mon Sep 17 00:00:00 2001 From: Wayne Werner Date: Tue, 12 May 2020 12:40:51 -0500 Subject: [PATCH 5/8] black/isort --- tests/unit/utils/test_templates.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/tests/unit/utils/test_templates.py b/tests/unit/utils/test_templates.py index 705b317bffbf..16adcab7641a 100644 --- a/tests/unit/utils/test_templates.py +++ b/tests/unit/utils/test_templates.py @@ -11,7 +11,6 @@ import pytest import salt.utils.files import salt.utils.templates -from tests.support import mock from tests.support.helpers import with_tempdir from tests.support.mock import patch from tests.support.unit import TestCase, skipIf @@ -254,7 +253,7 @@ def _test_generated_sls_context(self, tmplpath, sls, **expected): actual = salt.utils.templates._generate_sls_context(tmplpath, sls) self.assertDictContainsAll(actual, **expected) - @mock.patch("salt.utils.templates.generate_sls_context") + @patch("salt.utils.templates.generate_sls_context") @with_tempdir() def test_sls_context_call(self, tempdir, generate_sls_context): """ Check that generate_sls_context is called with proper parameters""" @@ -270,7 +269,7 @@ def test_sls_context_call(self, tempdir, generate_sls_context): res = wrapped(slsfile, context=context, tmplpath=tmplpath) generate_sls_context.assert_called_with(tmplpath, sls) - @mock.patch("salt.utils.templates.generate_sls_context") + @patch("salt.utils.templates.generate_sls_context") @with_tempdir() def test_sls_context_no_call(self, tempdir, generate_sls_context): """ Check that generate_sls_context is not called if sls is not set""" @@ -418,11 +417,11 @@ def test_generate_sls_context__two_level_repeating(self): slspath="foo/foo", ) - @mock.patch( + @patch( "salt.utils.templates._generate_sls_context_legacy", return_value="legacy" ) - @mock.patch("salt.utils.templates._generate_sls_context", return_value="new") - @mock.patch("salt.utils.templates.features.get", return_value=True) + @patch("salt.utils.templates._generate_sls_context", return_value="new") + @patch("salt.utils.templates.features.get", return_value=True) def test_feature_flag_on(self, feature_get, new_impl, legacy_impl): """ Test feature flag selection with FF on""" tplpath = "tplpath" @@ -431,11 +430,11 @@ def test_feature_flag_on(self, feature_get, new_impl, legacy_impl): new_impl.assert_called_with(tplpath, sls) legacy_impl.assert_not_called() - @mock.patch( + @patch( "salt.utils.templates._generate_sls_context_legacy", return_value="legacy" ) - @mock.patch("salt.utils.templates._generate_sls_context", return_value="new") - @mock.patch("salt.utils.templates.features.get", return_value=False) + @patch("salt.utils.templates._generate_sls_context", return_value="new") + @patch("salt.utils.templates.features.get", return_value=False) def test_feature_flag_off(self, feature_get, new_impl, legacy_impl): """ Test feature flag selection with FF on""" tplpath = "tplpath" From d1140b384757c9d063c11befff2ac5af2b49b878 Mon Sep 17 00:00:00 2001 From: Wayne Werner Date: Tue, 2 Jun 2020 15:33:15 -0500 Subject: [PATCH 6/8] Stop using jinja2's tojson We're already vendoring the filter, which also has less features than what we want to support. --- salt/utils/templates.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/salt/utils/templates.py b/salt/utils/templates.py index 812c3fcad49a..d95a9c0c0f6e 100644 --- a/salt/utils/templates.py +++ b/salt/utils/templates.py @@ -460,13 +460,9 @@ def opt_jinja_env_helper(opts, optname): undefined=jinja2.StrictUndefined, **env_args ) - tojson_filter = jinja_env.filters.get("tojson") indent_filter = jinja_env.filters.get("indent") jinja_env.tests.update(JinjaTest.salt_jinja_tests) jinja_env.filters.update(JinjaFilter.salt_jinja_filters) - if tojson_filter is not None: - # Use the existing tojson filter, if present (jinja2 >= 2.9) - jinja_env.filters["tojson"] = tojson_filter if salt.utils.jinja.JINJA_VERSION >= LooseVersion("2.11"): # Use the existing indent filter on Jinja versions where it's not broken jinja_env.filters["indent"] = indent_filter From 21d32c2c9d80315db108bf5c9f2bed251ea9727f Mon Sep 17 00:00:00 2001 From: Wayne Werner Date: Tue, 2 Jun 2020 15:36:16 -0500 Subject: [PATCH 7/8] Add changelog entry --- changelog/56012.added | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/56012.added diff --git a/changelog/56012.added b/changelog/56012.added new file mode 100644 index 000000000000..7be6c059e99a --- /dev/null +++ b/changelog/56012.added @@ -0,0 +1 @@ +Added arbitrary kwarg support for tojson filter. From c6acabe5d83f30e15b0ab684d6c05e02e86ab165 Mon Sep 17 00:00:00 2001 From: Wayne Werner Date: Thu, 4 Mar 2021 11:02:55 -0600 Subject: [PATCH 8/8] Change test to use OrderedDict I goofed and was not sorting in one test - that's fixed. The other test was failing because earlier versions of Python don't remember dict insertion/creation order. Without sorting the output order was accidental. On certain platforms they were accidental in a different order than expected, so this caused a problem. Changing to OrderedDict allows us to specify the order, and this should work on those older versions of Python. --- tests/unit/utils/test_templates.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/tests/unit/utils/test_templates.py b/tests/unit/utils/test_templates.py index 16adcab7641a..766318967e5b 100644 --- a/tests/unit/utils/test_templates.py +++ b/tests/unit/utils/test_templates.py @@ -6,6 +6,7 @@ import logging import os import sys +from collections import OrderedDict from pathlib import PurePath, PurePosixPath import pytest @@ -58,8 +59,8 @@ def test_render_jinja_variable(self): self.assertEqual(res, "OK") def test_render_jinja_tojson_sorted(self): - templ = """thing: {{ var|tojson(sort_keys=False) }}""" - expected = """thing: {"z": "zzz", "y": "yyy", "x": "xxx"}""" + templ = """thing: {{ var|tojson(sort_keys=True) }}""" + expected = """thing: {"x": "xxx", "y": "yyy", "z": "zzz"}""" with patch.dict(self.context, {"var": {"z": "zzz", "y": "yyy", "x": "xxx"}}): res = salt.utils.templates.render_jinja_tmpl(templ, self.context) @@ -68,9 +69,16 @@ def test_render_jinja_tojson_sorted(self): def test_render_jinja_tojson_unsorted(self): templ = """thing: {{ var|tojson(sort_keys=False) }}""" - expected = """thing: {"z": "zzz", "y": "yyy", "x": "xxx"}""" + expected = """thing: {"z": "zzz", "x": "xxx", "y": "yyy"}""" - with patch.dict(self.context, {"var": {"z": "zzz", "y": "yyy", "x": "xxx"}}): + # Values must be added to the dict in the expected order. This is + # only necessary for older Pythons that don't remember dict order. + d = OrderedDict() + d["z"] = "zzz" + d["x"] = "xxx" + d["y"] = "yyy" + + with patch.dict(self.context, {"var": d}): res = salt.utils.templates.render_jinja_tmpl(templ, self.context) assert res == expected @@ -417,9 +425,7 @@ def test_generate_sls_context__two_level_repeating(self): slspath="foo/foo", ) - @patch( - "salt.utils.templates._generate_sls_context_legacy", return_value="legacy" - ) + @patch("salt.utils.templates._generate_sls_context_legacy", return_value="legacy") @patch("salt.utils.templates._generate_sls_context", return_value="new") @patch("salt.utils.templates.features.get", return_value=True) def test_feature_flag_on(self, feature_get, new_impl, legacy_impl): @@ -430,9 +436,7 @@ def test_feature_flag_on(self, feature_get, new_impl, legacy_impl): new_impl.assert_called_with(tplpath, sls) legacy_impl.assert_not_called() - @patch( - "salt.utils.templates._generate_sls_context_legacy", return_value="legacy" - ) + @patch("salt.utils.templates._generate_sls_context_legacy", return_value="legacy") @patch("salt.utils.templates._generate_sls_context", return_value="new") @patch("salt.utils.templates.features.get", return_value=False) def test_feature_flag_off(self, feature_get, new_impl, legacy_impl):