[Resolve #1138] Bugfix for j2_environments (#1137)

Before this, the j2_environment feature that was added in #966 had been broken during the recent implementation of the `TemplateHandler` feature. The root cause was that the `_render_jinja_template` had been rewritten as `render_jinja_template` in `sceptre/template_handlers/helper.py` but the logic for J2 environments had not been copied.

The main change here is to hook the `stack_group_config` attribute of the `Template` class into this helper. 

As part of this I also cleaned up the old `_render_jinja_template` function which was no longer used and its tests.

.python-version

@@ -0,0 +1 @@
+3.6.9

sceptre/template.py

@@ -11,12 +11,7 @@
 import threading
 import botocore
 
-from jinja2 import Environment
-from jinja2 import FileSystemLoader
-from jinja2 import StrictUndefined
-from jinja2 import select_autoescape
 from sceptre.exceptions import TemplateHandlerNotFoundError
-from sceptre.config import strategies
 from pkg_resources import iter_entry_points
 
 
@@ -88,7 +83,8 @@ def body(self):
                 name=self.name,
                 arguments={k: v for k, v in self.handler_config.items() if k != "type"},
                 sceptre_user_data=self.sceptre_user_data,
-                connection_manager=self.connection_manager
+                connection_manager=self.connection_manager,
+                stack_group_config=self.stack_group_config
             )
             handler.validate()
             body = handler.handle()
@@ -237,38 +233,6 @@ def _bucket_region(self, bucket_name):
     def _domain_from_region(region):
         return "com.cn" if region.startswith("cn-") else "com"
 
-    def _render_jinja_template(self, template_dir, filename, jinja_vars):
-        """
-        Renders a jinja template.
-        Sceptre supports passing sceptre_user_data to JSON and YAML
-        CloudFormation templates using Jinja2 templating.
-        :param template_dir: The directory containing the template.
-        :type template_dir: str
-        :param filename: The name of the template file.
-        :type filename: str
-        :param jinja_vars: Dict of variables to render into the template.
-        :type jinja_vars: dict
-        :returns: The body of the CloudFormation template.
-        :rtype: str
-        """
-        logger = logging.getLogger(__name__)
-        logger.debug("%s Rendering CloudFormation template", filename)
-        default_j2_environment_config = {
-                "autoescape": select_autoescape(
-                    disabled_extensions=('j2',),
-                    default=True,
-                ),
-                "loader": FileSystemLoader(template_dir),
-                "undefined": StrictUndefined,
-            }
-        j2_environment_config = strategies.dict_merge(
-            default_j2_environment_config,
-            self.stack_group_config.get("j2_environment", {}))
-        j2_environment = Environment(**j2_environment_config)
-        template = j2_environment.get_template(filename)
-        body = template.render(**jinja_vars)
-        return body
-
     def _get_handler_of_type(self, type):
         """
         Gets a TemplateHandler type from the registry that can be used to get a string

sceptre/template_handlers/__init__.py

@@ -25,6 +25,9 @@ class TemplateHandler:
 
     :param connection_manager: Connection manager used to call AWS
     :type connection_manager: sceptre.connection_manager.ConnectionManager
+
+    :param stack_group_config: The Stack group config to use as defaults.
+    :type stack_group_config: dict
     """
 
     __metaclass__ = abc.ABCMeta
@@ -35,13 +38,17 @@ class TemplateHandler:
     supported_template_extensions = standard_template_extensions + \
         jinja_template_extensions + python_template_extensions
 
-    def __init__(self, name, arguments=None, sceptre_user_data=None, connection_manager=None):
+    def __init__(self, name, arguments=None, sceptre_user_data=None, connection_manager=None, stack_group_config=None):
         self.logger = logging.getLogger(__name__)
         self.name = name
         self.arguments = arguments
         self.sceptre_user_data = sceptre_user_data
         self.connection_manager = connection_manager
 
+        if stack_group_config is None:
+            stack_group_config = {}
+        self.stack_group_config = stack_group_config
+
     @abc.abstractmethod
     def schema(self):
         """

sceptre/template_handlers/file.py

@@ -35,7 +35,8 @@ def handle(self):
                 return helper.render_jinja_template(
                     os.path.dirname(path),
                     os.path.basename(path),
-                    {"sceptre_user_data": self.sceptre_user_data}
+                    {"sceptre_user_data": self.sceptre_user_data},
+                    self.stack_group_config.get("j2_environment", {})
                 )
             elif file_extension == ".py":
                 return helper.call_sceptre_handler(path,

sceptre/template_handlers/helper.py

@@ -6,6 +6,7 @@
 from importlib.machinery import SourceFileLoader
 from jinja2 import Environment, select_autoescape, FileSystemLoader, StrictUndefined
 from sceptre.exceptions import TemplateSceptreHandlerError
+from sceptre.config import strategies
 
 logger = logging.getLogger(__name__)
 
@@ -103,7 +104,7 @@ def _print_frame(filename, line, fcn, line_text):
         )
 
 
-def render_jinja_template(template_dir, filename, jinja_vars):
+def render_jinja_template(template_dir, filename, jinja_vars, j2_environment):
     """
     Renders a jinja template.
 
@@ -116,18 +117,28 @@ def render_jinja_template(template_dir, filename, jinja_vars):
     :type filename: str
     :param jinja_vars: Dict of variables to render into the template.
     :type jinja_vars: dict
+    :param j2_environment: The jinja2 environment.
+    :type stack_group_config: dict
+
     :returns: The body of the CloudFormation template.
     :rtype: str
     """
     logger.debug("%s Rendering CloudFormation template", filename)
-    env = Environment(
-        autoescape=select_autoescape(
+
+    default_j2_environment_config = {
+        "autoescape": select_autoescape(
             disabled_extensions=('j2',),
             default=True,
         ),
-        loader=FileSystemLoader(template_dir),
-        undefined=StrictUndefined
+        "loader": FileSystemLoader(template_dir),
+        "undefined": StrictUndefined
+    }
+    j2_environment_config = strategies.dict_merge(
+        default_j2_environment_config, j2_environment
     )
-    template = env.get_template(filename)
+    j2_environment = Environment(**j2_environment_config)
+
+    template = j2_environment.get_template(filename)
+
     body = template.render(**jinja_vars)
     return body

sceptre/template_handlers/http.py

@@ -55,7 +55,8 @@ def handle(self):
                         template = helper.render_jinja_template(
                             os.path.dirname(f.name),
                             os.path.basename(f.name),
-                            {"sceptre_user_data": self.sceptre_user_data}
+                            {"sceptre_user_data": self.sceptre_user_data},
+                            self.stack_group_config.get("j2_environment", {})
                         )
                     elif path.suffix in self.python_template_extensions:
                         template = helper.call_sceptre_handler(

sceptre/template_handlers/s3.py

@@ -59,7 +59,8 @@ def handle(self):
                         template = helper.render_jinja_template(
                             os.path.dirname(f.name),
                             os.path.basename(f.name),
-                            {"sceptre_user_data": self.sceptre_user_data}
+                            {"sceptre_user_data": self.sceptre_user_data},
+                            self.stack_group_config.get("j2_environment", {})
                         )
                     elif path.suffix in python_template_suffix:
                         template = helper.call_sceptre_handler(

tests/test_template.py

@@ -379,95 +379,3 @@ def test_template_handler_is_called(self):
 
         result = self.template.body
         assert result == "---\n" + str(sentinel.template_handler_argument)
-
-
-@pytest.mark.parametrize("filename,sceptre_user_data,expected", [
-    (
-        "vpc.j2",
-        {"vpc_id": "10.0.0.0/16"},
-        """Resources:
-  VPC:
-    Type: AWS::EC2::VPC
-    Properties:
-      CidrBlock: 10.0.0.0/16
-Outputs:
-  VpcId:
-    Value:
-      Ref: VPC"""
-    ),
-    (
-        "vpc.yaml.j2",
-        {"vpc_id": "10.0.0.0/16"},
-        """Resources:
-  VPC:
-    Type: AWS::EC2::VPC
-    Properties:
-      CidrBlock: 10.0.0.0/16
-Outputs:
-  VpcId:
-    Value:
-      Ref: VPC"""
-    ),
-    (
-        "sg.j2",
-        [
-            {"name": "sg_a", "inbound_ip": "10.0.0.0"},
-            {"name": "sg_b", "inbound_ip": "10.0.0.1"}
-        ],
-        """Resources:
-    sg_a:
-        Type: "AWS::EC2::SecurityGroup"
-        Properties:
-            InboundIp: 10.0.0.0
-    sg_b:
-        Type: "AWS::EC2::SecurityGroup"
-        Properties:
-            InboundIp: 10.0.0.1
-"""
-    )
-])
-def test_render_jinja_template(filename, sceptre_user_data, expected):
-    jinja_template_dir = os.path.join(
-        os.getcwd(),
-        "tests/fixtures/templates"
-    )
-    handler_config = {"type": "file", "path": filename}
-    template = Template(name="template_name",
-                        handler_config=handler_config,
-                        sceptre_user_data=sceptre_user_data,
-                        stack_group_config={})
-    result = template._render_jinja_template(
-        template_dir=jinja_template_dir,
-        filename=filename,
-        jinja_vars={"sceptre_user_data": sceptre_user_data}
-    )
-    expected_yaml = yaml.safe_load(expected)
-    result_yaml = yaml.safe_load(result)
-    assert expected_yaml == result_yaml
-
-
-@pytest.mark.parametrize("stack_group_config,expected_keys", [
-    ({}, ["autoescape", "loader", "undefined"]),
-    ({"j2_environment": {"lstrip_blocks": True}}, ["autoescape", "loader", "undefined", "lstrip_blocks"]),
-    ({"j2_environment": {"lstrip_blocks": True, "extensions": [
-     "test-ext"]}}, ["autoescape", "loader", "undefined", "lstrip_blocks", "extensions"])
-])
-@patch("sceptre.template.Environment")
-def test_render_jinja_template_j2_environment_config(mock_environment, stack_group_config, expected_keys):
-    filename = "vpc.j2"
-    sceptre_user_data = {"vpc_id": "10.0.0.0/16"}
-    handler_config = {"type": "file", "path": filename}
-    template = Template(name="template_name",
-                        handler_config=handler_config,
-                        sceptre_user_data=sceptre_user_data,
-                        stack_group_config=stack_group_config)
-    jinja_template_dir = os.path.join(
-        os.getcwd(),
-        "tests/fixtures/templates"
-    )
-    template._render_jinja_template(
-        template_dir=jinja_template_dir,
-        filename=filename,
-        jinja_vars={"sceptre_user_data": sceptre_user_data}
-    )
-    assert list(mock_environment.call_args.kwargs) == expected_keys

tests/test_template_handlers/test_file.py

@@ -1,8 +1,10 @@
 import os
+import sys
 import pytest
 import yaml
 
 import sceptre.template_handlers.helper as helper
+from unittest.mock import patch
 
 
 @pytest.mark.parametrize("filename,sceptre_user_data,expected", [
@@ -58,8 +60,34 @@ def test_render_jinja_template(filename, sceptre_user_data, expected):
     result = helper.render_jinja_template(
         template_dir=jinja_template_dir,
         filename=filename,
-        jinja_vars={"sceptre_user_data": sceptre_user_data}
+        jinja_vars={"sceptre_user_data": sceptre_user_data},
+        j2_environment={}
     )
     expected_yaml = yaml.safe_load(expected)
     result_yaml = yaml.safe_load(result)
     assert expected_yaml == result_yaml
+
+
+@pytest.mark.skipif(sys.version_info < (3, 8), reason="requires Python >= 3.8")
+@pytest.mark.parametrize("j2_environment,expected_keys", [
+    ({}, ["autoescape", "loader", "undefined"]),
+    ({"lstrip_blocks": True},
+     ["autoescape", "loader", "undefined", "lstrip_blocks"]),
+    ({"lstrip_blocks": True, "extensions": ["test-ext"]},
+     ["autoescape", "loader", "undefined", "lstrip_blocks", "extensions"])
+])
+@patch("sceptre.template_handlers.helper.Environment")
+def test_render_jinja_template_j2_environment_config(mock_environment, j2_environment, expected_keys):
+    filename = "vpc.j2"
+    sceptre_user_data = {"vpc_id": "10.0.0.0/16"}
+    jinja_template_dir = os.path.join(
+        os.getcwd(),
+        "tests/fixtures/templates"
+    )
+    _ = helper.render_jinja_template(
+        template_dir=jinja_template_dir,
+        filename=filename,
+        jinja_vars={"sceptre_user_data": sceptre_user_data},
+        j2_environment=j2_environment
+    )
+    assert list(mock_environment.call_args.kwargs) == expected_keys

tests/test_template_handlers/test_http.py

@@ -38,7 +38,7 @@ def test_handler_raw_template(self, mock_get_template, url):
 
     @patch('sceptre.template_handlers.helper.render_jinja_template')
     @patch('sceptre.template_handlers.http.Http._get_template')
-    def test_handler_jinja_template(slef, mock_get_template, mock_render_jinja_template):
+    def test_handler_jinja_template(self, mock_get_template, mock_render_jinja_template):
         mock_get_template_response = {
             "Description": "test template",
             "AWSTemplateFormatVersion": "2010-09-09",