Write rendered template to a temp file (#1321)

In the event that the reader class is unable to parse the rendered config, this adds logic to write the rendered config to a file to assist the caller to understand what went wrong.

Also, in the event that the reader is unable to even render the template due to Jinja syntax errors or missing vars, this also adds logic to write the template_vars to a similar file.

sceptre/cli/helpers.py

@@ -12,6 +12,7 @@
 from botocore.exceptions import BotoCoreError, ClientError
 from jinja2.exceptions import TemplateError
 
+from sceptre.helpers import logging_level
 from sceptre.exceptions import SceptreException
 from sceptre.stack_status import StackStatus
 from sceptre.stack_status_colourer import StackStatusColourer
@@ -28,10 +29,6 @@ def catch_exceptions(func):
     :returns: The decorated function.
     """
 
-    def logging_level():
-        logger = logging.getLogger(__name__)
-        return logger.getEffectiveLevel()
-
     @wraps(func)
     def decorated(*args, **kwargs):
         """

sceptre/config/reader.py

@@ -14,6 +14,8 @@
 import logging
 import sys
 import yaml
+import json
+import tempfile
 
 from os import environ, path, walk
 from typing import Set, Tuple
@@ -32,7 +34,7 @@
 from sceptre.exceptions import InvalidSceptreDirectoryError
 from sceptre.exceptions import VersionIncompatibleError
 from sceptre.exceptions import ConfigFileNotFoundError
-from sceptre.helpers import sceptreise_path
+from sceptre.helpers import sceptreise_path, logging_level
 from sceptre.stack import Stack
 from sceptre.config import strategies
 
@@ -405,6 +407,21 @@ def _recursive_read(
 
         return config
 
+    def _write_debug_file(self, content: str, prefix: str) -> str:
+        """
+        Write some content to a temp file for debug purposes.
+
+        :param content: the file content to write.
+        :returns: the full path to the temp file.
+        """
+        with tempfile.NamedTemporaryFile(
+            mode="w", delete=False, prefix=prefix
+        ) as temp_file:
+            temp_file.write(content)
+            temp_file.flush()
+
+        return temp_file.name
+
     def _render(self, directory_path, basename, stack_group_config):
         """
         Reads a configuration file, loads the config file as a template
@@ -421,49 +438,64 @@ def _render(self, directory_path, basename, stack_group_config):
         """
         config = {}
         abs_directory_path = path.join(self.full_config_path, directory_path)
-        if path.isfile(path.join(abs_directory_path, basename)):
-            default_j2_environment_config = {
-                "autoescape": select_autoescape(
-                    disabled_extensions=("yaml",),
-                    default=True,
-                ),
-                "loader": FileSystemLoader(abs_directory_path),
-                "undefined": StrictUndefined,
-            }
-            j2_environment_config = strategies.dict_merge(
-                default_j2_environment_config,
-                stack_group_config.get("j2_environment", {}),
+
+        if not path.isfile(path.join(abs_directory_path, basename)):
+            return
+
+        default_j2_environment_config = {
+            "autoescape": select_autoescape(
+                disabled_extensions=("yaml",),
+                default=True,
+            ),
+            "loader": FileSystemLoader(abs_directory_path),
+            "undefined": StrictUndefined,
+        }
+        j2_environment_config = strategies.dict_merge(
+            default_j2_environment_config,
+            stack_group_config.get("j2_environment", {}),
+        )
+        j2_environment = Environment(**j2_environment_config)
+
+        try:
+            template = j2_environment.get_template(basename)
+        except Exception as err:
+            raise SceptreException(
+                f"{Path(directory_path, basename).as_posix()} - {err}"
+            ) from err
+
+        self.templating_vars.update(stack_group_config)
+
+        try:
+            rendered_template = template.render(
+                self.templating_vars,
+                command_path=self.context.command_path.split(path.sep),
+                environment_variable=environ,
             )
-            j2_environment = Environment(**j2_environment_config)
-
-            try:
-                template = j2_environment.get_template(basename)
-            except Exception as err:
-                raise SceptreException(
-                    f"{Path(directory_path, basename).as_posix()} - {err}"
-                ) from err
-
-            self.templating_vars.update(stack_group_config)
-
-            try:
-                rendered_template = template.render(
-                    self.templating_vars,
-                    command_path=self.context.command_path.split(path.sep),
-                    environment_variable=environ,
+        except Exception as err:
+            message = f"{Path(directory_path, basename).as_posix()} - {err}"
+
+            if logging_level() == logging.DEBUG:
+                debug_file_path = self._write_debug_file(
+                    json.dumps(self.templating_vars), prefix="vars_"
                 )
-            except Exception as err:
-                raise SceptreException(
-                    f"{Path(directory_path, basename).as_posix()} - {err}"
-                ) from err
-
-            try:
-                config = yaml.safe_load(rendered_template)
-            except Exception as err:
-                raise ValueError(
-                    "Error parsing {}:\n{}".format(abs_directory_path, err)
+                message += f"\nTemplating vars saved to: {debug_file_path}"
+
+            raise SceptreException(message) from err
+
+        try:
+            config = yaml.safe_load(rendered_template)
+        except Exception as err:
+            message = f"Error parsing {abs_directory_path}{basename}:\n{err}"
+
+            if logging_level() == logging.DEBUG:
+                debug_file_path = self._write_debug_file(
+                    rendered_template, prefix="rendered_"
                 )
+                message += f"\nRendered template saved to: {debug_file_path}"
 
-            return config
+            raise ValueError(message)
+
+        return config
 
     @staticmethod
     def _check_valid_project_path(config_path):

sceptre/helpers.py

@@ -6,11 +6,20 @@
 
 import dateutil.parser
 import deprecation
+import logging
 
 from sceptre.exceptions import PathConversionError
 from sceptre import __version__
 
 
+def logging_level():
+    """
+    Return the logging level.
+    """
+    logger = logging.getLogger(__name__)
+    return logger.getEffectiveLevel()
+
+
 def get_external_stack_name(project_code, stack_name):
     """
     Returns the name given to a stack in CloudFormation.

tests/test_config_reader.py

@@ -6,17 +6,21 @@
 
 import pytest
 import yaml
+
 from click.testing import CliRunner
 from freezegun import freeze_time
+from glob import glob
 
 from sceptre.config.reader import ConfigReader
 from sceptre.context import SceptreContext
+
 from sceptre.exceptions import (
     DependencyDoesNotExistError,
     VersionIncompatibleError,
     ConfigFileNotFoundError,
     InvalidSceptreDirectoryError,
     InvalidConfigFileError,
+    SceptreException,
 )
 
 
@@ -581,3 +585,85 @@ def test_resolve_node_tag(self):
         new_node = config_reader.resolve_node_tag(mock_loader, mock_node)
 
         assert new_node.tag == "new_tag"
+
+    def test_render__missing_config_file__returns_none(self):
+        config_reader = ConfigReader(self.context)
+        directory_path = "configs"
+        basename = "missing_config.yaml"
+        stack_group_config = {}
+
+        result = config_reader._render(directory_path, basename, stack_group_config)
+        assert result is None
+
+    def test_render__existing_config_file__returns_dict(self):
+        with self.runner.isolated_filesystem():
+            project_path = os.path.abspath("./example")
+            config_dir = os.path.join(project_path, "config")
+            directory_path = os.path.join(config_dir, "configs")
+
+            os.makedirs(directory_path)
+
+            basename = "existing_config.yaml"
+            stack_group_config = {}
+
+            test_config_path = os.path.join(directory_path, basename)
+            test_config_content = "key: value"
+
+            with open(test_config_path, "w") as file:
+                file.write(test_config_content)
+
+            self.context.project_path = project_path
+            config_reader = ConfigReader(self.context)
+
+            result = config_reader._render("configs", basename, stack_group_config)
+
+            assert result == {"key": "value"}
+
+    def test_render__invalid_jinja_template__raises_and_creates_debug_file(self):
+        with self.runner.isolated_filesystem():
+            project_path = os.path.abspath("./example")
+            config_dir = os.path.join(project_path, "config")
+            directory_path = os.path.join(config_dir, "configs")
+
+            os.makedirs(directory_path)
+
+            basename = "invalid_jinja.yaml"
+            stack_group_config = {}
+
+            test_config_path = os.path.join(directory_path, basename)
+            test_config_content = "key: {{ invalid_var }}"
+
+            with open(test_config_path, "w") as file:
+                file.write(test_config_content)
+
+            self.context.project_path = project_path
+            config_reader = ConfigReader(self.context)
+
+            pattern = f"{os.path.join('configs', basename)} - .*"
+            with pytest.raises(SceptreException, match=pattern):
+                config_reader._render("configs", basename, stack_group_config)
+                assert len(glob("/tmp/vars_*")) == 1
+
+    def test_render_invalid_yaml__raises_and_creates_debug_file(self):
+        with self.runner.isolated_filesystem():
+            project_path = os.path.abspath("./example")
+            config_dir = os.path.join(project_path, "config")
+            directory_path = os.path.join(config_dir, "configs")
+
+            os.makedirs(directory_path)
+
+            basename = "invalid_yaml.yaml"
+            stack_group_config = {}
+
+            test_config_path = os.path.join(directory_path, basename)
+            test_config_content = "{ key: value"
+
+            with open(test_config_path, "w") as file:
+                file.write(test_config_content)
+
+            self.context.project_path = project_path
+            config_reader = ConfigReader(self.context)
+
+            with pytest.raises(ValueError, match="Error parsing .*"):
+                config_reader._render("configs", basename, stack_group_config)
+                assert len(glob("/tmp/rendered_*")) == 1