deprecate parse_yamltmpl. read nested jinja2 templates

.gitignore

@@ -131,6 +131,7 @@ dmypy.json
 # Editor backup files (Emacs, vim)
 *~
 *.sw[a-p]
+.DS_Store
 
 # Pycharm IDE files
 .idea/

src/wxflow/__init__.py

@@ -13,8 +13,7 @@
 from .task import Task
 from .template import Template, TemplateConstants
 from .timetools import *
-from .yaml_file import (YAMLFile, dump_as_yaml, parse_j2yaml, parse_yaml,
-                        parse_yamltmpl, save_as_yaml, vanilla_yaml)
+from .yaml_file import YAMLFile, save_as_yaml, dump_as_yaml, vanilla_yaml, parse_yaml, parse_j2yaml
 
 __docformat__ = "restructuredtext"
 __version__ = "0.1.0"

src/wxflow/jinja.py

@@ -1,8 +1,7 @@
-import io
 import os
 import sys
 from pathlib import Path
-from typing import Dict
+from typing import Dict, List
 
 import jinja2
 from markupsafe import Markup
@@ -48,54 +47,46 @@ class Jinja:
     A wrapper around jinja2 to render templates
     """
 
-    def __init__(self, template_path_or_string: str, data: Dict, allow_missing: bool = True):
+    def __init__(self, template_path_or_string: str,
+                 data: Dict,
+                 allow_missing: bool = True,
+                 searchpath: str | List = '/') -> None:
         """
         Description
         -----------
         Given a path to a (jinja2) template and a data object, substitute the
         template file with data.
         Allow for retaining missing or undefined variables.
+        Also provide additional search paths for templates that may be included in the main template
         Parameters
         ----------
         template_path_or_string : str
             Path to the template file or a templated string
         data : dict
             Data to be substituted into the template
+            TODO: make "data" optional so that the user can render the same template with different data
         allow_missing : bool
             If True, allow for missing or undefined variables
+        searchpath: str | list
+            Additional search paths for templates
         """
 
+        self.jinja2_version = jinja2.__version__
+
         self.data = data
         self.undefined = SilentUndefined if allow_missing else jinja2.StrictUndefined
+        self.template_searchpath = searchpath if isinstance(searchpath, list) else [searchpath]
 
         if os.path.isfile(template_path_or_string):
             self.template_type = 'file'
-            self.template_path = Path(template_path_or_string)
+            template_path = Path(template_path_or_string)
+            template_dir = template_path.parent
+            self.template_file = str(template_path.relative_to(template_dir))
+            self.template_searchpath.append(str(template_dir))
         else:
             self.template_type = 'stream'
             self.template_stream = template_path_or_string
 
-    @property
-    def render(self, data: Dict = None) -> str:
-        """
-        Description
-        -----------
-        Render the Jinja2 template with the data
-        Parameters
-        ----------
-        data: dict (optional)
-        Additional data to be used in the template
-        Not implemented yet.  Placed here for future use
-        Returns
-        -------
-        rendered: str
-        Rendered template into text
-        """
-
-        render_map = {'stream': self._render_stream,
-                      'file': self._render_file}
-        return render_map[self.template_type]()
-
     def get_set_env(self, loader: jinja2.BaseLoader, filters: Dict[str, callable] = None) -> jinja2.Environment:
         """
         Description
@@ -112,7 +103,7 @@ def get_set_env(self, loader: jinja2.BaseLoader, filters: Dict[str, callable] =
         to_YMD: convert a datetime object to a YYYYMMDD string
         to_julian: convert a datetime object to a julian day
         to_f90bool: convert a boolean to a fortran boolean
-        getenv: read variable from enviornment if defined, else UNDEFINED
+        getenv: read variable from environment if defined, else UNDEFINED
 
         Parameters
         ----------
@@ -139,17 +130,16 @@ def get_set_env(self, loader: jinja2.BaseLoader, filters: Dict[str, callable] =
         # Add any additional filters
         if filters is not None:
             for filter_name, filter_func in filters.items():
-                env.filters[filter_name] = filter_func
+                env = self.add_filter_to_env(env, filter_name, filter_func)
 
         return env
 
     @staticmethod
-    def add_filter_env(env: jinja2.Environment, filter_name: str, filter_func: callable):
+    def add_filter_to_env(env: jinja2.Environment, filter_name: str, filter_func: callable) -> jinja2.Environment:
         """
         Description
         -----------
         Add a custom filter to the jinja2 environment
-        Not implemented yet.  Placed here for future use
         Parameters
         ----------
         env: jinja2.Environment
@@ -168,57 +158,50 @@ def add_filter_env(env: jinja2.Environment, filter_name: str, filter_func: calla
 
         return env
 
-    def _render_stream(self, filters: Dict[str, callable] = None):
-        loader = jinja2.BaseLoader()
-        env = self.get_set_env(loader, filters)
-        template = env.from_string(self.template_stream)
-        return self._render_template(template)
-
-    def _render_file(self, data: Dict = None, filters: Dict[str, callable] = None):
-        template_dir = self.template_path.parent
-        template_file = self.template_path.relative_to(template_dir)
-
-        loader = jinja2.FileSystemLoader(template_dir)
-        env = self.get_set_env(loader, filters)
-        template = env.get_template(str(template_file))
-        return self._render_template(template)
-
-    def _render_template(self, template: jinja2.Template):
+    @property
+    def render(self) -> str:
         """
         Description
         -----------
-        Render a jinja2 template object
+        Render the Jinja2 template with the data
         Parameters
         ----------
-        template: jinja2.Template
-
+        None
         Returns
         -------
         rendered: str
+        Rendered template into text
         """
-        try:
-            rendered = template.render(**self.data)
-        except jinja2.UndefinedError as ee:
-            raise Exception(f"Undefined variable in Jinja2 template\n{ee}")
 
-        return rendered
+        render_map = {'stream': self._render_stream,
+                      'file': self._render_file}
+        return render_map[self.template_type]()
+
+    def _render_stream(self) -> str:
+        loader = jinja2.BaseLoader()
+        env = self.get_set_env(loader)
+        template = env.from_string(self.template_stream)
+        return self._render_template(template)
+
+    def _render_file(self) -> str:
+        loader = jinja2.FileSystemLoader(self.template_searchpath)
+        env = self.get_set_env(loader)
+        template = env.get_template(self.template_file)
+        return self._render_template(template)
 
-    def _render(self, template_name: str, loader: jinja2.BaseLoader) -> str:
+    def _render_template(self, template: jinja2.Template) -> str:
         """
         Description
         -----------
-        Internal method to render a jinja2 template
+        Render a jinja2 template object
         Parameters
         ----------
-        template_name: str
-        loader: jinja2.BaseLoader
+        template: jinja2.Template
+
         Returns
         -------
         rendered: str
-        rendered template
         """
-        env = jinja2.Environment(loader=loader, undefined=self.undefined)
-        template = env.get_template(template_name)
         try:
             rendered = template.render(**self.data)
         except jinja2.UndefinedError as ee:
@@ -242,6 +225,7 @@ def save(self, output_file: str) -> None:
         with open(output_file, 'wb') as fh:
             fh.write(self.render.encode("utf-8"))
 
+    @property
     def dump(self) -> None:
         """
         Description
@@ -251,5 +235,4 @@ def dump(self) -> None:
         -------
         None
         """
-        io.TextIOWrapper(sys.stdout.buffer,
-                         encoding="utf-8").write(self.render)
+        sys.stdout.write(self.render)

src/wxflow/yaml_file.py

@@ -1,16 +1,16 @@
 import datetime
 import json
 import os
+import sys
 import re
-from typing import Any, Dict
+from typing import Any, Dict, List
 
 import yaml
 
 from .attrdict import AttrDict
 from .jinja import Jinja
-from .template import Template, TemplateConstants
 
-__all__ = ['YAMLFile', 'parse_yaml', 'parse_yamltmpl', 'parse_j2yaml',
+__all__ = ['YAMLFile', 'parse_yaml', 'parse_j2yaml',
            'save_as_yaml', 'dump_as_yaml', 'vanilla_yaml']
 
 
@@ -39,9 +39,11 @@ def __init__(self, path=None, data=None):
     def save(self, target):
         save_as_yaml(self, target)
 
+    @property
     def dump(self):
         return dump_as_yaml(self)
 
+    @property
     def as_dict(self):
         return vanilla_yaml(self)
 
@@ -54,7 +56,7 @@ def save_as_yaml(data, target):
 
 
 def dump_as_yaml(data):
-    return yaml.dump(vanilla_yaml(data),
+    return yaml.dump(vanilla_yaml(data), sys.stdout, encoding='utf-8',
                      width=100000, sort_keys=False)
 
 
@@ -154,57 +156,25 @@ def vanilla_yaml(ctx):
         return ctx
 
 
-def parse_j2yaml(path: str, data: Dict) -> Dict[str, Any]:
+def parse_j2yaml(path: str, data: Dict, searchpath: str | List = '/') -> Dict[str, Any]:
     """
     Description
     -----------
     Load a compound jinja2-templated yaml file and resolve any templated variables.
     The jinja2 templates are first resolved and then the rendered template is parsed as a yaml.
-    Finally, any remaining $( ... ) templates are resolved
 
     Parameters
     ----------
     path : str
-        the path to the yaml file
+        the path to the jinja2 templated yaml file
     data : Dict[str, Any], optional
         the context for jinja2 templating
+    searchpath: str | List
+        additional search paths for included jinja2 templates
     Returns
     -------
     Dict[str, Any]
         the dict configuration
     """
-    jenv = Jinja(path, data)
-    yaml_file = jenv.render
-    yaml_dict = YAMLFile(data=yaml_file)
-    yaml_dict = Template.substitute_structure(
-        yaml_dict, TemplateConstants.DOLLAR_PARENTHESES, data.get)
 
-    # If the input yaml file included other yamls with jinja2 templates, then we need to re-parse the jinja2 templates in them
-    jenv2 = Jinja(json.dumps(yaml_dict, indent=4), data)
-    yaml_file2 = jenv2.render
-    yaml_dict = YAMLFile(data=yaml_file2)
-
-    return yaml_dict
-
-
-def parse_yamltmpl(path: str, data: Dict = None) -> Dict[str, Any]:
-    """
-    Description
-    -----------
-    Load a simple templated yaml file and then resolve any templated variables defined as $( ... )
-    Parameters
-    ----------
-    path : str
-        the path to the yaml file
-    data : Dict[str, Any], optional
-        the context for wxflow.Template templating
-    Returns
-    -------
-    Dict[str, Any]
-        the dict configuration
-    """
-    yaml_dict = YAMLFile(path=path)
-    if data is not None:
-        yaml_dict = Template.substitute_structure(yaml_dict, TemplateConstants.DOLLAR_PARENTHESES, data.get)
-
-    return yaml_dict
+    return YAMLFile(data=Jinja(path, data, searchpath=searchpath).render)

tests/test_jinja.py

@@ -6,6 +6,7 @@
 
 current_date = datetime.now()
 j2tmpl = """Hello {{ name }}! {{ greeting }} It is: {{ current_date | to_isotime }}"""
+j2includetmpl = """I am {{ my_name }}. {% include 'template.j2' %}"""
 
 
 @pytest.fixture
@@ -14,6 +15,10 @@ def create_template(tmp_path):
     with open(file_path, 'w') as fh:
         fh.write(j2tmpl)
 
+    file_path = tmp_path / 'include_template.j2'
+    with open(file_path, 'w') as fh:
+        fh.write(j2includetmpl)
+
 
 def test_render_stream():
     data = {"name": "John"}
@@ -35,3 +40,12 @@ def test_render_file(tmp_path, create_template):
     data = {"name": "Jane", "greeting": "How are you?", "current_date": current_date}
     j = Jinja(str(file_path), data, allow_missing=False)
     assert j.render == f"Hello Jane! How are you? It is: {to_isotime(current_date)}"
+
+
+def test_include(tmp_path, create_template):
+
+    file_path = tmp_path / 'include_template.j2'
+
+    data = {"my_name": "Jill", "name": "Joe", "greeting": "How are you?", "current_date": current_date}
+    j = Jinja(str(file_path), data, allow_missing=False)
+    assert j.render == f"I am Jill. Hello Joe! How are you? It is: {to_isotime(current_date)}"