From ef583a337a29fd4689d5c47e491292bff9469376 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Wed, 28 Feb 2024 15:36:52 -0500 Subject: [PATCH] deprecate parse_yamltmpl. read nested jinja2 templates --- .gitignore | 1 + src/wxflow/__init__.py | 3 +- src/wxflow/jinja.py | 107 +++++++++++++++++----------------------- src/wxflow/yaml_file.py | 52 +++++-------------- tests/test_jinja.py | 14 ++++++ tests/test_yaml_file.py | 36 ++------------ 6 files changed, 75 insertions(+), 138 deletions(-) diff --git a/.gitignore b/.gitignore index 03d0620..4286df6 100644 --- a/.gitignore +++ b/.gitignore @@ -131,6 +131,7 @@ dmypy.json # Editor backup files (Emacs, vim) *~ *.sw[a-p] +.DS_Store # Pycharm IDE files .idea/ diff --git a/src/wxflow/__init__.py b/src/wxflow/__init__.py index 692ba92..9a94733 100644 --- a/src/wxflow/__init__.py +++ b/src/wxflow/__init__.py @@ -13,8 +13,7 @@ from .task import Task from .template import Template, TemplateConstants from .timetools import * -from .yaml_file import (YAMLFile, dump_as_yaml, parse_j2yaml, parse_yaml, - parse_yamltmpl, save_as_yaml, vanilla_yaml) +from .yaml_file import YAMLFile, save_as_yaml, dump_as_yaml, vanilla_yaml, parse_yaml, parse_j2yaml __docformat__ = "restructuredtext" __version__ = "0.1.0" diff --git a/src/wxflow/jinja.py b/src/wxflow/jinja.py index f8dc39f..31e22ae 100644 --- a/src/wxflow/jinja.py +++ b/src/wxflow/jinja.py @@ -1,8 +1,7 @@ -import io import os import sys from pathlib import Path -from typing import Dict +from typing import Dict, List import jinja2 from markupsafe import Markup @@ -48,54 +47,46 @@ class Jinja: A wrapper around jinja2 to render templates """ - def __init__(self, template_path_or_string: str, data: Dict, allow_missing: bool = True): + def __init__(self, template_path_or_string: str, + data: Dict, + allow_missing: bool = True, + searchpath: str | List = '/') -> None: """ Description ----------- Given a path to a (jinja2) template and a data object, substitute the template file with data. Allow for retaining missing or undefined variables. + Also provide additional search paths for templates that may be included in the main template Parameters ---------- template_path_or_string : str Path to the template file or a templated string data : dict Data to be substituted into the template + TODO: make "data" optional so that the user can render the same template with different data allow_missing : bool If True, allow for missing or undefined variables + searchpath: str | list + Additional search paths for templates """ + self.jinja2_version = jinja2.__version__ + self.data = data self.undefined = SilentUndefined if allow_missing else jinja2.StrictUndefined + self.template_searchpath = searchpath if isinstance(searchpath, list) else [searchpath] if os.path.isfile(template_path_or_string): self.template_type = 'file' - self.template_path = Path(template_path_or_string) + template_path = Path(template_path_or_string) + template_dir = template_path.parent + self.template_file = str(template_path.relative_to(template_dir)) + self.template_searchpath.append(str(template_dir)) else: self.template_type = 'stream' self.template_stream = template_path_or_string - @property - def render(self, data: Dict = None) -> str: - """ - Description - ----------- - Render the Jinja2 template with the data - Parameters - ---------- - data: dict (optional) - Additional data to be used in the template - Not implemented yet. Placed here for future use - Returns - ------- - rendered: str - Rendered template into text - """ - - render_map = {'stream': self._render_stream, - 'file': self._render_file} - return render_map[self.template_type]() - def get_set_env(self, loader: jinja2.BaseLoader, filters: Dict[str, callable] = None) -> jinja2.Environment: """ Description @@ -112,7 +103,7 @@ def get_set_env(self, loader: jinja2.BaseLoader, filters: Dict[str, callable] = to_YMD: convert a datetime object to a YYYYMMDD string to_julian: convert a datetime object to a julian day to_f90bool: convert a boolean to a fortran boolean - getenv: read variable from enviornment if defined, else UNDEFINED + getenv: read variable from environment if defined, else UNDEFINED Parameters ---------- @@ -139,17 +130,16 @@ def get_set_env(self, loader: jinja2.BaseLoader, filters: Dict[str, callable] = # Add any additional filters if filters is not None: for filter_name, filter_func in filters.items(): - env.filters[filter_name] = filter_func + env = self.add_filter_to_env(env, filter_name, filter_func) return env @staticmethod - def add_filter_env(env: jinja2.Environment, filter_name: str, filter_func: callable): + def add_filter_to_env(env: jinja2.Environment, filter_name: str, filter_func: callable) -> jinja2.Environment: """ Description ----------- Add a custom filter to the jinja2 environment - Not implemented yet. Placed here for future use Parameters ---------- env: jinja2.Environment @@ -168,57 +158,50 @@ def add_filter_env(env: jinja2.Environment, filter_name: str, filter_func: calla return env - def _render_stream(self, filters: Dict[str, callable] = None): - loader = jinja2.BaseLoader() - env = self.get_set_env(loader, filters) - template = env.from_string(self.template_stream) - return self._render_template(template) - - def _render_file(self, data: Dict = None, filters: Dict[str, callable] = None): - template_dir = self.template_path.parent - template_file = self.template_path.relative_to(template_dir) - - loader = jinja2.FileSystemLoader(template_dir) - env = self.get_set_env(loader, filters) - template = env.get_template(str(template_file)) - return self._render_template(template) - - def _render_template(self, template: jinja2.Template): + @property + def render(self) -> str: """ Description ----------- - Render a jinja2 template object + Render the Jinja2 template with the data Parameters ---------- - template: jinja2.Template - + None Returns ------- rendered: str + Rendered template into text """ - try: - rendered = template.render(**self.data) - except jinja2.UndefinedError as ee: - raise Exception(f"Undefined variable in Jinja2 template\n{ee}") - return rendered + render_map = {'stream': self._render_stream, + 'file': self._render_file} + return render_map[self.template_type]() + + def _render_stream(self) -> str: + loader = jinja2.BaseLoader() + env = self.get_set_env(loader) + template = env.from_string(self.template_stream) + return self._render_template(template) + + def _render_file(self) -> str: + loader = jinja2.FileSystemLoader(self.template_searchpath) + env = self.get_set_env(loader) + template = env.get_template(self.template_file) + return self._render_template(template) - def _render(self, template_name: str, loader: jinja2.BaseLoader) -> str: + def _render_template(self, template: jinja2.Template) -> str: """ Description ----------- - Internal method to render a jinja2 template + Render a jinja2 template object Parameters ---------- - template_name: str - loader: jinja2.BaseLoader + template: jinja2.Template + Returns ------- rendered: str - rendered template """ - env = jinja2.Environment(loader=loader, undefined=self.undefined) - template = env.get_template(template_name) try: rendered = template.render(**self.data) except jinja2.UndefinedError as ee: @@ -242,6 +225,7 @@ def save(self, output_file: str) -> None: with open(output_file, 'wb') as fh: fh.write(self.render.encode("utf-8")) + @property def dump(self) -> None: """ Description @@ -251,5 +235,4 @@ def dump(self) -> None: ------- None """ - io.TextIOWrapper(sys.stdout.buffer, - encoding="utf-8").write(self.render) + sys.stdout.write(self.render) diff --git a/src/wxflow/yaml_file.py b/src/wxflow/yaml_file.py index bb601a3..f6acb23 100644 --- a/src/wxflow/yaml_file.py +++ b/src/wxflow/yaml_file.py @@ -1,16 +1,16 @@ import datetime import json import os +import sys import re -from typing import Any, Dict +from typing import Any, Dict, List import yaml from .attrdict import AttrDict from .jinja import Jinja -from .template import Template, TemplateConstants -__all__ = ['YAMLFile', 'parse_yaml', 'parse_yamltmpl', 'parse_j2yaml', +__all__ = ['YAMLFile', 'parse_yaml', 'parse_j2yaml', 'save_as_yaml', 'dump_as_yaml', 'vanilla_yaml'] @@ -39,9 +39,11 @@ def __init__(self, path=None, data=None): def save(self, target): save_as_yaml(self, target) + @property def dump(self): return dump_as_yaml(self) + @property def as_dict(self): return vanilla_yaml(self) @@ -54,7 +56,7 @@ def save_as_yaml(data, target): def dump_as_yaml(data): - return yaml.dump(vanilla_yaml(data), + return yaml.dump(vanilla_yaml(data), sys.stdout, encoding='utf-8', width=100000, sort_keys=False) @@ -154,57 +156,25 @@ def vanilla_yaml(ctx): return ctx -def parse_j2yaml(path: str, data: Dict) -> Dict[str, Any]: +def parse_j2yaml(path: str, data: Dict, searchpath: str | List = '/') -> Dict[str, Any]: """ Description ----------- Load a compound jinja2-templated yaml file and resolve any templated variables. The jinja2 templates are first resolved and then the rendered template is parsed as a yaml. - Finally, any remaining $( ... ) templates are resolved Parameters ---------- path : str - the path to the yaml file + the path to the jinja2 templated yaml file data : Dict[str, Any], optional the context for jinja2 templating + searchpath: str | List + additional search paths for included jinja2 templates Returns ------- Dict[str, Any] the dict configuration """ - jenv = Jinja(path, data) - yaml_file = jenv.render - yaml_dict = YAMLFile(data=yaml_file) - yaml_dict = Template.substitute_structure( - yaml_dict, TemplateConstants.DOLLAR_PARENTHESES, data.get) - # If the input yaml file included other yamls with jinja2 templates, then we need to re-parse the jinja2 templates in them - jenv2 = Jinja(json.dumps(yaml_dict, indent=4), data) - yaml_file2 = jenv2.render - yaml_dict = YAMLFile(data=yaml_file2) - - return yaml_dict - - -def parse_yamltmpl(path: str, data: Dict = None) -> Dict[str, Any]: - """ - Description - ----------- - Load a simple templated yaml file and then resolve any templated variables defined as $( ... ) - Parameters - ---------- - path : str - the path to the yaml file - data : Dict[str, Any], optional - the context for wxflow.Template templating - Returns - ------- - Dict[str, Any] - the dict configuration - """ - yaml_dict = YAMLFile(path=path) - if data is not None: - yaml_dict = Template.substitute_structure(yaml_dict, TemplateConstants.DOLLAR_PARENTHESES, data.get) - - return yaml_dict + return YAMLFile(data=Jinja(path, data, searchpath=searchpath).render) diff --git a/tests/test_jinja.py b/tests/test_jinja.py index 35dd6ca..2ba3751 100644 --- a/tests/test_jinja.py +++ b/tests/test_jinja.py @@ -6,6 +6,7 @@ current_date = datetime.now() j2tmpl = """Hello {{ name }}! {{ greeting }} It is: {{ current_date | to_isotime }}""" +j2includetmpl = """I am {{ my_name }}. {% include 'template.j2' %}""" @pytest.fixture @@ -14,6 +15,10 @@ def create_template(tmp_path): with open(file_path, 'w') as fh: fh.write(j2tmpl) + file_path = tmp_path / 'include_template.j2' + with open(file_path, 'w') as fh: + fh.write(j2includetmpl) + def test_render_stream(): data = {"name": "John"} @@ -35,3 +40,12 @@ def test_render_file(tmp_path, create_template): data = {"name": "Jane", "greeting": "How are you?", "current_date": current_date} j = Jinja(str(file_path), data, allow_missing=False) assert j.render == f"Hello Jane! How are you? It is: {to_isotime(current_date)}" + + +def test_include(tmp_path, create_template): + + file_path = tmp_path / 'include_template.j2' + + data = {"my_name": "Jill", "name": "Joe", "greeting": "How are you?", "current_date": current_date} + j = Jinja(str(file_path), data, allow_missing=False) + assert j.render == f"I am Jill. Hello Joe! How are you? It is: {to_isotime(current_date)}" diff --git a/tests/test_yaml_file.py b/tests/test_yaml_file.py index 9e5b5e2..91b8861 100644 --- a/tests/test_yaml_file.py +++ b/tests/test_yaml_file.py @@ -3,8 +3,7 @@ import pytest -from wxflow import (YAMLFile, dump_as_yaml, parse_j2yaml, parse_yamltmpl, - save_as_yaml) +from wxflow import (YAMLFile, parse_j2yaml, save_as_yaml) host_yaml = """ host: @@ -19,25 +18,14 @@ host_file: !INC ${TMP_PATH}/host.yaml """ -tmpl_yaml = """ -config: - config_file: !ENV ${TMP_PATH}/config.yaml - user: !ENV ${USER} - host_file: !INC ${TMP_PATH}/host.yaml -tmpl: - cdate: '{{PDY}}{{cyc}}' - homedir: /home/$(user) -""" -# Note the quotes ' ' around {{ }}. These quotes are necessary for yaml otherwise yaml will fail parsing - j2tmpl_yaml = """ config: config_file: !ENV ${TMP_PATH}/config.yaml user: !ENV ${USER} host_file: !INC ${TMP_PATH}/host.yaml tmpl: - cdate: '{{ current_cycle | to_YMD }}{{ current_cycle | strftime('%H') }}' - homedir: /home/$(user) + cdate: {{ current_cycle | to_YMD }}{{ current_cycle | strftime('%H') }} + homedir: /home/{{ user }} """ @@ -46,7 +34,6 @@ def create_template(tmpdir): """Create temporary templates for testing""" tmpdir.join('host.yaml').write(host_yaml) tmpdir.join('config.yaml').write(conf_yaml) - tmpdir.join('tmpl.yaml').write(tmpl_yaml) tmpdir.join('j2tmpl.yaml').write(j2tmpl_yaml) @@ -66,23 +53,6 @@ def test_yaml_file(tmp_path, create_template): assert yaml_in == conf -def test_yaml_file_with_templates(tmp_path, create_template): - - # Set env. variable - os.environ['TMP_PATH'] = str(tmp_path) - data = {'user': os.environ['USER']} - conf = parse_yamltmpl(path=str(tmp_path / 'tmpl.yaml'), data=data) - - # Write out yaml file - yaml_out = tmp_path / 'tmpl_output.yaml' - save_as_yaml(conf, yaml_out) - - # Read in the yaml file and compare w/ conf - yaml_in = YAMLFile(path=yaml_out) - - assert yaml_in == conf - - def test_yaml_file_with_j2templates(tmp_path, create_template): # Set env. variable