Python: implement async support for templates (#6486)

### Motivation and Context

<!-- Thank you for your contribution to the semantic-kernel repo!
Please help reviewers and future users, providing the following
information:
  1. Why is this change required?
  2. What problem does it solve?
  3. What scenario does it contribute to?
  4. If it fixes an open issue, please link to the issue here.
-->

Jinja2 templates now use async functions instead of nested ones!
With full testing

### Description

<!-- Describe your changes, the overall approach, the underlying design.
These notes will help understanding how your code works. Thanks! -->

### Contribution Checklist

<!-- Before submitting this PR, please make sure: -->

- [x] The code builds clean without any errors or warnings
- [x] The PR follows the [SK Contribution
Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md)
and the [pre-submission formatting
script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts)
raises no violations
- [x] All unit tests pass, and I have added new tests where possible
- [x] I didn't break anyone 😄

Author: eavanvalkenburg

python/semantic_kernel/prompt_template/jinja2_prompt_template.py

@@ -9,12 +9,12 @@
 from pydantic import PrivateAttr, field_validator
 
 from semantic_kernel.exceptions import Jinja2TemplateRenderException
-from semantic_kernel.exceptions.template_engine_exceptions import TemplateRenderException
 from semantic_kernel.functions.kernel_arguments import KernelArguments
 from semantic_kernel.prompt_template.const import JINJA2_TEMPLATE_FORMAT_NAME
 from semantic_kernel.prompt_template.prompt_template_base import PromptTemplateBase
 from semantic_kernel.prompt_template.prompt_template_config import PromptTemplateConfig
-from semantic_kernel.prompt_template.utils import JINJA2_SYSTEM_HELPERS, create_template_helper_from_function
+from semantic_kernel.prompt_template.utils import JINJA2_SYSTEM_HELPERS
+from semantic_kernel.prompt_template.utils.template_function_helpers import create_template_helper_from_function
 
 if TYPE_CHECKING:
     from semantic_kernel.kernel import Kernel
@@ -63,7 +63,7 @@ def model_post_init(self, _: Any) -> None:
         if not self.prompt_template_config.template:
             self._env = None
             return
-        self._env = ImmutableSandboxedEnvironment(loader=BaseLoader())
+        self._env = ImmutableSandboxedEnvironment(loader=BaseLoader(), enable_async=True)
 
     async def render(self, kernel: "Kernel", arguments: Optional["KernelArguments"] = None) -> str:
         """Render the prompt template.
@@ -97,16 +97,16 @@ async def render(self, kernel: "Kernel", arguments: Optional["KernelArguments"]
                         arguments,
                         self.prompt_template_config.template_format,
                         allow_unsafe_function_output,
+                        enable_async=True,
                     )
                     for function in plugin
                 }
             )
+        if self.prompt_template_config.template is None:
+            raise Jinja2TemplateRenderException("Error rendering template, template is None")
         try:
-            if self.prompt_template_config.template is None:
-                raise TemplateRenderException("Template is None")
             template = self._env.from_string(self.prompt_template_config.template, globals=helpers)
-            return template.render(**arguments)
-
+            return await template.render_async(**arguments)
         except TemplateError as exc:
             logger.error(
                 f"Error rendering prompt template: {self.prompt_template_config.template} with arguments: {arguments}"

python/semantic_kernel/prompt_template/utils/template_function_helpers.py

@@ -29,6 +29,50 @@ def create_template_helper_from_function(
     base_arguments: "KernelArguments",
     template_format: TEMPLATE_FORMAT_TYPES,
     allow_dangerously_set_content: bool = False,
+    enable_async: bool = False,
+) -> Callable[..., Any]:
+    """Create a helper function for both the Handlebars and Jinja2 templating engines from a kernel function.
+
+    Args:
+        function (KernelFunction): The kernel function to create a helper for.
+        kernel (Kernel): The kernel to use for invoking the function.
+        base_arguments (KernelArguments): The base arguments to use when invoking the function.
+        template_format (TEMPLATE_FORMAT_TYPES): The template format to create the helper for.
+        allow_dangerously_set_content (bool, optional): Return the content of the function result
+            without encoding it or not.
+        enable_async (bool, optional): Enable async helper function. Defaults to False.
+            Currently only works for Jinja2 templates.
+
+    Returns:
+        The function with args that are callable by the different templates.
+
+    Raises:
+        ValueError: If the template format is not supported.
+
+    """
+    if enable_async:
+        return _create_async_template_helper_from_function(
+            function=function,
+            kernel=kernel,
+            base_arguments=base_arguments,
+            template_format=template_format,
+            allow_dangerously_set_content=allow_dangerously_set_content,
+        )
+    return _create_sync_template_helper_from_function(
+        function=function,
+        kernel=kernel,
+        base_arguments=base_arguments,
+        template_format=template_format,
+        allow_dangerously_set_content=allow_dangerously_set_content,
+    )
+
+
+def _create_sync_template_helper_from_function(
+    function: "KernelFunction",
+    kernel: "Kernel",
+    base_arguments: "KernelArguments",
+    template_format: TEMPLATE_FORMAT_TYPES,
+    allow_dangerously_set_content: bool = False,
 ) -> Callable[..., Any]:
     """Create a helper function for both the Handlebars and Jinja2 templating engines from a kernel function."""
     if template_format not in [JINJA2_TEMPLATE_FORMAT_NAME, HANDLEBARS_TEMPLATE_FORMAT_NAME]:
@@ -67,3 +111,32 @@ def func(*args, **kwargs):
         return escape(str(result))
 
     return func
+
+
+def _create_async_template_helper_from_function(
+    function: "KernelFunction",
+    kernel: "Kernel",
+    base_arguments: "KernelArguments",
+    template_format: TEMPLATE_FORMAT_TYPES,
+    allow_dangerously_set_content: bool = False,
+) -> Callable[..., Any]:
+    """Create a async helper function for Jinja2 templating engines from a kernel function."""
+    if template_format not in [JINJA2_TEMPLATE_FORMAT_NAME]:
+        raise ValueError(f"Invalid template format: {template_format}")
+
+    async def func(*args, **kwargs):
+        arguments = KernelArguments()
+        if base_arguments and base_arguments.execution_settings:
+            arguments.execution_settings = base_arguments.execution_settings  # pragma: no cover
+        arguments.update(base_arguments)
+        arguments.update(kwargs)
+        logger.debug(
+            f"Invoking function {function.metadata.fully_qualified_name} "
+            f"with args: {arguments} and kwargs: {kwargs}."
+        )
+        result = await function.invoke(kernel=kernel, arguments=arguments)
+        if allow_dangerously_set_content:
+            return result
+        return escape(str(result))
+
+    return func

python/tests/unit/prompt_template/test_jinja2_prompt_template.py

@@ -92,6 +92,15 @@ async def test_it_renders_fail(kernel: Kernel):
         await target.render(kernel, KernelArguments())
 
 
+@pytest.mark.asyncio
+async def test_it_renders_fail_empty_template(kernel: Kernel):
+    template = "{{ plug-func 'test1'}}"
+    target = create_jinja2_prompt_template(template)
+    target.prompt_template_config.template = None
+    with pytest.raises(Jinja2TemplateRenderException):
+        await target.render(kernel, KernelArguments())
+
+
 @pytest.mark.asyncio
 async def test_it_renders_list(kernel: Kernel):
     template = "List: {% for item in items %}{{ item }}{% endfor %}"

python/tests/unit/prompt_template/test_template_helper.py

@@ -0,0 +1,54 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+import pytest
+
+from semantic_kernel.functions import kernel_function
+from semantic_kernel.functions.kernel_function_from_method import KernelFunctionFromMethod
+from semantic_kernel.kernel import Kernel
+from semantic_kernel.prompt_template.const import JINJA2_TEMPLATE_FORMAT_NAME
+from semantic_kernel.prompt_template.utils.template_function_helpers import create_template_helper_from_function
+
+
+def test_create_helpers(kernel: Kernel):
+    # Arrange
+    function = KernelFunctionFromMethod(kernel_function(lambda x: x + 1, name="test"), plugin_name="test")
+    base_arguments = {}
+    template_format = JINJA2_TEMPLATE_FORMAT_NAME
+    allow_dangerously_set_content = False
+    enable_async = False
+
+    # Act
+    result = create_template_helper_from_function(
+        function, kernel, base_arguments, template_format, allow_dangerously_set_content, enable_async
+    )
+
+    # Assert
+    assert int(str(result(x=1))) == 2
+
+
+@pytest.mark.parametrize(
+    "template_format, enable_async, exception",
+    [
+        ("jinja2", True, False),
+        ("jinja2", False, False),
+        ("handlebars", True, True),
+        ("handlebars", False, False),
+        ("semantic-kernel", False, True),
+        ("semantic-kernel", True, True),
+    ],
+)
+@pytest.mark.asyncio
+async def test_create_helpers_fail(kernel: Kernel, template_format: str, enable_async: bool, exception: bool):
+    # Arrange
+    function = KernelFunctionFromMethod(kernel_function(lambda x: x + 1, name="test"), plugin_name="test")
+
+    if exception:
+        with pytest.raises(ValueError):
+            create_template_helper_from_function(function, kernel, {}, template_format, False, enable_async)
+        return
+    result = create_template_helper_from_function(function, kernel, {}, template_format, False, enable_async)
+    if enable_async:
+        res = await result(x=1)
+        assert int(str(res)) == 2
+    else:
+        assert int(str(result(x=1))) == 2