Jinja2 optimization

docs/level-7-use-custom-jinja2-filter-test-n-global/custom-jj2-plugin/filter.py

@@ -1,7 +1,7 @@
 import sys
 import base64
 
-from moban.extensions import JinjaFilter
+from moban.jinja2.extensions import JinjaFilter
 
 
 @JinjaFilter()

docs/level-7-use-custom-jinja2-filter-test-n-global/custom-jj2-plugin/global.py

@@ -1,3 +1,3 @@
-from moban.extensions import jinja_global
+from moban.jinja2.extensions import jinja_global
 
 jinja_global('global', dict(hello='world'))

docs/level-7-use-custom-jinja2-filter-test-n-global/custom-jj2-plugin/test.py

@@ -1,4 +1,4 @@
-from moban.extensions import JinjaTest
+from moban.jinja2.extensions import JinjaTest
 
 
 @JinjaTest()

moban/engine_handlebars.py

@@ -22,6 +22,5 @@ def get_template(self, template_file):
 
     def apply_template(self, template, data, _):
         rendered_content = "".join(template(data))
-        rendered_content = utils.strip_off_trailing_new_lines(rendered_content)
-        rendered_content = rendered_content.encode("utf-8")
+        rendered_content = rendered_content
         return rendered_content

moban/jinja2/engine.py

@@ -1,9 +1,48 @@
 from jinja2 import Environment, FileSystemLoader
-from lml.plugin import PluginInfo
+from lml.loader import scan_plugins_regex
+from lml.plugin import PluginInfo, PluginManager
 
-import moban.utils as utils
 import moban.constants as constants
-from moban import plugins
+
+JINJA2_LIBRARIES = "^moban_jinja2_.+$"
+JINJA2_EXENSIONS = [
+    "moban.jinja2.filters.repr",
+    "moban.jinja2.filters.github",
+    "moban.jinja2.filters.text",
+    "moban.jinja2.tests.files",
+]
+
+
+class PluginMixin:
+    def get_all(self):
+        for name in self.registry.keys():
+            # only the first matching one is returned
+            the_filter = self.load_me_now(name)
+            yield (name, the_filter)
+
+
+class JinjaFilterManager(PluginManager, PluginMixin):
+    def __init__(self):
+        super(JinjaFilterManager, self).__init__(
+            constants.JINJA_FILTER_EXTENSION
+        )
+
+
+class JinjaTestManager(PluginManager, PluginMixin):
+    def __init__(self):
+        super(JinjaTestManager, self).__init__(constants.JINJA_TEST_EXTENSION)
+
+
+class JinjaGlobalsManager(PluginManager, PluginMixin):
+    def __init__(self):
+        super(JinjaGlobalsManager, self).__init__(
+            constants.JINJA_GLOBALS_EXTENSION
+        )
+
+
+FILTERS = JinjaFilterManager()
+TESTS = JinjaTestManager()
+GLOBALS = JinjaGlobalsManager()
 
 
 @PluginInfo(
@@ -18,6 +57,7 @@ def __init__(self, template_dirs):
 
         :param list temp_dirs: a list of template directories
         """
+        load_jinja2_extensions()
         self.template_dirs = template_dirs
         template_loader = FileSystemLoader(template_dirs)
         self.jj2_environment = Environment(
@@ -26,13 +66,13 @@ def __init__(self, template_dirs):
             trim_blocks=True,
             lstrip_blocks=True,
         )
-        for filter_name, filter_function in plugins.FILTERS.get_all():
+        for filter_name, filter_function in FILTERS.get_all():
             self.jj2_environment.filters[filter_name] = filter_function
 
-        for test_name, test_function in plugins.TESTS.get_all():
+        for test_name, test_function in TESTS.get_all():
             self.jj2_environment.tests[test_name] = test_function
 
-        for global_name, dict_obj in plugins.GLOBALS.get_all():
+        for global_name, dict_obj in GLOBALS.get_all():
             self.jj2_environment.globals[global_name] = dict_obj
 
     def get_template(self, template_file):
@@ -57,7 +97,7 @@ def get_template(self, template_file):
     def apply_template(self, template, data, output):
         """
         It is not expected this function to write content to file system.
-        Please just apply data inside the template and return utf-8 encoded
+        Please just apply data inside the template and return utf-8
         content.
 
         :param template: a jinja2 template from :class:`.get_template`
@@ -67,6 +107,9 @@ def apply_template(self, template, data, output):
         template.globals["__target__"] = output
         template.globals["__template__"] = template.name
         rendered_content = template.render(**data)
-        rendered_content = utils.strip_off_trailing_new_lines(rendered_content)
-        rendered_content = rendered_content.encode("utf-8")
+        rendered_content = rendered_content
         return rendered_content
+
+
+def load_jinja2_extensions():
+    scan_plugins_regex(JINJA2_LIBRARIES, "moban", None, JINJA2_EXENSIONS)

moban/jinja2/filters/github.py

@@ -1,6 +1,6 @@
 import re
 
-from moban.extensions import JinjaFilter
+from moban.jinja2.extensions import JinjaFilter
 
 GITHUB_REF_PATTERN = "`([^`]*?#[0-9]+)`"
 ISSUE = "^.*?" + GITHUB_REF_PATTERN + ".*?$"

moban/jinja2/filters/repr.py

@@ -1,4 +1,4 @@
-from moban.extensions import JinjaFilter
+from moban.jinja2.extensions import JinjaFilter
 
 
 @JinjaFilter()

moban/jinja2/filters/text.py

@@ -1,6 +1,6 @@
 import re
 
-from moban.extensions import JinjaFilter
+from moban.jinja2.extensions import JinjaFilter
 
 
 @JinjaFilter()
@@ -13,7 +13,7 @@ def split_length(input_line, length):
         yield line
     else:
         while True:
-            if " " in line[start : start + limit]:  # flake8: noqa
+            if " " in line[start : start + limit]:  # noqa
                 # go back and find a space
                 while limit > 0 and line[start + limit] != " ":
                     limit -= 1
@@ -25,7 +25,7 @@ def split_length(input_line, length):
                 ] != " ":
                     limit += 1
 
-            yield line[start : start + limit]  # flake8: noqa
+            yield line[start : start + limit]  # noqa
             start = start + limit + 1
             limit = length
             if len(line[start:]) < length or start + limit >= len(line):

moban/jinja2/tests/files.py

@@ -9,7 +9,7 @@
     samefile,
 )
 
-from moban.extensions import jinja_tests
+from moban.jinja2.extensions import jinja_tests
 
 jinja_tests(
     is_dir=isdir,

moban/main.py

@@ -24,11 +24,11 @@ def main():
     """
     program entry point
     """
-    plugins.refresh_plugins()
     parser = create_parser()
     options = vars(parser.parse_args())
     HASH_STORE.IGNORE_CACHE_FILE = options[constants.LABEL_FORCE]
     moban_file = options[constants.LABEL_MOBANFILE]
+    load_engine_factory_and_engines()  # Error: jinja2 if removed
     if moban_file is None:
         moban_file = mobanfile.find_default_moban_file()
     if moban_file:
@@ -147,3 +147,7 @@ def handle_command_line(options):
         engine.number_of_templated_files()
     )
     return exit_code
+
+
+def load_engine_factory_and_engines():
+    plugins.make_sure_all_pkg_are_loaded()

moban/plugins.py

@@ -8,46 +8,7 @@
 from moban.strategy import Strategy
 from moban.hashstore import HASH_STORE
 
-
-class PluginMixin:
-    def get_all(self):
-        for name in self.registry.keys():
-            # only the first matching one is returned
-            the_filter = self.load_me_now(name)
-            yield (name, the_filter)
-
-
-class JinjaFilterManager(PluginManager, PluginMixin):
-    def __init__(self):
-        super(JinjaFilterManager, self).__init__(
-            constants.JINJA_FILTER_EXTENSION
-        )
-
-
-class JinjaTestManager(PluginManager, PluginMixin):
-    def __init__(self):
-        super(JinjaTestManager, self).__init__(constants.JINJA_TEST_EXTENSION)
-
-
-class JinjaGlobalsManager(PluginManager, PluginMixin):
-    def __init__(self):
-        super(JinjaGlobalsManager, self).__init__(
-            constants.JINJA_GLOBALS_EXTENSION
-        )
-
-
-FILTERS = JinjaFilterManager()
-TESTS = JinjaTestManager()
-GLOBALS = JinjaGlobalsManager()
-
-BUILTIN_EXENSIONS = [
-    "moban.jinja2.filters.repr",
-    "moban.jinja2.filters.github",
-    "moban.jinja2.filters.text",
-    "moban.jinja2.tests.files",
-    "moban.jinja2.engine",
-    "moban.engine_handlebars",
-]
+BUILTIN_EXENSIONS = ["moban.jinja2.engine", "moban.engine_handlebars"]
 
 
 class LibraryManager(PluginManager):
@@ -61,7 +22,9 @@ def resource_path_of(self, library_name):
 
 class BaseEngine(object):
     def __init__(self, template_dirs, context_dirs, engine_cls):
-        refresh_plugins()
+        # pypi-moban-pkg cannot be found if removed
+        make_sure_all_pkg_are_loaded()
+
         template_dirs = list(expand_template_directories(template_dirs))
         verify_the_existence_of_directories(template_dirs)
         context_dirs = expand_template_directory(context_dirs)
@@ -100,12 +63,10 @@ def apply_template(self, template_abs_path, template, data, output_file):
             template, data, output_file
         )
         flag = HASH_STORE.is_file_changed(
-            output_file, rendered_content, template_abs_path
+            output_file, rendered_content.encode("utf-8"), template_abs_path
         )
         if flag:
-            utils.write_file_out(
-                output_file, rendered_content, strip=False, encode=False
-            )
+            utils.write_file_out(output_file, rendered_content)
             utils.file_permissions_copy(template_abs_path, output_file)
         return flag
 
@@ -230,7 +191,7 @@ def get_data(self, file_name):
         return data
 
 
-def refresh_plugins():
+def make_sure_all_pkg_are_loaded():
     scan_plugins_regex(constants.MOBAN_ALL, "moban", None, BUILTIN_EXENSIONS)
 
 

tests/test_engine.py

@@ -11,7 +11,6 @@
     BaseEngine,
     expand_template_directories,
 )
-from moban.extensions import jinja_global
 from moban.jinja2.engine import Engine
 from moban.engine_handlebars import EngineHandlebars
 
@@ -95,19 +94,6 @@ def test_handlebars_file_tests():
     os.unlink(output)
 
 
-def test_globals():
-    output = "globals.txt"
-    test_dict = dict(hello="world")
-    jinja_global("test", test_dict)
-    path = os.path.join("tests", "fixtures", "globals")
-    engine = BaseEngine([path], path, Engine)
-    engine.render_to_file("basic.template", "basic.yml", output)
-    with open(output, "r") as output_file:
-        content = output_file.read()
-        eq_(content, "world\n\ntest")
-    os.unlink(output)
-
-
 def test_global_template_variables():
     output = "test.txt"
     path = os.path.join("tests", "fixtures", "globals")

tests/test_handlebar_engine.py

@@ -10,5 +10,5 @@ def test_handlebars_template_not_found():
     template = engine.get_template("file_tests.template")
     data = dict(test="here")
     result = engine.apply_template(template, data, None)
-    expected = "yes\nhere".encode("utf-8")
+    expected = "yes\nhere"
     eq_(expected, result)

tests/test_jinja2_engine.py

@@ -10,5 +10,5 @@ def test_handlebars_template_not_found():
     template = engine.get_template("file_tests.template")
     data = dict(test="here")
     result = engine.apply_template(template, data, None)
-    expected = "here".encode("utf-8")
+    expected = "here"
     eq_(expected, result)

tests/test_jinja2_extensions.py

@@ -0,0 +1,19 @@
+import os
+
+from nose.tools import eq_
+from moban.plugins import BaseEngine
+from moban.jinja2.engine import Engine
+from moban.jinja2.extensions import jinja_global
+
+
+def test_globals():
+    output = "globals.txt"
+    test_dict = dict(hello="world")
+    jinja_global("test", test_dict)
+    path = os.path.join("tests", "fixtures", "globals")
+    engine = BaseEngine([path], path, Engine)
+    engine.render_to_file("basic.template", "basic.yml", output)
+    with open(output, "r") as output_file:
+        content = output_file.read()
+        eq_(content, "world\n\ntest")
+    os.unlink(output)