diff --git a/.azure-pipelines-steps-macos.yml b/.azure-pipelines-steps-macos.yml index 9611b057..81734822 100644 --- a/.azure-pipelines-steps-macos.yml +++ b/.azure-pipelines-steps-macos.yml @@ -3,6 +3,7 @@ steps: displayName: 'Use Python 3.x' - script: | python -m pip install --upgrade pip setuptools wheel + test ! -f rnd_requirements.txt || pip install -r rnd_requirements.txt pip install -r requirements.txt pip install -r tests/requirements.txt displayName: 'Setup dependencies' diff --git a/.azure-pipelines-steps.yml b/.azure-pipelines-steps.yml index f30f38df..4fc4925e 100644 --- a/.azure-pipelines-steps.yml +++ b/.azure-pipelines-steps.yml @@ -3,6 +3,7 @@ steps: displayName: 'Use Python 3.x' - script: | python -m pip install --upgrade pip setuptools wheel + if exist rnd_requirements.txt pip install -r rnd_requirements.txt pip install -r requirements.txt pip install -r tests\requirements.txt displayName: 'Setup dependencies' diff --git a/.moban.cd/changelog.yml b/.moban.cd/changelog.yml index a8b54c02..2f28e686 100644 --- a/.moban.cd/changelog.yml +++ b/.moban.cd/changelog.yml @@ -1,6 +1,13 @@ name: moban organisation: moremoban releases: +- changes: + - action: Updated + details: + - "`#90`: allow adding extra jinja2 extensions. `jinja2.ext.do`, `jinja2.ext.loopcontrols` are included by default. what's more, any other template enigne are eligible for extension additions." + - "`#158`: Empty file base_engine.py is finally removed" + date: 18-1-2019 + version: 0.3.9 - changes: - action: Updated details: diff --git a/.moban.cd/moban.yml b/.moban.cd/moban.yml index e354076a..e049536f 100644 --- a/.moban.cd/moban.yml +++ b/.moban.cd/moban.yml @@ -3,14 +3,14 @@ organisation: moremoban author: C. W. contact: wangc_2011@hotmail.com license: MIT -version: 0.3.8 -current_version: 0.3.8 -release: 0.3.8 +version: 0.3.9 +current_version: 0.3.9 +release: 0.3.9 branch: master command_line_interface: "moban" entry_point: "moban.main:main" company: Onni Software Ltd. -copyright_year: 2017-2018 +copyright_year: 2017-2019 keywords: - jinja2 - moban diff --git a/.travis.yml b/.travis.yml index 16330f5a..8295c536 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,7 @@ python: - 3.7-dev - 3.6 - 3.5 + - 3.4 - 2.7 before_install: - if [[ $TRAVIS_PYTHON_VERSION == "2.6" ]]; then pip install flake8==2.6.2; fi diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0b9b467e..ac728d82 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,19 @@ Change log ================================================================================ +0.3.9 - 18-1-2019 +-------------------------------------------------------------------------------- + +Updated +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +#. `#90 `_: allow adding extra + jinja2 extensions. `jinja2.ext.do`, `jinja2.ext.loopcontrols` are included by + default. what's more, any other template enigne are eligible for extension + additions. +#. `#158 `_: Empty file + base_engine.py is finally removed + 0.3.8 - 12-1-2019 -------------------------------------------------------------------------------- diff --git a/docs/conf.py b/docs/conf.py index ce753fd0..2049721a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -24,13 +24,13 @@ # -- Project information ----------------------------------------------------- project = u'moban' -copyright = u'2017-2018 Onni Software Ltd.' +copyright = u'2017-2019 Onni Software Ltd.' author = u'C. W.' # The short X.Y version -version = u'0.3.8' +version = u'0.3.9' # The full version, including alpha/beta/rc tags -release = u'0.3.8' +release = u'0.3.9' # -- General configuration --------------------------------------------------- diff --git a/docs/level-12-use-template-engine-extensions/.moban.yml b/docs/level-12-use-template-engine-extensions/.moban.yml new file mode 100644 index 00000000..63b4bc64 --- /dev/null +++ b/docs/level-12-use-template-engine-extensions/.moban.yml @@ -0,0 +1,6 @@ +targets: + - a.output: a.template + - b.output: b.template +extensions: + jinja2: + - jinja2.ext.with_ diff --git a/docs/level-12-use-template-engine-extensions/README.rst b/docs/level-12-use-template-engine-extensions/README.rst new file mode 100644 index 00000000..911a7266 --- /dev/null +++ b/docs/level-12-use-template-engine-extensions/README.rst @@ -0,0 +1,36 @@ +Level 12: use template engine extensions +================================================================================ + +jinja2 comes with a lot of extensions. In order not to be the blocker in the +middle, **extensions** is allowed in moban file to initialize jinja2 engine +with desired extensions. Two extensions, expression-statement and loop-controls +are enabled by default. + +The extensions syntax is:: + + extensions: + template_type: + - template.engine.specific.extension + +For example:: + + extensions: + jinja2: + - jinja2.ext.i18n + +Please also note that the following extensions are included by default: +`jinja2.ext.do`, `jinja2.ext.loopcontrols` + + +Evaluation +-------------------------------------------------------------------------------- +Please go to `docs/level-12-use-template-engine-extensions` directory. + +If you notice the file `a.template`, we are using a for loop control. This is +because moban comes with two default extensions loop-controls and +expression-statement. + +Now, let us try to use the extension `with`. To do that, we have to enable the +extension in the `.moban.yml` file following the above syntax. Now, the +extension can be used in the jinja2 templates. One such example is shown in the +`b.template` file. diff --git a/docs/level-12-use-template-engine-extensions/a.template b/docs/level-12-use-template-engine-extensions/a.template new file mode 100644 index 00000000..da43df97 --- /dev/null +++ b/docs/level-12-use-template-engine-extensions/a.template @@ -0,0 +1,3 @@ +{% for _ in range(1,5) %} +{{ hello }} +{% endfor %} diff --git a/docs/level-12-use-template-engine-extensions/b.template b/docs/level-12-use-template-engine-extensions/b.template new file mode 100644 index 00000000..72937b14 --- /dev/null +++ b/docs/level-12-use-template-engine-extensions/b.template @@ -0,0 +1,9 @@ +{% with %} + {% set foo = 142 %} +{{ foo }} + {% with %} + {% set foo = 42 %} +{{ foo }} + {% endwith %} +{{ foo }} +{% endwith %} diff --git a/docs/level-12-use-template-engine-extensions/data.yml b/docs/level-12-use-template-engine-extensions/data.yml new file mode 100644 index 00000000..bb56b055 --- /dev/null +++ b/docs/level-12-use-template-engine-extensions/data.yml @@ -0,0 +1 @@ +hello: world diff --git a/moban/_version.py b/moban/_version.py index 846cb583..bef32dc0 100644 --- a/moban/_version.py +++ b/moban/_version.py @@ -1,2 +1,2 @@ -__version__ = "0.3.8" +__version__ = "0.3.9" __author__ = "C. W." diff --git a/moban/base_engine.py b/moban/base_engine.py deleted file mode 100644 index e69de29b..00000000 diff --git a/moban/constants.py b/moban/constants.py index a667e874..b17eeaca 100644 --- a/moban/constants.py +++ b/moban/constants.py @@ -23,15 +23,13 @@ LABEL_TEMPLATE = "template" POSITIONAL_LABEL_TEMPLATE = "template_in_string" LABEL_TMPL_DIRS = "template_dir" +LABEL_EXIT_CODE = "exit-code" LABEL_OUTPUT = "output" LABEL_TEMPLATE_TYPE = "template_type" -LABEL_TARGETS = "targets" -LABEL_COPY = "copy" LABEL_OVERRIDES = "overrides" LABEL_MOBANFILE = "mobanfile" LABEL_FORCE = "force" -LABEL_REQUIRES = "requires" -LABEL_EXIT_CODE = "exit-code" + DEFAULT_CONFIGURATION_DIRNAME = ".moban.cd" DEFAULT_TEMPLATE_DIRNAME = ".moban.td" @@ -52,6 +50,11 @@ DEFAULT_MOBAN_VERSION = "1.0" MOBAN_DIR_NAME_UNDER_USER_HOME = ".moban" MOBAN_REPOS_DIR_NAME = "repos" +# moban file configuration +LABEL_REQUIRES = "requires" +LABEL_TARGETS = "targets" +LABEL_COPY = "copy" +LABEL_EXTENSIONS = "extensions" # error messages ERROR_DATA_FILE_NOT_FOUND = "Both %s and %s does not exist" diff --git a/moban/jinja2/engine.py b/moban/jinja2/engine.py index 10a59726..1244a816 100644 --- a/moban/jinja2/engine.py +++ b/moban/jinja2/engine.py @@ -12,6 +12,7 @@ "moban.jinja2.filters.text", "moban.jinja2.tests.files", ] +JINJA2_THIRD_PARTY_EXTENSIONS = ["jinja2.ext.do", "jinja2.ext.loopcontrols"] class PluginMixin: @@ -50,7 +51,7 @@ def __init__(self): constants.TEMPLATE_ENGINE_EXTENSION, tags=["jinja2", "jinja", "jj2", "j2"] ) class Engine(object): - def __init__(self, template_dirs): + def __init__(self, template_dirs, extensions=None): """ Contruct a jinja2 template engine @@ -61,12 +62,19 @@ def __init__(self, template_dirs): load_jinja2_extensions() self.template_dirs = template_dirs template_loader = FileSystemLoader(template_dirs) - self.jj2_environment = Environment( + env_params = dict( loader=template_loader, keep_trailing_newline=True, trim_blocks=True, lstrip_blocks=True, + extensions=[ + extension for extension in JINJA2_THIRD_PARTY_EXTENSIONS + ], # get a copy of this global variable ) + if is_extension_list_valid(extensions): + # because it is modified here + env_params["extensions"] += extensions + self.jj2_environment = Environment(**env_params) for filter_name, filter_function in FILTERS.get_all(): self.jj2_environment.filters[filter_name] = filter_function @@ -120,3 +128,11 @@ def apply_template(self, template, data, output): def load_jinja2_extensions(): scan_plugins_regex(JINJA2_LIBRARIES, "moban", None, JINJA2_EXENSIONS) + + +def is_extension_list_valid(extensions): + return ( + extensions is not None + and isinstance(extensions, list) + and len(extensions) > 0 + ) diff --git a/moban/mobanfile.py b/moban/mobanfile.py index 1223db29..cb43b03e 100644 --- a/moban/mobanfile.py +++ b/moban/mobanfile.py @@ -68,6 +68,10 @@ def handle_moban_file_v1(moban_file_configurations, command_line_options): if requires: handle_requires(requires) + extensions = moban_file_configurations.get(constants.LABEL_EXTENSIONS) + if extensions: + plugins.ENGINES.register_extensions(extensions) + if targets: # If template specified via CLI flag `-t: # 1. Only update the specified template diff --git a/moban/plugins.py b/moban/plugins.py index 5ba36cee..13488cc5 100644 --- a/moban/plugins.py +++ b/moban/plugins.py @@ -21,7 +21,9 @@ def resource_path_of(self, library_name): class BaseEngine(object): - def __init__(self, template_dirs, context_dirs, engine_cls): + def __init__( + self, template_dirs, context_dirs, engine_cls, engine_extensions=None + ): # pypi-moban-pkg cannot be found if removed make_sure_all_pkg_are_loaded() template_dirs = list(expand_template_directories(template_dirs)) @@ -29,7 +31,7 @@ def __init__(self, template_dirs, context_dirs, engine_cls): context_dirs = expand_template_directory(context_dirs) self.context = Context(context_dirs) self.template_dirs = template_dirs - self.engine = engine_cls(self.template_dirs) + self.engine = engine_cls(self.template_dirs, engine_extensions) self.engine_cls = engine_cls self.templated_count = 0 self.file_count = 0 @@ -143,10 +145,17 @@ def __init__(self): super(EngineFactory, self).__init__( constants.TEMPLATE_ENGINE_EXTENSION ) + self.extensions = {} + + def register_extensions(self, extensions): + self.extensions.update(extensions) def get_engine(self, template_type, template_dirs, context_dirs): engine_cls = self.load_me_now(template_type) - return BaseEngine(template_dirs, context_dirs, engine_cls) + engine_extensions = self.extensions.get(template_type) + return BaseEngine( + template_dirs, context_dirs, engine_cls, engine_extensions + ) def all_types(self): return list(self.registry.keys()) diff --git a/rnd_requirements.txt b/rnd_requirements.txt new file mode 100644 index 00000000..86b965ae --- /dev/null +++ b/rnd_requirements.txt @@ -0,0 +1 @@ +https://github.com/moremoban/moban-handlebars/archive/dev.zip diff --git a/setup.py b/setup.py index decfd6ff..4967639a 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ NAME = 'moban' AUTHOR = 'C. W.' -VERSION = '0.3.8' +VERSION = '0.3.9' EMAIL = 'wangc_2011@hotmail.com' LICENSE = 'MIT' ENTRY_POINTS = { @@ -25,7 +25,7 @@ 'Yet another jinja2 cli command for static text generation' ) URL = 'https://github.com/moremoban/moban' -DOWNLOAD_URL = '%s/archive/0.3.8.tar.gz' % URL +DOWNLOAD_URL = '%s/archive/0.3.9.tar.gz' % URL FILES = ['README.rst', 'CONTRIBUTORS.rst', 'CHANGELOG.rst'] KEYWORDS = [ 'python', @@ -60,8 +60,8 @@ # You do not need to read beyond this line PUBLISH_COMMAND = '{0} setup.py sdist bdist_wheel upload -r pypi'.format( sys.executable) -GS_COMMAND = ('gs moban v0.3.8 ' + - "Find 0.3.8 in changelog for more details") +GS_COMMAND = ('gs moban v0.3.9 ' + + "Find 0.3.9 in changelog for more details") NO_GS_MESSAGE = ('Automatic github release is disabled. ' + 'Please install gease to enable it.') UPLOAD_FAILED_MSG = ( diff --git a/tests/test_docs.py b/tests/test_docs.py index 1a4470b4..99c5bad1 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -107,6 +107,23 @@ def test_level_11(self): folder = "level-11-use-handlebars" self._raw_moban(["moban"], folder, expected, "a.output") + def test_level_12_a(self): + expected_a = """world +world +world +world +""" + folder = "level-12-use-template-engine-extensions" + self._raw_moban(["moban"], folder, expected_a, "a.output") + + def test_level_12_b(self): + expected_b = """142 +42 +142 +""" + folder = "level-12-use-template-engine-extensions" + self._raw_moban(["moban"], folder, expected_b, "b.output") + def test_misc_1(self): expected = "test file\n" diff --git a/tests/test_engine.py b/tests/test_engine.py index 9ce98862..8674c950 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -11,7 +11,7 @@ BaseEngine, expand_template_directories, ) -from moban.jinja2.engine import Engine +from moban.jinja2.engine import Engine, is_extension_list_valid USER_HOME = os.path.join("user", "home", ".moban", "repos") @@ -44,7 +44,7 @@ def test_default_template_type(): class FakeEngine: - def __init__(self, template_dirs): + def __init__(self, template_dirs, extensions=None): pass @@ -77,7 +77,7 @@ def test_non_existent_ctx_directries(): def test_file_tests(): output = "test.txt" path = os.path.join("tests", "fixtures", "jinja_tests") - engine = BaseEngine([path], path, Engine) + engine = ENGINES.get_engine("jinja2", [path], path) engine.render_to_file("file_tests.template", "file_tests.yml", output) with open(output, "r") as output_file: content = output_file.read() @@ -88,7 +88,7 @@ def test_file_tests(): def test_global_template_variables(): output = "test.txt" path = os.path.join("tests", "fixtures", "globals") - engine = BaseEngine([path], path, Engine) + engine = ENGINES.get_engine("jinja2", [path], path) engine.render_to_file("variables.template", "variables.yml", output) with open(output, "r") as output_file: content = output_file.read() @@ -99,7 +99,7 @@ def test_global_template_variables(): def test_nested_global_template_variables(): output = "test.txt" path = os.path.join("tests", "fixtures", "globals") - engine = BaseEngine([path], path, Engine) + engine = ENGINES.get_engine("jinja2", [path], path) engine.render_to_file("nested.template", "variables.yml", output) with open(output, "r") as output_file: content = output_file.read() @@ -113,7 +113,7 @@ def test_environ_variables_as_data(): os.environ[test_var] = test_value output = "test.txt" path = os.path.join("tests", "fixtures", "environ_vars_as_data") - engine = BaseEngine([path], path, Engine) + engine = ENGINES.get_engine("jinja2", [path], path) engine.render_to_file("test.template", "this_does_not_exist.yml", output) with open(output, "r") as output_file: content = output_file.read() @@ -124,9 +124,19 @@ def test_environ_variables_as_data(): def test_string_template(): output = "test.txt" path = os.path.join("tests", "fixtures") - engine = BaseEngine([path], path, Engine) + engine = ENGINES.get_engine("jinja2", [path], path) engine.render_string_to_file("{{simple}}", "simple.yaml", output) with open(output, "r") as output_file: content = output_file.read() eq_(content, "yaml") os.unlink(output) + + +def test_extensions_validator(): + test_fixtures = [None, ["module1", "module2"], []] + expected = [False, True, False] + actual = [] + for fixture in test_fixtures: + actual.append(is_extension_list_valid(fixture)) + + eq_(expected, actual) diff --git a/tests/test_template.py b/tests/test_template.py index d5d5ed45..0829f330 100644 --- a/tests/test_template.py +++ b/tests/test_template.py @@ -2,8 +2,7 @@ from mock import patch -from moban.plugins import BaseEngine -from moban.jinja2.engine import Engine +from moban.plugins import ENGINES @patch("moban.plugins.BaseEngine._render_with_finding_data_first") @@ -24,7 +23,7 @@ def test_do_templates_1(_do_templates_with_more_shared_data): ("5.template", "6.output"), ] } - engine = BaseEngine(".", ".", Engine) + engine = ENGINES.get_engine("jinja2", ".", ".") engine.render_to_files(jobs) _do_templates_with_more_shared_data.assert_called_with(expected) if os.path.exists(".moban.hashes"): @@ -49,7 +48,7 @@ def test_do_templates_2(_do_templates_with_more_shared_templates): ("data5.yml", "6.output"), ] } - engine = BaseEngine(".", ".", Engine) + engine = ENGINES.get_engine("jinja2", ".", ".") engine.render_to_files(jobs) _do_templates_with_more_shared_templates.assert_called_with(expected) if os.path.exists(".moban.hashes"): @@ -58,7 +57,9 @@ def test_do_templates_2(_do_templates_with_more_shared_templates): def test_do_templates_with_more_shared_templates(): base_dir = os.path.join("tests", "fixtures") - engine = BaseEngine(base_dir, os.path.join(base_dir, "config"), Engine) + engine = ENGINES.get_engine( + "jinja2", base_dir, os.path.join(base_dir, "config") + ) engine._render_with_finding_template_first( {"a.jj2": [(os.path.join(base_dir, "child.yaml"), "test")]} ) @@ -72,7 +73,9 @@ def test_do_templates_with_more_shared_templates(): def test_do_templates_with_more_shared_data(): base_dir = os.path.join("tests", "fixtures") - engine = BaseEngine(base_dir, os.path.join(base_dir, "config"), Engine) + engine = ENGINES.get_engine( + "jinja2", base_dir, os.path.join(base_dir, "config") + ) engine._render_with_finding_data_first( {os.path.join(base_dir, "child.yaml"): [("a.jj2", "test")]} ) diff --git a/tests/test_utils.py b/tests/test_utils.py index 56ed331d..609629df 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -45,9 +45,9 @@ def test_file_permission_copy(): def file_permissions_disabled_on_windows(): - if sys.platform == 'win32': - permissions = file_permissions('abc') - eq_('no-permission-support', permissions) + if sys.platform == "win32": + permissions = file_permissions("abc") + eq_("no-permission-support", permissions) else: raise SkipTest("No test required")