diff --git a/.moban.cd/changelog.yml b/.moban.cd/changelog.yml index 57b09f8d..4634570c 100644 --- a/.moban.cd/changelog.yml +++ b/.moban.cd/changelog.yml @@ -1,6 +1,12 @@ name: moban organisation: moremoban releases: + - changes: + - action: Added + details: + - "`#313`: Non-textual source files should default to copy" + date: tbd + version: 0.7.8 - changes: - action: Added details: diff --git a/.moban.d/moban_readme.jj2 b/.moban.d/moban_readme.jj2 index 1f6aa649..30d880eb 100644 --- a/.moban.d/moban_readme.jj2 +++ b/.moban.d/moban_readme.jj2 @@ -201,7 +201,7 @@ installed, Can I write my own template engine? -------------------------------------- -Yes and please click `here `_ for more details. +Yes and please check for `more details `_. Given the following template type function, and saved in custom-plugin dir: @@ -321,7 +321,7 @@ organisation. Here is the primary use case of moban, as of now: Usage beyond command line ============================= -All use cases are documented `here `_ +All use cases are `documented `_ Support ================================================================================ diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e2cae7bc..6c0c6f1f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,14 @@ Change log ================================================================================ +0.7.8 - tbd +-------------------------------------------------------------------------------- + +**Added** + +#. `#313 `_: Non-textual source + files should default to copy + 0.7.7 - 24.5.2020 -------------------------------------------------------------------------------- diff --git a/README.rst b/README.rst index 97cfa8b5..7b5f9615 100644 --- a/README.rst +++ b/README.rst @@ -316,7 +316,7 @@ And given the following velocity.template: Can I write my own template engine? -------------------------------------- -Yes and please click `here `_ for more details. +Yes and please check for `more details `_. Given the following template type function, and saved in custom-plugin dir: @@ -532,7 +532,7 @@ organisation. Here is the primary use case of moban, as of now: Usage beyond command line ============================= -All use cases are documented `here `_ +All use cases are `documented `_ Support ================================================================================ diff --git a/docs/extension.rst b/docs/extension.rst index db85e5f8..8fe54bd8 100644 --- a/docs/extension.rst +++ b/docs/extension.rst @@ -102,7 +102,10 @@ is an example starting point for any template engine. class Engine(object): def __init__(self, template_fs, options=None): """ - A list template directories will be given to your engine class + an instance of fs.multifs.MultiFS will be given. + + :param fs.multifs.MultiFS template_fs: a MultiFS instance or a FS instance + :param dict options: a dictionary containing environmental parameters """ def get_template(self, template_file): @@ -111,6 +114,11 @@ is an example starting point for any template engine. the templating function in next function below """ + def get_template_from_string(self, string): + """ + Sometimes, user would pass on command line string as template + """ + def apply_template(self, template, data, output): """ Given the template object from `get_template` function, and data as python dictionary, @@ -122,6 +130,10 @@ After you will have finished the engine plugin, you can either place it in `plug in order to get it loaded, or make an installable python package. In the latter case, please refer to `yehua`_: doing that in less than 5 minutes. +When the template engine failed to obtain the template, i.e. UnicodeEncodingError, +TemplateSyntaxError, your engine extension shall raise `moban.exceptions.PassOn` +exception, and `moban` would replace your template engine with default engine. + Custom content processors for Moban ---------------------------------------- diff --git a/moban/core/moban_factory.py b/moban/core/moban_factory.py index f56b65cf..6413c022 100644 --- a/moban/core/moban_factory.py +++ b/moban/core/moban_factory.py @@ -12,6 +12,7 @@ from moban.core.context import Context from moban.core.strategy import Strategy from moban.core.hashstore import HASH_STORE +from moban.core.definitions import TemplateTarget from moban.externals.buffered_writer import BufferedWriter LOG = logging.getLogger(__name__) @@ -100,6 +101,7 @@ def __init__(self, template_fs, context_dirs, engine): "ACTION_IN_PAST_TENSE", constants.LABEL_MOBAN_ACTION_IN_PAST_TENSE, ) + self.fall_out_targets = [] def report(self): if self.templated_count == 0: @@ -198,46 +200,79 @@ def render_to_files(self, array_of_template_targets): def _render_with_finding_template_first(self, template_file_index): for (template_file, data_output_pairs) in template_file_index.items(): - template = self.engine.get_template(template_file) - template_abs_path = self.template_fs.geturl( - template_file, purpose="fs" - ) - for (data_file, output) in data_output_pairs: - data = self.context.get_data(data_file) - flag = self.apply_template( - template_abs_path, template, data, output + try: + template = self.engine.get_template(template_file) + template_abs_path = self.template_fs.geturl( + template_file, purpose="fs" ) - if flag: - reporter.report_templating( - self.engine_action, template_file, output + for (data_file, output) in data_output_pairs: + data = self.context.get_data(data_file) + flag = self.apply_template( + template_abs_path, template, data, output + ) + if flag: + reporter.report_templating( + self.engine_action, template_file, output + ) + self.templated_count += 1 + self.file_count += 1 + except exceptions.PassOn: + for (data_file, output) in data_output_pairs: + self.fall_out_targets.append( + TemplateTarget( + template_file, + data_file, + output, + template_type=constants.TEMPLATE_COPY, + ) + ) + reporter.report_info_message( + f"{self.engine_action} is switched to copy:" + + f" {template_file} to {output}" ) - self.templated_count += 1 - self.file_count += 1 + continue def _render_with_finding_data_first(self, data_file_index): for (data_file, template_output_pairs) in data_file_index.items(): data = self.context.get_data(data_file) for (template_file, output) in template_output_pairs: - template = self.engine.get_template(template_file) - if isinstance(template, bool): - if template: - reporter.report_templating( - self.engine_action, template_file, None + try: + template = self.engine.get_template(template_file) + + if isinstance(template, bool): + if template: + reporter.report_templating( + self.engine_action, template_file, None + ) + self.templated_count += 1 + else: + template_abs_path = self.template_fs.geturl( + template_file, purpose="fs" + ) + flag = self.apply_template( + template_abs_path, template, data, output + ) + if flag: + reporter.report_templating( + self.engine_action, template_file, output + ) + self.templated_count += 1 + self.file_count += 1 + except exceptions.PassOn: + self.fall_out_targets.append( + TemplateTarget( + template_file, + data_file, + output, + template_type=constants.TEMPLATE_COPY, ) - self.templated_count += 1 - else: - template_abs_path = self.template_fs.geturl( - template_file, purpose="fs" ) - flag = self.apply_template( - template_abs_path, template, data, output + + reporter.report_info_message( + f"{self.engine_action} is switched to copy:" + + f" {template_file} to {output}" ) - if flag: - reporter.report_templating( - self.engine_action, template_file, output - ) - self.templated_count += 1 - self.file_count += 1 + continue def expand_template_directories(dirs): diff --git a/moban/core/mobanfile/__init__.py b/moban/core/mobanfile/__init__.py index ae7e5cbf..31c23f5c 100644 --- a/moban/core/mobanfile/__init__.py +++ b/moban/core/mobanfile/__init__.py @@ -115,6 +115,7 @@ def handle_targets(merged_options, targets): jobs_for_each_engine[primary_template_type].append(target) count = 0 + fall_out_targets = [] for template_type in jobs_for_each_engine.keys(): engine = core.ENGINES.get_engine( template_type, @@ -124,4 +125,15 @@ def handle_targets(merged_options, targets): engine.render_to_files(jobs_for_each_engine[template_type]) engine.report() count = count + engine.number_of_templated_files() + fall_out_targets += engine.fall_out_targets + + if fall_out_targets: + copy_engine = core.ENGINES.get_engine( + constants.TEMPLATE_COPY, + merged_options[constants.LABEL_TMPL_DIRS], + merged_options[constants.LABEL_CONFIG_DIR], + ) + copy_engine.render_to_files(fall_out_targets) + copy_engine.report() + count = count + copy_engine.number_of_templated_files() return count diff --git a/moban/exceptions.py b/moban/exceptions.py index fcb96e60..80b21441 100644 --- a/moban/exceptions.py +++ b/moban/exceptions.py @@ -40,3 +40,13 @@ class NoPermissionsNeeded(Exception): class SingleHTTPURLConstraint(Exception): pass + + +class PassOn(Exception): + """ + Raised when template engine cannot do anything with the given template. + + i.e. given a png image :/ + """ + + pass diff --git a/moban/plugins/jinja2/engine.py b/moban/plugins/jinja2/engine.py index b909c39a..28ef6898 100644 --- a/moban/plugins/jinja2/engine.py +++ b/moban/plugins/jinja2/engine.py @@ -8,9 +8,9 @@ from lml.loader import scan_plugins_regex from lml.plugin import PluginInfo, PluginManager from jinja2_fsloader import FSLoader -from jinja2.exceptions import TemplateNotFound +from jinja2.exceptions import TemplateNotFound, TemplateSyntaxError -from moban import constants +from moban import constants, exceptions from moban.externals import file_system JINJA2_LIBRARIES = "^moban_jinja2_.+$" @@ -131,6 +131,8 @@ def get_template(self, template_file): return self.jj2_environment.from_string(content) except fs.errors.ResourceNotFound: return self.jj2_environment.from_string(template_file) + except (UnicodeDecodeError, TemplateSyntaxError) as e: + raise exceptions.PassOn(str(e)) def get_template_from_string(self, string): return Template(string) diff --git a/tests/regression_tests/regr-02-templating-failure-results-in-copy-action/.moban.yml b/tests/regression_tests/regr-02-templating-failure-results-in-copy-action/.moban.yml new file mode 100644 index 00000000..437c6712 --- /dev/null +++ b/tests/regression_tests/regr-02-templating-failure-results-in-copy-action/.moban.yml @@ -0,0 +1,7 @@ +configuration: + template_dir: + - copy-source +targets: + - output: regression-test.png + template: image.png + template_type: jj2 diff --git a/tests/regression_tests/regr-02-templating-failure-results-in-copy-action/copy-source/image.png b/tests/regression_tests/regr-02-templating-failure-results-in-copy-action/copy-source/image.png new file mode 100644 index 00000000..04d75535 Binary files /dev/null and b/tests/regression_tests/regr-02-templating-failure-results-in-copy-action/copy-source/image.png differ diff --git a/tests/test_regression.py b/tests/test_regression.py index 9fefce97..b2c3771d 100644 --- a/tests/test_regression.py +++ b/tests/test_regression.py @@ -24,6 +24,16 @@ def test_coping_binary_file(self): "regression-test.png", ) + def test_default_copy_as_error_handling_behavior(self): + folder = "regr-02-templating-failure-results-in-copy-action" + args = ["moban"] + self._raw_moban( + args, + folder, + fs.path.join("copy-source", "image.png"), + "regression-test.png", + ) + def test_level_7(self): expected = "YWJj\n"