From 7fc02031d16f4ad290983263f82060c3ff6026c6 Mon Sep 17 00:00:00 2001 From: Ayan Banerjee Date: Sat, 27 Apr 2019 20:03:50 +0530 Subject: [PATCH 1/3] :hammer: Ignore error if occurs Closes #265 --- moban/plugins/template.py | 3 ++- moban/utils.py | 6 +++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/moban/plugins/template.py b/moban/plugins/template.py index 33851e37..8aa86aa0 100644 --- a/moban/plugins/template.py +++ b/moban/plugins/template.py @@ -110,7 +110,8 @@ def apply_template(self, template_abs_path, template, data, output_file): template, data, output_file ) rendered_content = utils.strip_off_trailing_new_lines(rendered_content) - rendered_content = rendered_content.encode("utf-8") + if not isinstance(rendered_content, bytes): + rendered_content = rendered_content.encode("utf-8") try: flag = HASH_STORE.is_file_changed( output_file, rendered_content, template_abs_path diff --git a/moban/utils.py b/moban/utils.py index e99a6ebd..00b6cdeb 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -62,7 +62,11 @@ def file_permissions(afile): def strip_off_trailing_new_lines(content): if isinstance(content, bytes): - content = content.decode("utf-8") + try: + content = content.decode("utf-8") + return re.sub(r"(\n\s+)+$", r"\n", content) + except UnicodeDecodeError: + return content return re.sub(r"(\n\s+)+$", r"\n", content) From 0d9e7ee40ecceeb78191fdf0d0777b184c690278 Mon Sep 17 00:00:00 2001 From: Ayan Banerjee Date: Sun, 28 Apr 2019 16:30:41 +0530 Subject: [PATCH 2/3] :microscope: Add tests for encoding --- tests/fixtures/non-unicode.char | 1 + tests/test_copy_encoding.py | 4 ++++ 2 files changed, 5 insertions(+) create mode 100644 tests/fixtures/non-unicode.char diff --git a/tests/fixtures/non-unicode.char b/tests/fixtures/non-unicode.char new file mode 100644 index 00000000..f488f849 --- /dev/null +++ b/tests/fixtures/non-unicode.char @@ -0,0 +1 @@ +� \ No newline at end of file diff --git a/tests/test_copy_encoding.py b/tests/test_copy_encoding.py index 67cb96bb..344ec3a1 100644 --- a/tests/test_copy_encoding.py +++ b/tests/test_copy_encoding.py @@ -15,3 +15,7 @@ def test_encoding_of_template(self): with open("tests/fixtures/coala_color.svg", "r") as expected: expected = expected.read() eq_(expected, template_content.decode("utf-8").replace("\r", "")) + template_content = self.engine.get_template("non-unicode.char") + with open("tests/fixtures/non-unicode.char", "rb") as expected: + expected = expected.read() + eq_(expected, template_content) From 98c85e4c0f758747bcad9cd46e9aea01860b36b4 Mon Sep 17 00:00:00 2001 From: Ayan Banerjee Date: Sun, 28 Apr 2019 16:39:45 +0530 Subject: [PATCH 3/3] :microscope: Add tests for encoding --- tests/fixtures/non-unicode.char | 2 +- tests/test_copy_encoding.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/fixtures/non-unicode.char b/tests/fixtures/non-unicode.char index f488f849..25796857 100644 --- a/tests/fixtures/non-unicode.char +++ b/tests/fixtures/non-unicode.char @@ -1 +1 @@ -� \ No newline at end of file +� diff --git a/tests/test_copy_encoding.py b/tests/test_copy_encoding.py index 344ec3a1..f024a223 100644 --- a/tests/test_copy_encoding.py +++ b/tests/test_copy_encoding.py @@ -12,9 +12,9 @@ def setUp(self): def test_encoding_of_template(self): template_content = self.engine.get_template("coala_color.svg") - with open("tests/fixtures/coala_color.svg", "r") as expected: + with open("tests/fixtures/coala_color.svg", "rb") as expected: expected = expected.read() - eq_(expected, template_content.decode("utf-8").replace("\r", "")) + eq_(expected, template_content) template_content = self.engine.get_template("non-unicode.char") with open("tests/fixtures/non-unicode.char", "rb") as expected: expected = expected.read()