From 3a0536e9518117ae7d823f7108e3adcfa78a443d Mon Sep 17 00:00:00 2001 From: chfw Date: Thu, 28 Feb 2019 18:29:24 +0000 Subject: [PATCH 01/12] :microscope: let's start the test first. #117 --- tests/regression-pack/.moban.yml | 3 ++ .../symbolic-link-gets-junk/sym-link-to-file | 1 + .../symbolic-link-gets-junk/symbolic-link | 1 + tests/test_regression_pack.py | 51 +++++++++++++++++++ 4 files changed, 56 insertions(+) create mode 100644 tests/regression-pack/.moban.yml create mode 120000 tests/regression-pack/symbolic-link-gets-junk/sym-link-to-file create mode 120000 tests/regression-pack/symbolic-link-gets-junk/symbolic-link create mode 100644 tests/test_regression_pack.py diff --git a/tests/regression-pack/.moban.yml b/tests/regression-pack/.moban.yml new file mode 100644 index 00000000..55876b73 --- /dev/null +++ b/tests/regression-pack/.moban.yml @@ -0,0 +1,3 @@ +targets: + - a.output: symbolic-link + - b.output: sym-link-to-file diff --git a/tests/regression-pack/symbolic-link-gets-junk/sym-link-to-file b/tests/regression-pack/symbolic-link-gets-junk/sym-link-to-file new file mode 120000 index 00000000..ce03d0af --- /dev/null +++ b/tests/regression-pack/symbolic-link-gets-junk/sym-link-to-file @@ -0,0 +1 @@ +/tmp/adobegc.log \ No newline at end of file diff --git a/tests/regression-pack/symbolic-link-gets-junk/symbolic-link b/tests/regression-pack/symbolic-link-gets-junk/symbolic-link new file mode 120000 index 00000000..cad23091 --- /dev/null +++ b/tests/regression-pack/symbolic-link-gets-junk/symbolic-link @@ -0,0 +1 @@ +/tmp \ No newline at end of file diff --git a/tests/test_regression_pack.py b/tests/test_regression_pack.py new file mode 100644 index 00000000..56117bed --- /dev/null +++ b/tests/test_regression_pack.py @@ -0,0 +1,51 @@ +import os +import sys +from textwrap import dedent + +from mock import patch +from nose.tools import eq_, SkipTest + +from moban.main import main + + +def custom_dedent(long_texts): + refined = dedent(long_texts) + if refined.startswith("\n"): + refined = refined[1:] + return refined + + +class TestTutorial: + def setUp(self): + self.current = os.getcwd() + + def test_symbolic_link_on_windows(self): + if sys.platform != 'win32': + raise SkipTest("No need to test on linux nor macos") + + expected = "test file\n" + + folder = "symbolic-link-gets-junk" + self._raw_moban(["moban"], folder, expected, "simple.file") + + def _moban(self, folder, expected): + args = ["moban", "-c", "data.yml", "-t", "a.template"] + self._raw_moban(args, folder, expected, "moban.output") + + def _raw_moban(self, args, folder, expected, output): + os.chdir(os.path.join("tests", "regression-pack", folder)) + with patch.object(sys, "argv", args): + main() + _verify_content(output, expected) + os.unlink(output) + + def tearDown(self): + if os.path.exists(".moban.hashes"): + os.unlink(".moban.hashes") + os.chdir(self.current) + + +def _verify_content(file_name, expected): + with open(file_name, "r") as f: + content = f.read() + eq_(content, expected) From 165a1f2cd42cd2f881de74b06a945c5304479e7a Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 1 Mar 2019 07:11:20 +0000 Subject: [PATCH 02/12] :bug: fix import error --- tests/test_regression_pack.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_regression_pack.py b/tests/test_regression_pack.py index 56117bed..f8edcbb3 100644 --- a/tests/test_regression_pack.py +++ b/tests/test_regression_pack.py @@ -3,7 +3,8 @@ from textwrap import dedent from mock import patch -from nose.tools import eq_, SkipTest +from nose.tools import eq_ +from nose import SkipTest from moban.main import main From 64ede14eb865ee241096302547c5b5a9d71e9478 Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 1 Mar 2019 07:26:17 +0000 Subject: [PATCH 03/12] :microscope: show true failures on windows --- test.bat | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test.bat b/test.bat index 17ae36a6..7f79f8f4 100644 --- a/test.bat +++ b/test.bat @@ -1,4 +1,3 @@ pip freeze -nosetests --with-coverage --cover-package=moban --cover-package=tests -flake8 . --exclude=docs,.moban.d --ignore=E203,E121,E123,E126,E226,E24,E704,W503,W504 +nosetests --with-coverage --cover-package=moban --cover-package=tests && flake8 . --exclude=docs,.moban.d --ignore=E203,E121,E123,E126,E226,E24,E704,W503,W504 From 3d317c2205f3c43c611259bc08a9a477f979bc19 Mon Sep 17 00:00:00 2001 From: chfw Date: Tue, 5 Mar 2019 17:58:13 +0000 Subject: [PATCH 04/12] :bug:fix samefile not available on windows. https://github.com/python-rope/ropevim/pull/66 --- moban/jinja2/tests/files.py | 10 +++++++--- moban/jinja2/tests/win32.py | 5 +++++ 2 files changed, 12 insertions(+), 3 deletions(-) create mode 100644 moban/jinja2/tests/win32.py diff --git a/moban/jinja2/tests/files.py b/moban/jinja2/tests/files.py index 66357dcd..bece18b2 100644 --- a/moban/jinja2/tests/files.py +++ b/moban/jinja2/tests/files.py @@ -1,3 +1,4 @@ +import sys from os.path import ( isabs, isdir, @@ -5,12 +6,15 @@ isfile, islink, ismount, - lexists, - samefile, + lexists ) - from moban.jinja2.extensions import jinja_tests +if sys.platform == "win32": + from moban.jinja2.tests.win32 import samefile +else: + from os.path import samefile + jinja_tests( is_dir=isdir, directory=isdir, diff --git a/moban/jinja2/tests/win32.py b/moban/jinja2/tests/win32.py new file mode 100644 index 00000000..c560cc8f --- /dev/null +++ b/moban/jinja2/tests/win32.py @@ -0,0 +1,5 @@ +from os.path import normcase, normpath + + +def samefile(file1, file2): + return normcase(normpath(file1)) == normcase(normpath(file2)) From 29274e6d292fc43b6d3bc41cea98f8027bcefef2 Mon Sep 17 00:00:00 2001 From: chfw Date: Tue, 5 Mar 2019 17:59:18 +0000 Subject: [PATCH 05/12] :bug: fix windows os path problem --- moban/utils.py | 2 +- tests/mobanfile/test_templates.py | 12 ++++++------ tests/test_copy_engine.py | 3 ++- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/moban/utils.py b/moban/utils.py index 82896458..65330714 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -54,7 +54,7 @@ def file_permissions_copy(source, dest): def file_permissions(afile): if sys.platform == "win32": - return "no-permission-support" + return "0" if not os.path.exists(afile): raise exceptions.FileNotFound(afile) return stat.S_IMODE(os.stat(afile).st_mode) diff --git a/tests/mobanfile/test_templates.py b/tests/mobanfile/test_templates.py index 02479212..469817f6 100644 --- a/tests/mobanfile/test_templates.py +++ b/tests/mobanfile/test_templates.py @@ -35,8 +35,8 @@ def test_listing_dir(self): ) expected = [ ( - "copier-directory/level1-file1", - "/tmp/copy-a-directory/level1-file1", + os.path.join("copier-directory", "level1-file1"), + os.path.join("/tmp/copy-a-directory", "level1-file1"), None, ) ] @@ -49,13 +49,13 @@ def test_listing_dir_recusively(self): ) expected = [ ( - "copier-directory/copier-sample-dir/file1", - "/tmp/copy-a-directory/copier-sample-dir/file1", + os.path.join("copier-directory", "copier-sample-dir", "file1"), + os.path.join("/tmp/copy-a-directory", "copier-sample-dir", "file1"), None, ), ( - "copier-directory/level1-file1", - "/tmp/copy-a-directory/level1-file1", + os.path.join("copier-directory", "level1-file1"), + os.path.join("/tmp/copy-a-directory", "level1-file1"), None, ), ] diff --git a/tests/test_copy_engine.py b/tests/test_copy_engine.py index e0d014b7..55ce5030 100644 --- a/tests/test_copy_engine.py +++ b/tests/test_copy_engine.py @@ -12,7 +12,8 @@ def setUp(self): def test_get_template(self): template_content = self.engine.get_template("copier-test01.csv") - eq_("test 01\n", template_content) + # remove '\r' for windows + eq_("test 01\n", template_content.replace('\r', '')) def test_get_template_from_string(self): test_content = "simply forwarded" From 6361ac4d07fcbf776e235694a082d234a71ae1e8 Mon Sep 17 00:00:00 2001 From: chfw Date: Thu, 7 Mar 2019 20:49:38 +0000 Subject: [PATCH 06/12] :green_heart: make windows ci work --- docs/level-11-use-handlebars/b.output | 4 ---- docs/level-14-custom-data-loader/b.output | 7 ------- docs/level-6-complex-configuration/a.output | 9 --------- .../filter.output | 1 - moban/plugins/template.py | 3 ++- moban/utils.py | 4 ++-- tests/test_hash_store.py | 4 ++++ 7 files changed, 8 insertions(+), 24 deletions(-) delete mode 100644 docs/level-11-use-handlebars/b.output delete mode 100644 docs/level-14-custom-data-loader/b.output delete mode 100644 docs/level-6-complex-configuration/a.output delete mode 100644 docs/level-7-use-custom-jinja2-filter-test-n-global/filter.output diff --git a/docs/level-11-use-handlebars/b.output b/docs/level-11-use-handlebars/b.output deleted file mode 100644 index e91bcf2b..00000000 --- a/docs/level-11-use-handlebars/b.output +++ /dev/null @@ -1,4 +0,0 @@ - -world - -shijie diff --git a/docs/level-14-custom-data-loader/b.output b/docs/level-14-custom-data-loader/b.output deleted file mode 100644 index 48ab2cac..00000000 --- a/docs/level-14-custom-data-loader/b.output +++ /dev/null @@ -1,7 +0,0 @@ -========header============ - -world from override_custom.yaml - -shijie from parent.custom - -========footer============ diff --git a/docs/level-6-complex-configuration/a.output b/docs/level-6-complex-configuration/a.output deleted file mode 100644 index 9649d143..00000000 --- a/docs/level-6-complex-configuration/a.output +++ /dev/null @@ -1,9 +0,0 @@ -========header============ - -world - -shijie - -this demonstrates jinja2's include statement - -========footer============ diff --git a/docs/level-7-use-custom-jinja2-filter-test-n-global/filter.output b/docs/level-7-use-custom-jinja2-filter-test-n-global/filter.output deleted file mode 100644 index ad2667c7..00000000 --- a/docs/level-7-use-custom-jinja2-filter-test-n-global/filter.output +++ /dev/null @@ -1 +0,0 @@ -YWJj diff --git a/moban/plugins/template.py b/moban/plugins/template.py index 0a9f08e1..51405d03 100644 --- a/moban/plugins/template.py +++ b/moban/plugins/template.py @@ -179,8 +179,9 @@ def expand_template_directories(dirs): def expand_template_directory(directory): + log.debug("Expanding %s..." % directory) translated_directory = None - if ":" in directory: + if ":" in directory and directory[1] != ':': library_or_repo_name, relative_path = directory.split(":") potential_repo_path = os.path.join( repo.get_moban_home(), library_or_repo_name diff --git a/moban/utils.py b/moban/utils.py index 65330714..e6fc5072 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -53,10 +53,10 @@ def file_permissions_copy(source, dest): def file_permissions(afile): - if sys.platform == "win32": - return "0" if not os.path.exists(afile): raise exceptions.FileNotFound(afile) + if sys.platform == "win32": + return 0 return stat.S_IMODE(os.stat(afile).st_mode) diff --git a/tests/test_hash_store.py b/tests/test_hash_store.py index 51e43dac..ef329152 100644 --- a/tests/test_hash_store.py +++ b/tests/test_hash_store.py @@ -1,5 +1,7 @@ import os +import sys +from nose import SkipTest from moban.hashstore import HashStore @@ -77,6 +79,8 @@ def test_dest_file_file_permision_changed(self): Save as above, but this time, the generated file had file permision change """ + if sys.platform == "win32": + raise SkipTest("No file permission check on windows") hs = HashStore() flag = hs.is_file_changed(*self.fixture) if flag: From 199a97ead8be8f6a65a51f35dbf3244f4512a56d Mon Sep 17 00:00:00 2001 From: chfw Date: Thu, 7 Mar 2019 20:58:15 +0000 Subject: [PATCH 07/12] :green_heart: final touch on the windows ci failure --- .../regression-pack/{ => symbolic-link-gets-junk}/.moban.yml | 3 +++ tests/test_regression_pack.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) rename tests/regression-pack/{ => symbolic-link-gets-junk}/.moban.yml (63%) diff --git a/tests/regression-pack/.moban.yml b/tests/regression-pack/symbolic-link-gets-junk/.moban.yml similarity index 63% rename from tests/regression-pack/.moban.yml rename to tests/regression-pack/symbolic-link-gets-junk/.moban.yml index 55876b73..8d8613b5 100644 --- a/tests/regression-pack/.moban.yml +++ b/tests/regression-pack/symbolic-link-gets-junk/.moban.yml @@ -1,3 +1,6 @@ +configuration: + template_dir: + - . targets: - a.output: symbolic-link - b.output: sym-link-to-file diff --git a/tests/test_regression_pack.py b/tests/test_regression_pack.py index f8edcbb3..d155be89 100644 --- a/tests/test_regression_pack.py +++ b/tests/test_regression_pack.py @@ -24,10 +24,10 @@ def test_symbolic_link_on_windows(self): if sys.platform != 'win32': raise SkipTest("No need to test on linux nor macos") - expected = "test file\n" + expected = "/tmp" folder = "symbolic-link-gets-junk" - self._raw_moban(["moban"], folder, expected, "simple.file") + self._raw_moban(["moban"], folder, expected, "a.output") def _moban(self, folder, expected): args = ["moban", "-c", "data.yml", "-t", "a.template"] From 58f17b255124f89828eefeccec19fda5651bdbfd Mon Sep 17 00:00:00 2001 From: chfw Date: Thu, 7 Mar 2019 21:01:00 +0000 Subject: [PATCH 08/12] :lipstick: beautify the code using black and isort --- moban/jinja2/tests/files.py | 11 ++--------- moban/plugins/template.py | 2 +- tests/mobanfile/test_templates.py | 4 +++- tests/test_copy_engine.py | 2 +- tests/test_hash_store.py | 1 + tests/test_regression_pack.py | 4 ++-- 6 files changed, 10 insertions(+), 14 deletions(-) diff --git a/moban/jinja2/tests/files.py b/moban/jinja2/tests/files.py index bece18b2..f5977627 100644 --- a/moban/jinja2/tests/files.py +++ b/moban/jinja2/tests/files.py @@ -1,13 +1,6 @@ import sys -from os.path import ( - isabs, - isdir, - exists, - isfile, - islink, - ismount, - lexists -) +from os.path import isabs, isdir, exists, isfile, islink, ismount, lexists + from moban.jinja2.extensions import jinja_tests if sys.platform == "win32": diff --git a/moban/plugins/template.py b/moban/plugins/template.py index 51405d03..33851e37 100644 --- a/moban/plugins/template.py +++ b/moban/plugins/template.py @@ -181,7 +181,7 @@ def expand_template_directories(dirs): def expand_template_directory(directory): log.debug("Expanding %s..." % directory) translated_directory = None - if ":" in directory and directory[1] != ':': + if ":" in directory and directory[1] != ":": library_or_repo_name, relative_path = directory.split(":") potential_repo_path = os.path.join( repo.get_moban_home(), library_or_repo_name diff --git a/tests/mobanfile/test_templates.py b/tests/mobanfile/test_templates.py index 469817f6..c5a9de88 100644 --- a/tests/mobanfile/test_templates.py +++ b/tests/mobanfile/test_templates.py @@ -50,7 +50,9 @@ def test_listing_dir_recusively(self): expected = [ ( os.path.join("copier-directory", "copier-sample-dir", "file1"), - os.path.join("/tmp/copy-a-directory", "copier-sample-dir", "file1"), + os.path.join( + "/tmp/copy-a-directory", "copier-sample-dir", "file1" + ), None, ), ( diff --git a/tests/test_copy_engine.py b/tests/test_copy_engine.py index 55ce5030..6629660b 100644 --- a/tests/test_copy_engine.py +++ b/tests/test_copy_engine.py @@ -13,7 +13,7 @@ def setUp(self): def test_get_template(self): template_content = self.engine.get_template("copier-test01.csv") # remove '\r' for windows - eq_("test 01\n", template_content.replace('\r', '')) + eq_("test 01\n", template_content.replace("\r", "")) def test_get_template_from_string(self): test_content = "simply forwarded" diff --git a/tests/test_hash_store.py b/tests/test_hash_store.py index ef329152..6a69880b 100644 --- a/tests/test_hash_store.py +++ b/tests/test_hash_store.py @@ -2,6 +2,7 @@ import sys from nose import SkipTest + from moban.hashstore import HashStore diff --git a/tests/test_regression_pack.py b/tests/test_regression_pack.py index d155be89..b4e978be 100644 --- a/tests/test_regression_pack.py +++ b/tests/test_regression_pack.py @@ -3,8 +3,8 @@ from textwrap import dedent from mock import patch -from nose.tools import eq_ from nose import SkipTest +from nose.tools import eq_ from moban.main import main @@ -21,7 +21,7 @@ def setUp(self): self.current = os.getcwd() def test_symbolic_link_on_windows(self): - if sys.platform != 'win32': + if sys.platform != "win32": raise SkipTest("No need to test on linux nor macos") expected = "/tmp" From 01608c597c34f2f1bf9b89b6ba01c3803b9f6c0f Mon Sep 17 00:00:00 2001 From: chfw Date: Thu, 7 Mar 2019 21:06:34 +0000 Subject: [PATCH 09/12] :newspaper: show sub folder on windows --- .../templated-folder/.show_sub_folder_on_windows | 1 + 1 file changed, 1 insertion(+) create mode 100644 docs/level-8-pass-a-folder-full-of-templates/templated-folder/.show_sub_folder_on_windows diff --git a/docs/level-8-pass-a-folder-full-of-templates/templated-folder/.show_sub_folder_on_windows b/docs/level-8-pass-a-folder-full-of-templates/templated-folder/.show_sub_folder_on_windows new file mode 100644 index 00000000..a90b191b --- /dev/null +++ b/docs/level-8-pass-a-folder-full-of-templates/templated-folder/.show_sub_folder_on_windows @@ -0,0 +1 @@ +please \ No newline at end of file From b7f4736a400e6c186f327667f3ed10f044f6f5d6 Mon Sep 17 00:00:00 2001 From: chfw Date: Thu, 7 Mar 2019 21:08:41 +0000 Subject: [PATCH 10/12] :bug: jinja2 does not windows style file path --- .../show_sub_folder_on_windows.txt} | 0 moban/mobanfile/templates.py | 7 ++++++- 2 files changed, 6 insertions(+), 1 deletion(-) rename docs/level-8-pass-a-folder-full-of-templates/{templated-folder/.show_sub_folder_on_windows => template-folder/show_sub_folder_on_windows.txt} (100%) diff --git a/docs/level-8-pass-a-folder-full-of-templates/templated-folder/.show_sub_folder_on_windows b/docs/level-8-pass-a-folder-full-of-templates/template-folder/show_sub_folder_on_windows.txt similarity index 100% rename from docs/level-8-pass-a-folder-full-of-templates/templated-folder/.show_sub_folder_on_windows rename to docs/level-8-pass-a-folder-full-of-templates/template-folder/show_sub_folder_on_windows.txt diff --git a/moban/mobanfile/templates.py b/moban/mobanfile/templates.py index b7af666b..a8ace47f 100644 --- a/moban/mobanfile/templates.py +++ b/moban/mobanfile/templates.py @@ -43,7 +43,12 @@ def _list_dir_files(source, actual_source_path, dest): for file_name in os.listdir(actual_source_path): real_src_file = os.path.join(actual_source_path, file_name) if os.path.isfile(real_src_file): - src_file_under_dir = os.path.join(source, file_name) + # please note jinja2 does NOT like windows path + # hence the following statement looks like cross platform + # src_file_under_dir = os.path.join(source, file_name) + # but actually it breaks windows instead. + src_file_under_dir = '%s/%s' % (source, file_name) + dest_file_under_dir = os.path.join(dest, file_name) template_type = _get_template_type(src_file_under_dir) yield (src_file_under_dir, dest_file_under_dir, template_type) From fb5f966de28a8626085b83b39ff02b6c8dd32e3b Mon Sep 17 00:00:00 2001 From: chfw Date: Thu, 7 Mar 2019 21:28:56 +0000 Subject: [PATCH 11/12] :green_heart: fix last failure...sigh.. --- tests/mobanfile/test_templates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/mobanfile/test_templates.py b/tests/mobanfile/test_templates.py index c5a9de88..11812a1b 100644 --- a/tests/mobanfile/test_templates.py +++ b/tests/mobanfile/test_templates.py @@ -35,7 +35,7 @@ def test_listing_dir(self): ) expected = [ ( - os.path.join("copier-directory", "level1-file1"), + "copier-directory/level1-file1", os.path.join("/tmp/copy-a-directory", "level1-file1"), None, ) From 2960f23b38fa9e1e86fef10e215dfa89519ab4af Mon Sep 17 00:00:00 2001 From: chfw Date: Thu, 7 Mar 2019 21:35:45 +0000 Subject: [PATCH 12/12] :books: update change log --- .moban.cd/changelog.yml | 3 ++- CHANGELOG.rst | 8 +++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.moban.cd/changelog.yml b/.moban.cd/changelog.yml index fb80ce80..a776e582 100644 --- a/.moban.cd/changelog.yml +++ b/.moban.cd/changelog.yml @@ -7,7 +7,8 @@ releases: - "`#234`: Define template parameters on the fly inside `targets` section" - "`#180`: No longer two statistics will be shown in v0.4.x. legacy copy targets are injected into a normal targets. cli target is made a clear priority." - "`#62`: select a group target to run" - date: unreleased + - "`#244`: version 0.4.2 is first version which would work perfectly on windows since 17 Nov 2018. Note that: file permissions are not used on windows. Why the date? because samefile is not avaiable on windows, causing unit tests to fail hence it lead to my conclusion that moban version between 17 Nov 2018 and March 2019 wont work well on Windows." + date: 08.03.2019 version: 0.4.2 - changes: - action: Added diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8961f484..9f88d3ad 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,7 +1,7 @@ Change log ================================================================================ -0.4.2 - unreleased +0.4.2 - 08.03.2019 -------------------------------------------------------------------------------- Added @@ -14,6 +14,12 @@ Added normal targets. cli target is made a clear priority. #. `#62 `_: select a group target to run +#. `#244 `_: version 0.4.2 is + first version which would work perfectly on windows since 17 Nov 2018. Note + that: file permissions are not used on windows. Why the date? because + samefile is not avaiable on windows, causing unit tests to fail hence it lead + to my conclusion that moban version between 17 Nov 2018 and March 2019 wont + work well on Windows. 0.4.1 - 28.02.2019 --------------------------------------------------------------------------------