From b2e94c9372f2f1c87bbf11b724ef9aa062326518 Mon Sep 17 00:00:00 2001 From: chfw Date: Wed, 13 Jun 2018 08:57:04 +0100 Subject: [PATCH 01/13] :sparkles: avoid blind copying. fix #28 --- moban/copier.py | 10 ++++++---- moban/engine.py | 6 ++---- moban/hashstore.py | 25 ++++++++++++++++++++++++- moban/main.py | 6 ++++-- moban/reporter.py | 4 ++++ mobanfile | 2 ++ tests/test_hash_store.py | 21 ++++++++++----------- 7 files changed, 52 insertions(+), 22 deletions(-) diff --git a/moban/copier.py b/moban/copier.py index 7dd99155..0a0bfb8e 100644 --- a/moban/copier.py +++ b/moban/copier.py @@ -2,6 +2,7 @@ import shutil import moban.reporter as reporter +from moban.hashstore import HASH_STORE class Copier(object): @@ -15,9 +16,10 @@ def copy_files(self, file_list): for dest, src in dest_src_pair.items(): src_path = self._get_src_file(src) if src_path: - reporter.report_copying(src_path, dest) - shutil.copy(src_path, dest) - self._count = self._count + 1 + if HASH_STORE.are_two_file_different(src_path, dest): + reporter.report_copying(src_path, dest) + shutil.copy(src_path, dest) + self._count = self._count + 1 else: reporter.report_error_message( "{0} cannot be found".format(src) @@ -30,7 +32,7 @@ def report(self): if self._count: reporter.report_copying_summary(self._count) else: - reporter.report_no_action() + reporter.report_no_copying() def _get_src_file(self, src): for folder in self.template_dirs: diff --git a/moban/engine.py b/moban/engine.py index da31efc6..28d8497f 100644 --- a/moban/engine.py +++ b/moban/engine.py @@ -6,7 +6,7 @@ from lml.plugin import PluginManager, PluginInfo from lml.loader import scan_plugins -from moban.hashstore import HashStore +from moban.hashstore import HASH_STORE from moban.extensions import JinjaFilterManager, JinjaTestManager from moban.extensions import JinjaGlobalsManager import moban.utils as utils @@ -74,7 +74,6 @@ def __init__(self, template_dirs, context_dirs): self.context = Context(context_dirs) self.template_dirs = template_dirs - self.hash_store = HashStore() self.__file_count = 0 self.__templated_count = 0 @@ -95,7 +94,6 @@ def render_to_files(self, array_of_param_tuple): self._render_with_finding_data_first(sta.data_file_index) else: self._render_with_finding_template_first(sta.template_file_index) - self.hash_store.close() def report(self): if self.__templated_count == 0: @@ -136,7 +134,7 @@ def _apply_template(self, template, data, output): rendered_content = template.render(**data) rendered_content = utils.strip_off_trailing_new_lines(rendered_content) rendered_content = rendered_content.encode("utf-8") - flag = self.hash_store.is_file_changed( + flag = HASH_STORE.is_file_changed( output, rendered_content, template.filename ) if flag: diff --git a/moban/hashstore.py b/moban/hashstore.py index 5d7129f9..d6b496c9 100644 --- a/moban/hashstore.py +++ b/moban/hashstore.py @@ -21,6 +21,25 @@ def __init__(self): else: self.hashes = {} + def are_two_file_different(self, source_file, dest_file): + source_changed = True + dest_changed = True + source_hash = get_file_hash(source_file) + + previous_source_hash = self.hashes.get("copy:" + source_file) + if previous_source_hash: + if source_hash == previous_source_hash: + source_changed = False + else: + self.hashes["copy:" + source_file] = source_hash + + if os.path.exists(dest_file): + dest_hash = get_file_hash(dest_file) + if source_hash == dest_hash: + dest_changed = False + + return source_changed or dest_changed + def is_file_changed(self, file_name, file_content, source_template): changed = self._is_source_updated( file_name, file_content, source_template @@ -49,11 +68,15 @@ def _is_source_updated(self, file_name, file_content, source_template): return changed - def close(self): + def save_db(self): + import json with open(self.cache_file, "w") as f: json.dump(self.hashes, f) +HASH_STORE = HashStore() + + def get_file_hash(afile): with open(afile, "rb") as handle: content = handle.read() diff --git a/moban/main.py b/moban/main.py index 3bde7841..3cd2a7ea 100644 --- a/moban/main.py +++ b/moban/main.py @@ -13,7 +13,7 @@ import argparse from moban.utils import merge, open_yaml -from moban.hashstore import HashStore +from moban.hashstore import HASH_STORE from moban.engine import ENGINES import moban.constants as constants import moban.mobanfile as mobanfile @@ -27,7 +27,7 @@ def main(): """ parser = create_parser() options = vars(parser.parse_args()) - HashStore.IGNORE_CACHE_FILE = options[constants.LABEL_FORCE] + HASH_STORE.IGNORE_CACHE_FILE = options[constants.LABEL_FORCE] moban_file = options[constants.LABEL_MOBANFILE] if moban_file is None: moban_file = mobanfile.find_default_moban_file() @@ -122,6 +122,7 @@ def handle_moban_file(moban_file, options): raise exceptions.MobanfileGrammarException( constants.MESSAGE_FILE_VERSION_NOT_SUPPORTED % version ) + HASH_STORE.save_db() def handle_command_line(options): @@ -142,6 +143,7 @@ def handle_command_line(options): options[constants.LABEL_CONFIG], options[constants.LABEL_OUTPUT], ) + HASH_STORE.save_db() exit_code = reporter.convert_to_shell_exit_code( engine.number_of_templated_files() ) diff --git a/moban/reporter.py b/moban/reporter.py index e4bfc521..0cd52fbd 100644 --- a/moban/reporter.py +++ b/moban/reporter.py @@ -24,6 +24,10 @@ def report_no_action(): print(crayons.yellow(MESSAGE_NO_TEMPLATING, bold=True)) +def report_no_copying(): + print(crayons.yellow(MESSAGE_NO_COPY, bold=True)) + + def report_full_run(file_count): figure = crayons.green(str(file_count), bold=True) message = MESSAGE_TEMPLATED_ALL.format(figure) diff --git a/mobanfile b/mobanfile index 473e29d2..3ea5c25c 100644 --- a/mobanfile +++ b/mobanfile @@ -14,3 +14,5 @@ targets: - output: CHANGELOG.rst configuration: changelog.yml template: CHANGELOG.rst.jj2 +copy: + - setup.py.test: setup.py \ No newline at end of file diff --git a/tests/test_hash_store.py b/tests/test_hash_store.py index 9211a074..1783d498 100644 --- a/tests/test_hash_store.py +++ b/tests/test_hash_store.py @@ -16,17 +16,16 @@ def tearDown(self): def test_simple_use_case(self): hs = HashStore() flag = hs.is_file_changed(*self.fixture) + hs.save_db() assert flag is True - hs.close() def test_dest_file_does_not_exist(self): hs = HashStore() flag = hs.is_file_changed(*self.fixture) - hs.close() + hs.save_db() hs2 = HashStore() flag = hs2.is_file_changed(*self.fixture) assert flag is True - hs2.close() def test_dest_file_exist(self): hs = HashStore() @@ -34,11 +33,11 @@ def test_dest_file_exist(self): if flag: with open(self.fixture[0], "wb") as f: f.write(self.fixture[1]) - hs.close() + hs.save_db() hs2 = HashStore() flag = hs2.is_file_changed(*self.fixture) assert flag is False - hs2.close() + hs2.save_db() os.unlink(self.fixture[0]) def test_dest_file_changed(self): @@ -55,19 +54,19 @@ def test_dest_file_changed(self): if flag: with open(self.fixture[0], "wb") as f: f.write(self.fixture[1]) - hs.close() + hs.save_db() # no change hs2 = HashStore() flag = hs2.is_file_changed(*self.fixture) assert flag is False - hs2.close() + hs2.save_db() # now let update the generated file hs3 = HashStore() with open(self.fixture[0], "w") as f: f.write("hey changed") flag = hs3.is_file_changed(*self.fixture) assert flag is True - hs3.close() + hs3.save_db() os.unlink(self.fixture[0]) def test_dest_file_file_permision_changed(self): @@ -80,16 +79,16 @@ def test_dest_file_file_permision_changed(self): if flag: with open(self.fixture[0], "wb") as f: f.write(self.fixture[1]) - hs.close() + hs.save_db() # no change hs2 = HashStore() flag = hs2.is_file_changed(*self.fixture) assert flag is False - hs2.close() + hs2.save_db() # now let change file permision of generated file hs3 = HashStore() os.chmod(self.fixture[0], 0o766) flag = hs3.is_file_changed(*self.fixture) assert flag is True - hs3.close() + hs3.save_db() os.unlink(self.fixture[0]) From 44f108a00afb85e81e57f7c3d2d293b91aeece58 Mon Sep 17 00:00:00 2001 From: chfw Date: Wed, 13 Jun 2018 09:01:03 +0100 Subject: [PATCH 02/13] :fire: remove test lines --- mobanfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/mobanfile b/mobanfile index 3ea5c25c..473e29d2 100644 --- a/mobanfile +++ b/mobanfile @@ -14,5 +14,3 @@ targets: - output: CHANGELOG.rst configuration: changelog.yml template: CHANGELOG.rst.jj2 -copy: - - setup.py.test: setup.py \ No newline at end of file From 1b07b0adab2681a06fe1987206da39f522b80a94 Mon Sep 17 00:00:00 2001 From: chfw Date: Wed, 13 Jun 2018 18:06:35 +0100 Subject: [PATCH 03/13] :microscope: test lazy copying feature --- moban/hashstore.py | 14 +++++++++----- tests/fixtures/copier-test02.csv | 1 + tests/fixtures/copier-test03.csv | 1 + tests/fixtures/copier-test04.csv | 1 + tests/fixtures/copier-test05.csv | 1 + tests/test_copier.py | 25 +++++++++++++++++++++++-- 6 files changed, 36 insertions(+), 7 deletions(-) create mode 100644 tests/fixtures/copier-test02.csv create mode 100644 tests/fixtures/copier-test03.csv create mode 100644 tests/fixtures/copier-test04.csv create mode 100644 tests/fixtures/copier-test05.csv diff --git a/moban/hashstore.py b/moban/hashstore.py index d6b496c9..74edfc79 100644 --- a/moban/hashstore.py +++ b/moban/hashstore.py @@ -22,23 +22,27 @@ def __init__(self): self.hashes = {} def are_two_file_different(self, source_file, dest_file): - source_changed = True - dest_changed = True + + different = True source_hash = get_file_hash(source_file) previous_source_hash = self.hashes.get("copy:" + source_file) if previous_source_hash: if source_hash == previous_source_hash: - source_changed = False + different = False else: self.hashes["copy:" + source_file] = source_hash if os.path.exists(dest_file): dest_hash = get_file_hash(dest_file) if source_hash == dest_hash: - dest_changed = False + different = False + else: + different = True + else: + different = True - return source_changed or dest_changed + return different def is_file_changed(self, file_name, file_content, source_template): changed = self._is_source_updated( diff --git a/tests/fixtures/copier-test02.csv b/tests/fixtures/copier-test02.csv new file mode 100644 index 00000000..f601f203 --- /dev/null +++ b/tests/fixtures/copier-test02.csv @@ -0,0 +1 @@ +test 02 \ No newline at end of file diff --git a/tests/fixtures/copier-test03.csv b/tests/fixtures/copier-test03.csv new file mode 100644 index 00000000..87451e76 --- /dev/null +++ b/tests/fixtures/copier-test03.csv @@ -0,0 +1 @@ +test 3 \ No newline at end of file diff --git a/tests/fixtures/copier-test04.csv b/tests/fixtures/copier-test04.csv new file mode 100644 index 00000000..8d84c23f --- /dev/null +++ b/tests/fixtures/copier-test04.csv @@ -0,0 +1 @@ +test 4 \ No newline at end of file diff --git a/tests/fixtures/copier-test05.csv b/tests/fixtures/copier-test05.csv new file mode 100644 index 00000000..62e4069c --- /dev/null +++ b/tests/fixtures/copier-test05.csv @@ -0,0 +1 @@ +test 05 \ No newline at end of file diff --git a/tests/test_copier.py b/tests/test_copier.py index 4d6034e8..d5e0ba47 100644 --- a/tests/test_copier.py +++ b/tests/test_copier.py @@ -32,12 +32,33 @@ def test_copy_files_file_not_found(self, reporter): def test_number_of_files(self): copier = Copier([os.path.join("tests", "fixtures")]) - file_list = [{"/tmp/test": "copier-test01.csv"}] + file_list = [{"/tmp/test": "copier-test04.csv"}] copier.copy_files(file_list) eq_(copier.number_of_copied_files(), 1) def test_handle_copy(self): tmpl_dirs = [os.path.join("tests", "fixtures")] - copy_config = [{"/tmp/test": "copier-test01.csv"}] + copy_config = [{"/tmp/test": "copier-test05.csv"}] count = handle_copy(tmpl_dirs, copy_config) eq_(count, 1) + + +@patch("moban.reporter.report_copying") +def test_lazy_copy_files(reporter): + copier = Copier([os.path.join("tests", "fixtures")]) + file_list = [{"/tmp/test2": "copier-test02.csv"}] + copier.copy_files(file_list) + copier.copy_files(file_list) # not called the second time + eq_(reporter.call_count, 1) + os.unlink("/tmp/test2") + + +@patch("moban.reporter.report_copying") +def test_lazy_copy_the_same_file(reporter): + copier = Copier([os.path.join("tests", "fixtures")]) + dest_file = os.path.join("tests", "fixtures", "copier-test03.csv") + file_list = [{ + dest_file: "copier-test03.csv" + }] + copier.copy_files(file_list) + eq_(reporter.call_count, 0) From 4411aff63ca04b52b09008214cc872d868f5562a Mon Sep 17 00:00:00 2001 From: chfw Date: Wed, 13 Jun 2018 18:08:32 +0100 Subject: [PATCH 04/13] :hammer: code refactoring --- moban/copier.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/moban/copier.py b/moban/copier.py index 0a0bfb8e..e7a4fb20 100644 --- a/moban/copier.py +++ b/moban/copier.py @@ -15,15 +15,14 @@ def copy_files(self, file_list): for dest_src_pair in file_list: for dest, src in dest_src_pair.items(): src_path = self._get_src_file(src) - if src_path: - if HASH_STORE.are_two_file_different(src_path, dest): - reporter.report_copying(src_path, dest) - shutil.copy(src_path, dest) - self._count = self._count + 1 - else: + if src_path is None: reporter.report_error_message( "{0} cannot be found".format(src) ) + elif HASH_STORE.are_two_file_different(src_path, dest): + reporter.report_copying(src_path, dest) + shutil.copy(src_path, dest) + self._count = self._count + 1 def number_of_copied_files(self): return self._count From d2d7ffa7905698934020040f97a700451ff20d26 Mon Sep 17 00:00:00 2001 From: chfw Date: Wed, 13 Jun 2018 18:14:01 +0100 Subject: [PATCH 05/13] :hammer: code refactoring --- moban/hashstore.py | 11 +++++------ moban/main.py | 4 ++-- tests/test_hash_store.py | 20 ++++++++++---------- 3 files changed, 17 insertions(+), 18 deletions(-) diff --git a/moban/hashstore.py b/moban/hashstore.py index 74edfc79..109f1ae1 100644 --- a/moban/hashstore.py +++ b/moban/hashstore.py @@ -27,12 +27,12 @@ def are_two_file_different(self, source_file, dest_file): source_hash = get_file_hash(source_file) previous_source_hash = self.hashes.get("copy:" + source_file) - if previous_source_hash: - if source_hash == previous_source_hash: - different = False - else: + if previous_source_hash is None: self.hashes["copy:" + source_file] = source_hash + if source_hash == previous_source_hash: + different = False + if os.path.exists(dest_file): dest_hash = get_file_hash(dest_file) if source_hash == dest_hash: @@ -72,8 +72,7 @@ def _is_source_updated(self, file_name, file_content, source_template): return changed - def save_db(self): - import json + def save_hashes(self): with open(self.cache_file, "w") as f: json.dump(self.hashes, f) diff --git a/moban/main.py b/moban/main.py index 3cd2a7ea..f1a74ce3 100644 --- a/moban/main.py +++ b/moban/main.py @@ -122,7 +122,7 @@ def handle_moban_file(moban_file, options): raise exceptions.MobanfileGrammarException( constants.MESSAGE_FILE_VERSION_NOT_SUPPORTED % version ) - HASH_STORE.save_db() + HASH_STORE.save_hashes() def handle_command_line(options): @@ -143,7 +143,7 @@ def handle_command_line(options): options[constants.LABEL_CONFIG], options[constants.LABEL_OUTPUT], ) - HASH_STORE.save_db() + HASH_STORE.save_hashes() exit_code = reporter.convert_to_shell_exit_code( engine.number_of_templated_files() ) diff --git a/tests/test_hash_store.py b/tests/test_hash_store.py index 1783d498..716716d9 100644 --- a/tests/test_hash_store.py +++ b/tests/test_hash_store.py @@ -16,13 +16,13 @@ def tearDown(self): def test_simple_use_case(self): hs = HashStore() flag = hs.is_file_changed(*self.fixture) - hs.save_db() + hs.save_hashes() assert flag is True def test_dest_file_does_not_exist(self): hs = HashStore() flag = hs.is_file_changed(*self.fixture) - hs.save_db() + hs.save_hashes() hs2 = HashStore() flag = hs2.is_file_changed(*self.fixture) assert flag is True @@ -33,11 +33,11 @@ def test_dest_file_exist(self): if flag: with open(self.fixture[0], "wb") as f: f.write(self.fixture[1]) - hs.save_db() + hs.save_hashes() hs2 = HashStore() flag = hs2.is_file_changed(*self.fixture) assert flag is False - hs2.save_db() + hs2.save_hashes() os.unlink(self.fixture[0]) def test_dest_file_changed(self): @@ -54,19 +54,19 @@ def test_dest_file_changed(self): if flag: with open(self.fixture[0], "wb") as f: f.write(self.fixture[1]) - hs.save_db() + hs.save_hashes() # no change hs2 = HashStore() flag = hs2.is_file_changed(*self.fixture) assert flag is False - hs2.save_db() + hs2.save_hashes() # now let update the generated file hs3 = HashStore() with open(self.fixture[0], "w") as f: f.write("hey changed") flag = hs3.is_file_changed(*self.fixture) assert flag is True - hs3.save_db() + hs3.save_hashes() os.unlink(self.fixture[0]) def test_dest_file_file_permision_changed(self): @@ -79,16 +79,16 @@ def test_dest_file_file_permision_changed(self): if flag: with open(self.fixture[0], "wb") as f: f.write(self.fixture[1]) - hs.save_db() + hs.save_hashes() # no change hs2 = HashStore() flag = hs2.is_file_changed(*self.fixture) assert flag is False - hs2.save_db() + hs2.save_hashes() # now let change file permision of generated file hs3 = HashStore() os.chmod(self.fixture[0], 0o766) flag = hs3.is_file_changed(*self.fixture) assert flag is True - hs3.save_db() + hs3.save_hashes() os.unlink(self.fixture[0]) From 79f49d92579c33d72cc088749f7b6a6e2453bd2d Mon Sep 17 00:00:00 2001 From: chfw Date: Wed, 13 Jun 2018 18:15:35 +0100 Subject: [PATCH 06/13] :art: append new lines --- tests/fixtures/copier-test01.csv | 1 + tests/fixtures/copier-test02.csv | 2 +- tests/fixtures/copier-test03.csv | 2 +- tests/fixtures/copier-test04.csv | 2 +- tests/fixtures/copier-test05.csv | 2 +- 5 files changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/fixtures/copier-test01.csv b/tests/fixtures/copier-test01.csv index e69de29b..ec821a7f 100644 --- a/tests/fixtures/copier-test01.csv +++ b/tests/fixtures/copier-test01.csv @@ -0,0 +1 @@ +test 01 diff --git a/tests/fixtures/copier-test02.csv b/tests/fixtures/copier-test02.csv index f601f203..6d6682b1 100644 --- a/tests/fixtures/copier-test02.csv +++ b/tests/fixtures/copier-test02.csv @@ -1 +1 @@ -test 02 \ No newline at end of file +test 02 diff --git a/tests/fixtures/copier-test03.csv b/tests/fixtures/copier-test03.csv index 87451e76..954a536f 100644 --- a/tests/fixtures/copier-test03.csv +++ b/tests/fixtures/copier-test03.csv @@ -1 +1 @@ -test 3 \ No newline at end of file +test 3 diff --git a/tests/fixtures/copier-test04.csv b/tests/fixtures/copier-test04.csv index 8d84c23f..1121e314 100644 --- a/tests/fixtures/copier-test04.csv +++ b/tests/fixtures/copier-test04.csv @@ -1 +1 @@ -test 4 \ No newline at end of file +test 4 diff --git a/tests/fixtures/copier-test05.csv b/tests/fixtures/copier-test05.csv index 62e4069c..f8b801bf 100644 --- a/tests/fixtures/copier-test05.csv +++ b/tests/fixtures/copier-test05.csv @@ -1 +1 @@ -test 05 \ No newline at end of file +test 05 From aece0f269006e111c15fcecc3df8aca21f5c3298 Mon Sep 17 00:00:00 2001 From: chfw Date: Wed, 13 Jun 2018 18:36:00 +0100 Subject: [PATCH 07/13] :microscope: more tests --- moban/hashstore.py | 14 +++++++------- tests/test_copier.py | 11 ----------- tests/test_hash_store.py | 25 +++++++++++++++++++++++++ 3 files changed, 32 insertions(+), 18 deletions(-) diff --git a/moban/hashstore.py b/moban/hashstore.py index 109f1ae1..76f42ab5 100644 --- a/moban/hashstore.py +++ b/moban/hashstore.py @@ -22,7 +22,6 @@ def __init__(self): self.hashes = {} def are_two_file_different(self, source_file, dest_file): - different = True source_hash = get_file_hash(source_file) @@ -33,14 +32,15 @@ def are_two_file_different(self, source_file, dest_file): if source_hash == previous_source_hash: different = False - if os.path.exists(dest_file): - dest_hash = get_file_hash(dest_file) - if source_hash == dest_hash: - different = False + if not different: + if os.path.exists(dest_file): + dest_hash = get_file_hash(dest_file) + if source_hash == dest_hash: + different = False + else: + different = True else: different = True - else: - different = True return different diff --git a/tests/test_copier.py b/tests/test_copier.py index d5e0ba47..acfaef80 100644 --- a/tests/test_copier.py +++ b/tests/test_copier.py @@ -51,14 +51,3 @@ def test_lazy_copy_files(reporter): copier.copy_files(file_list) # not called the second time eq_(reporter.call_count, 1) os.unlink("/tmp/test2") - - -@patch("moban.reporter.report_copying") -def test_lazy_copy_the_same_file(reporter): - copier = Copier([os.path.join("tests", "fixtures")]) - dest_file = os.path.join("tests", "fixtures", "copier-test03.csv") - file_list = [{ - dest_file: "copier-test03.csv" - }] - copier.copy_files(file_list) - eq_(reporter.call_count, 0) diff --git a/tests/test_hash_store.py b/tests/test_hash_store.py index 716716d9..bc71797f 100644 --- a/tests/test_hash_store.py +++ b/tests/test_hash_store.py @@ -1,4 +1,5 @@ import os + from moban.hashstore import HashStore @@ -92,3 +93,27 @@ def test_dest_file_file_permision_changed(self): assert flag is True hs3.save_hashes() os.unlink(self.fixture[0]) + + +class TestHashStore2: + + def setUp(self): + self.source_file = os.path.join("tests", "fixtures", "a.jj2") + self.dest_file = os.path.join("tests", "fixtures", "copier-test02.csv") + + def test_simple_use_case(self): + hs = HashStore() + flag = hs.are_two_file_different(self.source_file, '/tmp/abc') + assert flag is True + + def test_laziness_with_same_file(self): + hs = HashStore() + flag = hs.are_two_file_different(self.source_file, self.source_file) + assert flag is True # because we don't know it before + flag = hs.are_two_file_different(self.source_file, self.source_file) + assert flag is False + + def test_different_files(self): + hs = HashStore() + flag = hs.are_two_file_different(self.source_file, self.dest_file) + assert flag is True From 18bbf43ccd066c649aedc6a315b569af5bf73c63 Mon Sep 17 00:00:00 2001 From: chfw Date: Wed, 13 Jun 2018 18:39:13 +0100 Subject: [PATCH 08/13] :books: update change log --- .moban.cd/changelog.yml | 6 ++++++ .moban.cd/moban.yml | 4 ++-- CHANGELOG.rst | 9 +++++++++ setup.py | 2 +- 4 files changed, 18 insertions(+), 3 deletions(-) diff --git a/.moban.cd/changelog.yml b/.moban.cd/changelog.yml index 63bb4d61..70dfcf91 100644 --- a/.moban.cd/changelog.yml +++ b/.moban.cd/changelog.yml @@ -1,6 +1,12 @@ name: moban organisation: moremoban releases: +- changes: + - action: Updated + details: + - "`#28`: if a template has been copied once before, it is skipped in the next moban call" + date: unreleased + version: 0.2.2 - changes: - action: Updated details: diff --git a/.moban.cd/moban.yml b/.moban.cd/moban.yml index f5ac66b2..6a987b9c 100644 --- a/.moban.cd/moban.yml +++ b/.moban.cd/moban.yml @@ -3,8 +3,8 @@ organisation: moremoban author: C. W. contact: wangc_2011@hotmail.com license: MIT -version: 0.2.1 -current_version: 0.2.1 +version: 0.2.2 +current_version: 0.2.2 release: 0.2.1 branch: master command_line_interface: "moban" diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b79445a5..d4518024 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,15 @@ Change log ================================================================================ +0.2.2 - unreleased +-------------------------------------------------------------------------------- + +Updated +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +#. `#28 `_: if a template has been + copied once before, it is skipped in the next moban call + 0.2.1 - 13-06-2018 -------------------------------------------------------------------------------- diff --git a/setup.py b/setup.py index ce2be986..a1e881a2 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ NAME = 'moban' AUTHOR = 'C. W.' -VERSION = '0.2.1' +VERSION = '0.2.2' EMAIL = 'wangc_2011@hotmail.com' LICENSE = 'MIT' ENTRY_POINTS = { From b3c852c893a03847f12243e3c6c4dadfb61825f6 Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 15 Jun 2018 13:42:13 +0100 Subject: [PATCH 09/13] :sparkles: create directory when copying files to a folder that does not exist --- moban/copier.py | 3 +++ moban/utils.py | 11 +++++++++++ tests/test_utils.py | 9 +++++++++ 3 files changed, 23 insertions(+) diff --git a/moban/copier.py b/moban/copier.py index e7a4fb20..44163cdb 100644 --- a/moban/copier.py +++ b/moban/copier.py @@ -1,6 +1,7 @@ import os import shutil +import moban.utils as utils import moban.reporter as reporter from moban.hashstore import HASH_STORE @@ -20,6 +21,8 @@ def copy_files(self, file_list): "{0} cannot be found".format(src) ) elif HASH_STORE.are_two_file_different(src_path, dest): + dest_folder = os.path.dirname(dest) + utils.mkdir_p(dest_folder) reporter.report_copying(src_path, dest) shutil.copy(src_path, dest) self._count = self._count + 1 diff --git a/moban/utils.py b/moban/utils.py index 07555eb1..a08ba312 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -1,6 +1,7 @@ import os import re import stat +import errno import yaml @@ -103,3 +104,13 @@ def write_file_out(filename, content, strip=True, encode=True): if encode: content = content.encode("utf-8") out.write(content) + + +def mkdir_p(path): + try: + os.makedirs(path) + except OSError as exc: # Python >2.5 + if exc.errno == errno.EEXIST and os.path.isdir(path): + pass + else: + raise diff --git a/tests/test_utils.py b/tests/test_utils.py index 5956f9e8..d689e28e 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,11 +1,13 @@ import os import stat +from shutil import rmtree from nose.tools import eq_ from moban.utils import file_permissions_copy from moban.utils import write_file_out from moban.utils import strip_off_trailing_new_lines +from moban.utils import mkdir_p def create_file(test_file, permission): @@ -65,3 +67,10 @@ def test_strip_new_lines(): content = "test\n\n\n\n\n" actual = strip_off_trailing_new_lines(content) eq_(actual, "test\n") + + +def test_mkdir_p(): + test_path = 'a/b/c/d' + mkdir_p(test_path) + assert os.path.exists(test_path) + rmtree(test_path) From 41c30cd50a2727d7f541b45eb5e4db068a81fa45 Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 15 Jun 2018 23:19:47 +0100 Subject: [PATCH 10/13] :books: document copying feature --- docs/index.rst | 9 ++++++++- docs/misc-1-copying-templates/.moban.yml | 6 ++++++ docs/misc-1-copying-templates/README.rst | 7 +++++++ .../can-create-folder/if-not-exists.txt | 1 + .../file-in-template-sources-folder.txt | 1 + moban/copier.py | 7 +++++-- moban/reporter.py | 15 +++++++++++---- tests/test_docs.py | 6 ++++++ 8 files changed, 45 insertions(+), 7 deletions(-) create mode 100644 docs/misc-1-copying-templates/.moban.yml create mode 100644 docs/misc-1-copying-templates/README.rst create mode 100644 docs/misc-1-copying-templates/misc-1-copying/can-create-folder/if-not-exists.txt create mode 100644 docs/misc-1-copying-templates/template-sources/file-in-template-sources-folder.txt diff --git a/docs/index.rst b/docs/index.rst index 1b693796..45de2cc6 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -23,6 +23,14 @@ examples folder. level-6-complex-configuration/README.rst level-7-use-custom-jinja2-filter-test-n-global/README.rst +In pratice, the following use cases were found interesting to go along with. + +.. toctree:: + :maxdepth: 1 + + misc-1-copying-templates + + For more complex use case, please look at `its usage in pyexcel project `_ Developer Guide @@ -40,4 +48,3 @@ Indices and tables * :ref:`genindex` * :ref:`modindex` * :ref:`search` - diff --git a/docs/misc-1-copying-templates/.moban.yml b/docs/misc-1-copying-templates/.moban.yml new file mode 100644 index 00000000..095e52b8 --- /dev/null +++ b/docs/misc-1-copying-templates/.moban.yml @@ -0,0 +1,6 @@ +configuration: + template_dir: + - template-sources +copy: + - simple.file.copy: file-in-template-sources-folder.txt + - "misc-1-copying/can-create-folder/if-not-exists.txt": file-in-template-sources-folder.txt diff --git a/docs/misc-1-copying-templates/README.rst b/docs/misc-1-copying-templates/README.rst new file mode 100644 index 00000000..3e112b1c --- /dev/null +++ b/docs/misc-1-copying-templates/README.rst @@ -0,0 +1,7 @@ +Misc 1: copying templates +================================================================================ + +With `.moban.yml`, you can copy templates to your destination. + + + diff --git a/docs/misc-1-copying-templates/misc-1-copying/can-create-folder/if-not-exists.txt b/docs/misc-1-copying-templates/misc-1-copying/can-create-folder/if-not-exists.txt new file mode 100644 index 00000000..16b14f5d --- /dev/null +++ b/docs/misc-1-copying-templates/misc-1-copying/can-create-folder/if-not-exists.txt @@ -0,0 +1 @@ +test file diff --git a/docs/misc-1-copying-templates/template-sources/file-in-template-sources-folder.txt b/docs/misc-1-copying-templates/template-sources/file-in-template-sources-folder.txt new file mode 100644 index 00000000..16b14f5d --- /dev/null +++ b/docs/misc-1-copying-templates/template-sources/file-in-template-sources-folder.txt @@ -0,0 +1 @@ +test file diff --git a/moban/copier.py b/moban/copier.py index 44163cdb..2b9a8af5 100644 --- a/moban/copier.py +++ b/moban/copier.py @@ -10,11 +10,13 @@ class Copier(object): def __init__(self, template_dirs): self.template_dirs = template_dirs + self._file_count = 0 self._count = 0 def copy_files(self, file_list): for dest_src_pair in file_list: for dest, src in dest_src_pair.items(): + self._file_count += 1 src_path = self._get_src_file(src) if src_path is None: reporter.report_error_message( @@ -22,7 +24,8 @@ def copy_files(self, file_list): ) elif HASH_STORE.are_two_file_different(src_path, dest): dest_folder = os.path.dirname(dest) - utils.mkdir_p(dest_folder) + if dest_folder: + utils.mkdir_p(dest_folder) reporter.report_copying(src_path, dest) shutil.copy(src_path, dest) self._count = self._count + 1 @@ -32,7 +35,7 @@ def number_of_copied_files(self): def report(self): if self._count: - reporter.report_copying_summary(self._count) + reporter.report_copying_summary(self._file_count, self._count) else: reporter.report_no_copying() diff --git a/moban/reporter.py b/moban/reporter.py index 0cd52fbd..fe2356fb 100644 --- a/moban/reporter.py +++ b/moban/reporter.py @@ -9,6 +9,7 @@ MESSAGE_NO_TEMPLATING = "No templating" MESSAGE_REPORT = "Templated {0} out of {1} files." MESSAGE_TEMPLATED_ALL = "Templated {0} files." +MESSAGE_COPY_REPORT = "Copied {0} out of {1} files." MESSAGE_COPIED_ALL = "Copied {0} files." @@ -69,10 +70,16 @@ def report_no_copying_done(): print(crayons.red(MESSAGE_NO_COPY, bold=True)) -def report_copying_summary(file_count): - figure = crayons.green(str(file_count), bold=True) - message = MESSAGE_COPIED_ALL.format(figure) - print(_format_single(message, file_count)) +def report_copying_summary(total, copies): + if total == copies: + figure = crayons.green(str(total), bold=True) + message = MESSAGE_COPIED_ALL.format(figure) + print(_format_single(message, total)) + else: + figure = crayons.green(str(copies), bold=True) + total_figure = crayons.yellow(str(total), bold=True) + message = MESSAGE_COPY_REPORT.format(figure, total_figure) + print(_format_single(message, total)) def _format_single(message, count): diff --git a/tests/test_docs.py b/tests/test_docs.py index 12e6dddf..cb84e99f 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -87,6 +87,12 @@ def test_level_7(self): folder = "level-7-use-custom-jinja2-filter-test-n-global" self._raw_moban(["moban"], folder, expected, "test.output") + def test_misc_1(self): + expected = "test file\n" + + folder = "misc-1-copying-templates" + self._raw_moban(["moban"], folder, expected, "simple.file.copy") + def _moban(self, folder, expected): args = ["moban", "-c", "data.yml", "-t", "a.template"] self._raw_moban(args, folder, expected, "moban.output") From 343418371e7e3d81f3308390897ed52577eb18fc Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 15 Jun 2018 23:21:41 +0100 Subject: [PATCH 11/13] :books: update change log --- .moban.cd/changelog.yml | 3 +++ CHANGELOG.rst | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/.moban.cd/changelog.yml b/.moban.cd/changelog.yml index 70dfcf91..9ca75eb9 100644 --- a/.moban.cd/changelog.yml +++ b/.moban.cd/changelog.yml @@ -2,6 +2,9 @@ name: moban organisation: moremoban releases: - changes: + - action: Added + details: + - "`#31`: create directory if missing during copying" - action: Updated details: - "`#28`: if a template has been copied once before, it is skipped in the next moban call" diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d4518024..d7ea5a02 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,6 +4,12 @@ Change log 0.2.2 - unreleased -------------------------------------------------------------------------------- +Added +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +#. `#31 `_: create directory if + missing during copying + Updated ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ From 7d0f67e6d75d9e2ea0fb1d04596d69c52651258b Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 15 Jun 2018 23:23:13 +0100 Subject: [PATCH 12/13] :fire: remove generated file --- .../misc-1-copying/can-create-folder/if-not-exists.txt | 1 - 1 file changed, 1 deletion(-) delete mode 100644 docs/misc-1-copying-templates/misc-1-copying/can-create-folder/if-not-exists.txt diff --git a/docs/misc-1-copying-templates/misc-1-copying/can-create-folder/if-not-exists.txt b/docs/misc-1-copying-templates/misc-1-copying/can-create-folder/if-not-exists.txt deleted file mode 100644 index 16b14f5d..00000000 --- a/docs/misc-1-copying-templates/misc-1-copying/can-create-folder/if-not-exists.txt +++ /dev/null @@ -1 +0,0 @@ -test file From c12671665e4196c691cbfde25027deb36cd7d464 Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 16 Jun 2018 15:17:32 +0100 Subject: [PATCH 13/13] :egg: :ferris_wheel: release 0.2.2 --- .moban.cd/moban.yml | 4 ++-- setup.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.moban.cd/moban.yml b/.moban.cd/moban.yml index 6a987b9c..832038c1 100644 --- a/.moban.cd/moban.yml +++ b/.moban.cd/moban.yml @@ -5,7 +5,7 @@ contact: wangc_2011@hotmail.com license: MIT version: 0.2.2 current_version: 0.2.2 -release: 0.2.1 +release: 0.2.2 branch: master command_line_interface: "moban" entry_point: "moban.main:main" @@ -18,4 +18,4 @@ dependencies: - lml==0.0.3 - crayons description: Yet another jinja2 cli command for static text generation -scm_host: github.com \ No newline at end of file +scm_host: github.com diff --git a/setup.py b/setup.py index a1e881a2..6a9a565b 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ 'Yet another jinja2 cli command for static text generation' ) URL = 'https://github.com/moremoban/moban' -DOWNLOAD_URL = '%s/archive/0.2.1.tar.gz' % URL +DOWNLOAD_URL = '%s/archive/0.2.2.tar.gz' % URL FILES = ['README.rst', 'CHANGELOG.rst'] KEYWORDS = [ 'jinja2', @@ -58,8 +58,8 @@ # You do not need to read beyond this line PUBLISH_COMMAND = '{0} setup.py sdist bdist_wheel upload -r pypi'.format( sys.executable) -GS_COMMAND = ('gs moban v0.2.1 ' + - "Find 0.2.1 in changelog for more details") +GS_COMMAND = ('gs moban v0.2.2 ' + + "Find 0.2.2 in changelog for more details") NO_GS_MESSAGE = ('Automatic github release is disabled. ' + 'Please install gease to enable it.') UPLOAD_FAILED_MSG = (