diff --git a/.moban.cd/changelog.yml b/.moban.cd/changelog.yml index 63bb4d61..70dfcf91 100644 --- a/.moban.cd/changelog.yml +++ b/.moban.cd/changelog.yml @@ -1,6 +1,12 @@ name: moban organisation: moremoban releases: +- changes: + - action: Updated + details: + - "`#28`: if a template has been copied once before, it is skipped in the next moban call" + date: unreleased + version: 0.2.2 - changes: - action: Updated details: diff --git a/.moban.cd/moban.yml b/.moban.cd/moban.yml index f5ac66b2..6a987b9c 100644 --- a/.moban.cd/moban.yml +++ b/.moban.cd/moban.yml @@ -3,8 +3,8 @@ organisation: moremoban author: C. W. contact: wangc_2011@hotmail.com license: MIT -version: 0.2.1 -current_version: 0.2.1 +version: 0.2.2 +current_version: 0.2.2 release: 0.2.1 branch: master command_line_interface: "moban" diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b79445a5..d4518024 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,15 @@ Change log ================================================================================ +0.2.2 - unreleased +-------------------------------------------------------------------------------- + +Updated +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +#. `#28 `_: if a template has been + copied once before, it is skipped in the next moban call + 0.2.1 - 13-06-2018 -------------------------------------------------------------------------------- diff --git a/moban/copier.py b/moban/copier.py index 7dd99155..e7a4fb20 100644 --- a/moban/copier.py +++ b/moban/copier.py @@ -2,6 +2,7 @@ import shutil import moban.reporter as reporter +from moban.hashstore import HASH_STORE class Copier(object): @@ -14,14 +15,14 @@ def copy_files(self, file_list): for dest_src_pair in file_list: for dest, src in dest_src_pair.items(): src_path = self._get_src_file(src) - if src_path: - reporter.report_copying(src_path, dest) - shutil.copy(src_path, dest) - self._count = self._count + 1 - else: + if src_path is None: reporter.report_error_message( "{0} cannot be found".format(src) ) + elif HASH_STORE.are_two_file_different(src_path, dest): + reporter.report_copying(src_path, dest) + shutil.copy(src_path, dest) + self._count = self._count + 1 def number_of_copied_files(self): return self._count @@ -30,7 +31,7 @@ def report(self): if self._count: reporter.report_copying_summary(self._count) else: - reporter.report_no_action() + reporter.report_no_copying() def _get_src_file(self, src): for folder in self.template_dirs: diff --git a/moban/engine.py b/moban/engine.py index da31efc6..28d8497f 100644 --- a/moban/engine.py +++ b/moban/engine.py @@ -6,7 +6,7 @@ from lml.plugin import PluginManager, PluginInfo from lml.loader import scan_plugins -from moban.hashstore import HashStore +from moban.hashstore import HASH_STORE from moban.extensions import JinjaFilterManager, JinjaTestManager from moban.extensions import JinjaGlobalsManager import moban.utils as utils @@ -74,7 +74,6 @@ def __init__(self, template_dirs, context_dirs): self.context = Context(context_dirs) self.template_dirs = template_dirs - self.hash_store = HashStore() self.__file_count = 0 self.__templated_count = 0 @@ -95,7 +94,6 @@ def render_to_files(self, array_of_param_tuple): self._render_with_finding_data_first(sta.data_file_index) else: self._render_with_finding_template_first(sta.template_file_index) - self.hash_store.close() def report(self): if self.__templated_count == 0: @@ -136,7 +134,7 @@ def _apply_template(self, template, data, output): rendered_content = template.render(**data) rendered_content = utils.strip_off_trailing_new_lines(rendered_content) rendered_content = rendered_content.encode("utf-8") - flag = self.hash_store.is_file_changed( + flag = HASH_STORE.is_file_changed( output, rendered_content, template.filename ) if flag: diff --git a/moban/hashstore.py b/moban/hashstore.py index 5d7129f9..76f42ab5 100644 --- a/moban/hashstore.py +++ b/moban/hashstore.py @@ -21,6 +21,29 @@ def __init__(self): else: self.hashes = {} + def are_two_file_different(self, source_file, dest_file): + different = True + source_hash = get_file_hash(source_file) + + previous_source_hash = self.hashes.get("copy:" + source_file) + if previous_source_hash is None: + self.hashes["copy:" + source_file] = source_hash + + if source_hash == previous_source_hash: + different = False + + if not different: + if os.path.exists(dest_file): + dest_hash = get_file_hash(dest_file) + if source_hash == dest_hash: + different = False + else: + different = True + else: + different = True + + return different + def is_file_changed(self, file_name, file_content, source_template): changed = self._is_source_updated( file_name, file_content, source_template @@ -49,11 +72,14 @@ def _is_source_updated(self, file_name, file_content, source_template): return changed - def close(self): + def save_hashes(self): with open(self.cache_file, "w") as f: json.dump(self.hashes, f) +HASH_STORE = HashStore() + + def get_file_hash(afile): with open(afile, "rb") as handle: content = handle.read() diff --git a/moban/main.py b/moban/main.py index 3bde7841..f1a74ce3 100644 --- a/moban/main.py +++ b/moban/main.py @@ -13,7 +13,7 @@ import argparse from moban.utils import merge, open_yaml -from moban.hashstore import HashStore +from moban.hashstore import HASH_STORE from moban.engine import ENGINES import moban.constants as constants import moban.mobanfile as mobanfile @@ -27,7 +27,7 @@ def main(): """ parser = create_parser() options = vars(parser.parse_args()) - HashStore.IGNORE_CACHE_FILE = options[constants.LABEL_FORCE] + HASH_STORE.IGNORE_CACHE_FILE = options[constants.LABEL_FORCE] moban_file = options[constants.LABEL_MOBANFILE] if moban_file is None: moban_file = mobanfile.find_default_moban_file() @@ -122,6 +122,7 @@ def handle_moban_file(moban_file, options): raise exceptions.MobanfileGrammarException( constants.MESSAGE_FILE_VERSION_NOT_SUPPORTED % version ) + HASH_STORE.save_hashes() def handle_command_line(options): @@ -142,6 +143,7 @@ def handle_command_line(options): options[constants.LABEL_CONFIG], options[constants.LABEL_OUTPUT], ) + HASH_STORE.save_hashes() exit_code = reporter.convert_to_shell_exit_code( engine.number_of_templated_files() ) diff --git a/moban/reporter.py b/moban/reporter.py index e4bfc521..0cd52fbd 100644 --- a/moban/reporter.py +++ b/moban/reporter.py @@ -24,6 +24,10 @@ def report_no_action(): print(crayons.yellow(MESSAGE_NO_TEMPLATING, bold=True)) +def report_no_copying(): + print(crayons.yellow(MESSAGE_NO_COPY, bold=True)) + + def report_full_run(file_count): figure = crayons.green(str(file_count), bold=True) message = MESSAGE_TEMPLATED_ALL.format(figure) diff --git a/setup.py b/setup.py index ce2be986..a1e881a2 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ NAME = 'moban' AUTHOR = 'C. W.' -VERSION = '0.2.1' +VERSION = '0.2.2' EMAIL = 'wangc_2011@hotmail.com' LICENSE = 'MIT' ENTRY_POINTS = { diff --git a/tests/fixtures/copier-test01.csv b/tests/fixtures/copier-test01.csv index e69de29b..ec821a7f 100644 --- a/tests/fixtures/copier-test01.csv +++ b/tests/fixtures/copier-test01.csv @@ -0,0 +1 @@ +test 01 diff --git a/tests/fixtures/copier-test02.csv b/tests/fixtures/copier-test02.csv new file mode 100644 index 00000000..6d6682b1 --- /dev/null +++ b/tests/fixtures/copier-test02.csv @@ -0,0 +1 @@ +test 02 diff --git a/tests/fixtures/copier-test03.csv b/tests/fixtures/copier-test03.csv new file mode 100644 index 00000000..954a536f --- /dev/null +++ b/tests/fixtures/copier-test03.csv @@ -0,0 +1 @@ +test 3 diff --git a/tests/fixtures/copier-test04.csv b/tests/fixtures/copier-test04.csv new file mode 100644 index 00000000..1121e314 --- /dev/null +++ b/tests/fixtures/copier-test04.csv @@ -0,0 +1 @@ +test 4 diff --git a/tests/fixtures/copier-test05.csv b/tests/fixtures/copier-test05.csv new file mode 100644 index 00000000..f8b801bf --- /dev/null +++ b/tests/fixtures/copier-test05.csv @@ -0,0 +1 @@ +test 05 diff --git a/tests/test_copier.py b/tests/test_copier.py index 4d6034e8..acfaef80 100644 --- a/tests/test_copier.py +++ b/tests/test_copier.py @@ -32,12 +32,22 @@ def test_copy_files_file_not_found(self, reporter): def test_number_of_files(self): copier = Copier([os.path.join("tests", "fixtures")]) - file_list = [{"/tmp/test": "copier-test01.csv"}] + file_list = [{"/tmp/test": "copier-test04.csv"}] copier.copy_files(file_list) eq_(copier.number_of_copied_files(), 1) def test_handle_copy(self): tmpl_dirs = [os.path.join("tests", "fixtures")] - copy_config = [{"/tmp/test": "copier-test01.csv"}] + copy_config = [{"/tmp/test": "copier-test05.csv"}] count = handle_copy(tmpl_dirs, copy_config) eq_(count, 1) + + +@patch("moban.reporter.report_copying") +def test_lazy_copy_files(reporter): + copier = Copier([os.path.join("tests", "fixtures")]) + file_list = [{"/tmp/test2": "copier-test02.csv"}] + copier.copy_files(file_list) + copier.copy_files(file_list) # not called the second time + eq_(reporter.call_count, 1) + os.unlink("/tmp/test2") diff --git a/tests/test_hash_store.py b/tests/test_hash_store.py index 9211a074..bc71797f 100644 --- a/tests/test_hash_store.py +++ b/tests/test_hash_store.py @@ -1,4 +1,5 @@ import os + from moban.hashstore import HashStore @@ -16,17 +17,16 @@ def tearDown(self): def test_simple_use_case(self): hs = HashStore() flag = hs.is_file_changed(*self.fixture) + hs.save_hashes() assert flag is True - hs.close() def test_dest_file_does_not_exist(self): hs = HashStore() flag = hs.is_file_changed(*self.fixture) - hs.close() + hs.save_hashes() hs2 = HashStore() flag = hs2.is_file_changed(*self.fixture) assert flag is True - hs2.close() def test_dest_file_exist(self): hs = HashStore() @@ -34,11 +34,11 @@ def test_dest_file_exist(self): if flag: with open(self.fixture[0], "wb") as f: f.write(self.fixture[1]) - hs.close() + hs.save_hashes() hs2 = HashStore() flag = hs2.is_file_changed(*self.fixture) assert flag is False - hs2.close() + hs2.save_hashes() os.unlink(self.fixture[0]) def test_dest_file_changed(self): @@ -55,19 +55,19 @@ def test_dest_file_changed(self): if flag: with open(self.fixture[0], "wb") as f: f.write(self.fixture[1]) - hs.close() + hs.save_hashes() # no change hs2 = HashStore() flag = hs2.is_file_changed(*self.fixture) assert flag is False - hs2.close() + hs2.save_hashes() # now let update the generated file hs3 = HashStore() with open(self.fixture[0], "w") as f: f.write("hey changed") flag = hs3.is_file_changed(*self.fixture) assert flag is True - hs3.close() + hs3.save_hashes() os.unlink(self.fixture[0]) def test_dest_file_file_permision_changed(self): @@ -80,16 +80,40 @@ def test_dest_file_file_permision_changed(self): if flag: with open(self.fixture[0], "wb") as f: f.write(self.fixture[1]) - hs.close() + hs.save_hashes() # no change hs2 = HashStore() flag = hs2.is_file_changed(*self.fixture) assert flag is False - hs2.close() + hs2.save_hashes() # now let change file permision of generated file hs3 = HashStore() os.chmod(self.fixture[0], 0o766) flag = hs3.is_file_changed(*self.fixture) assert flag is True - hs3.close() + hs3.save_hashes() os.unlink(self.fixture[0]) + + +class TestHashStore2: + + def setUp(self): + self.source_file = os.path.join("tests", "fixtures", "a.jj2") + self.dest_file = os.path.join("tests", "fixtures", "copier-test02.csv") + + def test_simple_use_case(self): + hs = HashStore() + flag = hs.are_two_file_different(self.source_file, '/tmp/abc') + assert flag is True + + def test_laziness_with_same_file(self): + hs = HashStore() + flag = hs.are_two_file_different(self.source_file, self.source_file) + assert flag is True # because we don't know it before + flag = hs.are_two_file_different(self.source_file, self.source_file) + assert flag is False + + def test_different_files(self): + hs = HashStore() + flag = hs.are_two_file_different(self.source_file, self.dest_file) + assert flag is True