Lazy copy

.moban.cd/changelog.yml

@@ -1,6 +1,12 @@
 name: moban
 organisation: moremoban
 releases:
+- changes:
+  - action: Updated
+    details:
+    - "`#28`: if a template has been copied once before, it is skipped in the next moban call"
+  date: unreleased
+  version: 0.2.2
 - changes:
   - action: Updated
     details:

.moban.cd/moban.yml

@@ -3,8 +3,8 @@ organisation: moremoban
 author: C. W.
 contact: wangc_2011@hotmail.com
 license: MIT
-version: 0.2.1
-current_version: 0.2.1
+version: 0.2.2
+current_version: 0.2.2
 release: 0.2.1
 branch: master
 command_line_interface: "moban"

CHANGELOG.rst

@@ -1,6 +1,15 @@
 Change log
 ================================================================================
 
+0.2.2 - unreleased
+--------------------------------------------------------------------------------
+
+Updated
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+#. `#28 <https://github.com/moremoban/moban/issues/28>`_: if a template has been
+   copied once before, it is skipped in the next moban call
+
 0.2.1 - 13-06-2018
 --------------------------------------------------------------------------------
 

moban/copier.py

@@ -2,6 +2,7 @@
 import shutil
 
 import moban.reporter as reporter
+from moban.hashstore import HASH_STORE
 
 
 class Copier(object):
@@ -14,14 +15,14 @@ def copy_files(self, file_list):
         for dest_src_pair in file_list:
             for dest, src in dest_src_pair.items():
                 src_path = self._get_src_file(src)
-                if src_path:
-                    reporter.report_copying(src_path, dest)
-                    shutil.copy(src_path, dest)
-                    self._count = self._count + 1
-                else:
+                if src_path is None:
                     reporter.report_error_message(
                         "{0} cannot be found".format(src)
                     )
+                elif HASH_STORE.are_two_file_different(src_path, dest):
+                    reporter.report_copying(src_path, dest)
+                    shutil.copy(src_path, dest)
+                    self._count = self._count + 1
 
     def number_of_copied_files(self):
         return self._count
@@ -30,7 +31,7 @@ def report(self):
         if self._count:
             reporter.report_copying_summary(self._count)
         else:
-            reporter.report_no_action()
+            reporter.report_no_copying()
 
     def _get_src_file(self, src):
         for folder in self.template_dirs:

moban/engine.py

@@ -6,7 +6,7 @@
 from lml.plugin import PluginManager, PluginInfo
 from lml.loader import scan_plugins
 
-from moban.hashstore import HashStore
+from moban.hashstore import HASH_STORE
 from moban.extensions import JinjaFilterManager, JinjaTestManager
 from moban.extensions import JinjaGlobalsManager
 import moban.utils as utils
@@ -74,7 +74,6 @@ def __init__(self, template_dirs, context_dirs):
 
         self.context = Context(context_dirs)
         self.template_dirs = template_dirs
-        self.hash_store = HashStore()
         self.__file_count = 0
         self.__templated_count = 0
 
@@ -95,7 +94,6 @@ def render_to_files(self, array_of_param_tuple):
             self._render_with_finding_data_first(sta.data_file_index)
         else:
             self._render_with_finding_template_first(sta.template_file_index)
-        self.hash_store.close()
 
     def report(self):
         if self.__templated_count == 0:
@@ -136,7 +134,7 @@ def _apply_template(self, template, data, output):
         rendered_content = template.render(**data)
         rendered_content = utils.strip_off_trailing_new_lines(rendered_content)
         rendered_content = rendered_content.encode("utf-8")
-        flag = self.hash_store.is_file_changed(
+        flag = HASH_STORE.is_file_changed(
             output, rendered_content, template.filename
         )
         if flag:

moban/hashstore.py

@@ -21,6 +21,29 @@ def __init__(self):
         else:
             self.hashes = {}
 
+    def are_two_file_different(self, source_file, dest_file):
+        different = True
+        source_hash = get_file_hash(source_file)
+
+        previous_source_hash = self.hashes.get("copy:" + source_file)
+        if previous_source_hash is None:
+            self.hashes["copy:" + source_file] = source_hash
+
+        if source_hash == previous_source_hash:
+            different = False
+
+        if not different:
+            if os.path.exists(dest_file):
+                dest_hash = get_file_hash(dest_file)
+                if source_hash == dest_hash:
+                    different = False
+                else:
+                    different = True
+            else:
+                different = True
+
+        return different
+
     def is_file_changed(self, file_name, file_content, source_template):
         changed = self._is_source_updated(
             file_name, file_content, source_template
@@ -49,11 +72,14 @@ def _is_source_updated(self, file_name, file_content, source_template):
 
         return changed
 
-    def close(self):
+    def save_hashes(self):
         with open(self.cache_file, "w") as f:
             json.dump(self.hashes, f)
 
 
+HASH_STORE = HashStore()
+
+
 def get_file_hash(afile):
     with open(afile, "rb") as handle:
         content = handle.read()

moban/main.py

@@ -13,7 +13,7 @@
 import argparse
 
 from moban.utils import merge, open_yaml
-from moban.hashstore import HashStore
+from moban.hashstore import HASH_STORE
 from moban.engine import ENGINES
 import moban.constants as constants
 import moban.mobanfile as mobanfile
@@ -27,7 +27,7 @@ def main():
     """
     parser = create_parser()
     options = vars(parser.parse_args())
-    HashStore.IGNORE_CACHE_FILE = options[constants.LABEL_FORCE]
+    HASH_STORE.IGNORE_CACHE_FILE = options[constants.LABEL_FORCE]
     moban_file = options[constants.LABEL_MOBANFILE]
     if moban_file is None:
         moban_file = mobanfile.find_default_moban_file()
@@ -122,6 +122,7 @@ def handle_moban_file(moban_file, options):
         raise exceptions.MobanfileGrammarException(
             constants.MESSAGE_FILE_VERSION_NOT_SUPPORTED % version
         )
+    HASH_STORE.save_hashes()
 
 
 def handle_command_line(options):
@@ -142,6 +143,7 @@ def handle_command_line(options):
         options[constants.LABEL_CONFIG],
         options[constants.LABEL_OUTPUT],
     )
+    HASH_STORE.save_hashes()
     exit_code = reporter.convert_to_shell_exit_code(
         engine.number_of_templated_files()
     )

moban/reporter.py

@@ -24,6 +24,10 @@ def report_no_action():
     print(crayons.yellow(MESSAGE_NO_TEMPLATING, bold=True))
 
 
+def report_no_copying():
+    print(crayons.yellow(MESSAGE_NO_COPY, bold=True))
+
+
 def report_full_run(file_count):
     figure = crayons.green(str(file_count), bold=True)
     message = MESSAGE_TEMPLATED_ALL.format(figure)

setup.py

@@ -11,7 +11,7 @@
 
 NAME = 'moban'
 AUTHOR = 'C. W.'
-VERSION = '0.2.1'
+VERSION = '0.2.2'
 EMAIL = 'wangc_2011@hotmail.com'
 LICENSE = 'MIT'
 ENTRY_POINTS = {

tests/fixtures/copier-test01.csv

@@ -0,0 +1 @@
+test 01

tests/fixtures/copier-test02.csv

@@ -0,0 +1 @@
+test 02

tests/fixtures/copier-test03.csv

@@ -0,0 +1 @@
+test 3

tests/fixtures/copier-test04.csv

@@ -0,0 +1 @@
+test 4

tests/fixtures/copier-test05.csv

@@ -0,0 +1 @@
+test 05

tests/test_copier.py

@@ -32,12 +32,22 @@ def test_copy_files_file_not_found(self, reporter):
 
     def test_number_of_files(self):
         copier = Copier([os.path.join("tests", "fixtures")])
-        file_list = [{"/tmp/test": "copier-test01.csv"}]
+        file_list = [{"/tmp/test": "copier-test04.csv"}]
         copier.copy_files(file_list)
         eq_(copier.number_of_copied_files(), 1)
 
     def test_handle_copy(self):
         tmpl_dirs = [os.path.join("tests", "fixtures")]
-        copy_config = [{"/tmp/test": "copier-test01.csv"}]
+        copy_config = [{"/tmp/test": "copier-test05.csv"}]
         count = handle_copy(tmpl_dirs, copy_config)
         eq_(count, 1)
+
+
+@patch("moban.reporter.report_copying")
+def test_lazy_copy_files(reporter):
+    copier = Copier([os.path.join("tests", "fixtures")])
+    file_list = [{"/tmp/test2": "copier-test02.csv"}]
+    copier.copy_files(file_list)
+    copier.copy_files(file_list)  # not called the second time
+    eq_(reporter.call_count, 1)
+    os.unlink("/tmp/test2")

tests/test_hash_store.py

@@ -1,4 +1,5 @@
 import os
+
 from moban.hashstore import HashStore
 
 
@@ -16,29 +17,28 @@ def tearDown(self):
     def test_simple_use_case(self):
         hs = HashStore()
         flag = hs.is_file_changed(*self.fixture)
+        hs.save_hashes()
         assert flag is True
-        hs.close()
 
     def test_dest_file_does_not_exist(self):
         hs = HashStore()
         flag = hs.is_file_changed(*self.fixture)
-        hs.close()
+        hs.save_hashes()
         hs2 = HashStore()
         flag = hs2.is_file_changed(*self.fixture)
         assert flag is True
-        hs2.close()
 
     def test_dest_file_exist(self):
         hs = HashStore()
         flag = hs.is_file_changed(*self.fixture)
         if flag:
             with open(self.fixture[0], "wb") as f:
                 f.write(self.fixture[1])
-        hs.close()
+        hs.save_hashes()
         hs2 = HashStore()
         flag = hs2.is_file_changed(*self.fixture)
         assert flag is False
-        hs2.close()
+        hs2.save_hashes()
         os.unlink(self.fixture[0])
 
     def test_dest_file_changed(self):
@@ -55,19 +55,19 @@ def test_dest_file_changed(self):
         if flag:
             with open(self.fixture[0], "wb") as f:
                 f.write(self.fixture[1])
-        hs.close()
+        hs.save_hashes()
         # no change
         hs2 = HashStore()
         flag = hs2.is_file_changed(*self.fixture)
         assert flag is False
-        hs2.close()
+        hs2.save_hashes()
         # now let update the generated file
         hs3 = HashStore()
         with open(self.fixture[0], "w") as f:
             f.write("hey changed")
         flag = hs3.is_file_changed(*self.fixture)
         assert flag is True
-        hs3.close()
+        hs3.save_hashes()
         os.unlink(self.fixture[0])
 
     def test_dest_file_file_permision_changed(self):
@@ -80,16 +80,40 @@ def test_dest_file_file_permision_changed(self):
         if flag:
             with open(self.fixture[0], "wb") as f:
                 f.write(self.fixture[1])
-        hs.close()
+        hs.save_hashes()
         # no change
         hs2 = HashStore()
         flag = hs2.is_file_changed(*self.fixture)
         assert flag is False
-        hs2.close()
+        hs2.save_hashes()
         # now let change file permision of generated file
         hs3 = HashStore()
         os.chmod(self.fixture[0], 0o766)
         flag = hs3.is_file_changed(*self.fixture)
         assert flag is True
-        hs3.close()
+        hs3.save_hashes()
         os.unlink(self.fixture[0])
+
+
+class TestHashStore2:
+
+    def setUp(self):
+        self.source_file = os.path.join("tests", "fixtures", "a.jj2")
+        self.dest_file = os.path.join("tests", "fixtures", "copier-test02.csv")
+
+    def test_simple_use_case(self):
+        hs = HashStore()
+        flag = hs.are_two_file_different(self.source_file, '/tmp/abc')
+        assert flag is True
+
+    def test_laziness_with_same_file(self):
+        hs = HashStore()
+        flag = hs.are_two_file_different(self.source_file, self.source_file)
+        assert flag is True  # because we don't know it before
+        flag = hs.are_two_file_different(self.source_file, self.source_file)
+        assert flag is False
+
+    def test_different_files(self):
+        hs = HashStore()
+        flag = hs.are_two_file_different(self.source_file, self.dest_file)
+        assert flag is True