diff --git a/.moban.cd/changelog.yml b/.moban.cd/changelog.yml index 63bb4d61..9ca75eb9 100644 --- a/.moban.cd/changelog.yml +++ b/.moban.cd/changelog.yml @@ -1,6 +1,15 @@ name: moban organisation: moremoban releases: +- changes: + - action: Added + details: + - "`#31`: create directory if missing during copying" + - action: Updated + details: + - "`#28`: if a template has been copied once before, it is skipped in the next moban call" + date: unreleased + version: 0.2.2 - changes: - action: Updated details: diff --git a/.moban.cd/moban.yml b/.moban.cd/moban.yml index f5ac66b2..832038c1 100644 --- a/.moban.cd/moban.yml +++ b/.moban.cd/moban.yml @@ -3,9 +3,9 @@ organisation: moremoban author: C. W. contact: wangc_2011@hotmail.com license: MIT -version: 0.2.1 -current_version: 0.2.1 -release: 0.2.1 +version: 0.2.2 +current_version: 0.2.2 +release: 0.2.2 branch: master command_line_interface: "moban" entry_point: "moban.main:main" @@ -18,4 +18,4 @@ dependencies: - lml==0.0.3 - crayons description: Yet another jinja2 cli command for static text generation -scm_host: github.com \ No newline at end of file +scm_host: github.com diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b79445a5..d7ea5a02 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,21 @@ Change log ================================================================================ +0.2.2 - unreleased +-------------------------------------------------------------------------------- + +Added +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +#. `#31 `_: create directory if + missing during copying + +Updated +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +#. `#28 `_: if a template has been + copied once before, it is skipped in the next moban call + 0.2.1 - 13-06-2018 -------------------------------------------------------------------------------- diff --git a/docs/index.rst b/docs/index.rst index 1b693796..45de2cc6 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -23,6 +23,14 @@ examples folder. level-6-complex-configuration/README.rst level-7-use-custom-jinja2-filter-test-n-global/README.rst +In pratice, the following use cases were found interesting to go along with. + +.. toctree:: + :maxdepth: 1 + + misc-1-copying-templates + + For more complex use case, please look at `its usage in pyexcel project `_ Developer Guide @@ -40,4 +48,3 @@ Indices and tables * :ref:`genindex` * :ref:`modindex` * :ref:`search` - diff --git a/docs/misc-1-copying-templates/.moban.yml b/docs/misc-1-copying-templates/.moban.yml new file mode 100644 index 00000000..095e52b8 --- /dev/null +++ b/docs/misc-1-copying-templates/.moban.yml @@ -0,0 +1,6 @@ +configuration: + template_dir: + - template-sources +copy: + - simple.file.copy: file-in-template-sources-folder.txt + - "misc-1-copying/can-create-folder/if-not-exists.txt": file-in-template-sources-folder.txt diff --git a/docs/misc-1-copying-templates/README.rst b/docs/misc-1-copying-templates/README.rst new file mode 100644 index 00000000..3e112b1c --- /dev/null +++ b/docs/misc-1-copying-templates/README.rst @@ -0,0 +1,7 @@ +Misc 1: copying templates +================================================================================ + +With `.moban.yml`, you can copy templates to your destination. + + + diff --git a/docs/misc-1-copying-templates/template-sources/file-in-template-sources-folder.txt b/docs/misc-1-copying-templates/template-sources/file-in-template-sources-folder.txt new file mode 100644 index 00000000..16b14f5d --- /dev/null +++ b/docs/misc-1-copying-templates/template-sources/file-in-template-sources-folder.txt @@ -0,0 +1 @@ +test file diff --git a/moban/copier.py b/moban/copier.py index 7dd99155..2b9a8af5 100644 --- a/moban/copier.py +++ b/moban/copier.py @@ -1,36 +1,43 @@ import os import shutil +import moban.utils as utils import moban.reporter as reporter +from moban.hashstore import HASH_STORE class Copier(object): def __init__(self, template_dirs): self.template_dirs = template_dirs + self._file_count = 0 self._count = 0 def copy_files(self, file_list): for dest_src_pair in file_list: for dest, src in dest_src_pair.items(): + self._file_count += 1 src_path = self._get_src_file(src) - if src_path: - reporter.report_copying(src_path, dest) - shutil.copy(src_path, dest) - self._count = self._count + 1 - else: + if src_path is None: reporter.report_error_message( "{0} cannot be found".format(src) ) + elif HASH_STORE.are_two_file_different(src_path, dest): + dest_folder = os.path.dirname(dest) + if dest_folder: + utils.mkdir_p(dest_folder) + reporter.report_copying(src_path, dest) + shutil.copy(src_path, dest) + self._count = self._count + 1 def number_of_copied_files(self): return self._count def report(self): if self._count: - reporter.report_copying_summary(self._count) + reporter.report_copying_summary(self._file_count, self._count) else: - reporter.report_no_action() + reporter.report_no_copying() def _get_src_file(self, src): for folder in self.template_dirs: diff --git a/moban/engine.py b/moban/engine.py index da31efc6..28d8497f 100644 --- a/moban/engine.py +++ b/moban/engine.py @@ -6,7 +6,7 @@ from lml.plugin import PluginManager, PluginInfo from lml.loader import scan_plugins -from moban.hashstore import HashStore +from moban.hashstore import HASH_STORE from moban.extensions import JinjaFilterManager, JinjaTestManager from moban.extensions import JinjaGlobalsManager import moban.utils as utils @@ -74,7 +74,6 @@ def __init__(self, template_dirs, context_dirs): self.context = Context(context_dirs) self.template_dirs = template_dirs - self.hash_store = HashStore() self.__file_count = 0 self.__templated_count = 0 @@ -95,7 +94,6 @@ def render_to_files(self, array_of_param_tuple): self._render_with_finding_data_first(sta.data_file_index) else: self._render_with_finding_template_first(sta.template_file_index) - self.hash_store.close() def report(self): if self.__templated_count == 0: @@ -136,7 +134,7 @@ def _apply_template(self, template, data, output): rendered_content = template.render(**data) rendered_content = utils.strip_off_trailing_new_lines(rendered_content) rendered_content = rendered_content.encode("utf-8") - flag = self.hash_store.is_file_changed( + flag = HASH_STORE.is_file_changed( output, rendered_content, template.filename ) if flag: diff --git a/moban/hashstore.py b/moban/hashstore.py index 5d7129f9..76f42ab5 100644 --- a/moban/hashstore.py +++ b/moban/hashstore.py @@ -21,6 +21,29 @@ def __init__(self): else: self.hashes = {} + def are_two_file_different(self, source_file, dest_file): + different = True + source_hash = get_file_hash(source_file) + + previous_source_hash = self.hashes.get("copy:" + source_file) + if previous_source_hash is None: + self.hashes["copy:" + source_file] = source_hash + + if source_hash == previous_source_hash: + different = False + + if not different: + if os.path.exists(dest_file): + dest_hash = get_file_hash(dest_file) + if source_hash == dest_hash: + different = False + else: + different = True + else: + different = True + + return different + def is_file_changed(self, file_name, file_content, source_template): changed = self._is_source_updated( file_name, file_content, source_template @@ -49,11 +72,14 @@ def _is_source_updated(self, file_name, file_content, source_template): return changed - def close(self): + def save_hashes(self): with open(self.cache_file, "w") as f: json.dump(self.hashes, f) +HASH_STORE = HashStore() + + def get_file_hash(afile): with open(afile, "rb") as handle: content = handle.read() diff --git a/moban/main.py b/moban/main.py index 3bde7841..f1a74ce3 100644 --- a/moban/main.py +++ b/moban/main.py @@ -13,7 +13,7 @@ import argparse from moban.utils import merge, open_yaml -from moban.hashstore import HashStore +from moban.hashstore import HASH_STORE from moban.engine import ENGINES import moban.constants as constants import moban.mobanfile as mobanfile @@ -27,7 +27,7 @@ def main(): """ parser = create_parser() options = vars(parser.parse_args()) - HashStore.IGNORE_CACHE_FILE = options[constants.LABEL_FORCE] + HASH_STORE.IGNORE_CACHE_FILE = options[constants.LABEL_FORCE] moban_file = options[constants.LABEL_MOBANFILE] if moban_file is None: moban_file = mobanfile.find_default_moban_file() @@ -122,6 +122,7 @@ def handle_moban_file(moban_file, options): raise exceptions.MobanfileGrammarException( constants.MESSAGE_FILE_VERSION_NOT_SUPPORTED % version ) + HASH_STORE.save_hashes() def handle_command_line(options): @@ -142,6 +143,7 @@ def handle_command_line(options): options[constants.LABEL_CONFIG], options[constants.LABEL_OUTPUT], ) + HASH_STORE.save_hashes() exit_code = reporter.convert_to_shell_exit_code( engine.number_of_templated_files() ) diff --git a/moban/reporter.py b/moban/reporter.py index e4bfc521..fe2356fb 100644 --- a/moban/reporter.py +++ b/moban/reporter.py @@ -9,6 +9,7 @@ MESSAGE_NO_TEMPLATING = "No templating" MESSAGE_REPORT = "Templated {0} out of {1} files." MESSAGE_TEMPLATED_ALL = "Templated {0} files." +MESSAGE_COPY_REPORT = "Copied {0} out of {1} files." MESSAGE_COPIED_ALL = "Copied {0} files." @@ -24,6 +25,10 @@ def report_no_action(): print(crayons.yellow(MESSAGE_NO_TEMPLATING, bold=True)) +def report_no_copying(): + print(crayons.yellow(MESSAGE_NO_COPY, bold=True)) + + def report_full_run(file_count): figure = crayons.green(str(file_count), bold=True) message = MESSAGE_TEMPLATED_ALL.format(figure) @@ -65,10 +70,16 @@ def report_no_copying_done(): print(crayons.red(MESSAGE_NO_COPY, bold=True)) -def report_copying_summary(file_count): - figure = crayons.green(str(file_count), bold=True) - message = MESSAGE_COPIED_ALL.format(figure) - print(_format_single(message, file_count)) +def report_copying_summary(total, copies): + if total == copies: + figure = crayons.green(str(total), bold=True) + message = MESSAGE_COPIED_ALL.format(figure) + print(_format_single(message, total)) + else: + figure = crayons.green(str(copies), bold=True) + total_figure = crayons.yellow(str(total), bold=True) + message = MESSAGE_COPY_REPORT.format(figure, total_figure) + print(_format_single(message, total)) def _format_single(message, count): diff --git a/moban/utils.py b/moban/utils.py index 07555eb1..a08ba312 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -1,6 +1,7 @@ import os import re import stat +import errno import yaml @@ -103,3 +104,13 @@ def write_file_out(filename, content, strip=True, encode=True): if encode: content = content.encode("utf-8") out.write(content) + + +def mkdir_p(path): + try: + os.makedirs(path) + except OSError as exc: # Python >2.5 + if exc.errno == errno.EEXIST and os.path.isdir(path): + pass + else: + raise diff --git a/setup.py b/setup.py index ce2be986..6a9a565b 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ NAME = 'moban' AUTHOR = 'C. W.' -VERSION = '0.2.1' +VERSION = '0.2.2' EMAIL = 'wangc_2011@hotmail.com' LICENSE = 'MIT' ENTRY_POINTS = { @@ -23,7 +23,7 @@ 'Yet another jinja2 cli command for static text generation' ) URL = 'https://github.com/moremoban/moban' -DOWNLOAD_URL = '%s/archive/0.2.1.tar.gz' % URL +DOWNLOAD_URL = '%s/archive/0.2.2.tar.gz' % URL FILES = ['README.rst', 'CHANGELOG.rst'] KEYWORDS = [ 'jinja2', @@ -58,8 +58,8 @@ # You do not need to read beyond this line PUBLISH_COMMAND = '{0} setup.py sdist bdist_wheel upload -r pypi'.format( sys.executable) -GS_COMMAND = ('gs moban v0.2.1 ' + - "Find 0.2.1 in changelog for more details") +GS_COMMAND = ('gs moban v0.2.2 ' + + "Find 0.2.2 in changelog for more details") NO_GS_MESSAGE = ('Automatic github release is disabled. ' + 'Please install gease to enable it.') UPLOAD_FAILED_MSG = ( diff --git a/tests/fixtures/copier-test01.csv b/tests/fixtures/copier-test01.csv index e69de29b..ec821a7f 100644 --- a/tests/fixtures/copier-test01.csv +++ b/tests/fixtures/copier-test01.csv @@ -0,0 +1 @@ +test 01 diff --git a/tests/fixtures/copier-test02.csv b/tests/fixtures/copier-test02.csv new file mode 100644 index 00000000..6d6682b1 --- /dev/null +++ b/tests/fixtures/copier-test02.csv @@ -0,0 +1 @@ +test 02 diff --git a/tests/fixtures/copier-test03.csv b/tests/fixtures/copier-test03.csv new file mode 100644 index 00000000..954a536f --- /dev/null +++ b/tests/fixtures/copier-test03.csv @@ -0,0 +1 @@ +test 3 diff --git a/tests/fixtures/copier-test04.csv b/tests/fixtures/copier-test04.csv new file mode 100644 index 00000000..1121e314 --- /dev/null +++ b/tests/fixtures/copier-test04.csv @@ -0,0 +1 @@ +test 4 diff --git a/tests/fixtures/copier-test05.csv b/tests/fixtures/copier-test05.csv new file mode 100644 index 00000000..f8b801bf --- /dev/null +++ b/tests/fixtures/copier-test05.csv @@ -0,0 +1 @@ +test 05 diff --git a/tests/test_copier.py b/tests/test_copier.py index 4d6034e8..acfaef80 100644 --- a/tests/test_copier.py +++ b/tests/test_copier.py @@ -32,12 +32,22 @@ def test_copy_files_file_not_found(self, reporter): def test_number_of_files(self): copier = Copier([os.path.join("tests", "fixtures")]) - file_list = [{"/tmp/test": "copier-test01.csv"}] + file_list = [{"/tmp/test": "copier-test04.csv"}] copier.copy_files(file_list) eq_(copier.number_of_copied_files(), 1) def test_handle_copy(self): tmpl_dirs = [os.path.join("tests", "fixtures")] - copy_config = [{"/tmp/test": "copier-test01.csv"}] + copy_config = [{"/tmp/test": "copier-test05.csv"}] count = handle_copy(tmpl_dirs, copy_config) eq_(count, 1) + + +@patch("moban.reporter.report_copying") +def test_lazy_copy_files(reporter): + copier = Copier([os.path.join("tests", "fixtures")]) + file_list = [{"/tmp/test2": "copier-test02.csv"}] + copier.copy_files(file_list) + copier.copy_files(file_list) # not called the second time + eq_(reporter.call_count, 1) + os.unlink("/tmp/test2") diff --git a/tests/test_docs.py b/tests/test_docs.py index 12e6dddf..cb84e99f 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -87,6 +87,12 @@ def test_level_7(self): folder = "level-7-use-custom-jinja2-filter-test-n-global" self._raw_moban(["moban"], folder, expected, "test.output") + def test_misc_1(self): + expected = "test file\n" + + folder = "misc-1-copying-templates" + self._raw_moban(["moban"], folder, expected, "simple.file.copy") + def _moban(self, folder, expected): args = ["moban", "-c", "data.yml", "-t", "a.template"] self._raw_moban(args, folder, expected, "moban.output") diff --git a/tests/test_hash_store.py b/tests/test_hash_store.py index 9211a074..bc71797f 100644 --- a/tests/test_hash_store.py +++ b/tests/test_hash_store.py @@ -1,4 +1,5 @@ import os + from moban.hashstore import HashStore @@ -16,17 +17,16 @@ def tearDown(self): def test_simple_use_case(self): hs = HashStore() flag = hs.is_file_changed(*self.fixture) + hs.save_hashes() assert flag is True - hs.close() def test_dest_file_does_not_exist(self): hs = HashStore() flag = hs.is_file_changed(*self.fixture) - hs.close() + hs.save_hashes() hs2 = HashStore() flag = hs2.is_file_changed(*self.fixture) assert flag is True - hs2.close() def test_dest_file_exist(self): hs = HashStore() @@ -34,11 +34,11 @@ def test_dest_file_exist(self): if flag: with open(self.fixture[0], "wb") as f: f.write(self.fixture[1]) - hs.close() + hs.save_hashes() hs2 = HashStore() flag = hs2.is_file_changed(*self.fixture) assert flag is False - hs2.close() + hs2.save_hashes() os.unlink(self.fixture[0]) def test_dest_file_changed(self): @@ -55,19 +55,19 @@ def test_dest_file_changed(self): if flag: with open(self.fixture[0], "wb") as f: f.write(self.fixture[1]) - hs.close() + hs.save_hashes() # no change hs2 = HashStore() flag = hs2.is_file_changed(*self.fixture) assert flag is False - hs2.close() + hs2.save_hashes() # now let update the generated file hs3 = HashStore() with open(self.fixture[0], "w") as f: f.write("hey changed") flag = hs3.is_file_changed(*self.fixture) assert flag is True - hs3.close() + hs3.save_hashes() os.unlink(self.fixture[0]) def test_dest_file_file_permision_changed(self): @@ -80,16 +80,40 @@ def test_dest_file_file_permision_changed(self): if flag: with open(self.fixture[0], "wb") as f: f.write(self.fixture[1]) - hs.close() + hs.save_hashes() # no change hs2 = HashStore() flag = hs2.is_file_changed(*self.fixture) assert flag is False - hs2.close() + hs2.save_hashes() # now let change file permision of generated file hs3 = HashStore() os.chmod(self.fixture[0], 0o766) flag = hs3.is_file_changed(*self.fixture) assert flag is True - hs3.close() + hs3.save_hashes() os.unlink(self.fixture[0]) + + +class TestHashStore2: + + def setUp(self): + self.source_file = os.path.join("tests", "fixtures", "a.jj2") + self.dest_file = os.path.join("tests", "fixtures", "copier-test02.csv") + + def test_simple_use_case(self): + hs = HashStore() + flag = hs.are_two_file_different(self.source_file, '/tmp/abc') + assert flag is True + + def test_laziness_with_same_file(self): + hs = HashStore() + flag = hs.are_two_file_different(self.source_file, self.source_file) + assert flag is True # because we don't know it before + flag = hs.are_two_file_different(self.source_file, self.source_file) + assert flag is False + + def test_different_files(self): + hs = HashStore() + flag = hs.are_two_file_different(self.source_file, self.dest_file) + assert flag is True diff --git a/tests/test_utils.py b/tests/test_utils.py index 5956f9e8..d689e28e 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,11 +1,13 @@ import os import stat +from shutil import rmtree from nose.tools import eq_ from moban.utils import file_permissions_copy from moban.utils import write_file_out from moban.utils import strip_off_trailing_new_lines +from moban.utils import mkdir_p def create_file(test_file, permission): @@ -65,3 +67,10 @@ def test_strip_new_lines(): content = "test\n\n\n\n\n" actual = strip_off_trailing_new_lines(content) eq_(actual, "test\n") + + +def test_mkdir_p(): + test_path = 'a/b/c/d' + mkdir_p(test_path) + assert os.path.exists(test_path) + rmtree(test_path)